arsenetar
/
dupeguru
mirror of https://github.com/arsenetar/dupeguru.git
1
0
Fork 0
Code Releases Wiki Activity

Format files with black 

- Format all files with black
- Update tox.ini flake8 arguments to be compatible
- Add black to requirements-extra.txt
- Reduce ignored flake8 rules and fix a few violations
This commit is contained in:
Andrew Senetar 2019-12-31 20:16:27 -06:00
parent 359d6498f7
commit 7ba8aa3514
Signed by: arsenetar
GPG Key ID: C63300DCE48AB2F1
141 changed files with 5241 additions and 3648 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

143
build.py
View File

@ -13,129 +13,165 @@ from setuptools import setup, Extension
from hscommon import sphinxgen
from hscommon.build import (
add_to_pythonpath, print_and_do, move_all, fix_qt_resource_file,
add_to_pythonpath,
print_and_do,
move_all,
fix_qt_resource_file,
)
from hscommon import loc
def parse_args():
usage = "usage: %prog [options]"
parser = OptionParser(usage=usage)
parser.add_option(
'--clean', action='store_true', dest='clean',
help="Clean build folder before building"
"--clean",
action="store_true",
dest="clean",
help="Clean build folder before building",
)
parser.add_option(
'--doc', action='store_true', dest='doc',
help="Build only the help file"
"--doc", action="store_true", dest="doc", help="Build only the help file"
)
parser.add_option(
'--loc', action='store_true', dest='loc',
help="Build only localization"
"--loc", action="store_true", dest="loc", help="Build only localization"
)
parser.add_option(
'--updatepot', action='store_true', dest='updatepot',
help="Generate .pot files from source code."
"--updatepot",
action="store_true",
dest="updatepot",
help="Generate .pot files from source code.",
)
parser.add_option(
'--mergepot', action='store_true', dest='mergepot',
help="Update all .po files based on .pot files."
"--mergepot",
action="store_true",
dest="mergepot",
help="Update all .po files based on .pot files.",
)
parser.add_option(
'--normpo', action='store_true', dest='normpo',
help="Normalize all PO files (do this before commit)."
"--normpo",
action="store_true",
dest="normpo",
help="Normalize all PO files (do this before commit).",
)
(options, args) = parser.parse_args()
return options
def build_help():
print("Generating Help")
current_path = op.abspath('.')
help_basepath = op.join(current_path, 'help', 'en')
help_destpath = op.join(current_path, 'build', 'help')
changelog_path = op.join(current_path, 'help', 'changelog')
current_path = op.abspath(".")
help_basepath = op.join(current_path, "help", "en")
help_destpath = op.join(current_path, "build", "help")
changelog_path = op.join(current_path, "help", "changelog")
tixurl = "https://github.com/hsoft/dupeguru/issues/{}"
confrepl = {'language': 'en'}
changelogtmpl = op.join(current_path, 'help', 'changelog.tmpl')
conftmpl = op.join(current_path, 'help', 'conf.tmpl')
sphinxgen.gen(help_basepath, help_destpath, changelog_path, tixurl, confrepl, conftmpl, changelogtmpl)
confrepl = {"language": "en"}
changelogtmpl = op.join(current_path, "help", "changelog.tmpl")
conftmpl = op.join(current_path, "help", "conf.tmpl")
sphinxgen.gen(
help_basepath,
help_destpath,
changelog_path,
tixurl,
confrepl,
conftmpl,
changelogtmpl,
)
def build_qt_localizations():
loc.compile_all_po(op.join('qtlib', 'locale'))
loc.merge_locale_dir(op.join('qtlib', 'locale'), 'locale')
loc.compile_all_po(op.join("qtlib", "locale"))
loc.merge_locale_dir(op.join("qtlib", "locale"), "locale")
def build_localizations():
loc.compile_all_po('locale')
loc.compile_all_po("locale")
build_qt_localizations()
locale_dest = op.join('build', 'locale')
locale_dest = op.join("build", "locale")
if op.exists(locale_dest):
shutil.rmtree(locale_dest)
shutil.copytree('locale', locale_dest, ignore=shutil.ignore_patterns('*.po', '*.pot'))
shutil.copytree(
"locale", locale_dest, ignore=shutil.ignore_patterns("*.po", "*.pot")
)
def build_updatepot():
print("Building .pot files from source files")
print("Building core.pot")
loc.generate_pot(['core'], op.join('locale', 'core.pot'), ['tr'])
loc.generate_pot(["core"], op.join("locale", "core.pot"), ["tr"])
print("Building columns.pot")
loc.generate_pot(['core'], op.join('locale', 'columns.pot'), ['coltr'])
loc.generate_pot(["core"], op.join("locale", "columns.pot"), ["coltr"])
print("Building ui.pot")
# When we're not under OS X, we don't want to overwrite ui.pot because it contains Cocoa locs
# We want to merge the generated pot with the old pot in the most preserving way possible.
ui_packages = ['qt', op.join('cocoa', 'inter')]
loc.generate_pot(ui_packages, op.join('locale', 'ui.pot'), ['tr'], merge=True)
ui_packages = ["qt", op.join("cocoa", "inter")]
loc.generate_pot(ui_packages, op.join("locale", "ui.pot"), ["tr"], merge=True)
print("Building qtlib.pot")
loc.generate_pot(['qtlib'], op.join('qtlib', 'locale', 'qtlib.pot'), ['tr'])
loc.generate_pot(["qtlib"], op.join("qtlib", "locale", "qtlib.pot"), ["tr"])
def build_mergepot():
print("Updating .po files using .pot files")
loc.merge_pots_into_pos('locale')
loc.merge_pots_into_pos(op.join('qtlib', 'locale'))
loc.merge_pots_into_pos(op.join('cocoalib', 'locale'))
loc.merge_pots_into_pos("locale")
loc.merge_pots_into_pos(op.join("qtlib", "locale"))
loc.merge_pots_into_pos(op.join("cocoalib", "locale"))
def build_normpo():
loc.normalize_all_pos('locale')
loc.normalize_all_pos(op.join('qtlib', 'locale'))
loc.normalize_all_pos(op.join('cocoalib', 'locale'))
loc.normalize_all_pos("locale")
loc.normalize_all_pos(op.join("qtlib", "locale"))
loc.normalize_all_pos(op.join("cocoalib", "locale"))
def build_pe_modules():
print("Building PE Modules")
exts = [
Extension(
"_block",
[op.join('core', 'pe', 'modules', 'block.c'), op.join('core', 'pe', 'modules', 'common.c')]
[
op.join("core", "pe", "modules", "block.c"),
op.join("core", "pe", "modules", "common.c"),
],
),
Extension(
"_cache",
[op.join('core', 'pe', 'modules', 'cache.c'), op.join('core', 'pe', 'modules', 'common.c')]
[
op.join("core", "pe", "modules", "cache.c"),
op.join("core", "pe", "modules", "common.c"),
],
),
]
exts.append(Extension("_block_qt", [op.join('qt', 'pe', 'modules', 'block.c')]))
exts.append(Extension("_block_qt", [op.join("qt", "pe", "modules", "block.c")]))
setup(
script_args=['build_ext', '--inplace'],
ext_modules=exts,
script_args=["build_ext", "--inplace"], ext_modules=exts,
)
move_all('_block_qt*', op.join('qt', 'pe'))
move_all('_block*', op.join('core', 'pe'))
move_all('_cache*', op.join('core', 'pe'))
move_all("_block_qt*", op.join("qt", "pe"))
move_all("_block*", op.join("core", "pe"))
move_all("_cache*", op.join("core", "pe"))
def build_normal():
print("Building dupeGuru with UI qt")
add_to_pythonpath('.')
add_to_pythonpath(".")
print("Building dupeGuru")
build_pe_modules()
print("Building localizations")
build_localizations()
print("Building Qt stuff")
print_and_do("pyrcc5 {0} > {1}".format(op.join('qt', 'dg.qrc'), op.join('qt', 'dg_rc.py')))
fix_qt_resource_file(op.join('qt', 'dg_rc.py'))
print_and_do(
"pyrcc5 {0} > {1}".format(op.join("qt", "dg.qrc"), op.join("qt", "dg_rc.py"))
)
fix_qt_resource_file(op.join("qt", "dg_rc.py"))
build_help()
def main():
options = parse_args()
if options.clean:
if op.exists('build'):
shutil.rmtree('build')
if not op.exists('build'):
os.mkdir('build')
if op.exists("build"):
shutil.rmtree("build")
if not op.exists("build"):
os.mkdir("build")
if options.doc:
build_help()
elif options.loc:
@ -149,5 +185,6 @@ def main():
else:
build_normal()
if __name__ == '__main__':
if __name__ == "__main__":
main()

5
core/__init__.py
View File

@ -1,3 +1,2 @@
__version__ = '4.0.4'
__appname__ = 'dupeGuru'
__version__ = "4.0.4"
__appname__ = "dupeGuru"

213
core/app.py
View File

@ -34,8 +34,8 @@ from .gui.ignore_list_dialog import IgnoreListDialog
from .gui.problem_dialog import ProblemDialog
from .gui.stats_label import StatsLabel
HAD_FIRST_LAUNCH_PREFERENCE = 'HadFirstLaunch'
DEBUG_MODE_PREFERENCE = 'DebugMode'
HAD_FIRST_LAUNCH_PREFERENCE = "HadFirstLaunch"
DEBUG_MODE_PREFERENCE = "DebugMode"
MSG_NO_MARKED_DUPES = tr("There are no marked duplicates. Nothing has been done.")
MSG_NO_SELECTED_DUPES = tr("There are no selected duplicates. Nothing has been done.")
@ -44,23 +44,27 @@ MSG_MANY_FILES_TO_OPEN = tr(
"files are opened with, doing so can create quite a mess. Continue?"
)
class DestType:
Direct = 0
Relative = 1
Absolute = 2
class JobType:
Scan = 'job_scan'
Load = 'job_load'
Move = 'job_move'
Copy = 'job_copy'
Delete = 'job_delete'
Scan = "job_scan"
Load = "job_load"
Move = "job_move"
Copy = "job_copy"
Delete = "job_delete"
class AppMode:
Standard = 0
Music = 1
Picture = 2
JOBID2TITLE = {
JobType.Scan: tr("Scanning for duplicates"),
JobType.Load: tr("Loading"),
@ -69,6 +73,7 @@ JOBID2TITLE = {
JobType.Delete: tr("Sending to Trash"),
}
class DupeGuru(Broadcaster):
"""Holds everything together.
@ -100,6 +105,7 @@ class DupeGuru(Broadcaster):
Instance of :mod:`meta-gui <core.gui>` table listing the results from :attr:`results`
"""
# --- View interface
# get_default(key_name)
# set_default(key_name, value)
@ -116,7 +122,7 @@ class DupeGuru(Broadcaster):
NAME = PROMPT_NAME = "dupeGuru"
PICTURE_CACHE_TYPE = 'sqlite' # set to 'shelve' for a ShelveCache
PICTURE_CACHE_TYPE = "sqlite" # set to 'shelve' for a ShelveCache
def __init__(self, view):
if view.get_default(DEBUG_MODE_PREFERENCE):
@ -124,7 +130,9 @@ class DupeGuru(Broadcaster):
logging.debug("Debug mode enabled")
Broadcaster.__init__(self)
self.view = view
self.appdata = desktop.special_folder_path(desktop.SpecialFolder.AppData, appname=self.NAME)
self.appdata = desktop.special_folder_path(
desktop.SpecialFolder.AppData, appname=self.NAME
)
if not op.exists(self.appdata):
os.makedirs(self.appdata)
self.app_mode = AppMode.Standard
@ -136,11 +144,11 @@ class DupeGuru(Broadcaster):
# sent to the scanner. They don't have default values because those defaults values are
# defined in the scanner class.
self.options = {
'escape_filter_regexp': True,
'clean_empty_dirs': False,
'ignore_hardlink_matches': False,
'copymove_dest_type': DestType.Relative,
'picture_cache_type': self.PICTURE_CACHE_TYPE
"escape_filter_regexp": True,
"clean_empty_dirs": False,
"ignore_hardlink_matches": False,
"copymove_dest_type": DestType.Relative,
"picture_cache_type": self.PICTURE_CACHE_TYPE,
}
self.selected_dupes = []
self.details_panel = DetailsPanel(self)
@ -169,26 +177,30 @@ class DupeGuru(Broadcaster):
self.view.create_results_window()
def _get_picture_cache_path(self):
cache_type = self.options['picture_cache_type']
cache_name = 'cached_pictures.shelve' if cache_type == 'shelve' else 'cached_pictures.db'
cache_type = self.options["picture_cache_type"]
cache_name = (
"cached_pictures.shelve" if cache_type == "shelve" else "cached_pictures.db"
)
return op.join(self.appdata, cache_name)
def _get_dupe_sort_key(self, dupe, get_group, key, delta):
if self.app_mode in (AppMode.Music, AppMode.Picture):
if key == 'folder_path':
dupe_folder_path = getattr(dupe, 'display_folder_path', dupe.folder_path)
if key == "folder_path":
dupe_folder_path = getattr(
dupe, "display_folder_path", dupe.folder_path
)
return str(dupe_folder_path).lower()
if self.app_mode == AppMode.Picture:
if delta and key == 'dimensions':
if delta and key == "dimensions":
r = cmp_value(dupe, key)
ref_value = cmp_value(get_group().ref, key)
return get_delta_dimensions(r, ref_value)
if key == 'marked':
if key == "marked":
return self.results.is_marked(dupe)
if key == 'percentage':
if key == "percentage":
m = get_group().get_match_of(dupe)
return m.percentage
elif key == 'dupe_count':
elif key == "dupe_count":
return 0
else:
result = cmp_value(dupe, key)
@ -203,21 +215,25 @@ class DupeGuru(Broadcaster):
def _get_group_sort_key(self, group, key):
if self.app_mode in (AppMode.Music, AppMode.Picture):
if key == 'folder_path':
dupe_folder_path = getattr(group.ref, 'display_folder_path', group.ref.folder_path)
if key == "folder_path":
dupe_folder_path = getattr(
group.ref, "display_folder_path", group.ref.folder_path
)
return str(dupe_folder_path).lower()
if key == 'percentage':
if key == "percentage":
return group.percentage
if key == 'dupe_count':
if key == "dupe_count":
return len(group)
if key == 'marked':
if key == "marked":
return len([dupe for dupe in group.dupes if self.results.is_marked(dupe)])
return cmp_value(group.ref, key)
def _do_delete(self, j, link_deleted, use_hardlinks, direct_deletion):
def op(dupe):
j.add_progress()
return self._do_delete_dupe(dupe, link_deleted, use_hardlinks, direct_deletion)
return self._do_delete_dupe(
dupe, link_deleted, use_hardlinks, direct_deletion
)
j.start_job(self.results.mark_count)
self.results.perform_on_marked(op, True)
@ -258,8 +274,9 @@ class DupeGuru(Broadcaster):
def _get_export_data(self):
columns = [
col for col in self.result_table.columns.ordered_columns
if col.visible and col.name != 'marked'
col
for col in self.result_table.columns.ordered_columns
if col.visible and col.name != "marked"
]
colnames = [col.display for col in columns]
rows = []
@ -273,10 +290,11 @@ class DupeGuru(Broadcaster):
def _results_changed(self):
self.selected_dupes = [
d for d in self.selected_dupes
d
for d in self.selected_dupes
if self.results.get_group_of_duplicate(d) is not None
]
self.notify('results_changed')
self.notify("results_changed")
def _start_job(self, jobid, func, args=()):
title = JOBID2TITLE[jobid]
@ -310,7 +328,9 @@ class DupeGuru(Broadcaster):
msg = {
JobType.Copy: tr("All marked files were copied successfully."),
JobType.Move: tr("All marked files were moved successfully."),
JobType.Delete: tr("All marked files were successfully sent to Trash."),
JobType.Delete: tr(
"All marked files were successfully sent to Trash."
),
}[jobid]
self.view.show_message(msg)
@ -341,7 +361,7 @@ class DupeGuru(Broadcaster):
if dupes == self.selected_dupes:
return
self.selected_dupes = dupes
self.notify('dupes_selected')
self.notify("dupes_selected")
# --- Protected
def _get_fileclasses(self):
@ -370,7 +390,7 @@ class DupeGuru(Broadcaster):
"""
try:
self.directories.add_path(Path(d))
self.notify('directories_changed')
self.notify("directories_changed")
except directories.AlreadyThereError:
self.view.show_message(tr("'{}' already is in the list.").format(d))
except directories.InvalidPathError:
@ -383,7 +403,9 @@ class DupeGuru(Broadcaster):
if not dupes:
self.view.show_message(MSG_NO_SELECTED_DUPES)
return
msg = tr("All selected %d matches are going to be ignored in all subsequent scans. Continue?")
msg = tr(
"All selected %d matches are going to be ignored in all subsequent scans. Continue?"
)
if not self.view.ask_yes_no(msg % len(dupes)):
return
for dupe in dupes:
@ -400,15 +422,15 @@ class DupeGuru(Broadcaster):
:param str filter: filter to apply
"""
self.results.apply_filter(None)
if self.options['escape_filter_regexp']:
filter = escape(filter, set('()[]\\.|+?^'))
filter = escape(filter, '*', '.')
if self.options["escape_filter_regexp"]:
filter = escape(filter, set("()[]\\.|+?^"))
filter = escape(filter, "*", ".")
self.results.apply_filter(filter)
self._results_changed()
def clean_empty_dirs(self, path):
if self.options['clean_empty_dirs']:
while delete_if_empty(path, ['.DS_Store']):
if self.options["clean_empty_dirs"]:
while delete_if_empty(path, [".DS_Store"]):
path = path.parent()
def clear_picture_cache(self):
@ -444,6 +466,7 @@ class DupeGuru(Broadcaster):
:param bool copy: If True, duplicates will be copied instead of moved
"""
def do(j):
def op(dupe):
j.add_progress()
@ -459,7 +482,7 @@ class DupeGuru(Broadcaster):
prompt = tr("Select a directory to {} marked files to").format(opname)
destination = self.view.select_dest_folder(prompt)
if destination:
desttype = self.options['copymove_dest_type']
desttype = self.options["copymove_dest_type"]
jobid = JobType.Copy if copy else JobType.Move
self._start_job(jobid, do)
@ -472,8 +495,9 @@ class DupeGuru(Broadcaster):
if not self.deletion_options.show(self.results.mark_count):
return
args = [
self.deletion_options.link_deleted, self.deletion_options.use_hardlinks,
self.deletion_options.direct
self.deletion_options.link_deleted,
self.deletion_options.use_hardlinks,
self.deletion_options.direct,
]
logging.debug("Starting deletion job with args %r", args)
self._start_job(JobType.Delete, self._do_delete, args=args)
@ -495,7 +519,9 @@ class DupeGuru(Broadcaster):
The columns and their order in the resulting CSV file is determined in the same way as in
:meth:`export_to_xhtml`.
"""
dest_file = self.view.select_dest_file(tr("Select a destination for your exported CSV"), 'csv')
dest_file = self.view.select_dest_file(
tr("Select a destination for your exported CSV"), "csv"
)
if dest_file:
colnames, rows = self._get_export_data()
try:
@ -505,13 +531,16 @@ class DupeGuru(Broadcaster):
def get_display_info(self, dupe, group, delta=False):
def empty_data():
return {c.name: '---' for c in self.result_table.COLUMNS[1:]}
return {c.name: "---" for c in self.result_table.COLUMNS[1:]}
if (dupe is None) or (group is None):
return empty_data()
try:
return dupe.get_display_info(group, delta)
except Exception as e:
logging.warning("Exception on GetDisplayInfo for %s: %s", str(dupe.path), str(e))
logging.warning(
"Exception on GetDisplayInfo for %s: %s", str(dupe.path), str(e)
)
return empty_data()
def invoke_custom_command(self):
@ -521,9 +550,11 @@ class DupeGuru(Broadcaster):
is replaced with that dupe's ref file. If there's no selection, the command is not invoked.
If the dupe is a ref, ``%d`` and ``%r`` will be the same.
"""
cmd = self.view.get_default('CustomCommand')
cmd = self.view.get_default("CustomCommand")
if not cmd:
msg = tr("You have no custom command set up. Set it up in your preferences.")
msg = tr(
"You have no custom command set up. Set it up in your preferences."
)
self.view.show_message(msg)
return
if not self.selected_dupes:
@ -531,8 +562,8 @@ class DupeGuru(Broadcaster):
dupe = self.selected_dupes[0]
group = self.results.get_group_of_duplicate(dupe)
ref = group.ref
cmd = cmd.replace('%d', str(dupe.path))
cmd = cmd.replace('%r', str(ref.path))
cmd = cmd.replace("%d", str(dupe.path))
cmd = cmd.replace("%r", str(ref.path))
match = re.match(r'"([^"]+)"(.*)', cmd)
if match is not None:
# This code here is because subprocess. Popen doesn't seem to accept, under Windows,
@ -551,9 +582,9 @@ class DupeGuru(Broadcaster):
is persistent data, is the same as when the last session was closed (when :meth:`save` was
called).
"""
self.directories.load_from_file(op.join(self.appdata, 'last_directories.xml'))
self.notify('directories_changed')
p = op.join(self.appdata, 'ignore_list.xml')
self.directories.load_from_file(op.join(self.appdata, "last_directories.xml"))
self.notify("directories_changed")
p = op.join(self.appdata, "ignore_list.xml")
self.ignore_list.load_from_xml(p)
self.ignore_list_dialog.refresh()
@ -562,8 +593,10 @@ class DupeGuru(Broadcaster):
:param str filename: path of the XML file (created with :meth:`save_as`) to load
"""
def do(j):
self.results.load_from_xml(filename, self._get_file, j)
self._start_job(JobType.Load, do)
def make_selected_reference(self):
@ -588,35 +621,36 @@ class DupeGuru(Broadcaster):
if not self.result_table.power_marker:
if changed_groups:
self.selected_dupes = [
d for d in self.selected_dupes
d
for d in self.selected_dupes
if self.results.get_group_of_duplicate(d).ref is d
]
self.notify('results_changed')
self.notify("results_changed")
else:
# If we're in "Dupes Only" mode (previously called Power Marker), things are a bit
# different. The refs are not shown in the table, and if our operation is successful,
# this means that there's no way to follow our dupe selection. Then, the best thing to
# do is to keep our selection index-wise (different dupe selection, but same index
# selection).
self.notify('results_changed_but_keep_selection')
self.notify("results_changed_but_keep_selection")
def mark_all(self):
"""Set all dupes in the results as marked.
"""
self.results.mark_all()
self.notify('marking_changed')
self.notify("marking_changed")
def mark_none(self):
"""Set all dupes in the results as unmarked.
"""
self.results.mark_none()
self.notify('marking_changed')
self.notify("marking_changed")
def mark_invert(self):
"""Invert the marked state of all dupes in the results.
"""
self.results.mark_invert()
self.notify('marking_changed')
self.notify("marking_changed")
def mark_dupe(self, dupe, marked):
"""Change marked status of ``dupe``.
@ -629,7 +663,7 @@ class DupeGuru(Broadcaster):
self.results.mark(dupe)
else:
self.results.unmark(dupe)
self.notify('marking_changed')
self.notify("marking_changed")
def open_selected(self):
"""Open :attr:`selected_dupes` with their associated application.
@ -656,7 +690,7 @@ class DupeGuru(Broadcaster):
indexes = sorted(indexes, reverse=True)
for index in indexes:
del self.directories[index]
self.notify('directories_changed')
self.notify("directories_changed")
except IndexError:
pass
@ -669,7 +703,7 @@ class DupeGuru(Broadcaster):
:type duplicates: list of :class:`~core.fs.File`
"""
self.results.remove_duplicates(self.without_ref(duplicates))
self.notify('results_changed_but_keep_selection')
self.notify("results_changed_but_keep_selection")
def remove_marked(self):
"""Removed marked duplicates from the results (without touching the files themselves).
@ -724,7 +758,9 @@ class DupeGuru(Broadcaster):
if group.prioritize(key_func=sort_key):
count += 1
self._results_changed()
msg = tr("{} duplicate groups were changed by the re-prioritization.").format(count)
msg = tr("{} duplicate groups were changed by the re-prioritization.").format(
count
)
self.view.show_message(msg)
def reveal_selected(self):
@ -734,10 +770,10 @@ class DupeGuru(Broadcaster):
def save(self):
if not op.exists(self.appdata):
os.makedirs(self.appdata)
self.directories.save_to_file(op.join(self.appdata, 'last_directories.xml'))
p = op.join(self.appdata, 'ignore_list.xml')
self.directories.save_to_file(op.join(self.appdata, "last_directories.xml"))
p = op.join(self.appdata, "ignore_list.xml")
self.ignore_list.save_to_xml(p)
self.notify('save_session')
self.notify("save_session")
def save_as(self, filename):
"""Save results in ``filename``.
@ -756,7 +792,9 @@ class DupeGuru(Broadcaster):
"""
scanner = self.SCANNER_CLASS()
if not self.directories.has_any_file():
self.view.show_message(tr("The selected directories contain no scannable file."))
self.view.show_message(
tr("The selected directories contain no scannable file.")
)
return
# Send relevant options down to the scanner instance
for k, v in self.options.items():
@ -771,12 +809,16 @@ class DupeGuru(Broadcaster):
def do(j):
j.set_progress(0, tr("Collecting files to scan"))
if scanner.scan_type == ScanType.Folders:
files = list(self.directories.get_folders(folderclass=se.fs.Folder, j=j))
files = list(
self.directories.get_folders(folderclass=se.fs.Folder, j=j)
)
else:
files = list(self.directories.get_files(fileclasses=self.fileclasses, j=j))
if self.options['ignore_hardlink_matches']:
files = list(
self.directories.get_files(fileclasses=self.fileclasses, j=j)
)
if self.options["ignore_hardlink_matches"]:
files = self._remove_hardlink_dupes(files)
logging.info('Scanning %d files' % len(files))
logging.info("Scanning %d files" % len(files))
self.results.groups = scanner.get_dupe_groups(files, self.ignore_list, j)
self.discarded_file_count = scanner.discarded_file_count
@ -792,12 +834,16 @@ class DupeGuru(Broadcaster):
markfunc = self.results.mark
for dupe in selected:
markfunc(dupe)
self.notify('marking_changed')
self.notify("marking_changed")
def without_ref(self, dupes):
"""Returns ``dupes`` with all reference elements removed.
"""
return [dupe for dupe in dupes if self.results.get_group_of_duplicate(dupe).ref is not dupe]
return [
dupe
for dupe in dupes
if self.results.get_group_of_duplicate(dupe).ref is not dupe
]
def get_default(self, key, fallback_value=None):
result = nonone(self.view.get_default(key), fallback_value)
@ -836,12 +882,21 @@ class DupeGuru(Broadcaster):
@property
def METADATA_TO_READ(self):
if self.app_mode == AppMode.Picture:
return ['size', 'mtime', 'dimensions', 'exif_timestamp']
return ["size", "mtime", "dimensions", "exif_timestamp"]
elif self.app_mode == AppMode.Music:
return [
'size', 'mtime', 'duration', 'bitrate', 'samplerate', 'title', 'artist',
'album', 'genre', 'year', 'track', 'comment'
"size",
"mtime",
"duration",
"bitrate",
"samplerate",
"title",
"artist",
"album",
"genre",
"year",
"track",
"comment",
]
else:
return ['size', 'mtime']
return ["size", "mtime"]

57
core/directories.py
View File

@ -15,12 +15,13 @@ from hscommon.util import FileOrPath
from . import fs
__all__ = [
'Directories',
'DirectoryState',
'AlreadyThereError',
'InvalidPathError',
"Directories",
"DirectoryState",
"AlreadyThereError",
"InvalidPathError",
]
class DirectoryState:
"""Enum describing how a folder should be considered.
@ -28,16 +29,20 @@ class DirectoryState:
* DirectoryState.Reference: Scan files, but make sure never to delete any of them
* DirectoryState.Excluded: Don't scan this folder
"""
Normal = 0
Reference = 1
Excluded = 2
class AlreadyThereError(Exception):
"""The path being added is already in the directory list"""
class InvalidPathError(Exception):
"""The path being added is invalid"""
class Directories:
"""Holds user folder selection.
@ -47,6 +52,7 @@ class Directories:
Then, when the user starts the scan, :meth:`get_files` is called to retrieve all files (wrapped
in :mod:`core.fs`) that have to be scanned according to the chosen folders/states.
"""
# ---Override
def __init__(self):
self._dirs = []
@ -71,7 +77,7 @@ class Directories:
# ---Private
def _default_state_for_path(self, path):
# Override this in subclasses to specify the state of some special folders.
if path.name.startswith('.'): # hidden
if path.name.startswith("."): # hidden
return DirectoryState.Excluded
def _get_files(self, from_path, fileclasses, j):
@ -87,7 +93,9 @@ class Directories:
del dirs[:]
try:
if state != DirectoryState.Excluded:
found_files = [fs.get_file(root + f, fileclasses=fileclasses) for f in files]
found_files = [
fs.get_file(root + f, fileclasses=fileclasses) for f in files
]
found_files = [f for f in found_files if f is not None]
# In some cases, directories can be considered as files by dupeGuru, which is
# why we have this line below. In fact, there only one case: Bundle files under
@ -97,7 +105,11 @@ class Directories:
if f is not None:
found_files.append(f)
dirs.remove(d)
logging.debug("Collected %d files in folder %s", len(found_files), str(from_path))
logging.debug(
"Collected %d files in folder %s",
len(found_files),
str(from_path),
)
for file in found_files:
file.is_ref = state == DirectoryState.Reference
yield file
@ -212,21 +224,21 @@ class Directories:
root = ET.parse(infile).getroot()
except Exception:
return
for rdn in root.getiterator('root_directory'):
for rdn in root.getiterator("root_directory"):
attrib = rdn.attrib
if 'path' not in attrib:
if "path" not in attrib:
continue
path = attrib['path']
path = attrib["path"]
try:
self.add_path(Path(path))
except (AlreadyThereError, InvalidPathError):
pass
for sn in root.getiterator('state'):
for sn in root.getiterator("state"):
attrib = sn.attrib
if not ('path' in attrib and 'value' in attrib):
if not ("path" in attrib and "value" in attrib):
continue
path = attrib['path']
state = attrib['value']
path = attrib["path"]
state = attrib["value"]
self.states[Path(path)] = int(state)
def save_to_file(self, outfile):
@ -234,17 +246,17 @@ class Directories:
:param file outfile: path or file pointer to XML file to save to.
"""
with FileOrPath(outfile, 'wb') as fp:
root = ET.Element('directories')
with FileOrPath(outfile, "wb") as fp:
root = ET.Element("directories")
for root_path in self:
root_path_node = ET.SubElement(root, 'root_directory')
root_path_node.set('path', str(root_path))
root_path_node = ET.SubElement(root, "root_directory")
root_path_node.set("path", str(root_path))
for path, state in self.states.items():
state_node = ET.SubElement(root, 'state')
state_node.set('path', str(path))
state_node.set('value', str(state))
state_node = ET.SubElement(root, "state")
state_node.set("path", str(path))
state_node.set("value", str(state))
tree = ET.ElementTree(root)
tree.write(fp, encoding='utf-8')
tree.write(fp, encoding="utf-8")
def set_state(self, path, state):
"""Set the state of folder at ``path``.
@ -259,4 +271,3 @@ class Directories:
if path.is_parent_of(iter_path):
del self.states[iter_path]
self.states[path] = state

92
core/engine.py
View File

@ -17,25 +17,26 @@ from hscommon.util import flatten, multi_replace
from hscommon.trans import tr
from hscommon.jobprogress import job
(
WEIGHT_WORDS,
MATCH_SIMILAR_WORDS,
NO_FIELD_ORDER,
) = range(3)
(WEIGHT_WORDS, MATCH_SIMILAR_WORDS, NO_FIELD_ORDER,) = range(3)
JOB_REFRESH_RATE = 100
def getwords(s):
# We decompose the string so that ascii letters with accents can be part of the word.
s = normalize('NFD', s)
s = multi_replace(s, "-_&+():;\\[]{}.,<>/?~!@#$*", ' ').lower()
s = ''.join(c for c in s if c in string.ascii_letters + string.digits + string.whitespace)
return [_f for _f in s.split(' ') if _f] # remove empty elements
s = normalize("NFD", s)
s = multi_replace(s, "-_&+():;\\[]{}.,<>/?~!@#$*", " ").lower()
s = "".join(
c for c in s if c in string.ascii_letters + string.digits + string.whitespace
)
return [_f for _f in s.split(" ") if _f] # remove empty elements
def getfields(s):
fields = [getwords(field) for field in s.split(' - ')]
fields = [getwords(field) for field in s.split(" - ")]
return [_f for _f in fields if _f]
def unpack_fields(fields):
result = []
for field in fields:
@ -45,6 +46,7 @@ def unpack_fields(fields):
result.append(field)
return result
def compare(first, second, flags=()):
"""Returns the % of words that match between ``first`` and ``second``
@ -59,7 +61,7 @@ def compare(first, second, flags=()):
match_similar = MATCH_SIMILAR_WORDS in flags
weight_words = WEIGHT_WORDS in flags
joined = first + second
total_count = (sum(len(word) for word in joined) if weight_words else len(joined))
total_count = sum(len(word) for word in joined) if weight_words else len(joined)
match_count = 0
in_order = True
for word in first:
@ -71,12 +73,13 @@ def compare(first, second, flags=()):
if second[0] != word:
in_order = False
second.remove(word)
match_count += (len(word) if weight_words else 1)
match_count += len(word) if weight_words else 1
result = round(((match_count * 2) / total_count) * 100)
if (result == 100) and (not in_order):
result = 99 # We cannot consider a match exact unless the ordering is the same
return result
def compare_fields(first, second, flags=()):
"""Returns the score for the lowest matching :ref:`fields`.
@ -101,9 +104,12 @@ def compare_fields(first, second, flags=()):
if matched_field:
second.remove(matched_field)
else:
results = [compare(field1, field2, flags) for field1, field2 in zip(first, second)]
results = [
compare(field1, field2, flags) for field1, field2 in zip(first, second)
]
return min(results) if results else 0
def build_word_dict(objects, j=job.nulljob):
"""Returns a dict of objects mapped by their words.
@ -113,11 +119,14 @@ def build_word_dict(objects, j=job.nulljob):
The result will be a dict with words as keys, lists of objects as values.
"""
result = defaultdict(set)
for object in j.iter_with_progress(objects, 'Prepared %d/%d files', JOB_REFRESH_RATE):
for object in j.iter_with_progress(
objects, "Prepared %d/%d files", JOB_REFRESH_RATE
):
for word in unpack_fields(object.words):
result[word].add(object)
return result
def merge_similar_words(word_dict):
"""Take all keys in ``word_dict`` that are similar, and merge them together.
@ -138,6 +147,7 @@ def merge_similar_words(word_dict):
del word_dict[similar]
keys.remove(similar)
def reduce_common_words(word_dict, threshold):
"""Remove all objects from ``word_dict`` values where the object count >= ``threshold``
@ -146,7 +156,9 @@ def reduce_common_words(word_dict, threshold):
The exception to this removal are the objects where all the words of the object are common.
Because if we remove them, we will miss some duplicates!
"""
uncommon_words = set(word for word, objects in word_dict.items() if len(objects) < threshold)
uncommon_words = set(
word for word, objects in word_dict.items() if len(objects) < threshold
)
for word, objects in list(word_dict.items()):
if len(objects) < threshold:
continue
@ -159,11 +171,13 @@ def reduce_common_words(word_dict, threshold):
else:
del word_dict[word]
# Writing docstrings in a namedtuple is tricky. From Python 3.3, it's possible to set __doc__, but
# some research allowed me to find a more elegant solution, which is what is done here. See
# http://stackoverflow.com/questions/1606436/adding-docstrings-to-namedtuples-in-python
class Match(namedtuple('Match', 'first second percentage')):
class Match(namedtuple("Match", "first second percentage")):
"""Represents a match between two :class:`~core.fs.File`.
Regarless of the matching method, when two files are determined to match, a Match pair is created,
@ -182,16 +196,24 @@ class Match(namedtuple('Match', 'first second percentage')):
their match level according to the scan method which found the match. int from 1 to 100. For
exact scan methods, such as Contents scans, this will always be 100.
"""
__slots__ = ()
def get_match(first, second, flags=()):
# it is assumed here that first and second both have a "words" attribute
percentage = compare(first.words, second.words, flags)
return Match(first, second, percentage)
def getmatches(
objects, min_match_percentage=0, match_similar_words=False, weight_words=False,
no_field_order=False, j=job.nulljob):
objects,
min_match_percentage=0,
match_similar_words=False,
weight_words=False,
no_field_order=False,
j=job.nulljob,
):
"""Returns a list of :class:`Match` within ``objects`` after fuzzily matching their words.
:param objects: List of :class:`~core.fs.File` to match.
@ -206,7 +228,7 @@ def getmatches(
j = j.start_subjob(2)
sj = j.start_subjob(2)
for o in objects:
if not hasattr(o, 'words'):
if not hasattr(o, "words"):
o.words = getwords(o.name)
word_dict = build_word_dict(objects, sj)
reduce_common_words(word_dict, COMMON_WORD_THRESHOLD)
@ -242,10 +264,14 @@ def getmatches(
# This is the place where the memory usage is at its peak during the scan.
# Just continue the process with an incomplete list of matches.
del compared # This should give us enough room to call logging.
logging.warning('Memory Overflow. Matches: %d. Word dict: %d' % (len(result), len(word_dict)))
logging.warning(
"Memory Overflow. Matches: %d. Word dict: %d"
% (len(result), len(word_dict))
)
return result
return result
def getmatches_by_contents(files, j=job.nulljob):
"""Returns a list of :class:`Match` within ``files`` if their contents is the same.
@ -270,6 +296,7 @@ def getmatches_by_contents(files, j=job.nulljob):
j.add_progress(desc=tr("%d matches found") % len(result))
return result
class Group:
"""A group of :class:`~core.fs.File` that match together.
@ -297,6 +324,7 @@ class Group:
Average match percentage of match pairs containing :attr:`ref`.
"""
# ---Override
def __init__(self):
self._clear()
@ -339,6 +367,7 @@ class Group:
:param tuple match: pair of :class:`~core.fs.File` to add
"""
def add_candidate(item, match):
matches = self.candidates[item]
matches.add(match)
@ -362,7 +391,11 @@ class Group:
You can call this after the duplicate scanning process to free a bit of memory.
"""
discarded = set(m for m in self.matches if not all(obj in self.unordered for obj in [m.first, m.second]))
discarded = set(
m
for m in self.matches
if not all(obj in self.unordered for obj in [m.first, m.second])
)
self.matches -= discarded
self.candidates = defaultdict(set)
return discarded
@ -409,7 +442,9 @@ class Group:
self.unordered.remove(item)
self._percentage = None
self._matches_for_ref = None
if (len(self) > 1) and any(not getattr(item, 'is_ref', False) for item in self):
if (len(self) > 1) and any(
not getattr(item, "is_ref", False) for item in self
):
if discard_matches:
self.matches = set(m for m in self.matches if item not in m)
else:
@ -438,7 +473,9 @@ class Group:
if self._percentage is None:
if self.dupes:
matches = self._get_matches_for_ref()
self._percentage = sum(match.percentage for match in matches) // len(matches)
self._percentage = sum(match.percentage for match in matches) // len(
matches
)
else:
self._percentage = 0
return self._percentage
@ -485,7 +522,7 @@ def get_groups(matches):
del dupe2group
del matches
# should free enough memory to continue
logging.warning('Memory Overflow. Groups: {0}'.format(len(groups)))
logging.warning("Memory Overflow. Groups: {0}".format(len(groups)))
# Now that we have a group, we have to discard groups' matches and see if there're any "orphan"
# matches, that is, matches that were candidate in a group but that none of their 2 files were
# accepted in the group. With these orphan groups, it's safe to build additional groups
@ -493,9 +530,12 @@ def get_groups(matches):
orphan_matches = []
for group in groups:
orphan_matches += {
m for m in group.discard_matches()
m
for m in group.discard_matches()
if not any(obj in matched_files for obj in [m.first, m.second])
}
if groups and orphan_matches:
groups += get_groups(orphan_matches) # no job, as it isn't supposed to take a long time
groups += get_groups(
orphan_matches
) # no job, as it isn't supposed to take a long time
return groups

26
core/export.py
View File

@ -114,36 +114,42 @@ ROW_TEMPLATE = """
CELL_TEMPLATE = """<td>{value}</td>"""
def export_to_xhtml(colnames, rows):
# a row is a list of values with the first value being a flag indicating if the row should be indented
if rows:
assert len(rows[0]) == len(colnames) + 1 # + 1 is for the "indented" flag
colheaders = ''.join(COLHEADERS_TEMPLATE.format(name=name) for name in colnames)
colheaders = "".join(COLHEADERS_TEMPLATE.format(name=name) for name in colnames)
rendered_rows = []
previous_group_id = None
for row in rows:
# [2:] is to remove the indented flag + filename
if row[0] != previous_group_id:
# We've just changed dupe group, which means that this dupe is a ref. We don't indent it.
indented = ''
indented = ""
else:
indented = 'indented'
indented = "indented"
filename = row[1]
cells = ''.join(CELL_TEMPLATE.format(value=value) for value in row[2:])
rendered_rows.append(ROW_TEMPLATE.format(indented=indented, filename=filename, cells=cells))
cells = "".join(CELL_TEMPLATE.format(value=value) for value in row[2:])
rendered_rows.append(
ROW_TEMPLATE.format(indented=indented, filename=filename, cells=cells)
)
previous_group_id = row[0]
rendered_rows = ''.join(rendered_rows)
rendered_rows = "".join(rendered_rows)
# The main template can't use format because the css code uses {}
content = MAIN_TEMPLATE.replace('$colheaders', colheaders).replace('$rows', rendered_rows)
content = MAIN_TEMPLATE.replace("$colheaders", colheaders).replace(
"$rows", rendered_rows
)
folder = mkdtemp()
destpath = op.join(folder, 'export.htm')
fp = open(destpath, 'wt', encoding='utf-8')
destpath = op.join(folder, "export.htm")
fp = open(destpath, "wt", encoding="utf-8")
fp.write(content)
fp.close()
return destpath
def export_to_csv(dest, colnames, rows):
writer = csv.writer(open(dest, 'wt', encoding='utf-8'))
writer = csv.writer(open(dest, "wt", encoding="utf-8"))
writer.writerow(["Group ID"] + colnames)
for row in rows:
writer.writerow(row)

68
core/fs.py
View File

@ -17,19 +17,20 @@ import logging
from hscommon.util import nonone, get_file_ext
__all__ = [
'File',
'Folder',
'get_file',
'get_files',
'FSError',
'AlreadyExistsError',
'InvalidPath',
'InvalidDestinationError',
'OperationError',
"File",
"Folder",
"get_file",
"get_files",
"FSError",
"AlreadyExistsError",
"InvalidPath",
"InvalidDestinationError",
"OperationError",
]
NOT_SET = object()
class FSError(Exception):
cls_message = "An error has occured on '{name}' in '{parent}'"
@ -40,8 +41,8 @@ class FSError(Exception):
elif isinstance(fsobject, File):
name = fsobject.name
else:
name = ''
parentname = str(parent) if parent is not None else ''
name = ""
parentname = str(parent) if parent is not None else ""
Exception.__init__(self, message.format(name=name, parent=parentname))
@ -49,32 +50,39 @@ class AlreadyExistsError(FSError):
"The directory or file name we're trying to add already exists"
cls_message = "'{name}' already exists in '{parent}'"
class InvalidPath(FSError):
"The path of self is invalid, and cannot be worked with."
cls_message = "'{name}' is invalid."
class InvalidDestinationError(FSError):
"""A copy/move operation has been called, but the destination is invalid."""
cls_message = "'{name}' is an invalid destination for this operation."
class OperationError(FSError):
"""A copy/move/delete operation has been called, but the checkup after the
operation shows that it didn't work."""
cls_message = "Operation on '{name}' failed."
class File:
"""Represents a file and holds metadata to be used for scanning.
"""
INITIAL_INFO = {
'size': 0,
'mtime': 0,
'md5': '',
'md5partial': '',
"size": 0,
"mtime": 0,
"md5": "",
"md5partial": "",
}
# Slots for File make us save quite a bit of memory. In a memory test I've made with a lot of
# files, I saved 35% memory usage with "unread" files (no _read_info() call) and gains become
# even greater when we take into account read attributes (70%!). Yeah, it's worth it.
__slots__ = ('path', 'is_ref', 'words') + tuple(INITIAL_INFO.keys())
__slots__ = ("path", "is_ref", "words") + tuple(INITIAL_INFO.keys())
def __init__(self, path):
self.path = path
@ -90,7 +98,9 @@ class File:
try:
self._read_info(attrname)
except Exception as e:
logging.warning("An error '%s' was raised while decoding '%s'", e, repr(self.path))
logging.warning(
"An error '%s' was raised while decoding '%s'", e, repr(self.path)
)
result = object.__getattribute__(self, attrname)
if result is NOT_SET:
result = self.INITIAL_INFO[attrname]
@ -102,13 +112,13 @@ class File:
return (0x4000, 0x4000) # 16Kb
def _read_info(self, field):
if field in ('size', 'mtime'):
if field in ("size", "mtime"):
stats = self.path.stat()
self.size = nonone(stats.st_size, 0)
self.mtime = nonone(stats.st_mtime, 0)
elif field == 'md5partial':
elif field == "md5partial":
try:
fp = self.path.open('rb')
fp = self.path.open("rb")
offset, size = self._get_md5partial_offset_and_size()
fp.seek(offset)
partialdata = fp.read(size)
@ -117,9 +127,9 @@ class File:
fp.close()
except Exception:
pass
elif field == 'md5':
elif field == "md5":
try:
fp = self.path.open('rb')
fp = self.path.open("rb")
md5 = hashlib.md5()
# The goal here is to not run out of memory on really big files. However, the chunk
# size has to be large enough so that the python loop isn't too costly in terms of
@ -189,7 +199,8 @@ class Folder(File):
It has the size/md5 info of a File, but it's value are the sum of its subitems.
"""
__slots__ = File.__slots__ + ('_subfolders', )
__slots__ = File.__slots__ + ("_subfolders",)
def __init__(self, path):
File.__init__(self, path)
@ -201,12 +212,12 @@ class Folder(File):
return folders + files
def _read_info(self, field):
if field in {'size', 'mtime'}:
if field in {"size", "mtime"}:
size = sum((f.size for f in self._all_items()), 0)
self.size = size
stats = self.path.stat()
self.mtime = nonone(stats.st_mtime, 0)
elif field in {'md5', 'md5partial'}:
elif field in {"md5", "md5partial"}:
# What's sensitive here is that we must make sure that subfiles'
# md5 are always added up in the same order, but we also want a
# different md5 if a file gets moved in a different subdirectory.
@ -214,7 +225,7 @@ class Folder(File):
items = self._all_items()
items.sort(key=lambda f: f.path)
md5s = [getattr(f, field) for f in items]
return b''.join(md5s)
return b"".join(md5s)
md5 = hashlib.md5(get_dir_md5_concat())
digest = md5.digest()
@ -223,7 +234,9 @@ class Folder(File):
@property
def subfolders(self):
if self._subfolders is None:
subfolders = [p for p in self.path.listdir() if not p.islink() and p.isdir()]
subfolders = [
p for p in self.path.listdir() if not p.islink() and p.isdir()
]
self._subfolders = [self.__class__(p) for p in subfolders]
return self._subfolders
@ -244,6 +257,7 @@ def get_file(path, fileclasses=[File]):
if fileclass.can_handle(path):
return fileclass(path)
def get_files(path, fileclasses=[File]):
"""Returns a list of :class:`File` for each file contained in ``path``.

1
core/gui/__init__.py
View File

@ -13,4 +13,3 @@ blue, which is supposed to be orange, does the sorting logic, holds selection, e
.. _cross-toolkit: http://www.hardcoded.net/articles/cross-toolkit-software
"""

2
core/gui/base.py
View File

@ -8,6 +8,7 @@
from hscommon.notify import Listener
class DupeGuruGUIObject(Listener):
def __init__(self, app):
Listener.__init__(self, app)
@ -27,4 +28,3 @@ class DupeGuruGUIObject(Listener):
def results_changed_but_keep_selection(self):
pass

6
core/gui/deletion_options.py
View File

@ -10,6 +10,7 @@ import os
from hscommon.gui.base import GUIObject
from hscommon.trans import tr
class DeletionOptionsView:
"""Expected interface for :class:`DeletionOptions`'s view.
@ -26,6 +27,7 @@ class DeletionOptionsView:
Other than the flags, there's also a prompt message which has a dynamic content, defined by
:meth:`update_msg`.
"""
def update_msg(self, msg: str):
"""Update the dialog's prompt with ``str``.
"""
@ -40,12 +42,14 @@ class DeletionOptionsView:
"""Enable or disable the widget controlling :attr:`DeletionOptions.use_hardlinks`.
"""
class DeletionOptions(GUIObject):
"""Present the user with deletion options before proceeding.
When the user activates "Send to trash", we present him with a couple of options that changes
the behavior of that deletion operation.
"""
def __init__(self):
GUIObject.__init__(self)
#: Whether symlinks or hardlinks are used when doing :attr:`link_deleted`.
@ -103,5 +107,3 @@ class DeletionOptions(GUIObject):
self._link_deleted = value
hardlinks_enabled = value and self.supports_links()
self.view.set_hardlink_option_enabled(hardlinks_enabled)

6
core/gui/details_panel.py
View File

@ -9,6 +9,7 @@
from hscommon.gui.base import GUIObject
from .base import DupeGuruGUIObject
class DetailsPanel(GUIObject, DupeGuruGUIObject):
def __init__(self, app):
GUIObject.__init__(self, multibind=True)
@ -31,7 +32,9 @@ class DetailsPanel(GUIObject, DupeGuruGUIObject):
# we don't want the two sides of the table to display the stats for the same file
ref = group.ref if group is not None and group.ref is not dupe else None
data2 = self.app.get_display_info(ref, group, False)
columns = self.app.result_table.COLUMNS[1:] # first column is the 'marked' column
columns = self.app.result_table.COLUMNS[
1:
] # first column is the 'marked' column
self._table = [(c.display, data1[c.name], data2[c.name]) for c in columns]
# --- Public
@ -45,4 +48,3 @@ class DetailsPanel(GUIObject, DupeGuruGUIObject):
def dupes_selected(self):
self._refresh()
self.view.refresh()

6
core/gui/directory_tree.py
View File

@ -13,6 +13,7 @@ from .base import DupeGuruGUIObject
STATE_ORDER = [DirectoryState.Normal, DirectoryState.Reference, DirectoryState.Excluded]
# Lazily loads children
class DirectoryNode(Node):
def __init__(self, tree, path, name):
@ -35,7 +36,9 @@ class DirectoryNode(Node):
self._loaded = True
def update_all_states(self):
self._state = STATE_ORDER.index(self._tree.app.directories.get_state(self._directory_path))
self._state = STATE_ORDER.index(
self._tree.app.directories.get_state(self._directory_path)
)
for node in self:
node.update_all_states()
@ -104,4 +107,3 @@ class DirectoryTree(Tree, DupeGuruGUIObject):
def directories_changed(self):
self._refresh()
self.view.refresh()

6
core/gui/ignore_list_dialog.py
View File

@ -8,6 +8,7 @@
from hscommon.trans import tr
from .ignore_list_table import IgnoreListTable
class IgnoreListDialog:
# --- View interface
# show()
@ -21,7 +22,9 @@ class IgnoreListDialog:
def clear(self):
if not self.ignore_list:
return
msg = tr("Do you really want to remove all %d items from the ignore list?") % len(self.ignore_list)
msg = tr(
"Do you really want to remove all %d items from the ignore list?"
) % len(self.ignore_list)
if self.app.view.ask_yes_no(msg):
self.ignore_list.Clear()
self.refresh()
@ -36,4 +39,3 @@ class IgnoreListDialog:
def show(self):
self.view.show()

8
core/gui/ignore_list_table.py
View File

@ -10,13 +10,14 @@ from hscommon.gui.table import GUITable, Row
from hscommon.gui.column import Column, Columns
from hscommon.trans import trget
coltr = trget('columns')
coltr = trget("columns")
class IgnoreListTable(GUITable):
COLUMNS = [
# the str concat below saves us needless localization.
Column('path1', coltr("File Path") + " 1"),
Column('path2', coltr("File Path") + " 2"),
Column("path1", coltr("File Path") + " 1"),
Column("path2", coltr("File Path") + " 2"),
]
def __init__(self, ignore_list_dialog):
@ -38,4 +39,3 @@ class IgnoreListRow(Row):
self.path2_original = path2
self.path1 = str(path1)
self.path2 = str(path2)

3
core/gui/prioritize_dialog.py
View File

@ -9,6 +9,7 @@
from hscommon.gui.base import GUIObject
from hscommon.gui.selectable_list import GUISelectableList
class CriterionCategoryList(GUISelectableList):
def __init__(self, dialog):
self.dialog = dialog
@ -18,6 +19,7 @@ class CriterionCategoryList(GUISelectableList):
self.dialog.select_category(self.dialog.categories[self.selected_index])
GUISelectableList._update_selection(self)
class PrioritizationList(GUISelectableList):
def __init__(self, dialog):
self.dialog = dialog
@ -41,6 +43,7 @@ class PrioritizationList(GUISelectableList):
del prilist[i]
self._refresh_contents()
class PrioritizeDialog(GUIObject):
def __init__(self, app):
GUIObject.__init__(self)

2
core/gui/problem_dialog.py
View File

@ -10,6 +10,7 @@ from hscommon import desktop
from .problem_table import ProblemTable
class ProblemDialog:
def __init__(self, app):
self.app = app
@ -26,4 +27,3 @@ class ProblemDialog:
def select_dupe(self, dupe):
self._selected_dupe = dupe

8
core/gui/problem_table.py
View File

@ -10,12 +10,13 @@ from hscommon.gui.table import GUITable, Row
from hscommon.gui.column import Column, Columns
from hscommon.trans import trget
coltr = trget('columns')
coltr = trget("columns")
class ProblemTable(GUITable):
COLUMNS = [
Column('path', coltr("File Path")),
Column('msg', coltr("Error Message")),
Column("path", coltr("File Path")),
Column("msg", coltr("Error Message")),
]
def __init__(self, problem_dialog):
@ -41,4 +42,3 @@ class ProblemRow(Row):
self.dupe = dupe
self.msg = msg
self.path = str(dupe.path)

14
core/gui/result_table.py
View File

@ -13,6 +13,7 @@ from hscommon.gui.column import Columns
from .base import DupeGuruGUIObject
class DupeRow(Row):
def __init__(self, table, group, dupe):
Row.__init__(self, table)
@ -42,7 +43,9 @@ class DupeRow(Row):
dupe_info = self.data
ref_info = self._group.ref.get_display_info(group=self._group, delta=False)
for key, value in dupe_info.items():
if (key not in self._delta_columns) and (ref_info[key].lower() != value.lower()):
if (key not in self._delta_columns) and (
ref_info[key].lower() != value.lower()
):
self._delta_columns.add(key)
return column_name in self._delta_columns
@ -79,10 +82,10 @@ class ResultTable(GUITable, DupeGuruGUIObject):
def __init__(self, app):
GUITable.__init__(self)
DupeGuruGUIObject.__init__(self, app)
self.columns = Columns(self, prefaccess=app, savename='ResultTable')
self.columns = Columns(self, prefaccess=app, savename="ResultTable")
self._power_marker = False
self._delta_values = False
self._sort_descriptors = ('name', True)
self._sort_descriptors = ("name", True)
# --- Override
def _view_updated(self):
@ -96,7 +99,7 @@ class ResultTable(GUITable, DupeGuruGUIObject):
def _update_selection(self):
rows = self.selected_rows
self.app._select_dupes(list(map(attrgetter('_dupe'), rows)))
self.app._select_dupes(list(map(attrgetter("_dupe"), rows)))
def _fill(self):
if not self.power_marker:
@ -118,7 +121,7 @@ class ResultTable(GUITable, DupeGuruGUIObject):
try:
row = self[index]
except IndexError:
return '---'
return "---"
if self.delta_values:
return row.data_delta[column]
else:
@ -188,4 +191,3 @@ class ResultTable(GUITable, DupeGuruGUIObject):
def save_session(self):
self.columns.save_columns()

2
core/gui/stats_label.py
View File

@ -8,6 +8,7 @@
from .base import DupeGuruGUIObject
class StatsLabel(DupeGuruGUIObject):
def _view_updated(self):
self.view.refresh()
@ -18,4 +19,5 @@ class StatsLabel(DupeGuruGUIObject):
def results_changed(self):
self.view.refresh()
marking_changed = results_changed

26
core/ignore.py
View File

@ -10,12 +10,14 @@ from xml.etree import ElementTree as ET
from hscommon.util import FileOrPath
class IgnoreList:
"""An ignore list implementation that is iterable, filterable and exportable to XML.
Call Ignore to add an ignore list entry, and AreIgnore to check if 2 items are in the list.
When iterated, 2 sized tuples will be returned, the tuples containing 2 items ignored together.
"""
# ---Override
def __init__(self):
self._ignored = {}
@ -99,14 +101,14 @@ class IgnoreList:
root = ET.parse(infile).getroot()
except Exception:
return
file_elems = (e for e in root if e.tag == 'file')
file_elems = (e for e in root if e.tag == "file")
for fn in file_elems:
file_path = fn.get('path')
file_path = fn.get("path")
if not file_path:
continue
subfile_elems = (e for e in fn if e.tag == 'file')
subfile_elems = (e for e in fn if e.tag == "file")
for sfn in subfile_elems:
subfile_path = sfn.get('path')
subfile_path = sfn.get("path")
if subfile_path:
self.Ignore(file_path, subfile_path)
@ -115,15 +117,13 @@ class IgnoreList:
outfile can be a file object or a filename.
"""
root = ET.Element('ignore_list')
root = ET.Element("ignore_list")
for filename, subfiles in self._ignored.items():
file_node = ET.SubElement(root, 'file')
file_node.set('path', filename)
file_node = ET.SubElement(root, "file")
file_node.set("path", filename)
for subfilename in subfiles:
subfile_node = ET.SubElement(file_node, 'file')
subfile_node.set('path', subfilename)
subfile_node = ET.SubElement(file_node, "file")
subfile_node.set("path", subfilename)
tree = ET.ElementTree(root)
with FileOrPath(outfile, 'wb') as fp:
tree.write(fp, encoding='utf-8')
with FileOrPath(outfile, "wb") as fp:
tree.write(fp, encoding="utf-8")

2
core/markable.py
View File

@ -6,6 +6,7 @@
# which should be included with this package. The terms are also available at
# http://www.gnu.org/licenses/gpl-3.0.html
class Markable:
def __init__(self):
self.__marked = set()
@ -104,6 +105,7 @@ class Markable:
def mark_inverted(self):
return self.__inverted
class MarkableList(list, Markable):
def __init__(self):
list.__init__(self)

81
core/me/fs.py
View File

@ -13,25 +13,37 @@ from core.util import format_timestamp, format_perc, format_words, format_dupe_c
from core import fs
TAG_FIELDS = {
'audiosize', 'duration', 'bitrate', 'samplerate', 'title', 'artist',
'album', 'genre', 'year', 'track', 'comment'
"audiosize",
"duration",
"bitrate",
"samplerate",
"title",
"artist",
"album",
"genre",
"year",
"track",
"comment",
}
class MusicFile(fs.File):
INITIAL_INFO = fs.File.INITIAL_INFO.copy()
INITIAL_INFO.update({
'audiosize': 0,
'bitrate': 0,
'duration': 0,
'samplerate': 0,
'artist': '',
'album': '',
'title': '',
'genre': '',
'comment': '',
'year': '',
'track': 0,
})
INITIAL_INFO.update(
{
"audiosize": 0,
"bitrate": 0,
"duration": 0,
"samplerate": 0,
"artist": "",
"album": "",
"title": "",
"genre": "",
"comment": "",
"year": "",
"track": 0,
}
)
__slots__ = fs.File.__slots__ + tuple(INITIAL_INFO.keys())
@classmethod
@ -60,26 +72,26 @@ class MusicFile(fs.File):
else:
percentage = group.percentage
dupe_count = len(group.dupes)
dupe_folder_path = getattr(self, 'display_folder_path', self.folder_path)
dupe_folder_path = getattr(self, "display_folder_path", self.folder_path)
return {
'name': self.name,
'folder_path': str(dupe_folder_path),
'size': format_size(size, 2, 2, False),
'duration': format_time(duration, with_hours=False),
'bitrate': str(bitrate),
'samplerate': str(samplerate),
'extension': self.extension,
'mtime': format_timestamp(mtime, delta and m),
'title': self.title,
'artist': self.artist,
'album': self.album,
'genre': self.genre,
'year': self.year,
'track': str(self.track),
'comment': self.comment,
'percentage': format_perc(percentage),
'words': format_words(self.words) if hasattr(self, 'words') else '',
'dupe_count': format_dupe_count(dupe_count),
"name": self.name,
"folder_path": str(dupe_folder_path),
"size": format_size(size, 2, 2, False),
"duration": format_time(duration, with_hours=False),
"bitrate": str(bitrate),
"samplerate": str(samplerate),
"extension": self.extension,
"mtime": format_timestamp(mtime, delta and m),
"title": self.title,
"artist": self.artist,
"album": self.album,
"genre": self.genre,
"year": self.year,
"track": str(self.track),
"comment": self.comment,
"percentage": format_perc(percentage),
"words": format_words(self.words) if hasattr(self, "words") else "",
"dupe_count": format_dupe_count(dupe_count),
}
def _get_md5partial_offset_and_size(self):
@ -101,4 +113,3 @@ class MusicFile(fs.File):
self.comment = f.comment
self.year = f.year
self.track = f.track

25
core/me/prioritize.py
View File

@ -8,11 +8,16 @@
from hscommon.trans import trget
from core.prioritize import (
KindCategory, FolderCategory, FilenameCategory, NumericalCategory,
SizeCategory, MtimeCategory
KindCategory,
FolderCategory,
FilenameCategory,
NumericalCategory,
SizeCategory,
MtimeCategory,
)
coltr = trget('columns')
coltr = trget("columns")
class DurationCategory(NumericalCategory):
NAME = coltr("Duration")
@ -20,21 +25,29 @@ class DurationCategory(NumericalCategory):
def extract_value(self, dupe):
return dupe.duration
class BitrateCategory(NumericalCategory):
NAME = coltr("Bitrate")
def extract_value(self, dupe):
return dupe.bitrate
class SamplerateCategory(NumericalCategory):
NAME = coltr("Samplerate")
def extract_value(self, dupe):
return dupe.samplerate
def all_categories():
return [
KindCategory, FolderCategory, FilenameCategory, SizeCategory, DurationCategory,
BitrateCategory, SamplerateCategory, MtimeCategory
KindCategory,
FolderCategory,
FilenameCategory,
SizeCategory,
DurationCategory,
BitrateCategory,
SamplerateCategory,
MtimeCategory,
]

43
core/me/result_table.py
View File

@ -10,28 +10,29 @@ from hscommon.trans import trget
from core.gui.result_table import ResultTable as ResultTableBase
coltr = trget('columns')
coltr = trget("columns")
class ResultTable(ResultTableBase):
COLUMNS = [
Column('marked', ''),
Column('name', coltr("Filename")),
Column('folder_path', coltr("Folder"), visible=False, optional=True),
Column('size', coltr("Size (MB)"), optional=True),
Column('duration', coltr("Time"), optional=True),
Column('bitrate', coltr("Bitrate"), optional=True),
Column('samplerate', coltr("Sample Rate"), visible=False, optional=True),
Column('extension', coltr("Kind"), optional=True),
Column('mtime', coltr("Modification"), visible=False, optional=True),
Column('title', coltr("Title"), visible=False, optional=True),
Column('artist', coltr("Artist"), visible=False, optional=True),
Column('album', coltr("Album"), visible=False, optional=True),
Column('genre', coltr("Genre"), visible=False, optional=True),
Column('year', coltr("Year"), visible=False, optional=True),
Column('track', coltr("Track Number"), visible=False, optional=True),
Column('comment', coltr("Comment"), visible=False, optional=True),
Column('percentage', coltr("Match %"), optional=True),
Column('words', coltr("Words Used"), visible=False, optional=True),
Column('dupe_count', coltr("Dupe Count"), visible=False, optional=True),
Column("marked", ""),
Column("name", coltr("Filename")),
Column("folder_path", coltr("Folder"), visible=False, optional=True),
Column("size", coltr("Size (MB)"), optional=True),
Column("duration", coltr("Time"), optional=True),
Column("bitrate", coltr("Bitrate"), optional=True),
Column("samplerate", coltr("Sample Rate"), visible=False, optional=True),
Column("extension", coltr("Kind"), optional=True),
Column("mtime", coltr("Modification"), visible=False, optional=True),
Column("title", coltr("Title"), visible=False, optional=True),
Column("artist", coltr("Artist"), visible=False, optional=True),
Column("album", coltr("Album"), visible=False, optional=True),
Column("genre", coltr("Genre"), visible=False, optional=True),
Column("year", coltr("Year"), visible=False, optional=True),
Column("track", coltr("Track Number"), visible=False, optional=True),
Column("comment", coltr("Comment"), visible=False, optional=True),
Column("percentage", coltr("Match %"), optional=True),
Column("words", coltr("Words Used"), visible=False, optional=True),
Column("dupe_count", coltr("Dupe Count"), visible=False, optional=True),
]
DELTA_COLUMNS = {'size', 'duration', 'bitrate', 'samplerate', 'mtime'}
DELTA_COLUMNS = {"size", "duration", "bitrate", "samplerate", "mtime"}

3
core/me/scanner.py
View File

@ -8,6 +8,7 @@ from hscommon.trans import tr
from core.scanner import Scanner as ScannerBase, ScanOption, ScanType
class ScannerME(ScannerBase):
@staticmethod
def _key_func(dupe):
@ -22,5 +23,3 @@ class ScannerME(ScannerBase):
ScanOption(ScanType.Tag, tr("Tags")),
ScanOption(ScanType.Contents, tr("Contents")),
]

13
core/pe/__init__.py
View File

@ -1 +1,12 @@
from . import block, cache, exif, iphoto_plist, matchblock, matchexif, photo, prioritize, result_table, scanner # noqa
from . import ( # noqa
block,
cache,
exif,
iphoto_plist,
matchblock,
matchexif,
photo,
prioritize,
result_table,
scanner,
)

5
core/pe/cache.py
View File

@ -6,13 +6,15 @@
from ._cache import string_to_colors # noqa
def colors_to_string(colors):
"""Transform the 3 sized tuples 'colors' into a hex string.
[(0,100,255)] --> 0064ff
[(1,2,3),(4,5,6)] --> 010203040506
"""
return ''.join('%02x%02x%02x' % (r, g, b) for r, g, b in colors)
return "".join("%02x%02x%02x" % (r, g, b) for r, g, b in colors)
# This function is an important bottleneck of dupeGuru PE. It has been converted to C.
# def string_to_colors(s):
@ -23,4 +25,3 @@ def colors_to_string(colors):
# number = int(s[i:i+6], 16)
# result.append((number >> 16, (number >> 8) & 0xff, number & 0xff))
# return result

26
core/pe/cache_shelve.py
View File

@ -12,29 +12,36 @@ from collections import namedtuple
from .cache import string_to_colors, colors_to_string
def wrap_path(path):
return 'path:{}'.format(path)
return "path:{}".format(path)
def unwrap_path(key):
return key[5:]
def wrap_id(path):
return 'id:{}'.format(path)
return "id:{}".format(path)
def unwrap_id(key):
return int(key[3:])
CacheRow = namedtuple('CacheRow', 'id path blocks mtime')
CacheRow = namedtuple("CacheRow", "id path blocks mtime")
class ShelveCache:
"""A class to cache picture blocks in a shelve backend.
"""
def __init__(self, db=None, readonly=False):
self.istmp = db is None
if self.istmp:
self.dtmp = tempfile.mkdtemp()
self.ftmp = db = op.join(self.dtmp, 'tmpdb')
flag = 'r' if readonly else 'c'
self.ftmp = db = op.join(self.dtmp, "tmpdb")
flag = "r" if readonly else "c"
self.shelve = shelve.open(db, flag)
self.maxid = self._compute_maxid()
@ -54,10 +61,10 @@ class ShelveCache:
return string_to_colors(self.shelve[skey].blocks)
def __iter__(self):
return (unwrap_path(k) for k in self.shelve if k.startswith('path:'))
return (unwrap_path(k) for k in self.shelve if k.startswith("path:"))
def __len__(self):
return sum(1 for k in self.shelve if k.startswith('path:'))
return sum(1 for k in self.shelve if k.startswith("path:"))
def __setitem__(self, path_str, blocks):
blocks = colors_to_string(blocks)
@ -74,7 +81,9 @@ class ShelveCache:
self.shelve[wrap_id(rowid)] = wrap_path(path_str)
def _compute_maxid(self):
return max((unwrap_id(k) for k in self.shelve if k.startswith('id:')), default=1)
return max(
(unwrap_id(k) for k in self.shelve if k.startswith("id:")), default=1
)
def _get_new_id(self):
self.maxid += 1
@ -133,4 +142,3 @@ class ShelveCache:
# #402 and #439. I don't think it hurts to silently ignore the error, so that's
# what we do
pass

29
core/pe/cache_sqlite.py
View File

@ -11,10 +11,12 @@ import sqlite3 as sqlite
from .cache import string_to_colors, colors_to_string
class SqliteCache:
"""A class to cache picture blocks in a sqlite backend.
"""
def __init__(self, db=':memory:', readonly=False):
def __init__(self, db=":memory:", readonly=False):
# readonly is not used in the sqlite version of the cache
self.dbname = db
self.con = None
@ -67,16 +69,20 @@ class SqliteCache:
try:
self.con.execute(sql, [blocks, mtime, path_str])
except sqlite.OperationalError:
logging.warning('Picture cache could not set value for key %r', path_str)
logging.warning("Picture cache could not set value for key %r", path_str)
except sqlite.DatabaseError as e:
logging.warning('DatabaseError while setting value for key %r: %s', path_str, str(e))
logging.warning(
"DatabaseError while setting value for key %r: %s", path_str, str(e)
)
def _create_con(self, second_try=False):
def create_tables():
logging.debug("Creating picture cache tables.")
self.con.execute("drop table if exists pictures")
self.con.execute("drop index if exists idx_path")
self.con.execute("create table pictures(path TEXT, mtime INTEGER, blocks TEXT)")
self.con.execute(
"create table pictures(path TEXT, mtime INTEGER, blocks TEXT)"
)
self.con.execute("create index idx_path on pictures (path)")
self.con = sqlite.connect(self.dbname, isolation_level=None)
@ -87,14 +93,16 @@ class SqliteCache:
except sqlite.DatabaseError as e: # corrupted db
if second_try:
raise # Something really strange is happening
logging.warning('Could not create picture cache because of an error: %s', str(e))
logging.warning(
"Could not create picture cache because of an error: %s", str(e)
)
self.con.close()
os.remove(self.dbname)
self._create_con(second_try=True)
def clear(self):
self.close()
if self.dbname != ':memory:':
if self.dbname != ":memory:":
os.remove(self.dbname)
self._create_con()
@ -117,7 +125,9 @@ class SqliteCache:
raise ValueError(path)
def get_multiple(self, rowids):
sql = "select rowid, blocks from pictures where rowid in (%s)" % ','.join(map(str, rowids))
sql = "select rowid, blocks from pictures where rowid in (%s)" % ",".join(
map(str, rowids)
)
cur = self.con.execute(sql)
return ((rowid, string_to_colors(blocks)) for rowid, blocks in cur)
@ -138,6 +148,7 @@ class SqliteCache:
continue
todelete.append(rowid)
if todelete:
sql = "delete from pictures where rowid in (%s)" % ','.join(map(str, todelete))
sql = "delete from pictures where rowid in (%s)" % ",".join(
map(str, todelete)
)
self.con.execute(sql)

43
core/pe/exif.py
View File

@ -148,17 +148,18 @@ GPS_TA0GS = {
0x1B: "GPSProcessingMethod",
0x1C: "GPSAreaInformation",
0x1D: "GPSDateStamp",
0x1E: "GPSDifferential"
0x1E: "GPSDifferential",
}
INTEL_ENDIAN = ord('I')
MOTOROLA_ENDIAN = ord('M')
INTEL_ENDIAN = ord("I")
MOTOROLA_ENDIAN = ord("M")
# About MAX_COUNT: It's possible to have corrupted exif tags where the entry count is way too high
# and thus makes us loop, not endlessly, but for heck of a long time for nothing. Therefore, we put
# an arbitrary limit on the entry count we'll allow ourselves to read and any IFD reporting more
# entries than that will be considered corrupt.
MAX_COUNT = 0xffff
MAX_COUNT = 0xFFFF
def s2n_motorola(bytes):
x = 0
@ -166,6 +167,7 @@ def s2n_motorola(bytes):
x = (x << 8) | c
return x
def s2n_intel(bytes):
x = 0
y = 0
@ -174,13 +176,14 @@ def s2n_intel(bytes):
y = y + 8
return x
class Fraction:
def __init__(self, num, den):
self.num = num
self.den = den
def __repr__(self):
return '%d/%d' % (self.num, self.den)
return "%d/%d" % (self.num, self.den)
class TIFF_file:
@ -199,7 +202,13 @@ class TIFF_file:
val = val - (msb << 1)
if debug:
logging.debug(self.endian)
logging.debug("Slice for offset %d length %d: %r and value: %d", offset, length, slice, val)
logging.debug(
"Slice for offset %d length %d: %r and value: %d",
offset,
length,
slice,
val,
)
return val
def first_IFD(self):
@ -240,15 +249,16 @@ class TIFF_file:
offset = self.s2n(offset, 4)
if type == 2:
# Special case: nul-terminated ASCII string
values = str(self.data[offset:offset+count-1], encoding='latin-1')
values = str(self.data[offset : offset + count - 1], encoding="latin-1")
else:
values = []
signed = (type == 6 or type >= 8)
signed = type == 6 or type >= 8
for j in range(count):
if type in {5, 10}:
# The type is either 5 or 10
value_j = Fraction(self.s2n(offset, 4, signed),
self.s2n(offset+4, 4, signed))
value_j = Fraction(
self.s2n(offset, 4, signed), self.s2n(offset + 4, 4, signed)
)
else:
# Not a fraction
value_j = self.s2n(offset, typelen, signed)
@ -258,17 +268,19 @@ class TIFF_file:
a.append((tag, type, values))
return a
def read_exif_header(fp):
# If `fp`'s first bytes are not exif, it tries to find it in the next 4kb
def isexif(data):
return data[0:4] == b'\377\330\377\341' and data[6:10] == b'Exif'
return data[0:4] == b"\377\330\377\341" and data[6:10] == b"Exif"
data = fp.read(12)
if isexif(data):
return data
# ok, not exif, try to find it
large_data = fp.read(4096)
try:
index = large_data.index(b'Exif')
index = large_data.index(b"Exif")
data = large_data[index - 6 : index + 6]
# large_data omits the first 12 bytes, and the index is at the middle of the header, so we
# must seek index + 18
@ -277,13 +289,16 @@ def read_exif_header(fp):
except ValueError:
raise ValueError("Not an Exif file")
def get_fields(fp):
data = read_exif_header(fp)
length = data[4] * 256 + data[5]
logging.debug("Exif header length: %d bytes", length)
data = fp.read(length - 8)
data_format = data[0]
logging.debug("%s format", {INTEL_ENDIAN: 'Intel', MOTOROLA_ENDIAN: 'Motorola'}[data_format])
logging.debug(
"%s format", {INTEL_ENDIAN: "Intel", MOTOROLA_ENDIAN: "Motorola"}[data_format]
)
T = TIFF_file(data)
# There may be more than one IFD per file, but we only read the first one because others are
# most likely thumbnails.
@ -294,7 +309,7 @@ def get_fields(fp):
try:
stag = EXIF_TAGS[tag]
except KeyError:
stag = '0x%04X' % tag
stag = "0x%04X" % tag
if stag in result:
return # don't overwrite data
result[stag] = values

4
core/pe/iphoto_plist.py
View File

@ -8,17 +8,19 @@
import plistlib
class IPhotoPlistParser(plistlib._PlistParser):
"""A parser for iPhoto plists.
iPhoto plists tend to be malformed, so we have to subclass the built-in parser to be a bit more
lenient.
"""
def __init__(self):
plistlib._PlistParser.__init__(self, use_builtin_types=True, dict_type=dict)
# For debugging purposes, we remember the last bit of data to be analyzed so that we can
# log it in case of an exception
self.lastdata = ''
self.lastdata = ""
def get_data(self):
self.lastdata = plistlib._PlistParser.get_data(self)

59
core/pe/matchblock.py
View File

@ -48,14 +48,18 @@ except Exception:
logging.warning("Had problems to determine cpu count on launch.")
RESULTS_QUEUE_LIMIT = 8
def get_cache(cache_path, readonly=False):
if cache_path.endswith('shelve'):
if cache_path.endswith("shelve"):
from .cache_shelve import ShelveCache
return ShelveCache(cache_path, readonly=readonly)
else:
from .cache_sqlite import SqliteCache
return SqliteCache(cache_path, readonly=readonly)
def prepare_pictures(pictures, cache_path, with_dimensions, j=job.nulljob):
# The MemoryError handlers in there use logging without first caring about whether or not
# there is enough memory left to carry on the operation because it is assumed that the
@ -86,32 +90,45 @@ def prepare_pictures(pictures, cache_path, with_dimensions, j=job.nulljob):
except (IOError, ValueError) as e:
logging.warning(str(e))
except MemoryError:
logging.warning("Ran out of memory while reading %s of size %d", picture.unicode_path, picture.size)
if picture.size < 10 * 1024 * 1024: # We're really running out of memory
logging.warning(
"Ran out of memory while reading %s of size %d",
picture.unicode_path,
picture.size,
)
if (
picture.size < 10 * 1024 * 1024
): # We're really running out of memory
raise
except MemoryError:
logging.warning('Ran out of memory while preparing pictures')
logging.warning("Ran out of memory while preparing pictures")
cache.close()
return prepared
def get_chunks(pictures):
min_chunk_count = multiprocessing.cpu_count() * 2 # have enough chunks to feed all subprocesses
min_chunk_count = (
multiprocessing.cpu_count() * 2
) # have enough chunks to feed all subprocesses
chunk_count = len(pictures) // DEFAULT_CHUNK_SIZE
chunk_count = max(min_chunk_count, chunk_count)
chunk_size = (len(pictures) // chunk_count) + 1
chunk_size = max(MIN_CHUNK_SIZE, chunk_size)
logging.info(
"Creating %d chunks with a chunk size of %d for %d pictures", chunk_count,
chunk_size, len(pictures)
"Creating %d chunks with a chunk size of %d for %d pictures",
chunk_count,
chunk_size,
len(pictures),
)
chunks = [pictures[i : i + chunk_size] for i in range(0, len(pictures), chunk_size)]
return chunks
def get_match(first, second, percentage):
if percentage < 0:
percentage = 0
return Match(first, second, percentage)
def async_compare(ref_ids, other_ids, dbname, threshold, picinfo):
# The list of ids in ref_ids have to be compared to the list of ids in other_ids. other_ids
# can be None. In this case, ref_ids has to be compared with itself
@ -142,6 +159,7 @@ def async_compare(ref_ids, other_ids, dbname, threshold, picinfo):
cache.close()
return results
def getmatches(pictures, cache_path, threshold, match_scaled=False, j=job.nulljob):
def get_picinfo(p):
if match_scaled:
@ -160,11 +178,16 @@ def getmatches(pictures, cache_path, threshold, match_scaled=False, j=job.nulljo
async_results.remove(result)
comparison_count += 1
# About the NOQA below: I think there's a bug in pyflakes. To investigate...
progress_msg = tr("Performed %d/%d chunk matches") % (comparison_count, len(comparisons_to_do)) # NOQA
progress_msg = tr("Performed %d/%d chunk matches") % (
comparison_count,
len(comparisons_to_do),
) # NOQA
j.set_progress(comparison_count, progress_msg)
j = j.start_subjob([3, 7])
pictures = prepare_pictures(pictures, cache_path, with_dimensions=not match_scaled, j=j)
pictures = prepare_pictures(
pictures, cache_path, with_dimensions=not match_scaled, j=j
)
j = j.start_subjob([9, 1], tr("Preparing for matching"))
cache = get_cache(cache_path)
id2picture = {}
@ -175,7 +198,7 @@ def getmatches(pictures, cache_path, threshold, match_scaled=False, j=job.nulljo
except ValueError:
pass
cache.close()
pictures = [p for p in pictures if hasattr(p, 'cache_id')]
pictures = [p for p in pictures if hasattr(p, "cache_id")]
pool = multiprocessing.Pool()
async_results = []
matches = []
@ -203,9 +226,17 @@ def getmatches(pictures, cache_path, threshold, match_scaled=False, j=job.nulljo
# some wiggle room, log about the incident, and stop matching right here. We then process
# the matches we have. The rest of the process doesn't allocate much and we should be
# alright.
del comparisons_to_do, chunks, pictures # some wiggle room for the next statements
logging.warning("Ran out of memory when scanning! We had %d matches.", len(matches))
del matches[-len(matches)//3:] # some wiggle room to ensure we don't run out of memory again.
del (
comparisons_to_do,
chunks,
pictures,
) # some wiggle room for the next statements
logging.warning(
"Ran out of memory when scanning! We had %d matches.", len(matches)
)
del matches[
-len(matches) // 3 :
] # some wiggle room to ensure we don't run out of memory again.
pool.close()
result = []
myiter = j.iter_with_progress(
@ -225,5 +256,5 @@ def getmatches(pictures, cache_path, threshold, match_scaled=False, j=job.nulljo
result.append(get_match(ref, other, percentage))
return result
multiprocessing.freeze_support()
multiprocessing.freeze_support()

6
core/pe/matchexif.py
View File

@ -13,14 +13,15 @@ from hscommon.trans import tr
from core.engine import Match
def getmatches(files, match_scaled, j):
timestamp2pic = defaultdict(set)
for picture in j.iter_with_progress(files, tr("Read EXIF of %d/%d pictures")):
timestamp = picture.exif_timestamp
if timestamp:
timestamp2pic[timestamp].add(picture)
if '0000:00:00 00:00:00' in timestamp2pic: # very likely false matches
del timestamp2pic['0000:00:00 00:00:00']
if "0000:00:00 00:00:00" in timestamp2pic: # very likely false matches
del timestamp2pic["0000:00:00 00:00:00"]
matches = []
for pictures in timestamp2pic.values():
for p1, p2 in combinations(pictures, 2):
@ -28,4 +29,3 @@ def getmatches(files, match_scaled, j):
continue
matches.append(Match(p1, p2, 100))
return matches

48
core/pe/photo.py
View File

@ -14,8 +14,10 @@ from . import exif
# This global value is set by the platform-specific subclasser of the Photo base class
PLAT_SPECIFIC_PHOTO_CLASS = None
def format_dimensions(dimensions):
return '%d x %d' % (dimensions[0], dimensions[1])
return "%d x %d" % (dimensions[0], dimensions[1])
def get_delta_dimensions(value, ref_value):
return (value[0] - ref_value[0], value[1] - ref_value[1])
@ -23,14 +25,11 @@ def get_delta_dimensions(value, ref_value):
class Photo(fs.File):
INITIAL_INFO = fs.File.INITIAL_INFO.copy()
INITIAL_INFO.update({
'dimensions': (0, 0),
'exif_timestamp': '',
})
INITIAL_INFO.update({"dimensions": (0, 0), "exif_timestamp": ""})
__slots__ = fs.File.__slots__ + tuple(INITIAL_INFO.keys())
# These extensions are supported on all platforms
HANDLED_EXTS = {'png', 'jpg', 'jpeg', 'gif', 'bmp', 'tiff', 'tif'}
HANDLED_EXTS = {"png", "jpg", "jpeg", "gif", "bmp", "tiff", "tif"}
def _plat_get_dimensions(self):
raise NotImplementedError()
@ -39,12 +38,12 @@ class Photo(fs.File):
raise NotImplementedError()
def _get_orientation(self):
if not hasattr(self, '_cached_orientation'):
if not hasattr(self, "_cached_orientation"):
try:
with self.path.open('rb') as fp:
with self.path.open("rb") as fp:
exifdata = exif.get_fields(fp)
# the value is a list (probably one-sized) of ints
orientations = exifdata['Orientation']
orientations = exifdata["Orientation"]
self._cached_orientation = orientations[0]
except Exception: # Couldn't read EXIF data, no transforms
self._cached_orientation = 0
@ -52,12 +51,12 @@ class Photo(fs.File):
def _get_exif_timestamp(self):
try:
with self.path.open('rb') as fp:
with self.path.open("rb") as fp:
exifdata = exif.get_fields(fp)
return exifdata['DateTimeOriginal']
return exifdata["DateTimeOriginal"]
except Exception:
logging.info("Couldn't read EXIF of picture: %s", self.path)
return ''
return ""
@classmethod
def can_handle(cls, path):
@ -79,28 +78,27 @@ class Photo(fs.File):
else:
percentage = group.percentage
dupe_count = len(group.dupes)
dupe_folder_path = getattr(self, 'display_folder_path', self.folder_path)
dupe_folder_path = getattr(self, "display_folder_path", self.folder_path)
return {
'name': self.name,
'folder_path': str(dupe_folder_path),
'size': format_size(size, 0, 1, False),
'extension': self.extension,
'dimensions': format_dimensions(dimensions),
'exif_timestamp': self.exif_timestamp,
'mtime': format_timestamp(mtime, delta and m),
'percentage': format_perc(percentage),
'dupe_count': format_dupe_count(dupe_count),
"name": self.name,
"folder_path": str(dupe_folder_path),
"size": format_size(size, 0, 1, False),
"extension": self.extension,
"dimensions": format_dimensions(dimensions),
"exif_timestamp": self.exif_timestamp,
"mtime": format_timestamp(mtime, delta and m),
"percentage": format_perc(percentage),
"dupe_count": format_dupe_count(dupe_count),
}
def _read_info(self, field):
fs.File._read_info(self, field)
if field == 'dimensions':
if field == "dimensions":
self.dimensions = self._plat_get_dimensions()
if self._get_orientation() in {5, 6, 7, 8}:
self.dimensions = (self.dimensions[1], self.dimensions[0])
elif field == 'exif_timestamp':
elif field == "exif_timestamp":
self.exif_timestamp = self._get_exif_timestamp()
def get_blocks(self, block_count_per_side):
return self._plat_get_blocks(block_count_per_side, self._get_orientation())

20
core/pe/prioritize.py
View File

@ -8,11 +8,16 @@
from hscommon.trans import trget
from core.prioritize import (
KindCategory, FolderCategory, FilenameCategory, NumericalCategory,
SizeCategory, MtimeCategory
KindCategory,
FolderCategory,
FilenameCategory,
NumericalCategory,
SizeCategory,
MtimeCategory,
)
coltr = trget('columns')
coltr = trget("columns")
class DimensionsCategory(NumericalCategory):
NAME = coltr("Dimensions")
@ -24,8 +29,13 @@ class DimensionsCategory(NumericalCategory):
width, height = value
return (-width, -height)
def all_categories():
return [
KindCategory, FolderCategory, FilenameCategory, SizeCategory, DimensionsCategory,
MtimeCategory
KindCategory,
FolderCategory,
FilenameCategory,
SizeCategory,
DimensionsCategory,
MtimeCategory,
]

25
core/pe/result_table.py
View File

@ -10,19 +10,20 @@ from hscommon.trans import trget
from core.gui.result_table import ResultTable as ResultTableBase
coltr = trget('columns')
coltr = trget("columns")
class ResultTable(ResultTableBase):
COLUMNS = [
Column('marked', ''),
Column('name', coltr("Filename")),
Column('folder_path', coltr("Folder"), optional=True),
Column('size', coltr("Size (KB)"), optional=True),
Column('extension', coltr("Kind"), visible=False, optional=True),
Column('dimensions', coltr("Dimensions"), optional=True),
Column('exif_timestamp', coltr("EXIF Timestamp"), visible=False, optional=True),
Column('mtime', coltr("Modification"), visible=False, optional=True),
Column('percentage', coltr("Match %"), optional=True),
Column('dupe_count', coltr("Dupe Count"), visible=False, optional=True),
Column("marked", ""),
Column("name", coltr("Filename")),
Column("folder_path", coltr("Folder"), optional=True),
Column("size", coltr("Size (KB)"), optional=True),
Column("extension", coltr("Kind"), visible=False, optional=True),
Column("dimensions", coltr("Dimensions"), optional=True),
Column("exif_timestamp", coltr("EXIF Timestamp"), visible=False, optional=True),
Column("mtime", coltr("Modification"), visible=False, optional=True),
Column("percentage", coltr("Match %"), optional=True),
Column("dupe_count", coltr("Dupe Count"), visible=False, optional=True),
]
DELTA_COLUMNS = {'size', 'dimensions', 'mtime'}
DELTA_COLUMNS = {"size", "dimensions", "mtime"}

4
core/pe/scanner.py
View File

@ -10,6 +10,7 @@ from core.scanner import Scanner, ScanType, ScanOption
from . import matchblock, matchexif
class ScannerPE(Scanner):
cache_path = None
match_scaled = False
@ -28,10 +29,9 @@ class ScannerPE(Scanner):
cache_path=self.cache_path,
threshold=self.min_match_percentage,
match_scaled=self.match_scaled,
j=j
j=j,
)
elif self.scan_type == ScanType.ExifTimestamp:
return matchexif.getmatches(files, self.match_scaled, j)
else:
raise Exception("Invalid scan type")

17
core/prioritize.py
View File

@ -9,7 +9,8 @@
from hscommon.util import dedupe, flatten, rem_file_ext
from hscommon.trans import trget, tr
coltr = trget('columns')
coltr = trget("columns")
class CriterionCategory:
NAME = "Undefined"
@ -30,6 +31,7 @@ class CriterionCategory:
def criteria_list(self):
raise NotImplementedError()
class Criterion:
def __init__(self, category, value):
self.category = category
@ -68,6 +70,7 @@ class KindCategory(ValueListCategory):
value = tr("None")
return value
class FolderCategory(ValueListCategory):
NAME = coltr("Folder")
@ -84,6 +87,7 @@ class FolderCategory(ValueListCategory):
else:
return 1
class FilenameCategory(CriterionCategory):
NAME = coltr("Filename")
ENDS_WITH_NUMBER = 0
@ -117,12 +121,16 @@ class FilenameCategory(CriterionCategory):
return value
def criteria_list(self):
return [Criterion(self, crit_value) for crit_value in [
return [
Criterion(self, crit_value)
for crit_value in [
self.ENDS_WITH_NUMBER,
self.DOESNT_END_WITH_NUMBER,
self.LONGEST,
self.SHORTEST,
]]
]
]
class NumericalCategory(CriterionCategory):
HIGHEST = 0
@ -143,12 +151,14 @@ class NumericalCategory(CriterionCategory):
def criteria_list(self):
return [Criterion(self, self.HIGHEST), Criterion(self, self.LOWEST)]
class SizeCategory(NumericalCategory):
NAME = coltr("Size")
def extract_value(self, dupe):
return dupe.size
class MtimeCategory(NumericalCategory):
NAME = coltr("Modification")
@ -158,5 +168,6 @@ class MtimeCategory(NumericalCategory):
def format_criterion_value(self, value):
return tr("Newest") if value == self.HIGHEST else tr("Oldest")
def all_categories():
return [KindCategory, FolderCategory, FilenameCategory, SizeCategory, MtimeCategory]

91
core/results.py
View File

@ -20,6 +20,7 @@ from hscommon.trans import tr
from . import engine
from .markable import Markable
class Results(Markable):
"""Manages a collection of duplicate :class:`~core.engine.Group`.
@ -34,6 +35,7 @@ class Results(Markable):
A list of all duplicates (:class:`~core.fs.File` instances), without ref, contained in the
currently managed :attr:`groups`.
"""
# ---Override
def __init__(self, app):
Markable.__init__(self)
@ -98,10 +100,13 @@ class Results(Markable):
# This is debug logging to try to figure out #44
logging.warning(
"There is a None value in the Results' dupe list. dupes: %r groups: %r",
self.__dupes, self.groups
self.__dupes,
self.groups,
)
if self.__filtered_dupes:
self.__dupes = [dupe for dupe in self.__dupes if dupe in self.__filtered_dupes]
self.__dupes = [
dupe for dupe in self.__dupes if dupe in self.__filtered_dupes
]
sd = self.__dupes_sort_descriptor
if sd:
self.sort_dupes(sd[0], sd[1], sd[2])
@ -120,10 +125,18 @@ class Results(Markable):
total_count = self.__total_count
total_size = self.__total_size
else:
mark_count = len([dupe for dupe in self.__filtered_dupes if self.is_marked(dupe)])
marked_size = sum(dupe.size for dupe in self.__filtered_dupes if self.is_marked(dupe))
total_count = len([dupe for dupe in self.__filtered_dupes if self.is_markable(dupe)])
total_size = sum(dupe.size for dupe in self.__filtered_dupes if self.is_markable(dupe))
mark_count = len(
[dupe for dupe in self.__filtered_dupes if self.is_marked(dupe)]
)
marked_size = sum(
dupe.size for dupe in self.__filtered_dupes if self.is_marked(dupe)
)
total_count = len(
[dupe for dupe in self.__filtered_dupes if self.is_markable(dupe)]
)
total_size = sum(
dupe.size for dupe in self.__filtered_dupes if self.is_markable(dupe)
)
if self.mark_inverted:
marked_size = self.__total_size - marked_size
result = tr("%d / %d (%s / %s) duplicates marked.") % (
@ -133,7 +146,7 @@ class Results(Markable):
format_size(total_size, 2),
)
if self.__filters:
result += tr(" filter: %s") % ' --> '.join(self.__filters)
result += tr(" filter: %s") % " --> ".join(self.__filters)
return result
def __recalculate_stats(self):
@ -151,7 +164,7 @@ class Results(Markable):
for g in self.__groups:
for dupe in g:
self.__group_of_duplicate[dupe] = g
if not hasattr(dupe, 'is_ref'):
if not hasattr(dupe, "is_ref"):
dupe.is_ref = False
self.is_modified = bool(self.__groups)
old_filters = nonone(self.__filters, [])
@ -186,7 +199,11 @@ class Results(Markable):
self.__filters.append(filter_str)
if self.__filtered_dupes is None:
self.__filtered_dupes = flatten(g[:] for g in self.groups)
self.__filtered_dupes = set(dupe for dupe in self.__filtered_dupes if filter_re.search(str(dupe.path)))
self.__filtered_dupes = set(
dupe
for dupe in self.__filtered_dupes
if filter_re.search(str(dupe.path))
)
filtered_groups = set()
for dupe in self.__filtered_dupes:
filtered_groups.add(self.get_group_of_duplicate(dupe))
@ -214,6 +231,7 @@ class Results(Markable):
:param get_file: a function f(path) returning a :class:`~core.fs.File` wrapping the path.
:param j: A :ref:`job progress instance <jobs>`.
"""
def do_match(ref_file, other_files, group):
if not other_files:
return
@ -223,31 +241,31 @@ class Results(Markable):
self.apply_filter(None)
root = ET.parse(infile).getroot()
group_elems = list(root.getiterator('group'))
group_elems = list(root.getiterator("group"))
groups = []
marked = set()
for group_elem in j.iter_with_progress(group_elems, every=100):
group = engine.Group()
dupes = []
for file_elem in group_elem.getiterator('file'):
path = file_elem.get('path')
words = file_elem.get('words', '')
for file_elem in group_elem.getiterator("file"):
path = file_elem.get("path")
words = file_elem.get("words", "")
if not path:
continue
file = get_file(path)
if file is None:
continue
file.words = words.split(',')
file.is_ref = file_elem.get('is_ref') == 'y'
file.words = words.split(",")
file.is_ref = file_elem.get("is_ref") == "y"
dupes.append(file)
if file_elem.get('marked') == 'y':
if file_elem.get("marked") == "y":
marked.add(file)
for match_elem in group_elem.getiterator('match'):
for match_elem in group_elem.getiterator("match"):
try:
attrs = match_elem.attrib
first_file = dupes[int(attrs['first'])]
second_file = dupes[int(attrs['second'])]
percentage = int(attrs['percentage'])
first_file = dupes[int(attrs["first"])]
second_file = dupes[int(attrs["second"])]
percentage = int(attrs["percentage"])
group.add_match(engine.Match(first_file, second_file, percentage))
except (IndexError, KeyError, ValueError):
# Covers missing attr, non-int values and indexes out of bounds
@ -339,9 +357,9 @@ class Results(Markable):
:param outfile: file object or path.
"""
self.apply_filter(None)
root = ET.Element('results')
root = ET.Element("results")
for g in self.groups:
group_elem = ET.SubElement(root, 'group')
group_elem = ET.SubElement(root, "group")
dupe2index = {}
for index, d in enumerate(g):
dupe2index[d] = index
@ -349,24 +367,24 @@ class Results(Markable):
words = engine.unpack_fields(d.words)
except AttributeError:
words = ()
file_elem = ET.SubElement(group_elem, 'file')
file_elem = ET.SubElement(group_elem, "file")
try:
file_elem.set('path', str(d.path))
file_elem.set('words', ','.join(words))
file_elem.set("path", str(d.path))
file_elem.set("words", ",".join(words))
except ValueError: # If there's an invalid character, just skip the file
file_elem.set('path', '')
file_elem.set('is_ref', ('y' if d.is_ref else 'n'))
file_elem.set('marked', ('y' if self.is_marked(d) else 'n'))
file_elem.set("path", "")
file_elem.set("is_ref", ("y" if d.is_ref else "n"))
file_elem.set("marked", ("y" if self.is_marked(d) else "n"))
for match in g.matches:
match_elem = ET.SubElement(group_elem, 'match')
match_elem.set('first', str(dupe2index[match.first]))
match_elem.set('second', str(dupe2index[match.second]))
match_elem.set('percentage', str(int(match.percentage)))
match_elem = ET.SubElement(group_elem, "match")
match_elem.set("first", str(dupe2index[match.first]))
match_elem.set("second", str(dupe2index[match.second]))
match_elem.set("percentage", str(int(match.percentage)))
tree = ET.ElementTree(root)
def do_write(outfile):
with FileOrPath(outfile, 'wb') as fp:
tree.write(fp, encoding='utf-8')
with FileOrPath(outfile, "wb") as fp:
tree.write(fp, encoding="utf-8")
try:
do_write(outfile)
@ -392,7 +410,9 @@ class Results(Markable):
"""
if not self.__dupes:
self.__get_dupe_list()
keyfunc = lambda d: self.app._get_dupe_sort_key(d, lambda: self.get_group_of_duplicate(d), key, delta)
keyfunc = lambda d: self.app._get_dupe_sort_key(
d, lambda: self.get_group_of_duplicate(d), key, delta
)
self.__dupes.sort(key=keyfunc, reverse=not asc)
self.__dupes_sort_descriptor = (key, asc, delta)
@ -412,4 +432,3 @@ class Results(Markable):
dupes = property(__get_dupe_list)
groups = property(__get_groups, __set_groups)
stat_line = property(__get_stat_line)

69
core/scanner.py
View File

@ -19,6 +19,7 @@ from . import engine
# there will be some nasty bugs popping up (ScanType is used in core when in should exclusively be
# used in core_*). One day I'll clean this up.
class ScanType:
Filename = 0
Fields = 1
@ -31,11 +32,13 @@ class ScanType:
FuzzyBlock = 10
ExifTimestamp = 11
ScanOption = namedtuple('ScanOption', 'scan_type label')
SCANNABLE_TAGS = ['track', 'artist', 'album', 'title', 'genre', 'year']
ScanOption = namedtuple("ScanOption", "scan_type label")
SCANNABLE_TAGS = ["track", "artist", "album", "title", "genre", "year"]
RE_DIGIT_ENDING = re.compile(r"\d+|\(\d+\)|\[\d+\]|{\d+}")
RE_DIGIT_ENDING = re.compile(r'\d+|\(\d+\)|\[\d+\]|{\d+}')
def is_same_with_digit(name, refname):
# Returns True if name is the same as refname, but with digits (with brackets or not) at the end
@ -44,6 +47,7 @@ def is_same_with_digit(name, refname):
end = name[len(refname) :].strip()
return RE_DIGIT_ENDING.match(end) is not None
def remove_dupe_paths(files):
# Returns files with duplicates-by-path removed. Files with the exact same path are considered
# duplicates and only the first file to have a path is kept. In certain cases, we have files
@ -67,12 +71,16 @@ def remove_dupe_paths(files):
result.append(f)
return result
class Scanner:
def __init__(self):
self.discarded_file_count = 0
def _getmatches(self, files, j):
if self.size_threshold or self.scan_type in {ScanType.Contents, ScanType.Folders}:
if self.size_threshold or self.scan_type in {
ScanType.Contents,
ScanType.Folders,
}:
j = j.start_subjob([2, 8])
for f in j.iter_with_progress(files, tr("Read size of %d/%d files")):
f.size # pre-read, makes a smoother progress if read here (especially for bundles)
@ -83,12 +91,12 @@ class Scanner:
else:
j = j.start_subjob([2, 8])
kw = {}
kw['match_similar_words'] = self.match_similar_words
kw['weight_words'] = self.word_weighting
kw['min_match_percentage'] = self.min_match_percentage
kw["match_similar_words"] = self.match_similar_words
kw["weight_words"] = self.word_weighting
kw["min_match_percentage"] = self.min_match_percentage
if self.scan_type == ScanType.FieldsNoOrder:
self.scan_type = ScanType.Fields
kw['no_field_order'] = True
kw["no_field_order"] = True
func = {
ScanType.Filename: lambda f: engine.getwords(rem_file_ext(f.name)),
ScanType.Fields: lambda f: engine.getfields(rem_file_ext(f.name)),
@ -111,9 +119,9 @@ class Scanner:
def _tie_breaker(ref, dupe):
refname = rem_file_ext(ref.name).lower()
dupename = rem_file_ext(dupe.name).lower()
if 'copy' in dupename:
if "copy" in dupename:
return False
if 'copy' in refname:
if "copy" in refname:
return True
if is_same_with_digit(dupename, refname):
return False
@ -130,12 +138,12 @@ class Scanner:
raise NotImplementedError()
def get_dupe_groups(self, files, ignore_list=None, j=job.nulljob):
for f in (f for f in files if not hasattr(f, 'is_ref')):
for f in (f for f in files if not hasattr(f, "is_ref")):
f.is_ref = False
files = remove_dupe_paths(files)
logging.info("Getting matches. Scan type: %d", self.scan_type)
matches = self._getmatches(files, j)
logging.info('Found %d matches' % len(matches))
logging.info("Found %d matches" % len(matches))
j.set_progress(100, tr("Almost done! Fiddling with results..."))
# In removing what we call here "false matches", we first want to remove, if we scan by
# folders, we want to remove folder matches for which the parent is also in a match (they're
@ -153,20 +161,38 @@ class Scanner:
toremove.add(p)
else:
last_parent_path = p
matches = [m for m in matches if m.first.path not in toremove or m.second.path not in toremove]
matches = [
m
for m in matches
if m.first.path not in toremove or m.second.path not in toremove
]
if not self.mix_file_kind:
matches = [m for m in matches if get_file_ext(m.first.name) == get_file_ext(m.second.name)]
matches = [m for m in matches if m.first.path.exists() and m.second.path.exists()]
matches = [
m
for m in matches
if get_file_ext(m.first.name) == get_file_ext(m.second.name)
]
matches = [
m for m in matches if m.first.path.exists() and m.second.path.exists()
]
matches = [m for m in matches if not (m.first.is_ref and m.second.is_ref)]
if ignore_list:
matches = [
m for m in matches
m
for m in matches
if not ignore_list.AreIgnored(str(m.first.path), str(m.second.path))
]
logging.info('Grouping matches')
logging.info("Grouping matches")
groups = engine.get_groups(matches)
if self.scan_type in {ScanType.Filename, ScanType.Fields, ScanType.FieldsNoOrder, ScanType.Tag}:
matched_files = dedupe([m.first for m in matches] + [m.second for m in matches])
if self.scan_type in {
ScanType.Filename,
ScanType.Fields,
ScanType.FieldsNoOrder,
ScanType.Tag,
}:
matched_files = dedupe(
[m.first for m in matches] + [m.second for m in matches]
)
self.discarded_file_count = len(matched_files) - sum(len(g) for g in groups)
else:
# Ticket #195
@ -181,7 +207,7 @@ class Scanner:
# reporting discarded matches.
self.discarded_file_count = 0
groups = [g for g in groups if any(not f.is_ref for f in g)]
logging.info('Created %d groups' % len(groups))
logging.info("Created %d groups" % len(groups))
for g in groups:
g.prioritize(self._key_func, self._tie_breaker)
return groups
@ -190,7 +216,6 @@ class Scanner:
min_match_percentage = 80
mix_file_kind = True
scan_type = ScanType.Filename
scanned_tags = {'artist', 'title'}
scanned_tags = {"artist", "title"}
size_threshold = 0
word_weighting = False

19
core/se/fs.py
View File

@ -11,6 +11,7 @@ from hscommon.util import format_size
from core import fs
from core.util import format_timestamp, format_perc, format_words, format_dupe_count
def get_display_info(dupe, group, delta):
size = dupe.size
mtime = dupe.mtime
@ -26,16 +27,17 @@ def get_display_info(dupe, group, delta):
percentage = group.percentage
dupe_count = len(group.dupes)
return {
'name': dupe.name,
'folder_path': str(dupe.folder_path),
'size': format_size(size, 0, 1, False),
'extension': dupe.extension,
'mtime': format_timestamp(mtime, delta and m),
'percentage': format_perc(percentage),
'words': format_words(dupe.words) if hasattr(dupe, 'words') else '',
'dupe_count': format_dupe_count(dupe_count),
"name": dupe.name,
"folder_path": str(dupe.folder_path),
"size": format_size(size, 0, 1, False),
"extension": dupe.extension,
"mtime": format_timestamp(mtime, delta and m),
"percentage": format_perc(percentage),
"words": format_words(dupe.words) if hasattr(dupe, "words") else "",
"dupe_count": format_dupe_count(dupe_count),
}
class File(fs.File):
def get_display_info(self, group, delta):
return get_display_info(self, group, delta)
@ -44,4 +46,3 @@ class File(fs.File):
class Folder(fs.Folder):
def get_display_info(self, group, delta):
return get_display_info(self, group, delta)

23
core/se/result_table.py
View File

@ -10,18 +10,19 @@ from hscommon.trans import trget
from core.gui.result_table import ResultTable as ResultTableBase
coltr = trget('columns')
coltr = trget("columns")
class ResultTable(ResultTableBase):
COLUMNS = [
Column('marked', ''),
Column('name', coltr("Filename")),
Column('folder_path', coltr("Folder"), optional=True),
Column('size', coltr("Size (KB)"), optional=True),
Column('extension', coltr("Kind"), visible=False, optional=True),
Column('mtime', coltr("Modification"), visible=False, optional=True),
Column('percentage', coltr("Match %"), optional=True),
Column('words', coltr("Words Used"), visible=False, optional=True),
Column('dupe_count', coltr("Dupe Count"), visible=False, optional=True),
Column("marked", ""),
Column("name", coltr("Filename")),
Column("folder_path", coltr("Folder"), optional=True),
Column("size", coltr("Size (KB)"), optional=True),
Column("extension", coltr("Kind"), visible=False, optional=True),
Column("mtime", coltr("Modification"), visible=False, optional=True),
Column("percentage", coltr("Match %"), optional=True),
Column("words", coltr("Words Used"), visible=False, optional=True),
Column("dupe_count", coltr("Dupe Count"), visible=False, optional=True),
]
DELTA_COLUMNS = {'size', 'mtime'}
DELTA_COLUMNS = {"size", "mtime"}

2
core/se/scanner.py
View File

@ -8,6 +8,7 @@ from hscommon.trans import tr
from core.scanner import Scanner as ScannerBase, ScanOption, ScanType
class ScannerSE(ScannerBase):
@staticmethod
def get_scan_options():
@ -16,4 +17,3 @@ class ScannerSE(ScannerBase):
ScanOption(ScanType.Contents, tr("Contents")),
ScanOption(ScanType.Folders, tr("Folders")),
]

211
core/tests/app_test.py
View File

@ -20,77 +20,90 @@ from .results_test import GetTestGroups
from .. import app, fs, engine
from ..scanner import ScanType
def add_fake_files_to_directories(directories, files):
directories.get_files = lambda j=None: iter(files)
directories._dirs.append('this is just so Scan() doesnt return 3')
directories._dirs.append("this is just so Scan() doesnt return 3")
class TestCaseDupeGuru:
def test_apply_filter_calls_results_apply_filter(self, monkeypatch):
dgapp = TestApp().app
monkeypatch.setattr(dgapp.results, 'apply_filter', log_calls(dgapp.results.apply_filter))
dgapp.apply_filter('foo')
monkeypatch.setattr(
dgapp.results, "apply_filter", log_calls(dgapp.results.apply_filter)
)
dgapp.apply_filter("foo")
eq_(2, len(dgapp.results.apply_filter.calls))
call = dgapp.results.apply_filter.calls[0]
assert call['filter_str'] is None
assert call["filter_str"] is None
call = dgapp.results.apply_filter.calls[1]
eq_('foo', call['filter_str'])
eq_("foo", call["filter_str"])
def test_apply_filter_escapes_regexp(self, monkeypatch):
dgapp = TestApp().app
monkeypatch.setattr(dgapp.results, 'apply_filter', log_calls(dgapp.results.apply_filter))
dgapp.apply_filter('()[]\\.|+?^abc')
monkeypatch.setattr(
dgapp.results, "apply_filter", log_calls(dgapp.results.apply_filter)
)
dgapp.apply_filter("()[]\\.|+?^abc")
call = dgapp.results.apply_filter.calls[1]
eq_('\\(\\)\\[\\]\\\\\\.\\|\\+\\?\\^abc', call['filter_str'])
dgapp.apply_filter('(*)') # In "simple mode", we want the * to behave as a wilcard
eq_("\\(\\)\\[\\]\\\\\\.\\|\\+\\?\\^abc", call["filter_str"])
dgapp.apply_filter(
"(*)"
) # In "simple mode", we want the * to behave as a wilcard
call = dgapp.results.apply_filter.calls[3]
eq_(r'\(.*\)', call['filter_str'])
dgapp.options['escape_filter_regexp'] = False
dgapp.apply_filter('(abc)')
eq_(r"\(.*\)", call["filter_str"])
dgapp.options["escape_filter_regexp"] = False
dgapp.apply_filter("(abc)")
call = dgapp.results.apply_filter.calls[5]
eq_('(abc)', call['filter_str'])
eq_("(abc)", call["filter_str"])
def test_copy_or_move(self, tmpdir, monkeypatch):
# The goal here is just to have a test for a previous blowup I had. I know my test coverage
# for this unit is pathetic. What's done is done. My approach now is to add tests for
# every change I want to make. The blowup was caused by a missing import.
p = Path(str(tmpdir))
p['foo'].open('w').close()
monkeypatch.setattr(hscommon.conflict, 'smart_copy', log_calls(lambda source_path, dest_path: None))
p["foo"].open("w").close()
monkeypatch.setattr(
hscommon.conflict,
"smart_copy",
log_calls(lambda source_path, dest_path: None),
)
# XXX This monkeypatch is temporary. will be fixed in a better monkeypatcher.
monkeypatch.setattr(app, 'smart_copy', hscommon.conflict.smart_copy)
monkeypatch.setattr(os, 'makedirs', lambda path: None) # We don't want the test to create that fake directory
monkeypatch.setattr(app, "smart_copy", hscommon.conflict.smart_copy)
monkeypatch.setattr(
os, "makedirs", lambda path: None
) # We don't want the test to create that fake directory
dgapp = TestApp().app
dgapp.directories.add_path(p)
[f] = dgapp.directories.get_files()
dgapp.copy_or_move(f, True, 'some_destination', 0)
dgapp.copy_or_move(f, True, "some_destination", 0)
eq_(1, len(hscommon.conflict.smart_copy.calls))
call = hscommon.conflict.smart_copy.calls[0]
eq_(call['dest_path'], op.join('some_destination', 'foo'))
eq_(call['source_path'], f.path)
eq_(call["dest_path"], op.join("some_destination", "foo"))
eq_(call["source_path"], f.path)
def test_copy_or_move_clean_empty_dirs(self, tmpdir, monkeypatch):
tmppath = Path(str(tmpdir))
sourcepath = tmppath['source']
sourcepath = tmppath["source"]
sourcepath.mkdir()
sourcepath['myfile'].open('w')
sourcepath["myfile"].open("w")
app = TestApp().app
app.directories.add_path(tmppath)
[myfile] = app.directories.get_files()
monkeypatch.setattr(app, 'clean_empty_dirs', log_calls(lambda path: None))
app.copy_or_move(myfile, False, tmppath['dest'], 0)
monkeypatch.setattr(app, "clean_empty_dirs", log_calls(lambda path: None))
app.copy_or_move(myfile, False, tmppath["dest"], 0)
calls = app.clean_empty_dirs.calls
eq_(1, len(calls))
eq_(sourcepath, calls[0]['path'])
eq_(sourcepath, calls[0]["path"])
def test_Scan_with_objects_evaluating_to_false(self):
class FakeFile(fs.File):
def __bool__(self):
return False
# At some point, any() was used in a wrong way that made Scan() wrongly return 1
app = TestApp().app
f1, f2 = [FakeFile('foo') for i in range(2)]
f1, f2 = [FakeFile("foo") for i in range(2)]
f1.is_ref, f2.is_ref = (False, False)
assert not (bool(f1) and bool(f2))
add_fake_files_to_directories(app.directories, [f1, f2])
@ -101,12 +114,12 @@ class TestCaseDupeGuru:
# If the ignore_hardlink_matches option is set, don't match files hardlinking to the same
# inode.
tmppath = Path(str(tmpdir))
tmppath['myfile'].open('w').write('foo')
os.link(str(tmppath['myfile']), str(tmppath['hardlink']))
tmppath["myfile"].open("w").write("foo")
os.link(str(tmppath["myfile"]), str(tmppath["hardlink"]))
app = TestApp().app
app.directories.add_path(tmppath)
app.options['scan_type'] = ScanType.Contents
app.options['ignore_hardlink_matches'] = True
app.options["scan_type"] = ScanType.Contents
app.options["ignore_hardlink_matches"] = True
app.start_scanning()
eq_(len(app.results.groups), 0)
@ -116,27 +129,32 @@ class TestCaseDupeGuru:
# making the selected row None. Don't crash when it happens.
dgapp = TestApp().app
# selected_row is None because there's no result.
assert not dgapp.result_table.rename_selected('foo') # no crash
assert not dgapp.result_table.rename_selected("foo") # no crash
class TestCaseDupeGuru_clean_empty_dirs:
def pytest_funcarg__do_setup(self, request):
monkeypatch = request.getfuncargvalue('monkeypatch')
monkeypatch.setattr(hscommon.util, 'delete_if_empty', log_calls(lambda path, files_to_delete=[]: None))
monkeypatch = request.getfuncargvalue("monkeypatch")
monkeypatch.setattr(
hscommon.util,
"delete_if_empty",
log_calls(lambda path, files_to_delete=[]: None),
)
# XXX This monkeypatch is temporary. will be fixed in a better monkeypatcher.
monkeypatch.setattr(app, 'delete_if_empty', hscommon.util.delete_if_empty)
monkeypatch.setattr(app, "delete_if_empty", hscommon.util.delete_if_empty)
self.app = TestApp().app
def test_option_off(self, do_setup):
self.app.clean_empty_dirs(Path('/foo/bar'))
self.app.clean_empty_dirs(Path("/foo/bar"))
eq_(0, len(hscommon.util.delete_if_empty.calls))
def test_option_on(self, do_setup):
self.app.options['clean_empty_dirs'] = True
self.app.clean_empty_dirs(Path('/foo/bar'))
self.app.options["clean_empty_dirs"] = True
self.app.clean_empty_dirs(Path("/foo/bar"))
calls = hscommon.util.delete_if_empty.calls
eq_(1, len(calls))
eq_(Path('/foo/bar'), calls[0]['path'])
eq_(['.DS_Store'], calls[0]['files_to_delete'])
eq_(Path("/foo/bar"), calls[0]["path"])
eq_([".DS_Store"], calls[0]["files_to_delete"])
def test_recurse_up(self, do_setup, monkeypatch):
# delete_if_empty must be recursively called up in the path until it returns False
@ -144,16 +162,16 @@ class TestCaseDupeGuru_clean_empty_dirs:
def mock_delete_if_empty(path, files_to_delete=[]):
return len(path) > 1
monkeypatch.setattr(hscommon.util, 'delete_if_empty', mock_delete_if_empty)
monkeypatch.setattr(hscommon.util, "delete_if_empty", mock_delete_if_empty)
# XXX This monkeypatch is temporary. will be fixed in a better monkeypatcher.
monkeypatch.setattr(app, 'delete_if_empty', mock_delete_if_empty)
self.app.options['clean_empty_dirs'] = True
self.app.clean_empty_dirs(Path('not-empty/empty/empty'))
monkeypatch.setattr(app, "delete_if_empty", mock_delete_if_empty)
self.app.options["clean_empty_dirs"] = True
self.app.clean_empty_dirs(Path("not-empty/empty/empty"))
calls = hscommon.util.delete_if_empty.calls
eq_(3, len(calls))
eq_(Path('not-empty/empty/empty'), calls[0]['path'])
eq_(Path('not-empty/empty'), calls[1]['path'])
eq_(Path('not-empty'), calls[2]['path'])
eq_(Path("not-empty/empty/empty"), calls[0]["path"])
eq_(Path("not-empty/empty"), calls[1]["path"])
eq_(Path("not-empty"), calls[2]["path"])
class TestCaseDupeGuruWithResults:
@ -166,10 +184,10 @@ class TestCaseDupeGuruWithResults:
self.dtree = app.dtree
self.rtable = app.rtable
self.rtable.refresh()
tmpdir = request.getfuncargvalue('tmpdir')
tmpdir = request.getfuncargvalue("tmpdir")
tmppath = Path(str(tmpdir))
tmppath['foo'].mkdir()
tmppath['bar'].mkdir()
tmppath["foo"].mkdir()
tmppath["bar"].mkdir()
self.app.directories.add_path(tmppath)
def test_GetObjects(self, do_setup):
@ -188,7 +206,7 @@ class TestCaseDupeGuruWithResults:
def test_GetObjects_after_sort(self, do_setup):
objects = self.objects
groups = self.groups[:] # we need an un-sorted reference
self.rtable.sort('name', False)
self.rtable.sort("name", False)
r = self.rtable[1]
assert r._group is groups[1]
assert r._dupe is objects[4]
@ -221,7 +239,7 @@ class TestCaseDupeGuruWithResults:
app = self.app
objects = self.objects
groups = self.groups[:] # To keep the old order in memory
self.rtable.sort('name', False) #0
self.rtable.sort("name", False) # 0
# Now, the group order is supposed to be reversed
self.rtable.select([1, 2, 3])
eq_(len(app.selected_dupes), 3)
@ -248,7 +266,7 @@ class TestCaseDupeGuruWithResults:
app = self.app
objects = self.objects
self.rtable.power_marker = True
self.rtable.sort('name', False)
self.rtable.sort("name", False)
self.rtable.select([0, 1, 2])
eq_(len(app.selected_dupes), 3)
assert app.selected_dupes[0] is objects[4]
@ -285,11 +303,11 @@ class TestCaseDupeGuruWithResults:
def test_refreshDetailsWithSelected(self, do_setup):
self.rtable.select([1, 4])
eq_(self.dpanel.row(0), ('Filename', 'bar bleh', 'foo bar'))
self.dpanel.view.check_gui_calls(['refresh'])
eq_(self.dpanel.row(0), ("Filename", "bar bleh", "foo bar"))
self.dpanel.view.check_gui_calls(["refresh"])
self.rtable.select([])
eq_(self.dpanel.row(0), ('Filename', '---', '---'))
self.dpanel.view.check_gui_calls(['refresh'])
eq_(self.dpanel.row(0), ("Filename", "---", "---"))
self.dpanel.view.check_gui_calls(["refresh"])
def test_makeSelectedReference(self, do_setup):
app = self.app
@ -300,7 +318,9 @@ class TestCaseDupeGuruWithResults:
assert groups[0].ref is objects[1]
assert groups[1].ref is objects[4]
def test_makeSelectedReference_by_selecting_two_dupes_in_the_same_group(self, do_setup):
def test_makeSelectedReference_by_selecting_two_dupes_in_the_same_group(
self, do_setup
):
app = self.app
objects = self.objects
groups = self.groups
@ -336,7 +356,7 @@ class TestCaseDupeGuruWithResults:
def test_addDirectory_does_not_exist(self, do_setup):
app = self.app
app.add_directory('/does_not_exist')
app.add_directory("/does_not_exist")
eq_(len(app.view.messages), 1)
assert "exist" in app.view.messages[0]
@ -352,11 +372,11 @@ class TestCaseDupeGuruWithResults:
def test_purgeIgnoreList(self, do_setup, tmpdir):
app = self.app
p1 = str(tmpdir.join('file1'))
p2 = str(tmpdir.join('file2'))
open(p1, 'w').close()
open(p2, 'w').close()
dne = '/does_not_exist'
p1 = str(tmpdir.join("file1"))
p2 = str(tmpdir.join("file2"))
open(p1, "w").close()
open(p2, "w").close()
dne = "/does_not_exist"
app.ignore_list.Ignore(dne, p1)
app.ignore_list.Ignore(p2, dne)
app.ignore_list.Ignore(p1, p2)
@ -382,7 +402,9 @@ class TestCaseDupeGuruWithResults:
# results table.
app = self.app
app.JOB = Job(1, lambda *args, **kw: False) # Cancels the task
add_fake_files_to_directories(app.directories, self.objects) # We want the scan to at least start
add_fake_files_to_directories(
app.directories, self.objects
) # We want the scan to at least start
app.start_scanning() # will be cancelled immediately
eq_(len(app.result_table), 0)
@ -401,21 +423,21 @@ class TestCaseDupeGuruWithResults:
# Ref #238
self.rtable.delta_values = True
self.rtable.power_marker = True
self.rtable.sort('dupe_count', False)
self.rtable.sort("dupe_count", False)
# don't crash
self.rtable.sort('percentage', False)
self.rtable.sort("percentage", False)
# don't crash
class TestCaseDupeGuru_renameSelected:
def pytest_funcarg__do_setup(self, request):
tmpdir = request.getfuncargvalue('tmpdir')
tmpdir = request.getfuncargvalue("tmpdir")
p = Path(str(tmpdir))
fp = open(str(p['foo bar 1']), mode='w')
fp = open(str(p["foo bar 1"]), mode="w")
fp.close()
fp = open(str(p['foo bar 2']), mode='w')
fp = open(str(p["foo bar 2"]), mode="w")
fp.close()
fp = open(str(p['foo bar 3']), mode='w')
fp = open(str(p["foo bar 3"]), mode="w")
fp.close()
files = fs.get_files(p)
for f in files:
@ -437,46 +459,46 @@ class TestCaseDupeGuru_renameSelected:
app = self.app
g = self.groups[0]
self.rtable.select([1])
assert app.rename_selected('renamed')
assert app.rename_selected("renamed")
names = [p.name for p in self.p.listdir()]
assert 'renamed' in names
assert 'foo bar 2' not in names
eq_(g.dupes[0].name, 'renamed')
assert "renamed" in names
assert "foo bar 2" not in names
eq_(g.dupes[0].name, "renamed")
def test_none_selected(self, do_setup, monkeypatch):
app = self.app
g = self.groups[0]
self.rtable.select([])
monkeypatch.setattr(logging, 'warning', log_calls(lambda msg: None))
assert not app.rename_selected('renamed')
msg = logging.warning.calls[0]['msg']
eq_('dupeGuru Warning: list index out of range', msg)
monkeypatch.setattr(logging, "warning", log_calls(lambda msg: None))
assert not app.rename_selected("renamed")
msg = logging.warning.calls[0]["msg"]
eq_("dupeGuru Warning: list index out of range", msg)
names = [p.name for p in self.p.listdir()]
assert 'renamed' not in names
assert 'foo bar 2' in names
eq_(g.dupes[0].name, 'foo bar 2')
assert "renamed" not in names
assert "foo bar 2" in names
eq_(g.dupes[0].name, "foo bar 2")
def test_name_already_exists(self, do_setup, monkeypatch):
app = self.app
g = self.groups[0]
self.rtable.select([1])
monkeypatch.setattr(logging, 'warning', log_calls(lambda msg: None))
assert not app.rename_selected('foo bar 1')
msg = logging.warning.calls[0]['msg']
assert msg.startswith('dupeGuru Warning: \'foo bar 1\' already exists in')
monkeypatch.setattr(logging, "warning", log_calls(lambda msg: None))
assert not app.rename_selected("foo bar 1")
msg = logging.warning.calls[0]["msg"]
assert msg.startswith("dupeGuru Warning: 'foo bar 1' already exists in")
names = [p.name for p in self.p.listdir()]
assert 'foo bar 1' in names
assert 'foo bar 2' in names
eq_(g.dupes[0].name, 'foo bar 2')
assert "foo bar 1" in names
assert "foo bar 2" in names
eq_(g.dupes[0].name, "foo bar 2")
class TestAppWithDirectoriesInTree:
def pytest_funcarg__do_setup(self, request):
tmpdir = request.getfuncargvalue('tmpdir')
tmpdir = request.getfuncargvalue("tmpdir")
p = Path(str(tmpdir))
p['sub1'].mkdir()
p['sub2'].mkdir()
p['sub3'].mkdir()
p["sub1"].mkdir()
p["sub2"].mkdir()
p["sub3"].mkdir()
app = TestApp()
self.app = app.app
self.dtree = app.dtree
@ -494,5 +516,4 @@ class TestAppWithDirectoriesInTree:
eq_(len(node), 3)
subnode = node[0]
eq_(subnode.state, 1)
self.dtree.view.check_gui_calls(['refresh_states'])
self.dtree.view.check_gui_calls(["refresh_states"])

43
core/tests/base.py
View File

@ -17,6 +17,7 @@ from ..app import DupeGuru as DupeGuruBase
from ..gui.result_table import ResultTable as ResultTableBase
from ..gui.prioritize_dialog import PrioritizeDialog
class DupeGuruView:
JOB = nulljob
@ -44,23 +45,27 @@ class DupeGuruView:
def create_results_window(self):
pass
class ResultTable(ResultTableBase):
COLUMNS = [
Column('marked', ''),
Column('name', 'Filename'),
Column('folder_path', 'Directory'),
Column('size', 'Size (KB)'),
Column('extension', 'Kind'),
Column("marked", ""),
Column("name", "Filename"),
Column("folder_path", "Directory"),
Column("size", "Size (KB)"),
Column("extension", "Kind"),
]
DELTA_COLUMNS = {'size', }
DELTA_COLUMNS = {
"size",
}
class DupeGuru(DupeGuruBase):
NAME = 'dupeGuru'
METADATA_TO_READ = ['size']
NAME = "dupeGuru"
METADATA_TO_READ = ["size"]
def __init__(self):
DupeGuruBase.__init__(self, DupeGuruView())
self.appdata = '/tmp'
self.appdata = "/tmp"
self._recreate_result_table()
def _prioritization_categories(self):
@ -78,7 +83,7 @@ class NamedObject:
def __init__(self, name="foobar", with_words=False, size=1, folder=None):
self.name = name
if folder is None:
folder = 'basepath'
folder = "basepath"
self._folder = Path(folder)
self.size = size
self.md5partial = name
@ -97,10 +102,10 @@ class NamedObject:
r = group.ref
size -= r.size
return {
'name': self.name,
'folder_path': str(self.folder_path),
'size': format_size(size, 0, 1, False),
'extension': self.extension if hasattr(self, 'extension') else '---',
"name": self.name,
"folder_path": str(self.folder_path),
"size": format_size(size, 0, 1, False),
"extension": self.extension if hasattr(self, "extension") else "---",
}
@property
@ -115,6 +120,7 @@ class NamedObject:
def extension(self):
return get_file_ext(self.name)
# Returns a group set that looks like that:
# "foo bar" (1)
# "bar bleh" (1024)
@ -127,21 +133,24 @@ def GetTestGroups():
NamedObject("bar bleh"),
NamedObject("foo bleh"),
NamedObject("ibabtu"),
NamedObject("ibabtu")
NamedObject("ibabtu"),
]
objects[1].size = 1024
matches = engine.getmatches(objects) # we should have 5 matches
groups = engine.get_groups(matches) # We should have 2 groups
for g in groups:
g.prioritize(lambda x: objects.index(x)) #We want the dupes to be in the same order as the list is
g.prioritize(
lambda x: objects.index(x)
) # We want the dupes to be in the same order as the list is
groups.sort(key=len, reverse=True) # We want the group with 3 members to be first.
return (objects, matches, groups)
class TestApp(TestAppBase):
def __init__(self):
def link_gui(gui):
gui.view = self.make_logger()
if hasattr(gui, 'columns'): # tables
if hasattr(gui, "columns"): # tables
gui.columns.view = self.make_logger()
return gui

30
core/tests/block_test.py
View File

@ -13,13 +13,18 @@ try:
except ImportError:
skip("Can't import the block module, probably hasn't been compiled.")
def my_avgdiff(first, second, limit=768, min_iter=3): # this is so I don't have to re-write every call
def my_avgdiff(
first, second, limit=768, min_iter=3
): # this is so I don't have to re-write every call
return avgdiff(first, second, limit, min_iter)
BLACK = (0, 0, 0)
RED = (0xff, 0, 0)
GREEN = (0, 0xff, 0)
BLUE = (0, 0, 0xff)
RED = (0xFF, 0, 0)
GREEN = (0, 0xFF, 0)
BLUE = (0, 0, 0xFF)
class FakeImage:
def __init__(self, size, data):
@ -37,16 +42,20 @@ class FakeImage:
pixels.append(pixel)
return FakeImage((box[2] - box[0], box[3] - box[1]), pixels)
def empty():
return FakeImage((0, 0), [])
def single_pixel(): # one red pixel
return FakeImage((1, 1), [(0xff, 0, 0)])
return FakeImage((1, 1), [(0xFF, 0, 0)])
def four_pixels():
pixels = [RED, (0, 0x80, 0xff), (0x80, 0, 0), (0, 0x40, 0x80)]
pixels = [RED, (0, 0x80, 0xFF), (0x80, 0, 0), (0, 0x40, 0x80)]
return FakeImage((2, 2), pixels)
class TestCasegetblock:
def test_single_pixel(self):
im = single_pixel()
@ -60,9 +69,9 @@ class TestCasegetblock:
def test_four_pixels(self):
im = four_pixels()
[b] = getblocks2(im, 1)
meanred = (0xff + 0x80) // 4
meanred = (0xFF + 0x80) // 4
meangreen = (0x80 + 0x40) // 4
meanblue = (0xff + 0x80) // 4
meanblue = (0xFF + 0x80) // 4
eq_((meanred, meangreen, meanblue), b)
@ -158,6 +167,7 @@ class TestCasegetblock:
# eq_(BLACK, blocks[3])
#
class TestCasegetblocks2:
def test_empty_image(self):
im = empty()
@ -169,9 +179,9 @@ class TestCasegetblocks2:
blocks = getblocks2(im, 1)
eq_(1, len(blocks))
block = blocks[0]
meanred = (0xff + 0x80) // 4
meanred = (0xFF + 0x80) // 4
meangreen = (0x80 + 0x40) // 4
meanblue = (0xff + 0x80) // 4
meanblue = (0xFF + 0x80) // 4
eq_((meanred, meangreen, meanblue), block)
def test_four_blocks_all_black(self):

86
core/tests/cache_test.py
View File

@ -16,34 +16,35 @@ try:
except ImportError:
skip("Can't import the cache module, probably hasn't been compiled.")
class TestCasecolors_to_string:
def test_no_color(self):
eq_('', colors_to_string([]))
eq_("", colors_to_string([]))
def test_single_color(self):
eq_('000000', colors_to_string([(0, 0, 0)]))
eq_('010101', colors_to_string([(1, 1, 1)]))
eq_('0a141e', colors_to_string([(10, 20, 30)]))
eq_("000000", colors_to_string([(0, 0, 0)]))
eq_("010101", colors_to_string([(1, 1, 1)]))
eq_("0a141e", colors_to_string([(10, 20, 30)]))
def test_two_colors(self):
eq_('000102030405', colors_to_string([(0, 1, 2), (3, 4, 5)]))
eq_("000102030405", colors_to_string([(0, 1, 2), (3, 4, 5)]))
class TestCasestring_to_colors:
def test_empty(self):
eq_([], string_to_colors(''))
eq_([], string_to_colors(""))
def test_single_color(self):
eq_([(0, 0, 0)], string_to_colors('000000'))
eq_([(2, 3, 4)], string_to_colors('020304'))
eq_([(10, 20, 30)], string_to_colors('0a141e'))
eq_([(0, 0, 0)], string_to_colors("000000"))
eq_([(2, 3, 4)], string_to_colors("020304"))
eq_([(10, 20, 30)], string_to_colors("0a141e"))
def test_two_colors(self):
eq_([(10, 20, 30), (40, 50, 60)], string_to_colors('0a141e28323c'))
eq_([(10, 20, 30), (40, 50, 60)], string_to_colors("0a141e28323c"))
def test_incomplete_color(self):
# don't return anything if it's not a complete color
eq_([], string_to_colors('102'))
eq_([], string_to_colors("102"))
class BaseTestCaseCache:
@ -54,58 +55,58 @@ class BaseTestCaseCache:
c = self.get_cache()
eq_(0, len(c))
with raises(KeyError):
c['foo']
c["foo"]
def test_set_then_retrieve_blocks(self):
c = self.get_cache()
b = [(0, 0, 0), (1, 2, 3)]
c['foo'] = b
eq_(b, c['foo'])
c["foo"] = b
eq_(b, c["foo"])
def test_delitem(self):
c = self.get_cache()
c['foo'] = ''
del c['foo']
assert 'foo' not in c
c["foo"] = ""
del c["foo"]
assert "foo" not in c
with raises(KeyError):
del c['foo']
del c["foo"]
def test_persistance(self, tmpdir):
DBNAME = tmpdir.join('hstest.db')
DBNAME = tmpdir.join("hstest.db")
c = self.get_cache(str(DBNAME))
c['foo'] = [(1, 2, 3)]
c["foo"] = [(1, 2, 3)]
del c
c = self.get_cache(str(DBNAME))
eq_([(1, 2, 3)], c['foo'])
eq_([(1, 2, 3)], c["foo"])
def test_filter(self):
c = self.get_cache()
c['foo'] = ''
c['bar'] = ''
c['baz'] = ''
c.filter(lambda p: p != 'bar') #only 'bar' is removed
c["foo"] = ""
c["bar"] = ""
c["baz"] = ""
c.filter(lambda p: p != "bar") # only 'bar' is removed
eq_(2, len(c))
assert 'foo' in c
assert 'baz' in c
assert 'bar' not in c
assert "foo" in c
assert "baz" in c
assert "bar" not in c
def test_clear(self):
c = self.get_cache()
c['foo'] = ''
c['bar'] = ''
c['baz'] = ''
c["foo"] = ""
c["bar"] = ""
c["baz"] = ""
c.clear()
eq_(0, len(c))
assert 'foo' not in c
assert 'baz' not in c
assert 'bar' not in c
assert "foo" not in c
assert "baz" not in c
assert "bar" not in c
def test_by_id(self):
# it's possible to use the cache by referring to the files by their row_id
c = self.get_cache()
b = [(0, 0, 0), (1, 2, 3)]
c['foo'] = b
foo_id = c.get_id('foo')
c["foo"] = b
foo_id = c.get_id("foo")
eq_(c[foo_id], b)
@ -120,16 +121,16 @@ class TestCaseSqliteCache(BaseTestCaseCache):
# If we don't do this monkeypatching, we get a weird exception about trying to flush a
# closed file. I've tried setting logging level and stuff, but nothing worked. So, there we
# go, a dirty monkeypatch.
monkeypatch.setattr(logging, 'warning', lambda *args, **kw: None)
dbname = str(tmpdir.join('foo.db'))
fp = open(dbname, 'w')
fp.write('invalid sqlite content')
monkeypatch.setattr(logging, "warning", lambda *args, **kw: None)
dbname = str(tmpdir.join("foo.db"))
fp = open(dbname, "w")
fp.write("invalid sqlite content")
fp.close()
c = self.get_cache(dbname) # should not raise a DatabaseError
c['foo'] = [(1, 2, 3)]
c["foo"] = [(1, 2, 3)]
del c
c = self.get_cache(dbname)
eq_(c['foo'], [(1, 2, 3)])
eq_(c["foo"], [(1, 2, 3)])
class TestCaseShelveCache(BaseTestCaseCache):
@ -161,4 +162,3 @@ class TestCaseCacheSQLEscape:
del c["foo'bar"]
except KeyError:
assert False

176
core/tests/directories_test.py
View File

@ -14,91 +14,105 @@ from hscommon.path import Path
from hscommon.testutil import eq_
from ..fs import File
from ..directories import Directories, DirectoryState, AlreadyThereError, InvalidPathError
from ..directories import (
Directories,
DirectoryState,
AlreadyThereError,
InvalidPathError,
)
def create_fake_fs(rootpath):
# We have it as a separate function because other units are using it.
rootpath = rootpath['fs']
rootpath = rootpath["fs"]
rootpath.mkdir()
rootpath['dir1'].mkdir()
rootpath['dir2'].mkdir()
rootpath['dir3'].mkdir()
fp = rootpath['file1.test'].open('w')
fp.write('1')
rootpath["dir1"].mkdir()
rootpath["dir2"].mkdir()
rootpath["dir3"].mkdir()
fp = rootpath["file1.test"].open("w")
fp.write("1")
fp.close()
fp = rootpath['file2.test'].open('w')
fp.write('12')
fp = rootpath["file2.test"].open("w")
fp.write("12")
fp.close()
fp = rootpath['file3.test'].open('w')
fp.write('123')
fp = rootpath["file3.test"].open("w")
fp.write("123")
fp.close()
fp = rootpath['dir1']['file1.test'].open('w')
fp.write('1')
fp = rootpath["dir1"]["file1.test"].open("w")
fp.write("1")
fp.close()
fp = rootpath['dir2']['file2.test'].open('w')
fp.write('12')
fp = rootpath["dir2"]["file2.test"].open("w")
fp.write("12")
fp.close()
fp = rootpath['dir3']['file3.test'].open('w')
fp.write('123')
fp = rootpath["dir3"]["file3.test"].open("w")
fp.write("123")
fp.close()
return rootpath
testpath = None
def setup_module(module):
# In this unit, we have tests depending on two directory structure. One with only one file in it
# and another with a more complex structure.
testpath = Path(tempfile.mkdtemp())
module.testpath = testpath
rootpath = testpath['onefile']
rootpath = testpath["onefile"]
rootpath.mkdir()
fp = rootpath['test.txt'].open('w')
fp.write('test_data')
fp = rootpath["test.txt"].open("w")
fp.write("test_data")
fp.close()
create_fake_fs(testpath)
def teardown_module(module):
shutil.rmtree(str(module.testpath))
def test_empty():
d = Directories()
eq_(len(d), 0)
assert 'foobar' not in d
assert "foobar" not in d
def test_add_path():
d = Directories()
p = testpath['onefile']
p = testpath["onefile"]
d.add_path(p)
eq_(1, len(d))
assert p in d
assert (p['foobar']) in d
assert (p["foobar"]) in d
assert p.parent() not in d
p = testpath['fs']
p = testpath["fs"]
d.add_path(p)
eq_(2, len(d))
assert p in d
def test_AddPath_when_path_is_already_there():
d = Directories()
p = testpath['onefile']
p = testpath["onefile"]
d.add_path(p)
with raises(AlreadyThereError):
d.add_path(p)
with raises(AlreadyThereError):
d.add_path(p['foobar'])
d.add_path(p["foobar"])
eq_(1, len(d))
def test_add_path_containing_paths_already_there():
d = Directories()
d.add_path(testpath['onefile'])
d.add_path(testpath["onefile"])
eq_(1, len(d))
d.add_path(testpath)
eq_(len(d), 1)
eq_(d[0], testpath)
def test_AddPath_non_latin(tmpdir):
p = Path(str(tmpdir))
to_add = p['unicode\u201a']
to_add = p["unicode\u201a"]
os.mkdir(str(to_add))
d = Directories()
try:
@ -106,63 +120,69 @@ def test_AddPath_non_latin(tmpdir):
except UnicodeDecodeError:
assert False
def test_del():
d = Directories()
d.add_path(testpath['onefile'])
d.add_path(testpath["onefile"])
try:
del d[1]
assert False
except IndexError:
pass
d.add_path(testpath['fs'])
d.add_path(testpath["fs"])
del d[1]
eq_(1, len(d))
def test_states():
d = Directories()
p = testpath['onefile']
p = testpath["onefile"]
d.add_path(p)
eq_(DirectoryState.Normal, d.get_state(p))
d.set_state(p, DirectoryState.Reference)
eq_(DirectoryState.Reference, d.get_state(p))
eq_(DirectoryState.Reference, d.get_state(p['dir1']))
eq_(DirectoryState.Reference, d.get_state(p["dir1"]))
eq_(1, len(d.states))
eq_(p, list(d.states.keys())[0])
eq_(DirectoryState.Reference, d.states[p])
def test_get_state_with_path_not_there():
# When the path's not there, just return DirectoryState.Normal
d = Directories()
d.add_path(testpath['onefile'])
d.add_path(testpath["onefile"])
eq_(d.get_state(testpath), DirectoryState.Normal)
def test_states_overwritten_when_larger_directory_eat_smaller_ones():
# ref #248
# When setting the state of a folder, we overwrite previously set states for subfolders.
d = Directories()
p = testpath['onefile']
p = testpath["onefile"]
d.add_path(p)
d.set_state(p, DirectoryState.Excluded)
d.add_path(testpath)
d.set_state(testpath, DirectoryState.Reference)
eq_(d.get_state(p), DirectoryState.Reference)
eq_(d.get_state(p['dir1']), DirectoryState.Reference)
eq_(d.get_state(p["dir1"]), DirectoryState.Reference)
eq_(d.get_state(testpath), DirectoryState.Reference)
def test_get_files():
d = Directories()
p = testpath['fs']
p = testpath["fs"]
d.add_path(p)
d.set_state(p['dir1'], DirectoryState.Reference)
d.set_state(p['dir2'], DirectoryState.Excluded)
d.set_state(p["dir1"], DirectoryState.Reference)
d.set_state(p["dir2"], DirectoryState.Excluded)
files = list(d.get_files())
eq_(5, len(files))
for f in files:
if f.path.parent() == p['dir1']:
if f.path.parent() == p["dir1"]:
assert f.is_ref
else:
assert not f.is_ref
def test_get_files_with_folders():
# When fileclasses handle folders, return them and stop recursing!
class FakeFile(File):
@ -171,106 +191,115 @@ def test_get_files_with_folders():
return True
d = Directories()
p = testpath['fs']
p = testpath["fs"]
d.add_path(p)
files = list(d.get_files(fileclasses=[FakeFile]))
# We have the 3 root files and the 3 root dirs
eq_(6, len(files))
def test_get_folders():
d = Directories()
p = testpath['fs']
p = testpath["fs"]
d.add_path(p)
d.set_state(p['dir1'], DirectoryState.Reference)
d.set_state(p['dir2'], DirectoryState.Excluded)
d.set_state(p["dir1"], DirectoryState.Reference)
d.set_state(p["dir2"], DirectoryState.Excluded)
folders = list(d.get_folders())
eq_(len(folders), 3)
ref = [f for f in folders if f.is_ref]
not_ref = [f for f in folders if not f.is_ref]
eq_(len(ref), 1)
eq_(ref[0].path, p['dir1'])
eq_(ref[0].path, p["dir1"])
eq_(len(not_ref), 2)
eq_(ref[0].size, 1)
def test_get_files_with_inherited_exclusion():
d = Directories()
p = testpath['onefile']
p = testpath["onefile"]
d.add_path(p)
d.set_state(p, DirectoryState.Excluded)
eq_([], list(d.get_files()))
def test_save_and_load(tmpdir):
d1 = Directories()
d2 = Directories()
p1 = Path(str(tmpdir.join('p1')))
p1 = Path(str(tmpdir.join("p1")))
p1.mkdir()
p2 = Path(str(tmpdir.join('p2')))
p2 = Path(str(tmpdir.join("p2")))
p2.mkdir()
d1.add_path(p1)
d1.add_path(p2)
d1.set_state(p1, DirectoryState.Reference)
d1.set_state(p1['dir1'], DirectoryState.Excluded)
tmpxml = str(tmpdir.join('directories_testunit.xml'))
d1.set_state(p1["dir1"], DirectoryState.Excluded)
tmpxml = str(tmpdir.join("directories_testunit.xml"))
d1.save_to_file(tmpxml)
d2.load_from_file(tmpxml)
eq_(2, len(d2))
eq_(DirectoryState.Reference, d2.get_state(p1))
eq_(DirectoryState.Excluded, d2.get_state(p1['dir1']))
eq_(DirectoryState.Excluded, d2.get_state(p1["dir1"]))
def test_invalid_path():
d = Directories()
p = Path('does_not_exist')
p = Path("does_not_exist")
with raises(InvalidPathError):
d.add_path(p)
eq_(0, len(d))
def test_set_state_on_invalid_path():
d = Directories()
try:
d.set_state(Path('foobar',), DirectoryState.Normal)
d.set_state(Path("foobar",), DirectoryState.Normal)
except LookupError:
assert False
def test_load_from_file_with_invalid_path(tmpdir):
# This test simulates a load from file resulting in a
# InvalidPath raise. Other directories must be loaded.
d1 = Directories()
d1.add_path(testpath['onefile'])
d1.add_path(testpath["onefile"])
# Will raise InvalidPath upon loading
p = Path(str(tmpdir.join('toremove')))
p = Path(str(tmpdir.join("toremove")))
p.mkdir()
d1.add_path(p)
p.rmdir()
tmpxml = str(tmpdir.join('directories_testunit.xml'))
tmpxml = str(tmpdir.join("directories_testunit.xml"))
d1.save_to_file(tmpxml)
d2 = Directories()
d2.load_from_file(tmpxml)
eq_(1, len(d2))
def test_unicode_save(tmpdir):
d = Directories()
p1 = Path(str(tmpdir))['hello\xe9']
p1 = Path(str(tmpdir))["hello\xe9"]
p1.mkdir()
p1['foo\xe9'].mkdir()
p1["foo\xe9"].mkdir()
d.add_path(p1)
d.set_state(p1['foo\xe9'], DirectoryState.Excluded)
tmpxml = str(tmpdir.join('directories_testunit.xml'))
d.set_state(p1["foo\xe9"], DirectoryState.Excluded)
tmpxml = str(tmpdir.join("directories_testunit.xml"))
try:
d.save_to_file(tmpxml)
except UnicodeDecodeError:
assert False
def test_get_files_refreshes_its_directories():
d = Directories()
p = testpath['fs']
p = testpath["fs"]
d.add_path(p)
files = d.get_files()
eq_(6, len(list(files)))
time.sleep(1)
os.remove(str(p['dir1']['file1.test']))
os.remove(str(p["dir1"]["file1.test"]))
files = d.get_files()
eq_(5, len(list(files)))
def test_get_files_does_not_choke_on_non_existing_directories(tmpdir):
d = Directories()
p = Path(str(tmpdir))
@ -278,36 +307,37 @@ def test_get_files_does_not_choke_on_non_existing_directories(tmpdir):
p.rmtree()
eq_([], list(d.get_files()))
def test_get_state_returns_excluded_by_default_for_hidden_directories(tmpdir):
d = Directories()
p = Path(str(tmpdir))
hidden_dir_path = p['.foo']
p['.foo'].mkdir()
hidden_dir_path = p[".foo"]
p[".foo"].mkdir()
d.add_path(p)
eq_(d.get_state(hidden_dir_path), DirectoryState.Excluded)
# But it can be overriden
d.set_state(hidden_dir_path, DirectoryState.Normal)
eq_(d.get_state(hidden_dir_path), DirectoryState.Normal)
def test_default_path_state_override(tmpdir):
# It's possible for a subclass to override the default state of a path
class MyDirectories(Directories):
def _default_state_for_path(self, path):
if 'foobar' in path:
if "foobar" in path:
return DirectoryState.Excluded
d = MyDirectories()
p1 = Path(str(tmpdir))
p1['foobar'].mkdir()
p1['foobar/somefile'].open('w').close()
p1['foobaz'].mkdir()
p1['foobaz/somefile'].open('w').close()
p1["foobar"].mkdir()
p1["foobar/somefile"].open("w").close()
p1["foobaz"].mkdir()
p1["foobaz/somefile"].open("w").close()
d.add_path(p1)
eq_(d.get_state(p1['foobaz']), DirectoryState.Normal)
eq_(d.get_state(p1['foobar']), DirectoryState.Excluded)
eq_(d.get_state(p1["foobaz"]), DirectoryState.Normal)
eq_(d.get_state(p1["foobar"]), DirectoryState.Excluded)
eq_(len(list(d.get_files())), 1) # only the 'foobaz' file is there
# However, the default state can be changed
d.set_state(p1['foobar'], DirectoryState.Normal)
eq_(d.get_state(p1['foobar']), DirectoryState.Normal)
d.set_state(p1["foobar"], DirectoryState.Normal)
eq_(d.get_state(p1["foobar"]), DirectoryState.Normal)
eq_(len(list(d.get_files())), 2)

405
core/tests/engine_test.py
View File

@ -13,13 +13,28 @@ from hscommon.testutil import eq_, log_calls
from .base import NamedObject
from .. import engine
from ..engine import (
get_match, getwords, Group, getfields, unpack_fields, compare_fields, compare, WEIGHT_WORDS,
MATCH_SIMILAR_WORDS, NO_FIELD_ORDER, build_word_dict, get_groups, getmatches, Match,
getmatches_by_contents, merge_similar_words, reduce_common_words
get_match,
getwords,
Group,
getfields,
unpack_fields,
compare_fields,
compare,
WEIGHT_WORDS,
MATCH_SIMILAR_WORDS,
NO_FIELD_ORDER,
build_word_dict,
get_groups,
getmatches,
Match,
getmatches_by_contents,
merge_similar_words,
reduce_common_words,
)
no = NamedObject
def get_match_triangle():
o1 = NamedObject(with_words=True)
o2 = NamedObject(with_words=True)
@ -29,6 +44,7 @@ def get_match_triangle():
m3 = get_match(o2, o3)
return [m1, m2, m3]
def get_test_group():
m1, m2, m3 = get_match_triangle()
result = Group()
@ -37,6 +53,7 @@ def get_test_group():
result.add_match(m3)
return result
def assert_match(m, name1, name2):
# When testing matches, whether objects are in first or second position very often doesn't
# matter. This function makes this test more convenient.
@ -46,53 +63,54 @@ def assert_match(m, name1, name2):
eq_(m.first.name, name2)
eq_(m.second.name, name1)
class TestCasegetwords:
def test_spaces(self):
eq_(['a', 'b', 'c', 'd'], getwords("a b c d"))
eq_(['a', 'b', 'c', 'd'], getwords(" a b c d "))
eq_(["a", "b", "c", "d"], getwords("a b c d"))
eq_(["a", "b", "c", "d"], getwords(" a b c d "))
def test_splitter_chars(self):
eq_(
[chr(i) for i in range(ord('a'), ord('z')+1)],
getwords("a-b_c&d+e(f)g;h\\i[j]k{l}m:n.o,p<q>r/s?t~u!v@w#x$y*z")
[chr(i) for i in range(ord("a"), ord("z") + 1)],
getwords("a-b_c&d+e(f)g;h\\i[j]k{l}m:n.o,p<q>r/s?t~u!v@w#x$y*z"),
)
def test_joiner_chars(self):
eq_(["aec"], getwords("a'e\u0301c"))
def test_empty(self):
eq_([], getwords(''))
eq_([], getwords(""))
def test_returns_lowercase(self):
eq_(['foo', 'bar'], getwords('FOO BAR'))
eq_(["foo", "bar"], getwords("FOO BAR"))
def test_decompose_unicode(self):
eq_(getwords('foo\xe9bar'), ['fooebar'])
eq_(getwords("foo\xe9bar"), ["fooebar"])
class TestCasegetfields:
def test_simple(self):
eq_([['a', 'b'], ['c', 'd', 'e']], getfields('a b - c d e'))
eq_([["a", "b"], ["c", "d", "e"]], getfields("a b - c d e"))
def test_empty(self):
eq_([], getfields(''))
eq_([], getfields(""))
def test_cleans_empty_fields(self):
expected = [['a', 'bc', 'def']]
actual = getfields(' - a bc def')
expected = [["a", "bc", "def"]]
actual = getfields(" - a bc def")
eq_(expected, actual)
expected = [['bc', 'def']]
expected = [["bc", "def"]]
class TestCaseunpack_fields:
def test_with_fields(self):
expected = ['a', 'b', 'c', 'd', 'e', 'f']
actual = unpack_fields([['a'], ['b', 'c'], ['d', 'e', 'f']])
expected = ["a", "b", "c", "d", "e", "f"]
actual = unpack_fields([["a"], ["b", "c"], ["d", "e", "f"]])
eq_(expected, actual)
def test_without_fields(self):
expected = ['a', 'b', 'c', 'd', 'e', 'f']
actual = unpack_fields(['a', 'b', 'c', 'd', 'e', 'f'])
expected = ["a", "b", "c", "d", "e", "f"]
actual = unpack_fields(["a", "b", "c", "d", "e", "f"])
eq_(expected, actual)
def test_empty(self):
@ -101,134 +119,151 @@ class TestCaseunpack_fields:
class TestCaseWordCompare:
def test_list(self):
eq_(100, compare(['a', 'b', 'c', 'd'], ['a', 'b', 'c', 'd']))
eq_(86, compare(['a', 'b', 'c', 'd'], ['a', 'b', 'c']))
eq_(100, compare(["a", "b", "c", "d"], ["a", "b", "c", "d"]))
eq_(86, compare(["a", "b", "c", "d"], ["a", "b", "c"]))
def test_unordered(self):
# Sometimes, users don't want fuzzy matching too much When they set the slider
# to 100, they don't expect a filename with the same words, but not the same order, to match.
# Thus, we want to return 99 in that case.
eq_(99, compare(['a', 'b', 'c', 'd'], ['d', 'b', 'c', 'a']))
eq_(99, compare(["a", "b", "c", "d"], ["d", "b", "c", "a"]))
def test_word_occurs_twice(self):
# if a word occurs twice in first, but once in second, we want the word to be only counted once
eq_(89, compare(['a', 'b', 'c', 'd', 'a'], ['d', 'b', 'c', 'a']))
eq_(89, compare(["a", "b", "c", "d", "a"], ["d", "b", "c", "a"]))
def test_uses_copy_of_lists(self):
first = ['foo', 'bar']
second = ['bar', 'bleh']
first = ["foo", "bar"]
second = ["bar", "bleh"]
compare(first, second)
eq_(['foo', 'bar'], first)
eq_(['bar', 'bleh'], second)
eq_(["foo", "bar"], first)
eq_(["bar", "bleh"], second)
def test_word_weight(self):
eq_(int((6.0 / 13.0) * 100), compare(['foo', 'bar'], ['bar', 'bleh'], (WEIGHT_WORDS, )))
eq_(
int((6.0 / 13.0) * 100),
compare(["foo", "bar"], ["bar", "bleh"], (WEIGHT_WORDS,)),
)
def test_similar_words(self):
eq_(100, compare(['the', 'white', 'stripes'], ['the', 'whites', 'stripe'], (MATCH_SIMILAR_WORDS, )))
eq_(
100,
compare(
["the", "white", "stripes"],
["the", "whites", "stripe"],
(MATCH_SIMILAR_WORDS,),
),
)
def test_empty(self):
eq_(0, compare([], []))
def test_with_fields(self):
eq_(67, compare([['a', 'b'], ['c', 'd', 'e']], [['a', 'b'], ['c', 'd', 'f']]))
eq_(67, compare([["a", "b"], ["c", "d", "e"]], [["a", "b"], ["c", "d", "f"]]))
def test_propagate_flags_with_fields(self, monkeypatch):
def mock_compare(first, second, flags):
eq_((0, 1, 2, 3, 5), flags)
monkeypatch.setattr(engine, 'compare_fields', mock_compare)
compare([['a']], [['a']], (0, 1, 2, 3, 5))
monkeypatch.setattr(engine, "compare_fields", mock_compare)
compare([["a"]], [["a"]], (0, 1, 2, 3, 5))
class TestCaseWordCompareWithFields:
def test_simple(self):
eq_(67, compare_fields([['a', 'b'], ['c', 'd', 'e']], [['a', 'b'], ['c', 'd', 'f']]))
eq_(
67,
compare_fields(
[["a", "b"], ["c", "d", "e"]], [["a", "b"], ["c", "d", "f"]]
),
)
def test_empty(self):
eq_(0, compare_fields([], []))
def test_different_length(self):
eq_(0, compare_fields([['a'], ['b']], [['a'], ['b'], ['c']]))
eq_(0, compare_fields([["a"], ["b"]], [["a"], ["b"], ["c"]]))
def test_propagates_flags(self, monkeypatch):
def mock_compare(first, second, flags):
eq_((0, 1, 2, 3, 5), flags)
monkeypatch.setattr(engine, 'compare_fields', mock_compare)
compare_fields([['a']], [['a']], (0, 1, 2, 3, 5))
monkeypatch.setattr(engine, "compare_fields", mock_compare)
compare_fields([["a"]], [["a"]], (0, 1, 2, 3, 5))
def test_order(self):
first = [['a', 'b'], ['c', 'd', 'e']]
second = [['c', 'd', 'f'], ['a', 'b']]
first = [["a", "b"], ["c", "d", "e"]]
second = [["c", "d", "f"], ["a", "b"]]
eq_(0, compare_fields(first, second))
def test_no_order(self):
first = [['a', 'b'], ['c', 'd', 'e']]
second = [['c', 'd', 'f'], ['a', 'b']]
first = [["a", "b"], ["c", "d", "e"]]
second = [["c", "d", "f"], ["a", "b"]]
eq_(67, compare_fields(first, second, (NO_FIELD_ORDER,)))
first = [['a', 'b'], ['a', 'b']] #a field can only be matched once.
second = [['c', 'd', 'f'], ['a', 'b']]
first = [["a", "b"], ["a", "b"]] # a field can only be matched once.
second = [["c", "d", "f"], ["a", "b"]]
eq_(0, compare_fields(first, second, (NO_FIELD_ORDER,)))
first = [['a', 'b'], ['a', 'b', 'c']]
second = [['c', 'd', 'f'], ['a', 'b']]
first = [["a", "b"], ["a", "b", "c"]]
second = [["c", "d", "f"], ["a", "b"]]
eq_(33, compare_fields(first, second, (NO_FIELD_ORDER,)))
def test_compare_fields_without_order_doesnt_alter_fields(self):
# The NO_ORDER comp type altered the fields!
first = [['a', 'b'], ['c', 'd', 'e']]
second = [['c', 'd', 'f'], ['a', 'b']]
first = [["a", "b"], ["c", "d", "e"]]
second = [["c", "d", "f"], ["a", "b"]]
eq_(67, compare_fields(first, second, (NO_FIELD_ORDER,)))
eq_([['a', 'b'], ['c', 'd', 'e']], first)
eq_([['c', 'd', 'f'], ['a', 'b']], second)
eq_([["a", "b"], ["c", "d", "e"]], first)
eq_([["c", "d", "f"], ["a", "b"]], second)
class TestCasebuild_word_dict:
def test_with_standard_words(self):
l = [NamedObject('foo bar', True)]
l.append(NamedObject('bar baz', True))
l.append(NamedObject('baz bleh foo', True))
d = build_word_dict(l)
itemList = [NamedObject("foo bar", True)]
itemList.append(NamedObject("bar baz", True))
itemList.append(NamedObject("baz bleh foo", True))
d = build_word_dict(itemList)
eq_(4, len(d))
eq_(2, len(d['foo']))
assert l[0] in d['foo']
assert l[2] in d['foo']
eq_(2, len(d['bar']))
assert l[0] in d['bar']
assert l[1] in d['bar']
eq_(2, len(d['baz']))
assert l[1] in d['baz']
assert l[2] in d['baz']
eq_(1, len(d['bleh']))
assert l[2] in d['bleh']
eq_(2, len(d["foo"]))
assert itemList[0] in d["foo"]
assert itemList[2] in d["foo"]
eq_(2, len(d["bar"]))
assert itemList[0] in d["bar"]
assert itemList[1] in d["bar"]
eq_(2, len(d["baz"]))
assert itemList[1] in d["baz"]
assert itemList[2] in d["baz"]
eq_(1, len(d["bleh"]))
assert itemList[2] in d["bleh"]
def test_unpack_fields(self):
o = NamedObject('')
o.words = [['foo', 'bar'], ['baz']]
o = NamedObject("")
o.words = [["foo", "bar"], ["baz"]]
d = build_word_dict([o])
eq_(3, len(d))
eq_(1, len(d['foo']))
eq_(1, len(d["foo"]))
def test_words_are_unaltered(self):
o = NamedObject('')
o.words = [['foo', 'bar'], ['baz']]
o = NamedObject("")
o.words = [["foo", "bar"], ["baz"]]
build_word_dict([o])
eq_([['foo', 'bar'], ['baz']], o.words)
eq_([["foo", "bar"], ["baz"]], o.words)
def test_object_instances_can_only_be_once_in_words_object_list(self):
o = NamedObject('foo foo', True)
o = NamedObject("foo foo", True)
d = build_word_dict([o])
eq_(1, len(d['foo']))
eq_(1, len(d["foo"]))
def test_job(self):
def do_progress(p, d=''):
def do_progress(p, d=""):
self.log.append(p)
return True
j = job.Job(1, do_progress)
self.log = []
s = "foo bar"
build_word_dict([NamedObject(s, True), NamedObject(s, True), NamedObject(s, True)], j)
build_word_dict(
[NamedObject(s, True), NamedObject(s, True), NamedObject(s, True)], j
)
# We don't have intermediate log because iter_with_progress is called with every > 1
eq_(0, self.log[0])
eq_(100, self.log[1])
@ -237,51 +272,56 @@ class TestCasebuild_word_dict:
class TestCasemerge_similar_words:
def test_some_similar_words(self):
d = {
'foobar': set([1]),
'foobar1': set([2]),
'foobar2': set([3]),
"foobar": set([1]),
"foobar1": set([2]),
"foobar2": set([3]),
}
merge_similar_words(d)
eq_(1, len(d))
eq_(3, len(d['foobar']))
eq_(3, len(d["foobar"]))
class TestCasereduce_common_words:
def test_typical(self):
d = {
'foo': set([NamedObject('foo bar', True) for i in range(50)]),
'bar': set([NamedObject('foo bar', True) for i in range(49)])
"foo": set([NamedObject("foo bar", True) for i in range(50)]),
"bar": set([NamedObject("foo bar", True) for i in range(49)]),
}
reduce_common_words(d, 50)
assert 'foo' not in d
eq_(49, len(d['bar']))
assert "foo" not in d
eq_(49, len(d["bar"]))
def test_dont_remove_objects_with_only_common_words(self):
d = {
'common': set([NamedObject("common uncommon", True) for i in range(50)] + [NamedObject("common", True)]),
'uncommon': set([NamedObject("common uncommon", True)])
"common": set(
[NamedObject("common uncommon", True) for i in range(50)]
+ [NamedObject("common", True)]
),
"uncommon": set([NamedObject("common uncommon", True)]),
}
reduce_common_words(d, 50)
eq_(1, len(d['common']))
eq_(1, len(d['uncommon']))
eq_(1, len(d["common"]))
eq_(1, len(d["uncommon"]))
def test_values_still_are_set_instances(self):
d = {
'common': set([NamedObject("common uncommon", True) for i in range(50)] + [NamedObject("common", True)]),
'uncommon': set([NamedObject("common uncommon", True)])
"common": set(
[NamedObject("common uncommon", True) for i in range(50)]
+ [NamedObject("common", True)]
),
"uncommon": set([NamedObject("common uncommon", True)]),
}
reduce_common_words(d, 50)
assert isinstance(d['common'], set)
assert isinstance(d['uncommon'], set)
assert isinstance(d["common"], set)
assert isinstance(d["uncommon"], set)
def test_dont_raise_KeyError_when_a_word_has_been_removed(self):
# If a word has been removed by the reduce, an object in a subsequent common word that
# contains the word that has been removed would cause a KeyError.
d = {
'foo': set([NamedObject('foo bar baz', True) for i in range(50)]),
'bar': set([NamedObject('foo bar baz', True) for i in range(50)]),
'baz': set([NamedObject('foo bar baz', True) for i in range(49)])
"foo": set([NamedObject("foo bar baz", True) for i in range(50)]),
"bar": set([NamedObject("foo bar baz", True) for i in range(50)]),
"baz": set([NamedObject("foo bar baz", True) for i in range(49)]),
}
try:
reduce_common_words(d, 50)
@ -291,13 +331,11 @@ class TestCasereduce_common_words:
def test_unpack_fields(self):
# object.words may be fields.
def create_it():
o = NamedObject('')
o.words = [['foo', 'bar'], ['baz']]
o = NamedObject("")
o.words = [["foo", "bar"], ["baz"]]
return o
d = {
'foo': set([create_it() for i in range(50)])
}
d = {"foo": set([create_it() for i in range(50)])}
try:
reduce_common_words(d, 50)
except TypeError:
@ -308,16 +346,20 @@ class TestCasereduce_common_words:
# be counted as a common word for subsequent words. For example, if 'foo' is processed
# as a common word, keeping a "foo bar" file in it, and the 'bar' is processed, "foo bar"
# would not stay in 'bar' because 'foo' is not a common word anymore.
only_common = NamedObject('foo bar', True)
only_common = NamedObject("foo bar", True)
d = {
'foo': set([NamedObject('foo bar baz', True) for i in range(49)] + [only_common]),
'bar': set([NamedObject('foo bar baz', True) for i in range(49)] + [only_common]),
'baz': set([NamedObject('foo bar baz', True) for i in range(49)])
"foo": set(
[NamedObject("foo bar baz", True) for i in range(49)] + [only_common]
),
"bar": set(
[NamedObject("foo bar baz", True) for i in range(49)] + [only_common]
),
"baz": set([NamedObject("foo bar baz", True) for i in range(49)]),
}
reduce_common_words(d, 50)
eq_(1, len(d['foo']))
eq_(1, len(d['bar']))
eq_(49, len(d['baz']))
eq_(1, len(d["foo"]))
eq_(1, len(d["bar"]))
eq_(49, len(d["baz"]))
class TestCaseget_match:
@ -326,8 +368,8 @@ class TestCaseget_match:
o2 = NamedObject("bar bleh", True)
m = get_match(o1, o2)
eq_(50, m.percentage)
eq_(['foo', 'bar'], m.first.words)
eq_(['bar', 'bleh'], m.second.words)
eq_(["foo", "bar"], m.first.words)
eq_(["bar", "bleh"], m.second.words)
assert m.first is o1
assert m.second is o2
@ -340,7 +382,9 @@ class TestCaseget_match:
assert object() not in m
def test_word_weight(self):
m = get_match(NamedObject("foo bar", True), NamedObject("bar bleh", True), (WEIGHT_WORDS, ))
m = get_match(
NamedObject("foo bar", True), NamedObject("bar bleh", True), (WEIGHT_WORDS,)
)
eq_(m.percentage, int((6.0 / 13.0) * 100))
@ -349,35 +393,40 @@ class TestCaseGetMatches:
eq_(getmatches([]), [])
def test_simple(self):
l = [NamedObject("foo bar"), NamedObject("bar bleh"), NamedObject("a b c foo")]
r = getmatches(l)
itemList = [NamedObject("foo bar"), NamedObject("bar bleh"), NamedObject("a b c foo")]
r = getmatches(itemList)
eq_(2, len(r))
m = first(m for m in r if m.percentage == 50) # "foo bar" and "bar bleh"
assert_match(m, 'foo bar', 'bar bleh')
assert_match(m, "foo bar", "bar bleh")
m = first(m for m in r if m.percentage == 33) # "foo bar" and "a b c foo"
assert_match(m, 'foo bar', 'a b c foo')
assert_match(m, "foo bar", "a b c foo")
def test_null_and_unrelated_objects(self):
l = [NamedObject("foo bar"), NamedObject("bar bleh"), NamedObject(""), NamedObject("unrelated object")]
r = getmatches(l)
itemList = [
NamedObject("foo bar"),
NamedObject("bar bleh"),
NamedObject(""),
NamedObject("unrelated object"),
]
r = getmatches(itemList)
eq_(len(r), 1)
m = r[0]
eq_(m.percentage, 50)
assert_match(m, 'foo bar', 'bar bleh')
assert_match(m, "foo bar", "bar bleh")
def test_twice_the_same_word(self):
l = [NamedObject("foo foo bar"), NamedObject("bar bleh")]
r = getmatches(l)
itemList = [NamedObject("foo foo bar"), NamedObject("bar bleh")]
r = getmatches(itemList)
eq_(1, len(r))
def test_twice_the_same_word_when_preworded(self):
l = [NamedObject("foo foo bar", True), NamedObject("bar bleh", True)]
r = getmatches(l)
itemList = [NamedObject("foo foo bar", True), NamedObject("bar bleh", True)]
r = getmatches(itemList)
eq_(1, len(r))
def test_two_words_match(self):
l = [NamedObject("foo bar"), NamedObject("foo bar bleh")]
r = getmatches(l)
itemList = [NamedObject("foo bar"), NamedObject("foo bar bleh")]
r = getmatches(itemList)
eq_(1, len(r))
def test_match_files_with_only_common_words(self):
@ -385,18 +434,18 @@ class TestCaseGetMatches:
# The problem with the common_word_threshold is that the files containing only common
# words will never be matched together. We *should* match them.
# This test assumes that the common word threashold const is 50
l = [NamedObject("foo") for i in range(50)]
r = getmatches(l)
itemList = [NamedObject("foo") for i in range(50)]
r = getmatches(itemList)
eq_(1225, len(r))
def test_use_words_already_there_if_there(self):
o1 = NamedObject('foo')
o2 = NamedObject('bar')
o2.words = ['foo']
o1 = NamedObject("foo")
o2 = NamedObject("bar")
o2.words = ["foo"]
eq_(1, len(getmatches([o1, o2])))
def test_job(self):
def do_progress(p, d=''):
def do_progress(p, d=""):
self.log.append(p)
return True
@ -409,28 +458,28 @@ class TestCaseGetMatches:
eq_(100, self.log[-1])
def test_weight_words(self):
l = [NamedObject("foo bar"), NamedObject("bar bleh")]
m = getmatches(l, weight_words=True)[0]
itemList = [NamedObject("foo bar"), NamedObject("bar bleh")]
m = getmatches(itemList, weight_words=True)[0]
eq_(int((6.0 / 13.0) * 100), m.percentage)
def test_similar_word(self):
l = [NamedObject("foobar"), NamedObject("foobars")]
eq_(len(getmatches(l, match_similar_words=True)), 1)
eq_(getmatches(l, match_similar_words=True)[0].percentage, 100)
l = [NamedObject("foobar"), NamedObject("foo")]
eq_(len(getmatches(l, match_similar_words=True)), 0) #too far
l = [NamedObject("bizkit"), NamedObject("bizket")]
eq_(len(getmatches(l, match_similar_words=True)), 1)
l = [NamedObject("foobar"), NamedObject("foosbar")]
eq_(len(getmatches(l, match_similar_words=True)), 1)
itemList = [NamedObject("foobar"), NamedObject("foobars")]
eq_(len(getmatches(itemList, match_similar_words=True)), 1)
eq_(getmatches(itemList, match_similar_words=True)[0].percentage, 100)
itemList = [NamedObject("foobar"), NamedObject("foo")]
eq_(len(getmatches(itemList, match_similar_words=True)), 0) # too far
itemList = [NamedObject("bizkit"), NamedObject("bizket")]
eq_(len(getmatches(itemList, match_similar_words=True)), 1)
itemList = [NamedObject("foobar"), NamedObject("foosbar")]
eq_(len(getmatches(itemList, match_similar_words=True)), 1)
def test_single_object_with_similar_words(self):
l = [NamedObject("foo foos")]
eq_(len(getmatches(l, match_similar_words=True)), 0)
itemList = [NamedObject("foo foos")]
eq_(len(getmatches(itemList, match_similar_words=True)), 0)
def test_double_words_get_counted_only_once(self):
l = [NamedObject("foo bar foo bleh"), NamedObject("foo bar bleh bar")]
m = getmatches(l)[0]
itemList = [NamedObject("foo bar foo bleh"), NamedObject("foo bar bleh bar")]
m = getmatches(itemList)[0]
eq_(75, m.percentage)
def test_with_fields(self):
@ -450,13 +499,13 @@ class TestCaseGetMatches:
eq_(m.percentage, 50)
def test_only_match_similar_when_the_option_is_set(self):
l = [NamedObject("foobar"), NamedObject("foobars")]
eq_(len(getmatches(l, match_similar_words=False)), 0)
itemList = [NamedObject("foobar"), NamedObject("foobars")]
eq_(len(getmatches(itemList, match_similar_words=False)), 0)
def test_dont_recurse_do_match(self):
# with nosetests, the stack is increased. The number has to be high enough not to be failing falsely
sys.setrecursionlimit(200)
files = [NamedObject('foo bar') for i in range(201)]
files = [NamedObject("foo bar") for i in range(201)]
try:
getmatches(files)
except RuntimeError:
@ -465,8 +514,8 @@ class TestCaseGetMatches:
sys.setrecursionlimit(1000)
def test_min_match_percentage(self):
l = [NamedObject("foo bar"), NamedObject("bar bleh"), NamedObject("a b c foo")]
r = getmatches(l, min_match_percentage=50)
itemList = [NamedObject("foo bar"), NamedObject("bar bleh"), NamedObject("a b c foo")]
r = getmatches(itemList, min_match_percentage=50)
eq_(1, len(r)) # Only "foo bar" / "bar bleh" should match
def test_MemoryError(self, monkeypatch):
@ -477,11 +526,11 @@ class TestCaseGetMatches:
return Match(first, second, 0)
objects = [NamedObject() for i in range(10)] # results in 45 matches
monkeypatch.setattr(engine, 'get_match', mocked_match)
monkeypatch.setattr(engine, "get_match", mocked_match)
try:
r = getmatches(objects)
except MemoryError:
self.fail('MemorryError must be handled')
self.fail("MemorryError must be handled")
eq_(42, len(r))
@ -599,7 +648,7 @@ class TestCaseGroup:
eq_([o1], g.dupes)
g.switch_ref(o2)
assert o2 is g.ref
g.switch_ref(NamedObject('', True))
g.switch_ref(NamedObject("", True))
assert o2 is g.ref
def test_switch_ref_from_ref_dir(self):
@ -620,7 +669,7 @@ class TestCaseGroup:
m = g.get_match_of(o)
assert g.ref in m
assert o in m
assert g.get_match_of(NamedObject('', True)) is None
assert g.get_match_of(NamedObject("", True)) is None
assert g.get_match_of(g.ref) is None
def test_percentage(self):
@ -651,9 +700,9 @@ class TestCaseGroup:
o1 = m1.first
o2 = m1.second
o3 = m2.second
o1.name = 'c'
o2.name = 'b'
o3.name = 'a'
o1.name = "c"
o2.name = "b"
o3.name = "a"
g = Group()
g.add_match(m1)
g.add_match(m2)
@ -709,9 +758,9 @@ class TestCaseGroup:
def test_prioritize_nothing_changes(self):
# prioritize() returns False when nothing changes in the group.
g = get_test_group()
g[0].name = 'a'
g[1].name = 'b'
g[2].name = 'c'
g[0].name = "a"
g[1].name = "b"
g[2].name = "c"
assert not g.prioritize(lambda x: x.name)
def test_list_like(self):
@ -723,7 +772,11 @@ class TestCaseGroup:
def test_discard_matches(self):
g = Group()
o1, o2, o3 = (NamedObject("foo", True), NamedObject("bar", True), NamedObject("baz", True))
o1, o2, o3 = (
NamedObject("foo", True),
NamedObject("bar", True),
NamedObject("baz", True),
)
g.add_match(get_match(o1, o2))
g.add_match(get_match(o1, o3))
g.discard_matches()
@ -737,8 +790,8 @@ class TestCaseget_groups:
eq_([], r)
def test_simple(self):
l = [NamedObject("foo bar"), NamedObject("bar bleh")]
matches = getmatches(l)
itemList = [NamedObject("foo bar"), NamedObject("bar bleh")]
matches = getmatches(itemList)
m = matches[0]
r = get_groups(matches)
eq_(1, len(r))
@ -748,27 +801,38 @@ class TestCaseget_groups:
def test_group_with_multiple_matches(self):
# This results in 3 matches
l = [NamedObject("foo"), NamedObject("foo"), NamedObject("foo")]
matches = getmatches(l)
itemList = [NamedObject("foo"), NamedObject("foo"), NamedObject("foo")]
matches = getmatches(itemList)
r = get_groups(matches)
eq_(1, len(r))
g = r[0]
eq_(3, len(g))
def test_must_choose_a_group(self):
l = [NamedObject("a b"), NamedObject("a b"), NamedObject("b c"), NamedObject("c d"), NamedObject("c d")]
itemList = [
NamedObject("a b"),
NamedObject("a b"),
NamedObject("b c"),
NamedObject("c d"),
NamedObject("c d"),
]
# There will be 2 groups here: group "a b" and group "c d"
# "b c" can go either of them, but not both.
matches = getmatches(l)
matches = getmatches(itemList)
r = get_groups(matches)
eq_(2, len(r))
eq_(5, len(r[0]) + len(r[1]))
def test_should_all_go_in_the_same_group(self):
l = [NamedObject("a b"), NamedObject("a b"), NamedObject("a b"), NamedObject("a b")]
itemList = [
NamedObject("a b"),
NamedObject("a b"),
NamedObject("a b"),
NamedObject("a b"),
]
# There will be 2 groups here: group "a b" and group "c d"
# "b c" can fit in both, but it must be in only one of them
matches = getmatches(l)
matches = getmatches(itemList)
r = get_groups(matches)
eq_(1, len(r))
@ -787,8 +851,8 @@ class TestCaseget_groups:
assert o3 in g
def test_four_sized_group(self):
l = [NamedObject("foobar") for i in range(4)]
m = getmatches(l)
itemList = [NamedObject("foobar") for i in range(4)]
m = getmatches(itemList)
r = get_groups(m)
eq_(1, len(r))
eq_(4, len(r[0]))
@ -811,7 +875,9 @@ class TestCaseget_groups:
m1 = Match(A, B, 90) # This is the strongest "A" match
m2 = Match(A, C, 80) # Because C doesn't match with B, it won't be in the group
m3 = Match(A, D, 80) # Same thing for D
m4 = Match(C, D, 70) # However, because C and D match, they should have their own group.
m4 = Match(
C, D, 70
) # However, because C and D match, they should have their own group.
groups = get_groups([m1, m2, m3, m4])
eq_(len(groups), 2)
g1, g2 = groups
@ -819,4 +885,3 @@ class TestCaseget_groups:
assert B in g1
assert C in g2
assert D in g2

17
core/tests/fs_test.py
View File

@ -14,23 +14,25 @@ from core.tests.directories_test import create_fake_fs
from .. import fs
def test_size_aggregates_subfiles(tmpdir):
p = create_fake_fs(Path(str(tmpdir)))
b = fs.Folder(p)
eq_(b.size, 12)
def test_md5_aggregate_subfiles_sorted(tmpdir):
# dir.allfiles can return child in any order. Thus, bundle.md5 must aggregate
# all files' md5 it contains, but it must make sure that it does so in the
# same order everytime.
p = create_fake_fs(Path(str(tmpdir)))
b = fs.Folder(p)
md51 = fs.File(p['dir1']['file1.test']).md5
md52 = fs.File(p['dir2']['file2.test']).md5
md53 = fs.File(p['dir3']['file3.test']).md5
md54 = fs.File(p['file1.test']).md5
md55 = fs.File(p['file2.test']).md5
md56 = fs.File(p['file3.test']).md5
md51 = fs.File(p["dir1"]["file1.test"]).md5
md52 = fs.File(p["dir2"]["file2.test"]).md5
md53 = fs.File(p["dir3"]["file3.test"]).md5
md54 = fs.File(p["file1.test"]).md5
md55 = fs.File(p["file2.test"]).md5
md56 = fs.File(p["file3.test"]).md5
# The expected md5 is the md5 of md5s for folders and the direct md5 for files
folder_md51 = hashlib.md5(md51).digest()
folder_md52 = hashlib.md5(md52).digest()
@ -38,8 +40,9 @@ def test_md5_aggregate_subfiles_sorted(tmpdir):
md5 = hashlib.md5(folder_md51 + folder_md52 + folder_md53 + md54 + md55 + md56)
eq_(b.md5, md5.digest())
def test_has_file_attrs(tmpdir):
# a Folder must behave like a file, so it must have mtime attributes
b = fs.Folder(Path(str(tmpdir)))
assert b.mtime > 0
eq_(b.extension, '')
eq_(b.extension, "")

130
core/tests/ignore_test.py
View File

@ -12,152 +12,172 @@ from hscommon.testutil import eq_
from ..ignore import IgnoreList
def test_empty():
il = IgnoreList()
eq_(0, len(il))
assert not il.AreIgnored('foo', 'bar')
assert not il.AreIgnored("foo", "bar")
def test_simple():
il = IgnoreList()
il.Ignore('foo', 'bar')
assert il.AreIgnored('foo', 'bar')
assert il.AreIgnored('bar', 'foo')
assert not il.AreIgnored('foo', 'bleh')
assert not il.AreIgnored('bleh', 'bar')
il.Ignore("foo", "bar")
assert il.AreIgnored("foo", "bar")
assert il.AreIgnored("bar", "foo")
assert not il.AreIgnored("foo", "bleh")
assert not il.AreIgnored("bleh", "bar")
eq_(1, len(il))
def test_multiple():
il = IgnoreList()
il.Ignore('foo', 'bar')
il.Ignore('foo', 'bleh')
il.Ignore('bleh', 'bar')
il.Ignore('aybabtu', 'bleh')
assert il.AreIgnored('foo', 'bar')
assert il.AreIgnored('bar', 'foo')
assert il.AreIgnored('foo', 'bleh')
assert il.AreIgnored('bleh', 'bar')
assert not il.AreIgnored('aybabtu', 'bar')
il.Ignore("foo", "bar")
il.Ignore("foo", "bleh")
il.Ignore("bleh", "bar")
il.Ignore("aybabtu", "bleh")
assert il.AreIgnored("foo", "bar")
assert il.AreIgnored("bar", "foo")
assert il.AreIgnored("foo", "bleh")
assert il.AreIgnored("bleh", "bar")
assert not il.AreIgnored("aybabtu", "bar")
eq_(4, len(il))
def test_clear():
il = IgnoreList()
il.Ignore('foo', 'bar')
il.Ignore("foo", "bar")
il.Clear()
assert not il.AreIgnored('foo', 'bar')
assert not il.AreIgnored('bar', 'foo')
assert not il.AreIgnored("foo", "bar")
assert not il.AreIgnored("bar", "foo")
eq_(0, len(il))
def test_add_same_twice():
il = IgnoreList()
il.Ignore('foo', 'bar')
il.Ignore('bar', 'foo')
il.Ignore("foo", "bar")
il.Ignore("bar", "foo")
eq_(1, len(il))
def test_save_to_xml():
il = IgnoreList()
il.Ignore('foo', 'bar')
il.Ignore('foo', 'bleh')
il.Ignore('bleh', 'bar')
il.Ignore("foo", "bar")
il.Ignore("foo", "bleh")
il.Ignore("bleh", "bar")
f = io.BytesIO()
il.save_to_xml(f)
f.seek(0)
doc = ET.parse(f)
root = doc.getroot()
eq_(root.tag, 'ignore_list')
eq_(root.tag, "ignore_list")
eq_(len(root), 2)
eq_(len([c for c in root if c.tag == 'file']), 2)
eq_(len([c for c in root if c.tag == "file"]), 2)
f1, f2 = root[:]
subchildren = [c for c in f1 if c.tag == 'file'] + [c for c in f2 if c.tag == 'file']
subchildren = [c for c in f1 if c.tag == "file"] + [
c for c in f2 if c.tag == "file"
]
eq_(len(subchildren), 3)
def test_SaveThenLoad():
il = IgnoreList()
il.Ignore('foo', 'bar')
il.Ignore('foo', 'bleh')
il.Ignore('bleh', 'bar')
il.Ignore('\u00e9', 'bar')
il.Ignore("foo", "bar")
il.Ignore("foo", "bleh")
il.Ignore("bleh", "bar")
il.Ignore("\u00e9", "bar")
f = io.BytesIO()
il.save_to_xml(f)
f.seek(0)
il = IgnoreList()
il.load_from_xml(f)
eq_(4, len(il))
assert il.AreIgnored('\u00e9', 'bar')
assert il.AreIgnored("\u00e9", "bar")
def test_LoadXML_with_empty_file_tags():
f = io.BytesIO()
f.write(b'<?xml version="1.0" encoding="utf-8"?><ignore_list><file><file/></file></ignore_list>')
f.write(
b'<?xml version="1.0" encoding="utf-8"?><ignore_list><file><file/></file></ignore_list>'
)
f.seek(0)
il = IgnoreList()
il.load_from_xml(f)
eq_(0, len(il))
def test_AreIgnore_works_when_a_child_is_a_key_somewhere_else():
il = IgnoreList()
il.Ignore('foo', 'bar')
il.Ignore('bar', 'baz')
assert il.AreIgnored('bar', 'foo')
il.Ignore("foo", "bar")
il.Ignore("bar", "baz")
assert il.AreIgnored("bar", "foo")
def test_no_dupes_when_a_child_is_a_key_somewhere_else():
il = IgnoreList()
il.Ignore('foo', 'bar')
il.Ignore('bar', 'baz')
il.Ignore('bar', 'foo')
il.Ignore("foo", "bar")
il.Ignore("bar", "baz")
il.Ignore("bar", "foo")
eq_(2, len(il))
def test_iterate():
# It must be possible to iterate through ignore list
il = IgnoreList()
expected = [('foo', 'bar'), ('bar', 'baz'), ('foo', 'baz')]
expected = [("foo", "bar"), ("bar", "baz"), ("foo", "baz")]
for i in expected:
il.Ignore(i[0], i[1])
for i in il:
expected.remove(i) # No exception should be raised
assert not expected # expected should be empty
def test_filter():
il = IgnoreList()
il.Ignore('foo', 'bar')
il.Ignore('bar', 'baz')
il.Ignore('foo', 'baz')
il.Filter(lambda f, s: f == 'bar')
il.Ignore("foo", "bar")
il.Ignore("bar", "baz")
il.Ignore("foo", "baz")
il.Filter(lambda f, s: f == "bar")
eq_(1, len(il))
assert not il.AreIgnored('foo', 'bar')
assert il.AreIgnored('bar', 'baz')
assert not il.AreIgnored("foo", "bar")
assert il.AreIgnored("bar", "baz")
def test_save_with_non_ascii_items():
il = IgnoreList()
il.Ignore('\xac', '\xbf')
il.Ignore("\xac", "\xbf")
f = io.BytesIO()
try:
il.save_to_xml(f)
except Exception as e:
raise AssertionError(str(e))
def test_len():
il = IgnoreList()
eq_(0, len(il))
il.Ignore('foo', 'bar')
il.Ignore("foo", "bar")
eq_(1, len(il))
def test_nonzero():
il = IgnoreList()
assert not il
il.Ignore('foo', 'bar')
il.Ignore("foo", "bar")
assert il
def test_remove():
il = IgnoreList()
il.Ignore('foo', 'bar')
il.Ignore('foo', 'baz')
il.remove('bar', 'foo')
il.Ignore("foo", "bar")
il.Ignore("foo", "baz")
il.remove("bar", "foo")
eq_(len(il), 1)
assert not il.AreIgnored('foo', 'bar')
assert not il.AreIgnored("foo", "bar")
def test_remove_non_existant():
il = IgnoreList()
il.Ignore('foo', 'bar')
il.Ignore('foo', 'baz')
il.Ignore("foo", "bar")
il.Ignore("foo", "baz")
with raises(ValueError):
il.remove('foo', 'bleh')
il.remove("foo", "bleh")

48
core/tests/markable_test.py
View File

@ -8,33 +8,39 @@ from hscommon.testutil import eq_
from ..markable import MarkableList, Markable
def gen():
ml = MarkableList()
ml.extend(list(range(10)))
return ml
def test_unmarked():
ml = gen()
for i in ml:
assert not ml.is_marked(i)
def test_mark():
ml = gen()
assert ml.mark(3)
assert ml.is_marked(3)
assert not ml.is_marked(2)
def test_unmark():
ml = gen()
ml.mark(4)
assert ml.unmark(4)
assert not ml.is_marked(4)
def test_unmark_unmarked():
ml = gen()
assert not ml.unmark(4)
assert not ml.is_marked(4)
def test_mark_twice_and_unmark():
ml = gen()
assert ml.mark(5)
@ -42,6 +48,7 @@ def test_mark_twice_and_unmark():
ml.unmark(5)
assert not ml.is_marked(5)
def test_mark_toggle():
ml = gen()
ml.mark_toggle(6)
@ -51,22 +58,25 @@ def test_mark_toggle():
ml.mark_toggle(6)
assert ml.is_marked(6)
def test_is_markable():
class Foobar(Markable):
def _is_markable(self, o):
return o == 'foobar'
return o == "foobar"
f = Foobar()
assert not f.is_marked('foobar')
assert not f.mark('foo')
assert not f.is_marked('foo')
f.mark_toggle('foo')
assert not f.is_marked('foo')
f.mark('foobar')
assert f.is_marked('foobar')
assert not f.is_marked("foobar")
assert not f.mark("foo")
assert not f.is_marked("foo")
f.mark_toggle("foo")
assert not f.is_marked("foo")
f.mark("foobar")
assert f.is_marked("foobar")
ml = gen()
ml.mark(11)
assert not ml.is_marked(11)
def test_change_notifications():
class Foobar(Markable):
def _did_mark(self, o):
@ -77,13 +87,14 @@ def test_change_notifications():
f = Foobar()
f.log = []
f.mark('foo')
f.mark('foo')
f.mark_toggle('bar')
f.unmark('foo')
f.unmark('foo')
f.mark_toggle('bar')
eq_([(True, 'foo'), (True, 'bar'), (False, 'foo'), (False, 'bar')], f.log)
f.mark("foo")
f.mark("foo")
f.mark_toggle("bar")
f.unmark("foo")
f.unmark("foo")
f.mark_toggle("bar")
eq_([(True, "foo"), (True, "bar"), (False, "foo"), (False, "bar")], f.log)
def test_mark_count():
ml = gen()
@ -93,6 +104,7 @@ def test_mark_count():
ml.mark(11)
eq_(1, ml.mark_count)
def test_mark_none():
log = []
ml = gen()
@ -104,6 +116,7 @@ def test_mark_none():
eq_(0, ml.mark_count)
eq_([1, 2], log)
def test_mark_all():
ml = gen()
eq_(0, ml.mark_count)
@ -111,6 +124,7 @@ def test_mark_all():
eq_(10, ml.mark_count)
assert ml.is_marked(1)
def test_mark_invert():
ml = gen()
ml.mark(1)
@ -118,6 +132,7 @@ def test_mark_invert():
assert not ml.is_marked(1)
assert ml.is_marked(2)
def test_mark_while_inverted():
log = []
ml = gen()
@ -134,6 +149,7 @@ def test_mark_while_inverted():
eq_(7, ml.mark_count)
eq_([(True, 1), (False, 1), (True, 2), (True, 1), (True, 3)], log)
def test_remove_mark_flag():
ml = gen()
ml.mark(1)
@ -145,10 +161,12 @@ def test_remove_mark_flag():
ml._remove_mark_flag(1)
assert ml.is_marked(1)
def test_is_marked_returns_false_if_object_not_markable():
class MyMarkableList(MarkableList):
def _is_markable(self, o):
return o != 4
ml = MyMarkableList()
ml.extend(list(range(10)))
ml.mark_invert()

68
core/tests/prioritize_test.py
View File

@ -14,6 +14,7 @@ from ..engine import Group, Match
no = NamedObject
def app_with_dupes(dupes):
# Creates an app with specified dupes. dupes is a list of lists, each list in the list being
# a dupe group. We cheat a little bit by creating dupe groups manually instead of running a
@ -29,22 +30,25 @@ def app_with_dupes(dupes):
app.app._results_changed()
return app
# ---
def app_normal_results():
# Just some results, with different extensions and size, for good measure.
dupes = [
[
no('foo1.ext1', size=1, folder='folder1'),
no('foo2.ext2', size=2, folder='folder2')
no("foo1.ext1", size=1, folder="folder1"),
no("foo2.ext2", size=2, folder="folder2"),
],
]
return app_with_dupes(dupes)
@with_app(app_normal_results)
def test_kind_subcrit(app):
# The subcriteria of the "Kind" criteria is a list of extensions contained in the dupes.
app.select_pri_criterion("Kind")
eq_(app.pdialog.criteria_list[:], ['ext1', 'ext2'])
eq_(app.pdialog.criteria_list[:], ["ext1", "ext2"])
@with_app(app_normal_results)
def test_kind_reprioritization(app):
@ -54,12 +58,14 @@ def test_kind_reprioritization(app):
app.pdialog.criteria_list.select([1]) # ext2
app.pdialog.add_selected()
app.pdialog.perform_reprioritization()
eq_(app.rtable[0].data['name'], 'foo2.ext2')
eq_(app.rtable[0].data["name"], "foo2.ext2")
@with_app(app_normal_results)
def test_folder_subcrit(app):
app.select_pri_criterion("Folder")
eq_(app.pdialog.criteria_list[:], ['folder1', 'folder2'])
eq_(app.pdialog.criteria_list[:], ["folder1", "folder2"])
@with_app(app_normal_results)
def test_folder_reprioritization(app):
@ -67,7 +73,8 @@ def test_folder_reprioritization(app):
app.pdialog.criteria_list.select([1]) # folder2
app.pdialog.add_selected()
app.pdialog.perform_reprioritization()
eq_(app.rtable[0].data['name'], 'foo2.ext2')
eq_(app.rtable[0].data["name"], "foo2.ext2")
@with_app(app_normal_results)
def test_prilist_display(app):
@ -88,10 +95,12 @@ def test_prilist_display(app):
]
eq_(app.pdialog.prioritization_list[:], expected)
@with_app(app_normal_results)
def test_size_subcrit(app):
app.select_pri_criterion("Size")
eq_(app.pdialog.criteria_list[:], ['Highest', 'Lowest'])
eq_(app.pdialog.criteria_list[:], ["Highest", "Lowest"])
@with_app(app_normal_results)
def test_size_reprioritization(app):
@ -99,7 +108,8 @@ def test_size_reprioritization(app):
app.pdialog.criteria_list.select([0]) # highest
app.pdialog.add_selected()
app.pdialog.perform_reprioritization()
eq_(app.rtable[0].data['name'], 'foo2.ext2')
eq_(app.rtable[0].data["name"], "foo2.ext2")
@with_app(app_normal_results)
def test_reorder_prioritizations(app):
@ -112,6 +122,7 @@ def test_reorder_prioritizations(app):
]
eq_(app.pdialog.prioritization_list[:], expected)
@with_app(app_normal_results)
def test_remove_crit_from_list(app):
app.add_pri_criterion("Kind", 0)
@ -123,46 +134,43 @@ def test_remove_crit_from_list(app):
]
eq_(app.pdialog.prioritization_list[:], expected)
@with_app(app_normal_results)
def test_add_crit_without_selection(app):
# Adding a criterion without having made a selection doesn't cause a crash.
app.pdialog.add_selected() # no crash
# ---
def app_one_name_ends_with_number():
dupes = [
[
no('foo.ext'),
no('foo1.ext'),
],
[no("foo.ext"), no("foo1.ext")],
]
return app_with_dupes(dupes)
@with_app(app_one_name_ends_with_number)
def test_filename_reprioritization(app):
app.add_pri_criterion("Filename", 0) # Ends with a number
app.pdialog.perform_reprioritization()
eq_(app.rtable[0].data['name'], 'foo1.ext')
eq_(app.rtable[0].data["name"], "foo1.ext")
# ---
def app_with_subfolders():
dupes = [
[
no('foo1', folder='baz'),
no('foo2', folder='foo/bar'),
],
[
no('foo3', folder='baz'),
no('foo4', folder='foo'),
],
[no("foo1", folder="baz"), no("foo2", folder="foo/bar")],
[no("foo3", folder="baz"), no("foo4", folder="foo")],
]
return app_with_dupes(dupes)
@with_app(app_with_subfolders)
def test_folder_crit_is_sorted(app):
# Folder subcriteria are sorted.
app.select_pri_criterion("Folder")
eq_(app.pdialog.criteria_list[:], ['baz', 'foo', op.join('foo', 'bar')])
eq_(app.pdialog.criteria_list[:], ["baz", "foo", op.join("foo", "bar")])
@with_app(app_with_subfolders)
def test_folder_crit_includes_subfolders(app):
@ -171,27 +179,27 @@ def test_folder_crit_includes_subfolders(app):
app.add_pri_criterion("Folder", 1) # foo
app.pdialog.perform_reprioritization()
# Both foo and foo/bar dupes will be prioritized
eq_(app.rtable[0].data['name'], 'foo2')
eq_(app.rtable[2].data['name'], 'foo4')
eq_(app.rtable[0].data["name"], "foo2")
eq_(app.rtable[2].data["name"], "foo4")
@with_app(app_with_subfolders)
def test_display_something_on_empty_extensions(app):
# When there's no extension, display "None" instead of nothing at all.
app.select_pri_criterion("Kind")
eq_(app.pdialog.criteria_list[:], ['None'])
eq_(app.pdialog.criteria_list[:], ["None"])
# ---
def app_one_name_longer_than_the_other():
dupes = [
[
no('shortest.ext'),
no('loooongest.ext'),
],
[no("shortest.ext"), no("loooongest.ext")],
]
return app_with_dupes(dupes)
@with_app(app_one_name_longer_than_the_other)
def test_longest_filename_prioritization(app):
app.add_pri_criterion("Filename", 2) # Longest
app.pdialog.perform_reprioritization()
eq_(app.rtable[0].data['name'], 'loooongest.ext')
eq_(app.rtable[0].data["name"], "loooongest.ext")

21
core/tests/result_table_test.py
View File

@ -8,6 +8,7 @@
from .base import TestApp, GetTestGroups
def app_with_results():
app = TestApp()
objects, matches, groups = GetTestGroups()
@ -15,23 +16,26 @@ def app_with_results():
app.rtable.refresh()
return app
def test_delta_flags_delta_mode_off():
app = app_with_results()
# When the delta mode is off, we never have delta values flags
app.rtable.delta_values = False
# Ref file, always false anyway
assert not app.rtable[0].is_cell_delta('size')
assert not app.rtable[0].is_cell_delta("size")
# False because delta mode is off
assert not app.rtable[1].is_cell_delta('size')
assert not app.rtable[1].is_cell_delta("size")
def test_delta_flags_delta_mode_on_delta_columns():
# When the delta mode is on, delta columns always have a delta flag, except for ref rows
app = app_with_results()
app.rtable.delta_values = True
# Ref file, always false anyway
assert not app.rtable[0].is_cell_delta('size')
assert not app.rtable[0].is_cell_delta("size")
# But for a dupe, the flag is on
assert app.rtable[1].is_cell_delta('size')
assert app.rtable[1].is_cell_delta("size")
def test_delta_flags_delta_mode_on_non_delta_columns():
# When the delta mode is on, non-delta columns have a delta flag if their value differs from
@ -39,11 +43,12 @@ def test_delta_flags_delta_mode_on_non_delta_columns():
app = app_with_results()
app.rtable.delta_values = True
# "bar bleh" != "foo bar", flag on
assert app.rtable[1].is_cell_delta('name')
assert app.rtable[1].is_cell_delta("name")
# "ibabtu" row, but it's a ref, flag off
assert not app.rtable[3].is_cell_delta('name')
assert not app.rtable[3].is_cell_delta("name")
# "ibabtu" == "ibabtu", flag off
assert not app.rtable[4].is_cell_delta('name')
assert not app.rtable[4].is_cell_delta("name")
def test_delta_flags_delta_mode_on_non_delta_columns_case_insensitive():
# Comparison that occurs for non-numeric columns to check whether they're delta is case
@ -53,4 +58,4 @@ def test_delta_flags_delta_mode_on_non_delta_columns_case_insensitive():
app.app.results.groups[1].dupes[0].name = "IBaBTU"
app.rtable.delta_values = True
# "ibAbtu" == "IBaBTU", flag off
assert not app.rtable[4].is_cell_delta('name')
assert not app.rtable[4].is_cell_delta("name")

251
core/tests/results_test.py
View File

@ -17,6 +17,7 @@ from .. import engine
from .base import NamedObject, GetTestGroups, DupeGuru
from ..results import Results
class TestCaseResultsEmpty:
def setup_method(self, method):
self.app = DupeGuru()
@ -24,7 +25,7 @@ class TestCaseResultsEmpty:
def test_apply_invalid_filter(self):
# If the applied filter is an invalid regexp, just ignore the filter.
self.results.apply_filter('[') # invalid
self.results.apply_filter("[") # invalid
self.test_stat_line() # make sure that the stats line isn't saying we applied a '[' filter
def test_stat_line(self):
@ -34,7 +35,7 @@ class TestCaseResultsEmpty:
eq_(0, len(self.results.groups))
def test_get_group_of_duplicate(self):
assert self.results.get_group_of_duplicate('foo') is None
assert self.results.get_group_of_duplicate("foo") is None
def test_save_to_xml(self):
f = io.BytesIO()
@ -42,7 +43,7 @@ class TestCaseResultsEmpty:
f.seek(0)
doc = ET.parse(f)
root = doc.getroot()
eq_('results', root.tag)
eq_("results", root.tag)
def test_is_modified(self):
assert not self.results.is_modified
@ -59,10 +60,10 @@ class TestCaseResultsEmpty:
# would have been some kind of feedback to the user, but the work involved for something
# that simply never happens (I never received a report of this crash, I experienced it
# while fooling around) is too much. Instead, use standard name conflict resolution.
folderpath = tmpdir.join('foo')
folderpath = tmpdir.join("foo")
folderpath.mkdir()
self.results.save_to_xml(str(folderpath)) # no crash
assert tmpdir.join('[000] foo').check()
assert tmpdir.join("[000] foo").check()
class TestCaseResultsWithSomeGroups:
@ -116,18 +117,22 @@ class TestCaseResultsWithSomeGroups:
assert d is g.ref
def test_sort_groups(self):
self.results.make_ref(self.objects[1]) #We want to make the 1024 sized object to go ref.
self.results.make_ref(
self.objects[1]
) # We want to make the 1024 sized object to go ref.
g1, g2 = self.groups
self.results.sort_groups('size')
self.results.sort_groups("size")
assert self.results.groups[0] is g2
assert self.results.groups[1] is g1
self.results.sort_groups('size', False)
self.results.sort_groups("size", False)
assert self.results.groups[0] is g1
assert self.results.groups[1] is g2
def test_set_groups_when_sorted(self):
self.results.make_ref(self.objects[1]) #We want to make the 1024 sized object to go ref.
self.results.sort_groups('size')
self.results.make_ref(
self.objects[1]
) # We want to make the 1024 sized object to go ref.
self.results.sort_groups("size")
objects, matches, groups = GetTestGroups()
g1, g2 = groups
g1.switch_ref(objects[1])
@ -158,9 +163,9 @@ class TestCaseResultsWithSomeGroups:
o3.size = 3
o4.size = 2
o5.size = 1
self.results.sort_dupes('size')
self.results.sort_dupes("size")
eq_([o5, o3, o2], self.results.dupes)
self.results.sort_dupes('size', False)
self.results.sort_dupes("size", False)
eq_([o2, o3, o5], self.results.dupes)
def test_dupe_list_remember_sort(self):
@ -170,7 +175,7 @@ class TestCaseResultsWithSomeGroups:
o3.size = 3
o4.size = 2
o5.size = 1
self.results.sort_dupes('size')
self.results.sort_dupes("size")
self.results.make_ref(o2)
eq_([o5, o3, o1], self.results.dupes)
@ -181,14 +186,14 @@ class TestCaseResultsWithSomeGroups:
o3.size = 3 # -7
o4.size = 20
o5.size = 1 # -19
self.results.sort_dupes('size', delta=True)
self.results.sort_dupes("size", delta=True)
eq_([o5, o2, o3], self.results.dupes)
def test_sort_empty_list(self):
# There was an infinite loop when sorting an empty list.
app = DupeGuru()
r = app.results
r.sort_dupes('name')
r.sort_dupes("name")
eq_([], r.dupes)
def test_dupe_list_update_on_remove_duplicates(self):
@ -236,7 +241,7 @@ class TestCaseResultsWithSomeGroups:
# "aaa" makes our dupe go first in alphabetical order, but since we have the same value as
# ref, we're going last.
g2r.name = g2d1.name = "aaa"
self.results.sort_dupes('name', delta=True)
self.results.sort_dupes("name", delta=True)
eq_("aaa", self.results.dupes[2].name)
def test_dupe_list_sort_delta_values_nonnumeric_case_insensitive(self):
@ -244,9 +249,10 @@ class TestCaseResultsWithSomeGroups:
g1r, g1d1, g1d2, g2r, g2d1 = self.objects
g2r.name = "AaA"
g2d1.name = "aAa"
self.results.sort_dupes('name', delta=True)
self.results.sort_dupes("name", delta=True)
eq_("aAa", self.results.dupes[2].name)
class TestCaseResultsWithSavedResults:
def setup_method(self, method):
self.app = DupeGuru()
@ -335,7 +341,7 @@ class TestCaseResultsMarkings:
def log_object(o):
log.append(o)
if o is self.objects[1]:
raise EnvironmentError('foobar')
raise EnvironmentError("foobar")
log = []
self.results.mark_all()
@ -350,7 +356,7 @@ class TestCaseResultsMarkings:
eq_(len(self.results.problems), 1)
dupe, msg = self.results.problems[0]
assert dupe is self.objects[1]
eq_(msg, 'foobar')
eq_(msg, "foobar")
def test_perform_on_marked_with_ref(self):
def log_object(o):
@ -408,20 +414,20 @@ class TestCaseResultsMarkings:
f.seek(0)
doc = ET.parse(f)
root = doc.getroot()
g1, g2 = root.getiterator('group')
d1, d2, d3 = g1.getiterator('file')
eq_('n', d1.get('marked'))
eq_('n', d2.get('marked'))
eq_('y', d3.get('marked'))
d1, d2 = g2.getiterator('file')
eq_('n', d1.get('marked'))
eq_('y', d2.get('marked'))
g1, g2 = root.getiterator("group")
d1, d2, d3 = g1.getiterator("file")
eq_("n", d1.get("marked"))
eq_("n", d2.get("marked"))
eq_("y", d3.get("marked"))
d1, d2 = g2.getiterator("file")
eq_("n", d1.get("marked"))
eq_("y", d2.get("marked"))
def test_LoadXML(self):
def get_file(path):
return [f for f in self.objects if str(f.path) == path][0]
self.objects[4].name = 'ibabtu 2' #we can't have 2 files with the same path
self.objects[4].name = "ibabtu 2" # we can't have 2 files with the same path
self.results.mark(self.objects[1])
self.results.mark_invert()
f = io.BytesIO()
@ -449,46 +455,46 @@ class TestCaseResultsXML:
def test_save_to_xml(self):
self.objects[0].is_ref = True
self.objects[0].words = [['foo', 'bar']]
self.objects[0].words = [["foo", "bar"]]
f = io.BytesIO()
self.results.save_to_xml(f)
f.seek(0)
doc = ET.parse(f)
root = doc.getroot()
eq_('results', root.tag)
eq_("results", root.tag)
eq_(2, len(root))
eq_(2, len([c for c in root if c.tag == 'group']))
eq_(2, len([c for c in root if c.tag == "group"]))
g1, g2 = root
eq_(6, len(g1))
eq_(3, len([c for c in g1 if c.tag == 'file']))
eq_(3, len([c for c in g1 if c.tag == 'match']))
d1, d2, d3 = [c for c in g1 if c.tag == 'file']
eq_(op.join('basepath', 'foo bar'), d1.get('path'))
eq_(op.join('basepath', 'bar bleh'), d2.get('path'))
eq_(op.join('basepath', 'foo bleh'), d3.get('path'))
eq_('y', d1.get('is_ref'))
eq_('n', d2.get('is_ref'))
eq_('n', d3.get('is_ref'))
eq_('foo,bar', d1.get('words'))
eq_('bar,bleh', d2.get('words'))
eq_('foo,bleh', d3.get('words'))
eq_(3, len([c for c in g1 if c.tag == "file"]))
eq_(3, len([c for c in g1 if c.tag == "match"]))
d1, d2, d3 = [c for c in g1 if c.tag == "file"]
eq_(op.join("basepath", "foo bar"), d1.get("path"))
eq_(op.join("basepath", "bar bleh"), d2.get("path"))
eq_(op.join("basepath", "foo bleh"), d3.get("path"))
eq_("y", d1.get("is_ref"))
eq_("n", d2.get("is_ref"))
eq_("n", d3.get("is_ref"))
eq_("foo,bar", d1.get("words"))
eq_("bar,bleh", d2.get("words"))
eq_("foo,bleh", d3.get("words"))
eq_(3, len(g2))
eq_(2, len([c for c in g2 if c.tag == 'file']))
eq_(1, len([c for c in g2 if c.tag == 'match']))
d1, d2 = [c for c in g2 if c.tag == 'file']
eq_(op.join('basepath', 'ibabtu'), d1.get('path'))
eq_(op.join('basepath', 'ibabtu'), d2.get('path'))
eq_('n', d1.get('is_ref'))
eq_('n', d2.get('is_ref'))
eq_('ibabtu', d1.get('words'))
eq_('ibabtu', d2.get('words'))
eq_(2, len([c for c in g2 if c.tag == "file"]))
eq_(1, len([c for c in g2 if c.tag == "match"]))
d1, d2 = [c for c in g2 if c.tag == "file"]
eq_(op.join("basepath", "ibabtu"), d1.get("path"))
eq_(op.join("basepath", "ibabtu"), d2.get("path"))
eq_("n", d1.get("is_ref"))
eq_("n", d2.get("is_ref"))
eq_("ibabtu", d1.get("words"))
eq_("ibabtu", d2.get("words"))
def test_LoadXML(self):
def get_file(path):
return [f for f in self.objects if str(f.path) == path][0]
self.objects[0].is_ref = True
self.objects[4].name = 'ibabtu 2' #we can't have 2 files with the same path
self.objects[4].name = "ibabtu 2" # we can't have 2 files with the same path
f = io.BytesIO()
self.results.save_to_xml(f)
f.seek(0)
@ -504,23 +510,23 @@ class TestCaseResultsXML:
assert g1[0] is self.objects[0]
assert g1[1] is self.objects[1]
assert g1[2] is self.objects[2]
eq_(['foo', 'bar'], g1[0].words)
eq_(['bar', 'bleh'], g1[1].words)
eq_(['foo', 'bleh'], g1[2].words)
eq_(["foo", "bar"], g1[0].words)
eq_(["bar", "bleh"], g1[1].words)
eq_(["foo", "bleh"], g1[2].words)
eq_(2, len(g2))
assert not g2[0].is_ref
assert not g2[1].is_ref
assert g2[0] is self.objects[3]
assert g2[1] is self.objects[4]
eq_(['ibabtu'], g2[0].words)
eq_(['ibabtu'], g2[1].words)
eq_(["ibabtu"], g2[0].words)
eq_(["ibabtu"], g2[1].words)
def test_LoadXML_with_filename(self, tmpdir):
def get_file(path):
return [f for f in self.objects if str(f.path) == path][0]
filename = str(tmpdir.join('dupeguru_results.xml'))
self.objects[4].name = 'ibabtu 2' #we can't have 2 files with the same path
filename = str(tmpdir.join("dupeguru_results.xml"))
self.objects[4].name = "ibabtu 2" # we can't have 2 files with the same path
self.results.save_to_xml(filename)
app = DupeGuru()
r = Results(app)
@ -529,11 +535,11 @@ class TestCaseResultsXML:
def test_LoadXML_with_some_files_that_dont_exist_anymore(self):
def get_file(path):
if path.endswith('ibabtu 2'):
if path.endswith("ibabtu 2"):
return None
return [f for f in self.objects if str(f.path) == path][0]
self.objects[4].name = 'ibabtu 2' #we can't have 2 files with the same path
self.objects[4].name = "ibabtu 2" # we can't have 2 files with the same path
f = io.BytesIO()
self.results.save_to_xml(f)
f.seek(0)
@ -547,36 +553,36 @@ class TestCaseResultsXML:
def get_file(path):
return [f for f in self.objects if str(f.path) == path][0]
root = ET.Element('foobar') #The root element shouldn't matter, really.
group_node = ET.SubElement(root, 'group')
dupe_node = ET.SubElement(group_node, 'file') #Perfectly correct file
dupe_node.set('path', op.join('basepath', 'foo bar'))
dupe_node.set('is_ref', 'y')
dupe_node.set('words', 'foo, bar')
dupe_node = ET.SubElement(group_node, 'file') #is_ref missing, default to 'n'
dupe_node.set('path', op.join('basepath', 'foo bleh'))
dupe_node.set('words', 'foo, bleh')
dupe_node = ET.SubElement(group_node, 'file') #words are missing, valid.
dupe_node.set('path', op.join('basepath', 'bar bleh'))
dupe_node = ET.SubElement(group_node, 'file') #path is missing, invalid.
dupe_node.set('words', 'foo, bleh')
dupe_node = ET.SubElement(group_node, 'foobar') #Invalid element name
dupe_node.set('path', op.join('basepath', 'bar bleh'))
dupe_node.set('is_ref', 'y')
dupe_node.set('words', 'bar, bleh')
match_node = ET.SubElement(group_node, 'match') # match pointing to a bad index
match_node.set('first', '42')
match_node.set('second', '45')
match_node = ET.SubElement(group_node, 'match') # match with missing attrs
match_node = ET.SubElement(group_node, 'match') # match with non-int values
match_node.set('first', 'foo')
match_node.set('second', 'bar')
match_node.set('percentage', 'baz')
group_node = ET.SubElement(root, 'foobar') #invalid group
group_node = ET.SubElement(root, 'group') #empty group
root = ET.Element("foobar") # The root element shouldn't matter, really.
group_node = ET.SubElement(root, "group")
dupe_node = ET.SubElement(group_node, "file") # Perfectly correct file
dupe_node.set("path", op.join("basepath", "foo bar"))
dupe_node.set("is_ref", "y")
dupe_node.set("words", "foo, bar")
dupe_node = ET.SubElement(group_node, "file") # is_ref missing, default to 'n'
dupe_node.set("path", op.join("basepath", "foo bleh"))
dupe_node.set("words", "foo, bleh")
dupe_node = ET.SubElement(group_node, "file") # words are missing, valid.
dupe_node.set("path", op.join("basepath", "bar bleh"))
dupe_node = ET.SubElement(group_node, "file") # path is missing, invalid.
dupe_node.set("words", "foo, bleh")
dupe_node = ET.SubElement(group_node, "foobar") # Invalid element name
dupe_node.set("path", op.join("basepath", "bar bleh"))
dupe_node.set("is_ref", "y")
dupe_node.set("words", "bar, bleh")
match_node = ET.SubElement(group_node, "match") # match pointing to a bad index
match_node.set("first", "42")
match_node.set("second", "45")
match_node = ET.SubElement(group_node, "match") # match with missing attrs
match_node = ET.SubElement(group_node, "match") # match with non-int values
match_node.set("first", "foo")
match_node.set("second", "bar")
match_node.set("percentage", "baz")
group_node = ET.SubElement(root, "foobar") # invalid group
group_node = ET.SubElement(root, "group") # empty group
f = io.BytesIO()
tree = ET.ElementTree(root)
tree.write(f, encoding='utf-8')
tree.write(f, encoding="utf-8")
f.seek(0)
app = DupeGuru()
r = Results(app)
@ -586,16 +592,18 @@ class TestCaseResultsXML:
def test_xml_non_ascii(self):
def get_file(path):
if path == op.join('basepath', '\xe9foo bar'):
if path == op.join("basepath", "\xe9foo bar"):
return objects[0]
if path == op.join('basepath', 'bar bleh'):
if path == op.join("basepath", "bar bleh"):
return objects[1]
objects = [NamedObject("\xe9foo bar", True), NamedObject("bar bleh", True)]
matches = engine.getmatches(objects) # we should have 5 matches
groups = engine.get_groups(matches) # We should have 2 groups
for g in groups:
g.prioritize(lambda x: objects.index(x)) #We want the dupes to be in the same order as the list is
g.prioritize(
lambda x: objects.index(x)
) # We want the dupes to be in the same order as the list is
app = DupeGuru()
results = Results(app)
results.groups = groups
@ -607,11 +615,11 @@ class TestCaseResultsXML:
r.load_from_xml(f, get_file)
g = r.groups[0]
eq_("\xe9foo bar", g[0].name)
eq_(['efoo', 'bar'], g[0].words)
eq_(["efoo", "bar"], g[0].words)
def test_load_invalid_xml(self):
f = io.BytesIO()
f.write(b'<this is invalid')
f.write(b"<this is invalid")
f.seek(0)
app = DupeGuru()
r = Results(app)
@ -623,7 +631,7 @@ class TestCaseResultsXML:
app = DupeGuru()
r = Results(app)
with raises(IOError):
r.load_from_xml('does_not_exist.xml', None)
r.load_from_xml("does_not_exist.xml", None)
eq_(0, len(r.groups))
def test_remember_match_percentage(self):
@ -661,12 +669,12 @@ class TestCaseResultsXML:
def test_apply_filter_works_on_paths(self):
# apply_filter() searches on the whole path, not just on the filename.
self.results.apply_filter('basepath')
self.results.apply_filter("basepath")
eq_(len(self.results.groups), 2)
def test_save_xml_with_invalid_characters(self):
# Don't crash when saving files that have invalid xml characters in their path
self.objects[0].name = 'foo\x19'
self.objects[0].name = "foo\x19"
self.results.save_to_xml(io.BytesIO()) # don't crash
@ -676,7 +684,7 @@ class TestCaseResultsFilter:
self.results = self.app.results
self.objects, self.matches, self.groups = GetTestGroups()
self.results.groups = self.groups
self.results.apply_filter(r'foo')
self.results.apply_filter(r"foo")
def test_groups(self):
eq_(1, len(self.results.groups))
@ -702,23 +710,23 @@ class TestCaseResultsFilter:
def test_include_ref_dupes_in_filter(self):
# When only the ref of a group match the filter, include it in the group
self.results.apply_filter(None)
self.results.apply_filter(r'foo bar')
self.results.apply_filter(r"foo bar")
eq_(1, len(self.results.groups))
eq_(0, len(self.results.dupes))
def test_filters_build_on_one_another(self):
self.results.apply_filter(r'bar')
self.results.apply_filter(r"bar")
eq_(1, len(self.results.groups))
eq_(0, len(self.results.dupes))
def test_stat_line(self):
expected = '0 / 1 (0.00 B / 1.00 B) duplicates marked. filter: foo'
expected = "0 / 1 (0.00 B / 1.00 B) duplicates marked. filter: foo"
eq_(expected, self.results.stat_line)
self.results.apply_filter(r'bar')
expected = '0 / 0 (0.00 B / 0.00 B) duplicates marked. filter: foo --> bar'
self.results.apply_filter(r"bar")
expected = "0 / 0 (0.00 B / 0.00 B) duplicates marked. filter: foo --> bar"
eq_(expected, self.results.stat_line)
self.results.apply_filter(None)
expected = '0 / 3 (0.00 B / 1.01 KB) duplicates marked.'
expected = "0 / 3 (0.00 B / 1.01 KB) duplicates marked."
eq_(expected, self.results.stat_line)
def test_mark_count_is_filtered_as_well(self):
@ -726,8 +734,8 @@ class TestCaseResultsFilter:
# We don't want to perform mark_all() because we want the mark list to contain objects
for dupe in self.results.dupes:
self.results.mark(dupe)
self.results.apply_filter(r'foo')
expected = '1 / 1 (1.00 B / 1.00 B) duplicates marked. filter: foo'
self.results.apply_filter(r"foo")
expected = "1 / 1 (1.00 B / 1.00 B) duplicates marked. filter: foo"
eq_(expected, self.results.stat_line)
def test_mark_all_only_affects_filtered_items(self):
@ -741,15 +749,15 @@ class TestCaseResultsFilter:
self.results.apply_filter(None)
self.results.make_ref(self.objects[1]) # to have the 1024 b obkect as ref
g1, g2 = self.groups
self.results.apply_filter('a') # Matches both group
self.results.sort_groups('size')
self.results.apply_filter("a") # Matches both group
self.results.sort_groups("size")
assert self.results.groups[0] is g2
assert self.results.groups[1] is g1
self.results.apply_filter(None)
assert self.results.groups[0] is g2
assert self.results.groups[1] is g1
self.results.sort_groups('size', False)
self.results.apply_filter('a')
self.results.sort_groups("size", False)
self.results.apply_filter("a")
assert self.results.groups[1] is g2
assert self.results.groups[0] is g1
@ -764,12 +772,12 @@ class TestCaseResultsFilter:
def get_file(path):
return [f for f in self.objects if str(f.path) == path][0]
filename = str(tmpdir.join('dupeguru_results.xml'))
self.objects[4].name = 'ibabtu 2' #we can't have 2 files with the same path
filename = str(tmpdir.join("dupeguru_results.xml"))
self.objects[4].name = "ibabtu 2" # we can't have 2 files with the same path
self.results.save_to_xml(filename)
app = DupeGuru()
r = Results(app)
r.apply_filter('foo')
r.apply_filter("foo")
r.load_from_xml(filename, get_file)
eq_(2, len(r.groups))
@ -778,7 +786,7 @@ class TestCaseResultsFilter:
self.results.apply_filter(None)
eq_(2, len(self.results.groups))
eq_(2, len(self.results.dupes))
self.results.apply_filter('ibabtu')
self.results.apply_filter("ibabtu")
self.results.remove_duplicates([self.results.dupes[0]])
self.results.apply_filter(None)
eq_(1, len(self.results.groups))
@ -786,7 +794,7 @@ class TestCaseResultsFilter:
def test_filter_is_case_insensitive(self):
self.results.apply_filter(None)
self.results.apply_filter('FOO')
self.results.apply_filter("FOO")
eq_(1, len(self.results.dupes))
def test_make_ref_on_filtered_out_doesnt_mess_stats(self):
@ -797,10 +805,12 @@ class TestCaseResultsFilter:
bar_bleh = g1[1] # The "bar bleh" dupe is filtered out
self.results.make_ref(bar_bleh)
# Now the stats should display *2* markable dupes (instead of 1)
expected = '0 / 2 (0.00 B / 2.00 B) duplicates marked. filter: foo'
expected = "0 / 2 (0.00 B / 2.00 B) duplicates marked. filter: foo"
eq_(expected, self.results.stat_line)
self.results.apply_filter(None) # Now let's make sure our unfiltered results aren't fucked up
expected = '0 / 3 (0.00 B / 3.00 B) duplicates marked.'
self.results.apply_filter(
None
) # Now let's make sure our unfiltered results aren't fucked up
expected = "0 / 3 (0.00 B / 3.00 B) duplicates marked."
eq_(expected, self.results.stat_line)
@ -814,6 +824,5 @@ class TestCaseResultsRefFile:
self.results.groups = self.groups
def test_stat_line(self):
expected = '0 / 2 (0.00 B / 2.00 B) duplicates marked.'
expected = "0 / 2 (0.00 B / 2.00 B) duplicates marked."
eq_(expected, self.results.stat_line)

316
core/tests/scanner_test.py
View File

@ -14,6 +14,7 @@ from ..ignore import IgnoreList
from ..scanner import Scanner, ScanType
from ..me.scanner import ScannerME
class NamedObject:
def __init__(self, name="foobar", size=1, path=None):
if path is None:
@ -26,22 +27,25 @@ class NamedObject:
self.words = getwords(name)
def __repr__(self):
return '<NamedObject %r %r>' % (self.name, self.path)
return "<NamedObject %r %r>" % (self.name, self.path)
no = NamedObject
def pytest_funcarg__fake_fileexists(request):
# This is a hack to avoid invalidating all previous tests since the scanner started to test
# for file existence before doing the match grouping.
monkeypatch = request.getfuncargvalue('monkeypatch')
monkeypatch.setattr(Path, 'exists', lambda _: True)
monkeypatch = request.getfuncargvalue("monkeypatch")
monkeypatch.setattr(Path, "exists", lambda _: True)
def test_empty(fake_fileexists):
s = Scanner()
r = s.get_dupe_groups([])
eq_(r, [])
def test_default_settings(fake_fileexists):
s = Scanner()
eq_(s.min_match_percentage, 80)
@ -50,9 +54,10 @@ def test_default_settings(fake_fileexists):
eq_(s.word_weighting, False)
eq_(s.match_similar_words, False)
def test_simple_with_default_settings(fake_fileexists):
s = Scanner()
f = [no('foo bar', path='p1'), no('foo bar', path='p2'), no('foo bleh')]
f = [no("foo bar", path="p1"), no("foo bar", path="p2"), no("foo bleh")]
r = s.get_dupe_groups(f)
eq_(len(r), 1)
g = r[0]
@ -61,29 +66,42 @@ def test_simple_with_default_settings(fake_fileexists):
assert g.ref in f[:2]
assert g.dupes[0] in f[:2]
def test_simple_with_lower_min_match(fake_fileexists):
s = Scanner()
s.min_match_percentage = 50
f = [no('foo bar', path='p1'), no('foo bar', path='p2'), no('foo bleh')]
f = [no("foo bar", path="p1"), no("foo bar", path="p2"), no("foo bleh")]
r = s.get_dupe_groups(f)
eq_(len(r), 1)
g = r[0]
eq_(len(g), 3)
def test_trim_all_ref_groups(fake_fileexists):
# When all files of a group are ref, don't include that group in the results, but also don't
# count the files from that group as discarded.
s = Scanner()
f = [no('foo', path='p1'), no('foo', path='p2'), no('bar', path='p1'), no('bar', path='p2')]
f = [
no("foo", path="p1"),
no("foo", path="p2"),
no("bar", path="p1"),
no("bar", path="p2"),
]
f[2].is_ref = True
f[3].is_ref = True
r = s.get_dupe_groups(f)
eq_(len(r), 1)
eq_(s.discarded_file_count, 0)
def test_priorize(fake_fileexists):
s = Scanner()
f = [no('foo', path='p1'), no('foo', path='p2'), no('bar', path='p1'), no('bar', path='p2')]
f = [
no("foo", path="p1"),
no("foo", path="p2"),
no("bar", path="p1"),
no("bar", path="p2"),
]
f[1].size = 2
f[2].size = 3
f[3].is_ref = True
@ -94,18 +112,20 @@ def test_priorize(fake_fileexists):
assert f[3] in (g1.ref, g2.ref)
assert f[2] in (g1.dupes[0], g2.dupes[0])
def test_content_scan(fake_fileexists):
s = Scanner()
s.scan_type = ScanType.Contents
f = [no('foo'), no('bar'), no('bleh')]
f[0].md5 = f[0].md5partial = 'foobar'
f[1].md5 = f[1].md5partial = 'foobar'
f[2].md5 = f[2].md5partial = 'bleh'
f = [no("foo"), no("bar"), no("bleh")]
f[0].md5 = f[0].md5partial = "foobar"
f[1].md5 = f[1].md5partial = "foobar"
f[2].md5 = f[2].md5partial = "bleh"
r = s.get_dupe_groups(f)
eq_(len(r), 1)
eq_(len(r[0]), 2)
eq_(s.discarded_file_count, 0) # don't count the different md5 as discarded!
def test_content_scan_compare_sizes_first(fake_fileexists):
class MyFile(no):
@property
@ -114,16 +134,17 @@ def test_content_scan_compare_sizes_first(fake_fileexists):
s = Scanner()
s.scan_type = ScanType.Contents
f = [MyFile('foo', 1), MyFile('bar', 2)]
f = [MyFile("foo", 1), MyFile("bar", 2)]
eq_(len(s.get_dupe_groups(f)), 0)
def test_min_match_perc_doesnt_matter_for_content_scan(fake_fileexists):
s = Scanner()
s.scan_type = ScanType.Contents
f = [no('foo'), no('bar'), no('bleh')]
f[0].md5 = f[0].md5partial = 'foobar'
f[1].md5 = f[1].md5partial = 'foobar'
f[2].md5 = f[2].md5partial = 'bleh'
f = [no("foo"), no("bar"), no("bleh")]
f[0].md5 = f[0].md5partial = "foobar"
f[1].md5 = f[1].md5partial = "foobar"
f[2].md5 = f[2].md5partial = "bleh"
s.min_match_percentage = 101
r = s.get_dupe_groups(f)
eq_(len(r), 1)
@ -133,157 +154,180 @@ def test_min_match_perc_doesnt_matter_for_content_scan(fake_fileexists):
eq_(len(r), 1)
eq_(len(r[0]), 2)
def test_content_scan_doesnt_put_md5_in_words_at_the_end(fake_fileexists):
s = Scanner()
s.scan_type = ScanType.Contents
f = [no('foo'), no('bar')]
f[0].md5 = f[0].md5partial = '\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f'
f[1].md5 = f[1].md5partial = '\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f'
f = [no("foo"), no("bar")]
f[0].md5 = f[
0
].md5partial = "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
f[1].md5 = f[
1
].md5partial = "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
r = s.get_dupe_groups(f)
r[0]
def test_extension_is_not_counted_in_filename_scan(fake_fileexists):
s = Scanner()
s.min_match_percentage = 100
f = [no('foo.bar'), no('foo.bleh')]
f = [no("foo.bar"), no("foo.bleh")]
r = s.get_dupe_groups(f)
eq_(len(r), 1)
eq_(len(r[0]), 2)
def test_job(fake_fileexists):
def do_progress(progress, desc=''):
def do_progress(progress, desc=""):
log.append(progress)
return True
s = Scanner()
log = []
f = [no('foo bar'), no('foo bar'), no('foo bleh')]
f = [no("foo bar"), no("foo bar"), no("foo bleh")]
s.get_dupe_groups(f, j=job.Job(1, do_progress))
eq_(log[0], 0)
eq_(log[-1], 100)
def test_mix_file_kind(fake_fileexists):
s = Scanner()
s.mix_file_kind = False
f = [no('foo.1'), no('foo.2')]
f = [no("foo.1"), no("foo.2")]
r = s.get_dupe_groups(f)
eq_(len(r), 0)
def test_word_weighting(fake_fileexists):
s = Scanner()
s.min_match_percentage = 75
s.word_weighting = True
f = [no('foo bar'), no('foo bar bleh')]
f = [no("foo bar"), no("foo bar bleh")]
r = s.get_dupe_groups(f)
eq_(len(r), 1)
g = r[0]
m = g.get_match_of(g.dupes[0])
eq_(m.percentage, 75) # 16 letters, 12 matching
def test_similar_words(fake_fileexists):
s = Scanner()
s.match_similar_words = True
f = [no('The White Stripes'), no('The Whites Stripe'), no('Limp Bizkit'), no('Limp Bizkitt')]
f = [
no("The White Stripes"),
no("The Whites Stripe"),
no("Limp Bizkit"),
no("Limp Bizkitt"),
]
r = s.get_dupe_groups(f)
eq_(len(r), 2)
def test_fields(fake_fileexists):
s = Scanner()
s.scan_type = ScanType.Fields
f = [no('The White Stripes - Little Ghost'), no('The White Stripes - Little Acorn')]
f = [no("The White Stripes - Little Ghost"), no("The White Stripes - Little Acorn")]
r = s.get_dupe_groups(f)
eq_(len(r), 0)
def test_fields_no_order(fake_fileexists):
s = Scanner()
s.scan_type = ScanType.FieldsNoOrder
f = [no('The White Stripes - Little Ghost'), no('Little Ghost - The White Stripes')]
f = [no("The White Stripes - Little Ghost"), no("Little Ghost - The White Stripes")]
r = s.get_dupe_groups(f)
eq_(len(r), 1)
def test_tag_scan(fake_fileexists):
s = Scanner()
s.scan_type = ScanType.Tag
o1 = no('foo')
o2 = no('bar')
o1.artist = 'The White Stripes'
o1.title = 'The Air Near My Fingers'
o2.artist = 'The White Stripes'
o2.title = 'The Air Near My Fingers'
o1 = no("foo")
o2 = no("bar")
o1.artist = "The White Stripes"
o1.title = "The Air Near My Fingers"
o2.artist = "The White Stripes"
o2.title = "The Air Near My Fingers"
r = s.get_dupe_groups([o1, o2])
eq_(len(r), 1)
def test_tag_with_album_scan(fake_fileexists):
s = Scanner()
s.scan_type = ScanType.Tag
s.scanned_tags = set(['artist', 'album', 'title'])
o1 = no('foo')
o2 = no('bar')
o3 = no('bleh')
o1.artist = 'The White Stripes'
o1.title = 'The Air Near My Fingers'
o1.album = 'Elephant'
o2.artist = 'The White Stripes'
o2.title = 'The Air Near My Fingers'
o2.album = 'Elephant'
o3.artist = 'The White Stripes'
o3.title = 'The Air Near My Fingers'
o3.album = 'foobar'
s.scanned_tags = set(["artist", "album", "title"])
o1 = no("foo")
o2 = no("bar")
o3 = no("bleh")
o1.artist = "The White Stripes"
o1.title = "The Air Near My Fingers"
o1.album = "Elephant"
o2.artist = "The White Stripes"
o2.title = "The Air Near My Fingers"
o2.album = "Elephant"
o3.artist = "The White Stripes"
o3.title = "The Air Near My Fingers"
o3.album = "foobar"
r = s.get_dupe_groups([o1, o2, o3])
eq_(len(r), 1)
def test_that_dash_in_tags_dont_create_new_fields(fake_fileexists):
s = Scanner()
s.scan_type = ScanType.Tag
s.scanned_tags = set(['artist', 'album', 'title'])
s.scanned_tags = set(["artist", "album", "title"])
s.min_match_percentage = 50
o1 = no('foo')
o2 = no('bar')
o1.artist = 'The White Stripes - a'
o1.title = 'The Air Near My Fingers - a'
o1.album = 'Elephant - a'
o2.artist = 'The White Stripes - b'
o2.title = 'The Air Near My Fingers - b'
o2.album = 'Elephant - b'
o1 = no("foo")
o2 = no("bar")
o1.artist = "The White Stripes - a"
o1.title = "The Air Near My Fingers - a"
o1.album = "Elephant - a"
o2.artist = "The White Stripes - b"
o2.title = "The Air Near My Fingers - b"
o2.album = "Elephant - b"
r = s.get_dupe_groups([o1, o2])
eq_(len(r), 1)
def test_tag_scan_with_different_scanned(fake_fileexists):
s = Scanner()
s.scan_type = ScanType.Tag
s.scanned_tags = set(['track', 'year'])
o1 = no('foo')
o2 = no('bar')
o1.artist = 'The White Stripes'
o1.title = 'some title'
o1.track = 'foo'
o1.year = 'bar'
o2.artist = 'The White Stripes'
o2.title = 'another title'
o2.track = 'foo'
o2.year = 'bar'
s.scanned_tags = set(["track", "year"])
o1 = no("foo")
o2 = no("bar")
o1.artist = "The White Stripes"
o1.title = "some title"
o1.track = "foo"
o1.year = "bar"
o2.artist = "The White Stripes"
o2.title = "another title"
o2.track = "foo"
o2.year = "bar"
r = s.get_dupe_groups([o1, o2])
eq_(len(r), 1)
def test_tag_scan_only_scans_existing_tags(fake_fileexists):
s = Scanner()
s.scan_type = ScanType.Tag
s.scanned_tags = set(['artist', 'foo'])
o1 = no('foo')
o2 = no('bar')
o1.artist = 'The White Stripes'
o1.foo = 'foo'
o2.artist = 'The White Stripes'
o2.foo = 'bar'
s.scanned_tags = set(["artist", "foo"])
o1 = no("foo")
o2 = no("bar")
o1.artist = "The White Stripes"
o1.foo = "foo"
o2.artist = "The White Stripes"
o2.foo = "bar"
r = s.get_dupe_groups([o1, o2])
eq_(len(r), 1) # Because 'foo' is not scanned, they match
def test_tag_scan_converts_to_str(fake_fileexists):
s = Scanner()
s.scan_type = ScanType.Tag
s.scanned_tags = set(['track'])
o1 = no('foo')
o2 = no('bar')
s.scanned_tags = set(["track"])
o1 = no("foo")
o2 = no("bar")
o1.track = 42
o2.track = 42
try:
@ -292,28 +336,30 @@ def test_tag_scan_converts_to_str(fake_fileexists):
raise AssertionError()
eq_(len(r), 1)
def test_tag_scan_non_ascii(fake_fileexists):
s = Scanner()
s.scan_type = ScanType.Tag
s.scanned_tags = set(['title'])
o1 = no('foo')
o2 = no('bar')
o1.title = 'foobar\u00e9'
o2.title = 'foobar\u00e9'
s.scanned_tags = set(["title"])
o1 = no("foo")
o2 = no("bar")
o1.title = "foobar\u00e9"
o2.title = "foobar\u00e9"
try:
r = s.get_dupe_groups([o1, o2])
except UnicodeEncodeError:
raise AssertionError()
eq_(len(r), 1)
def test_ignore_list(fake_fileexists):
s = Scanner()
f1 = no('foobar')
f2 = no('foobar')
f3 = no('foobar')
f1.path = Path('dir1/foobar')
f2.path = Path('dir2/foobar')
f3.path = Path('dir3/foobar')
f1 = no("foobar")
f2 = no("foobar")
f3 = no("foobar")
f1.path = Path("dir1/foobar")
f2.path = Path("dir2/foobar")
f3.path = Path("dir3/foobar")
ignore_list = IgnoreList()
ignore_list.Ignore(str(f1.path), str(f2.path))
ignore_list.Ignore(str(f1.path), str(f3.path))
@ -327,16 +373,17 @@ def test_ignore_list(fake_fileexists):
# Ignored matches are not counted as discarded
eq_(s.discarded_file_count, 0)
def test_ignore_list_checks_for_unicode(fake_fileexists):
# scanner was calling path_str for ignore list checks. Since the Path changes, it must
# be unicode(path)
s = Scanner()
f1 = no('foobar')
f2 = no('foobar')
f3 = no('foobar')
f1.path = Path('foo1\u00e9')
f2.path = Path('foo2\u00e9')
f3.path = Path('foo3\u00e9')
f1 = no("foobar")
f2 = no("foobar")
f3 = no("foobar")
f1.path = Path("foo1\u00e9")
f2.path = Path("foo2\u00e9")
f3.path = Path("foo3\u00e9")
ignore_list = IgnoreList()
ignore_list.Ignore(str(f1.path), str(f2.path))
ignore_list.Ignore(str(f1.path), str(f3.path))
@ -348,6 +395,7 @@ def test_ignore_list_checks_for_unicode(fake_fileexists):
assert f2 in g
assert f3 in g
def test_file_evaluates_to_false(fake_fileexists):
# A very wrong way to use any() was added at some point, causing resulting group list
# to be empty.
@ -355,19 +403,19 @@ def test_file_evaluates_to_false(fake_fileexists):
def __bool__(self):
return False
s = Scanner()
f1 = FalseNamedObject('foobar', path='p1')
f2 = FalseNamedObject('foobar', path='p2')
f1 = FalseNamedObject("foobar", path="p1")
f2 = FalseNamedObject("foobar", path="p2")
r = s.get_dupe_groups([f1, f2])
eq_(len(r), 1)
def test_size_threshold(fake_fileexists):
# Only file equal or higher than the size_threshold in size are scanned
s = Scanner()
f1 = no('foo', 1, path='p1')
f2 = no('foo', 2, path='p2')
f3 = no('foo', 3, path='p3')
f1 = no("foo", 1, path="p1")
f2 = no("foo", 2, path="p2")
f3 = no("foo", 3, path="p3")
s.size_threshold = 2
groups = s.get_dupe_groups([f1, f2, f3])
eq_(len(groups), 1)
@ -377,48 +425,52 @@ def test_size_threshold(fake_fileexists):
assert f2 in group
assert f3 in group
def test_tie_breaker_path_deepness(fake_fileexists):
# If there is a tie in prioritization, path deepness is used as a tie breaker
s = Scanner()
o1, o2 = no('foo'), no('foo')
o1.path = Path('foo')
o2.path = Path('foo/bar')
o1, o2 = no("foo"), no("foo")
o1.path = Path("foo")
o2.path = Path("foo/bar")
[group] = s.get_dupe_groups([o1, o2])
assert group.ref is o2
def test_tie_breaker_copy(fake_fileexists):
# if copy is in the words used (even if it has a deeper path), it becomes a dupe
s = Scanner()
o1, o2 = no('foo bar Copy'), no('foo bar')
o1.path = Path('deeper/path')
o2.path = Path('foo')
o1, o2 = no("foo bar Copy"), no("foo bar")
o1.path = Path("deeper/path")
o2.path = Path("foo")
[group] = s.get_dupe_groups([o1, o2])
assert group.ref is o2
def test_tie_breaker_same_name_plus_digit(fake_fileexists):
# if ref has the same words as dupe, but has some just one extra word which is a digit, it
# becomes a dupe
s = Scanner()
o1 = no('foo bar 42')
o2 = no('foo bar [42]')
o3 = no('foo bar (42)')
o4 = no('foo bar {42}')
o5 = no('foo bar')
o1 = no("foo bar 42")
o2 = no("foo bar [42]")
o3 = no("foo bar (42)")
o4 = no("foo bar {42}")
o5 = no("foo bar")
# all numbered names have deeper paths, so they'll end up ref if the digits aren't correctly
# used as tie breakers
o1.path = Path('deeper/path')
o2.path = Path('deeper/path')
o3.path = Path('deeper/path')
o4.path = Path('deeper/path')
o5.path = Path('foo')
o1.path = Path("deeper/path")
o2.path = Path("deeper/path")
o3.path = Path("deeper/path")
o4.path = Path("deeper/path")
o5.path = Path("foo")
[group] = s.get_dupe_groups([o1, o2, o3, o4, o5])
assert group.ref is o5
def test_partial_group_match(fake_fileexists):
# Count the number of discarded matches (when a file doesn't match all other dupes of the
# group) in Scanner.discarded_file_count
s = Scanner()
o1, o2, o3 = no('a b'), no('a'), no('b')
o1, o2, o3 = no("a b"), no("a"), no("b")
s.min_match_percentage = 50
[group] = s.get_dupe_groups([o1, o2, o3])
eq_(len(group), 2)
@ -431,6 +483,7 @@ def test_partial_group_match(fake_fileexists):
assert o3 in group
eq_(s.discarded_file_count, 1)
def test_dont_group_files_that_dont_exist(tmpdir):
# when creating groups, check that files exist first. It's possible that these files have
# been moved during the scan by the user.
@ -439,8 +492,8 @@ def test_dont_group_files_that_dont_exist(tmpdir):
s = Scanner()
s.scan_type = ScanType.Contents
p = Path(str(tmpdir))
p['file1'].open('w').write('foo')
p['file2'].open('w').write('foo')
p["file1"].open("w").write("foo")
p["file2"].open("w").write("foo")
file1, file2 = fs.get_files(p)
def getmatches(*args, **kw):
@ -451,6 +504,7 @@ def test_dont_group_files_that_dont_exist(tmpdir):
assert not s.get_dupe_groups([file1, file2])
def test_folder_scan_exclude_subfolder_matches(fake_fileexists):
# when doing a Folders scan type, don't include matches for folders whose parent folder already
# match.
@ -458,31 +512,33 @@ def test_folder_scan_exclude_subfolder_matches(fake_fileexists):
s.scan_type = ScanType.Folders
topf1 = no("top folder 1", size=42)
topf1.md5 = topf1.md5partial = b"some_md5_1"
topf1.path = Path('/topf1')
topf1.path = Path("/topf1")
topf2 = no("top folder 2", size=42)
topf2.md5 = topf2.md5partial = b"some_md5_1"
topf2.path = Path('/topf2')
topf2.path = Path("/topf2")
subf1 = no("sub folder 1", size=41)
subf1.md5 = subf1.md5partial = b"some_md5_2"
subf1.path = Path('/topf1/sub')
subf1.path = Path("/topf1/sub")
subf2 = no("sub folder 2", size=41)
subf2.md5 = subf2.md5partial = b"some_md5_2"
subf2.path = Path('/topf2/sub')
subf2.path = Path("/topf2/sub")
eq_(len(s.get_dupe_groups([topf1, topf2, subf1, subf2])), 1) # only top folders
# however, if another folder matches a subfolder, keep in in the matches
otherf = no("other folder", size=41)
otherf.md5 = otherf.md5partial = b"some_md5_2"
otherf.path = Path('/otherfolder')
otherf.path = Path("/otherfolder")
eq_(len(s.get_dupe_groups([topf1, topf2, subf1, subf2, otherf])), 2)
def test_ignore_files_with_same_path(fake_fileexists):
# It's possible that the scanner is fed with two file instances pointing to the same path. One
# of these files has to be ignored
s = Scanner()
f1 = no('foobar', path='path1/foobar')
f2 = no('foobar', path='path1/foobar')
f1 = no("foobar", path="path1/foobar")
f2 = no("foobar", path="path1/foobar")
eq_(s.get_dupe_groups([f1, f2]), [])
def test_dont_count_ref_files_as_discarded(fake_fileexists):
# To speed up the scan, we don't bother comparing contents of files that are both ref files.
# However, this causes problems in "discarded" counting and we make sure here that we don't
@ -492,20 +548,20 @@ def test_dont_count_ref_files_as_discarded(fake_fileexists):
o1 = no("foo", path="p1")
o2 = no("foo", path="p2")
o3 = no("foo", path="p3")
o1.md5 = o1.md5partial = 'foobar'
o2.md5 = o2.md5partial = 'foobar'
o3.md5 = o3.md5partial = 'foobar'
o1.md5 = o1.md5partial = "foobar"
o2.md5 = o2.md5partial = "foobar"
o3.md5 = o3.md5partial = "foobar"
o1.is_ref = True
o2.is_ref = True
eq_(len(s.get_dupe_groups([o1, o2, o3])), 1)
eq_(s.discarded_file_count, 0)
def test_priorize_me(fake_fileexists):
# in ScannerME, bitrate goes first (right after is_ref) in priorization
s = ScannerME()
o1, o2 = no('foo', path='p1'), no('foo', path='p2')
o1, o2 = no("foo", path="p1"), no("foo", path="p2")
o1.bitrate = 1
o2.bitrate = 2
[group] = s.get_dupe_groups([o1, o2])
assert group.ref is o2

26
core/util.py
View File

@ -8,35 +8,41 @@ import time
from hscommon.util import format_time_decimal
def format_timestamp(t, delta):
if delta:
return format_time_decimal(t)
else:
if t > 0:
return time.strftime('%Y/%m/%d %H:%M:%S', time.localtime(t))
return time.strftime("%Y/%m/%d %H:%M:%S", time.localtime(t))
else:
return '---'
return "---"
def format_words(w):
def do_format(w):
if isinstance(w, list):
return '(%s)' % ', '.join(do_format(item) for item in w)
return "(%s)" % ", ".join(do_format(item) for item in w)
else:
return w.replace('\n', ' ')
return w.replace("\n", " ")
return ", ".join(do_format(item) for item in w)
return ', '.join(do_format(item) for item in w)
def format_perc(p):
return "%0.0f" % p
def format_dupe_count(c):
return str(c) if c else '---'
return str(c) if c else "---"
def cmp_value(dupe, attrname):
value = getattr(dupe, attrname, '')
value = getattr(dupe, attrname, "")
return value.lower() if isinstance(value, str) else value
def fix_surrogate_encoding(s, encoding='utf-8'):
def fix_surrogate_encoding(s, encoding="utf-8"):
# ref #210. It's possible to end up with file paths that, while correct unicode strings, are
# decoded with the 'surrogateescape' option, which make the string unencodable to utf-8. We fix
# these strings here by trying to encode them and, if it fails, we do an encode/decode dance
@ -49,8 +55,6 @@ def fix_surrogate_encoding(s, encoding='utf-8'):
try:
s.encode(encoding)
except UnicodeEncodeError:
return s.encode(encoding, 'replace').decode(encoding)
return s.encode(encoding, "replace").decode(encoding)
else:
return s

286
hscommon/build.py
View File

@ -26,7 +26,8 @@ import modulefinder
from setuptools import setup, Extension
from .plat import ISWINDOWS
from .util import modified_after, find_in_path, ensure_folder, delete_files_with_pattern
from .util import ensure_folder, delete_files_with_pattern
def print_and_do(cmd):
"""Prints ``cmd`` and executes it in the shell.
@ -35,6 +36,7 @@ def print_and_do(cmd):
p = Popen(cmd, shell=True)
return p.wait()
def _perform(src, dst, action, actionname):
if not op.lexists(src):
print("Copying %s failed: it doesn't exist." % src)
@ -44,26 +46,32 @@ def _perform(src, dst, action, actionname):
shutil.rmtree(dst)
else:
os.remove(dst)
print('%s %s --> %s' % (actionname, src, dst))
print("%s %s --> %s" % (actionname, src, dst))
action(src, dst)
def copy_file_or_folder(src, dst):
if op.isdir(src):
shutil.copytree(src, dst, symlinks=True)
else:
shutil.copy(src, dst)
def move(src, dst):
_perform(src, dst, os.rename, 'Moving')
_perform(src, dst, os.rename, "Moving")
def copy(src, dst):
_perform(src, dst, copy_file_or_folder, 'Copying')
_perform(src, dst, copy_file_or_folder, "Copying")
def symlink(src, dst):
_perform(src, dst, os.symlink, 'Symlinking')
_perform(src, dst, os.symlink, "Symlinking")
def hardlink(src, dst):
_perform(src, dst, os.link, 'Hardlinking')
_perform(src, dst, os.link, "Hardlinking")
def _perform_on_all(pattern, dst, action):
# pattern is a glob pattern, example "folder/foo*". The file is moved directly in dst, no folder
@ -73,12 +81,15 @@ def _perform_on_all(pattern, dst, action):
destpath = op.join(dst, op.basename(fn))
action(fn, destpath)
def move_all(pattern, dst):
_perform_on_all(pattern, dst, move)
def copy_all(pattern, dst):
_perform_on_all(pattern, dst, copy)
def ensure_empty_folder(path):
"""Make sure that the path exists and that it's an empty folder.
"""
@ -86,43 +97,54 @@ def ensure_empty_folder(path):
shutil.rmtree(path)
os.mkdir(path)
def filereplace(filename, outfilename=None, **kwargs):
"""Reads `filename`, replaces all {variables} in kwargs, and writes the result to `outfilename`.
"""
if outfilename is None:
outfilename = filename
fp = open(filename, 'rt', encoding='utf-8')
fp = open(filename, "rt", encoding="utf-8")
contents = fp.read()
fp.close()
# We can't use str.format() because in some files, there might be {} characters that mess with it.
for key, item in kwargs.items():
contents = contents.replace('{{{}}}'.format(key), item)
fp = open(outfilename, 'wt', encoding='utf-8')
contents = contents.replace("{{{}}}".format(key), item)
fp = open(outfilename, "wt", encoding="utf-8")
fp.write(contents)
fp.close()
def get_module_version(modulename):
mod = importlib.import_module(modulename)
return mod.__version__
def setup_package_argparser(parser):
parser.add_argument(
'--sign', dest='sign_identity',
help="Sign app under specified identity before packaging (OS X only)"
"--sign",
dest="sign_identity",
help="Sign app under specified identity before packaging (OS X only)",
)
parser.add_argument(
'--nosign', action='store_true', dest='nosign',
help="Don't sign the packaged app (OS X only)"
"--nosign",
action="store_true",
dest="nosign",
help="Don't sign the packaged app (OS X only)",
)
parser.add_argument(
'--src-pkg', action='store_true', dest='src_pkg',
help="Build a tar.gz of the current source."
"--src-pkg",
action="store_true",
dest="src_pkg",
help="Build a tar.gz of the current source.",
)
parser.add_argument(
'--arch-pkg', action='store_true', dest='arch_pkg',
help="Force Arch Linux packaging type, regardless of distro name."
"--arch-pkg",
action="store_true",
dest="arch_pkg",
help="Force Arch Linux packaging type, regardless of distro name.",
)
# `args` come from an ArgumentParser updated with setup_package_argparser()
def package_cocoa_app_in_dmg(app_path, destfolder, args):
# Rather than signing our app in XCode during the build phase, we sign it during the package
@ -130,7 +152,9 @@ def package_cocoa_app_in_dmg(app_path, destfolder, args):
# a valid signature.
if args.sign_identity:
sign_identity = "Developer ID Application: {}".format(args.sign_identity)
result = print_and_do('codesign --force --deep --sign "{}" "{}"'.format(sign_identity, app_path))
result = print_and_do(
'codesign --force --deep --sign "{}" "{}"'.format(sign_identity, app_path)
)
if result != 0:
print("ERROR: Signing failed. Aborting packaging.")
return
@ -139,23 +163,31 @@ def package_cocoa_app_in_dmg(app_path, destfolder, args):
return
build_dmg(app_path, destfolder)
def build_dmg(app_path, destfolder):
"""Builds a DMG volume with application at ``app_path`` and puts it in ``dest_path``.
The name of the resulting DMG volume is determined by the app's name and version.
"""
print(repr(op.join(app_path, 'Contents', 'Info.plist')))
plist = plistlib.readPlist(op.join(app_path, 'Contents', 'Info.plist'))
print(repr(op.join(app_path, "Contents", "Info.plist")))
plist = plistlib.readPlist(op.join(app_path, "Contents", "Info.plist"))
workpath = tempfile.mkdtemp()
dmgpath = op.join(workpath, plist['CFBundleName'])
dmgpath = op.join(workpath, plist["CFBundleName"])
os.mkdir(dmgpath)
print_and_do('cp -R "%s" "%s"' % (app_path, dmgpath))
print_and_do('ln -s /Applications "%s"' % op.join(dmgpath, 'Applications'))
dmgname = '%s_osx_%s.dmg' % (plist['CFBundleName'].lower().replace(' ', '_'), plist['CFBundleVersion'].replace('.', '_'))
print('Building %s' % dmgname)
print_and_do('ln -s /Applications "%s"' % op.join(dmgpath, "Applications"))
dmgname = "%s_osx_%s.dmg" % (
plist["CFBundleName"].lower().replace(" ", "_"),
plist["CFBundleVersion"].replace(".", "_"),
)
print("Building %s" % dmgname)
# UDBZ = bzip compression. UDZO (zip compression) was used before, but it compresses much less.
print_and_do('hdiutil create "%s" -format UDBZ -nocrossdev -srcdir "%s"' % (op.join(destfolder, dmgname), dmgpath))
print('Build Complete')
print_and_do(
'hdiutil create "%s" -format UDBZ -nocrossdev -srcdir "%s"'
% (op.join(destfolder, dmgname), dmgpath)
)
print("Build Complete")
def copy_sysconfig_files_for_embed(destpath):
# This normally shouldn't be needed for Python 3.3+.
@ -163,24 +195,28 @@ def copy_sysconfig_files_for_embed(destpath):
configh = sysconfig.get_config_h_filename()
shutil.copy(makefile, destpath)
shutil.copy(configh, destpath)
with open(op.join(destpath, 'site.py'), 'w') as fp:
fp.write("""
with open(op.join(destpath, "site.py"), "w") as fp:
fp.write(
"""
import os.path as op
from distutils import sysconfig
sysconfig.get_makefile_filename = lambda: op.join(op.dirname(__file__), 'Makefile')
sysconfig.get_config_h_filename = lambda: op.join(op.dirname(__file__), 'pyconfig.h')
""")
"""
)
def add_to_pythonpath(path):
"""Adds ``path`` to both ``PYTHONPATH`` env and ``sys.path``.
"""
abspath = op.abspath(path)
pythonpath = os.environ.get('PYTHONPATH', '')
pathsep = ';' if ISWINDOWS else ':'
pythonpath = os.environ.get("PYTHONPATH", "")
pathsep = ";" if ISWINDOWS else ":"
pythonpath = pathsep.join([abspath, pythonpath]) if pythonpath else abspath
os.environ['PYTHONPATH'] = pythonpath
os.environ["PYTHONPATH"] = pythonpath
sys.path.insert(1, abspath)
# This is a method to hack around those freakingly tricky data inclusion/exlusion rules
# in setuptools. We copy the packages *without data* in a build folder and then build the plugin
# from there.
@ -195,14 +231,16 @@ def copy_packages(packages_names, dest, create_links=False, extra_ignores=None):
create_links = False
if not extra_ignores:
extra_ignores = []
ignore = shutil.ignore_patterns('.hg*', 'tests', 'testdata', 'modules', 'docs', 'locale', *extra_ignores)
ignore = shutil.ignore_patterns(
".hg*", "tests", "testdata", "modules", "docs", "locale", *extra_ignores
)
for package_name in packages_names:
if op.exists(package_name):
source_path = package_name
else:
mod = __import__(package_name)
source_path = mod.__file__
if mod.__file__.endswith('__init__.py'):
if mod.__file__.endswith("__init__.py"):
source_path = op.dirname(source_path)
dest_name = op.basename(source_path)
dest_path = op.join(dest, dest_name)
@ -220,58 +258,81 @@ def copy_packages(packages_names, dest, create_links=False, extra_ignores=None):
else:
shutil.copy(source_path, dest_path)
def copy_qt_plugins(folder_names, dest): # This is only for Windows
from PyQt5.QtCore import QLibraryInfo
qt_plugin_dir = QLibraryInfo.location(QLibraryInfo.PluginsPath)
def ignore(path, names):
if path == qt_plugin_dir:
return [n for n in names if n not in folder_names]
else:
return [n for n in names if not n.endswith('.dll')]
return [n for n in names if not n.endswith(".dll")]
shutil.copytree(qt_plugin_dir, dest, ignore=ignore)
def build_debian_changelog(changelogpath, destfile, pkgname, from_version=None,
distribution='precise', fix_version=None):
def build_debian_changelog(
changelogpath,
destfile,
pkgname,
from_version=None,
distribution="precise",
fix_version=None,
):
"""Builds a debian changelog out of a YAML changelog.
Use fix_version to patch the top changelog to that version (if, for example, there was a
packaging error and you need to quickly fix it)
"""
def desc2list(desc):
# We take each item, enumerated with the '*' character, and transform it into a list.
desc = desc.replace('\n', ' ')
desc = desc.replace(' ', ' ')
result = desc.split('*')
desc = desc.replace("\n", " ")
desc = desc.replace(" ", " ")
result = desc.split("*")
return [s.strip() for s in result if s.strip()]
ENTRY_MODEL = "{pkg} ({version}-1) {distribution}; urgency=low\n\n{changes}\n -- Virgil Dupras <hsoft@hardcoded.net> {date}\n\n"
ENTRY_MODEL = (
"{pkg} ({version}-1) {distribution}; urgency=low\n\n{changes}\n "
"-- Virgil Dupras <hsoft@hardcoded.net> {date}\n\n"
)
CHANGE_MODEL = " * {description}\n"
changelogs = read_changelog_file(changelogpath)
if from_version:
# We only want logs from a particular version
for index, log in enumerate(changelogs):
if log['version'] == from_version:
if log["version"] == from_version:
changelogs = changelogs[: index + 1]
break
if fix_version:
changelogs[0]['version'] = fix_version
changelogs[0]["version"] = fix_version
rendered_logs = []
for log in changelogs:
version = log['version']
logdate = log['date']
desc = log['description']
rendered_date = logdate.strftime('%a, %d %b %Y 00:00:00 +0000')
version = log["version"]
logdate = log["date"]
desc = log["description"]
rendered_date = logdate.strftime("%a, %d %b %Y 00:00:00 +0000")
rendered_descs = [CHANGE_MODEL.format(description=d) for d in desc2list(desc)]
changes = ''.join(rendered_descs)
rendered_log = ENTRY_MODEL.format(pkg=pkgname, version=version, changes=changes,
date=rendered_date, distribution=distribution)
changes = "".join(rendered_descs)
rendered_log = ENTRY_MODEL.format(
pkg=pkgname,
version=version,
changes=changes,
date=rendered_date,
distribution=distribution,
)
rendered_logs.append(rendered_log)
result = ''.join(rendered_logs)
fp = open(destfile, 'w')
result = "".join(rendered_logs)
fp = open(destfile, "w")
fp.write(result)
fp.close()
re_changelog_header = re.compile(r'=== ([\d.b]*) \(([\d\-]*)\)')
re_changelog_header = re.compile(r"=== ([\d.b]*) \(([\d\-]*)\)")
def read_changelog_file(filename):
def iter_by_three(it):
while True:
@ -283,25 +344,31 @@ def read_changelog_file(filename):
return
yield version, date, description
with open(filename, 'rt', encoding='utf-8') as fp:
with open(filename, "rt", encoding="utf-8") as fp:
contents = fp.read()
splitted = re_changelog_header.split(contents)[1:] # the first item is empty
# splitted = [version1, date1, desc1, version2, date2, ...]
result = []
for version, date_str, description in iter_by_three(iter(splitted)):
date = datetime.strptime(date_str, '%Y-%m-%d').date()
d = {'date': date, 'date_str': date_str, 'version': version, 'description': description.strip()}
date = datetime.strptime(date_str, "%Y-%m-%d").date()
d = {
"date": date,
"date_str": date_str,
"version": version,
"description": description.strip(),
}
result.append(d)
return result
class OSXAppStructure:
def __init__(self, dest):
self.dest = dest
self.contents = op.join(dest, 'Contents')
self.macos = op.join(self.contents, 'MacOS')
self.resources = op.join(self.contents, 'Resources')
self.frameworks = op.join(self.contents, 'Frameworks')
self.infoplist = op.join(self.contents, 'Info.plist')
self.contents = op.join(dest, "Contents")
self.macos = op.join(self.contents, "MacOS")
self.resources = op.join(self.contents, "Resources")
self.frameworks = op.join(self.contents, "Frameworks")
self.infoplist = op.join(self.contents, "Info.plist")
def create(self, infoplist):
ensure_empty_folder(self.dest)
@ -309,11 +376,11 @@ class OSXAppStructure:
os.mkdir(self.resources)
os.mkdir(self.frameworks)
copy(infoplist, self.infoplist)
open(op.join(self.contents, 'PkgInfo'), 'wt').write("APPLxxxx")
open(op.join(self.contents, "PkgInfo"), "wt").write("APPLxxxx")
def copy_executable(self, executable):
info = plistlib.readPlist(self.infoplist)
self.executablename = info['CFBundleExecutable']
self.executablename = info["CFBundleExecutable"]
self.executablepath = op.join(self.macos, self.executablename)
copy(executable, self.executablepath)
@ -329,8 +396,14 @@ class OSXAppStructure:
copy(path, framework_dest)
def create_osx_app_structure(dest, executable, infoplist, resources=None, frameworks=None,
symlink_resources=False):
def create_osx_app_structure(
dest,
executable,
infoplist,
resources=None,
frameworks=None,
symlink_resources=False,
):
# `dest`: A path to the destination .app folder
# `executable`: the path of the executable file that goes in "MacOS"
# `infoplist`: The path to your Info.plist file.
@ -343,13 +416,14 @@ def create_osx_app_structure(dest, executable, infoplist, resources=None, framew
app.copy_resources(*resources, use_symlinks=symlink_resources)
app.copy_frameworks(*frameworks)
class OSXFrameworkStructure:
def __init__(self, dest):
self.dest = dest
self.contents = op.join(dest, 'Versions', 'A')
self.resources = op.join(self.contents, 'Resources')
self.headers = op.join(self.contents, 'Headers')
self.infoplist = op.join(self.resources, 'Info.plist')
self.contents = op.join(dest, "Versions", "A")
self.resources = op.join(self.contents, "Resources")
self.headers = op.join(self.contents, "Headers")
self.infoplist = op.join(self.resources, "Info.plist")
self._update_executable_path()
def _update_executable_path(self):
@ -357,7 +431,7 @@ class OSXFrameworkStructure:
self.executablename = self.executablepath = None
return
info = plistlib.readPlist(self.infoplist)
self.executablename = info['CFBundleExecutable']
self.executablename = info["CFBundleExecutable"]
self.executablepath = op.join(self.contents, self.executablename)
def create(self, infoplist):
@ -371,10 +445,10 @@ class OSXFrameworkStructure:
def create_symlinks(self):
# Only call this after create() and copy_executable()
rel = lambda path: op.relpath(path, self.dest)
os.symlink('A', op.join(self.dest, 'Versions', 'Current'))
os.symlink("A", op.join(self.dest, "Versions", "Current"))
os.symlink(rel(self.executablepath), op.join(self.dest, self.executablename))
os.symlink(rel(self.headers), op.join(self.dest, 'Headers'))
os.symlink(rel(self.resources), op.join(self.dest, 'Resources'))
os.symlink(rel(self.headers), op.join(self.dest, "Headers"))
os.symlink(rel(self.resources), op.join(self.dest, "Resources"))
def copy_executable(self, executable):
copy(executable, self.executablepath)
@ -393,23 +467,28 @@ class OSXFrameworkStructure:
def copy_embeddable_python_dylib(dst):
runtime = op.join(sysconfig.get_config_var('PYTHONFRAMEWORKPREFIX'), sysconfig.get_config_var('LDLIBRARY'))
filedest = op.join(dst, 'Python')
runtime = op.join(
sysconfig.get_config_var("PYTHONFRAMEWORKPREFIX"),
sysconfig.get_config_var("LDLIBRARY"),
)
filedest = op.join(dst, "Python")
shutil.copy(runtime, filedest)
os.chmod(filedest, 0o774) # We need write permission to use install_name_tool
cmd = 'install_name_tool -id @rpath/Python %s' % filedest
cmd = "install_name_tool -id @rpath/Python %s" % filedest
print_and_do(cmd)
def collect_stdlib_dependencies(script, dest_folder, extra_deps=None):
sysprefix = sys.prefix # could be a virtualenv
real_lib_prefix = sysconfig.get_config_var('LIBDEST')
real_lib_prefix = sysconfig.get_config_var("LIBDEST")
def is_stdlib_path(path):
# A module path is only a stdlib path if it's in either sys.prefix or
# sysconfig.get_config_var('prefix') (the 2 are different if we are in a virtualenv) and if
# there's no "site-package in the path.
if not path:
return False
if 'site-package' in path:
if "site-package" in path:
return False
if not (path.startswith(sysprefix) or path.startswith(real_lib_prefix)):
return False
@ -425,13 +504,17 @@ def collect_stdlib_dependencies(script, dest_folder, extra_deps=None):
relpath = op.relpath(p, real_lib_prefix)
elif p.startswith(sysprefix):
relpath = op.relpath(p, sysprefix)
assert relpath.startswith('lib/python3.') # we want to get rid of that lib/python3.x part
relpath = relpath[len('lib/python3.X/'):]
assert relpath.startswith(
"lib/python3."
) # we want to get rid of that lib/python3.x part
relpath = relpath[len("lib/python3.X/") :]
else:
raise AssertionError()
if relpath.startswith('lib-dynload'): # We copy .so files in lib-dynload directly in our dest
relpath = relpath[len('lib-dynload/'):]
if relpath.startswith('encodings') or relpath.startswith('distutils'):
if relpath.startswith(
"lib-dynload"
): # We copy .so files in lib-dynload directly in our dest
relpath = relpath[len("lib-dynload/") :]
if relpath.startswith("encodings") or relpath.startswith("distutils"):
# We force their inclusion later.
continue
dest_path = op.join(dest_folder, relpath)
@ -440,34 +523,47 @@ def collect_stdlib_dependencies(script, dest_folder, extra_deps=None):
# stringprep is used by encodings.
# We use real_lib_prefix with distutils because virtualenv messes with it and we need to refer
# to the original distutils folder.
FORCED_INCLUSION = ['encodings', 'stringprep', op.join(real_lib_prefix, 'distutils')]
FORCED_INCLUSION = [
"encodings",
"stringprep",
op.join(real_lib_prefix, "distutils"),
]
if extra_deps:
FORCED_INCLUSION += extra_deps
copy_packages(FORCED_INCLUSION, dest_folder)
# There's a couple of rather big exe files in the distutils folder that we absolutely don't
# need. Remove them.
delete_files_with_pattern(op.join(dest_folder, 'distutils'), '*.exe')
delete_files_with_pattern(op.join(dest_folder, "distutils"), "*.exe")
# And, finally, create an empty "site.py" that Python needs around on startup.
open(op.join(dest_folder, 'site.py'), 'w').close()
open(op.join(dest_folder, "site.py"), "w").close()
def fix_qt_resource_file(path):
# pyrcc5 under Windows, if the locale is non-english, can produce a source file with a date
# containing accented characters. If it does, the encoding is wrong and it prevents the file
# from being correctly frozen by cx_freeze. To work around that, we open the file, strip all
# comments, and save.
with open(path, 'rb') as fp:
with open(path, "rb") as fp:
contents = fp.read()
lines = contents.split(b'\n')
lines = [l for l in lines if not l.startswith(b'#')]
with open(path, 'wb') as fp:
fp.write(b'\n'.join(lines))
lines = contents.split(b"\n")
lines = [l for l in lines if not l.startswith(b"#")]
with open(path, "wb") as fp:
fp.write(b"\n".join(lines))
def build_cocoa_ext(extname, dest, source_files, extra_frameworks=(), extra_includes=()):
def build_cocoa_ext(
extname, dest, source_files, extra_frameworks=(), extra_includes=()
):
extra_link_args = ["-framework", "CoreFoundation", "-framework", "Foundation"]
for extra in extra_frameworks:
extra_link_args += ['-framework', extra]
ext = Extension(extname, source_files, extra_link_args=extra_link_args, include_dirs=extra_includes)
setup(script_args=['build_ext', '--inplace'], ext_modules=[ext])
extra_link_args += ["-framework", extra]
ext = Extension(
extname,
source_files,
extra_link_args=extra_link_args,
include_dirs=extra_includes,
)
setup(script_args=["build_ext", "--inplace"], ext_modules=[ext])
# Our problem here is to get the fully qualified filename of the resulting .so but I couldn't
# find a documented way to do so. The only thing I could find is this below :(
fn = ext._file_name

16
hscommon/build_ext.py
View File

@ -8,26 +8,24 @@ import argparse
from setuptools import setup, Extension
def get_parser():
parser = argparse.ArgumentParser(description="Build an arbitrary Python extension.")
parser.add_argument(
'source_files', nargs='+',
help="List of source files to compile"
)
parser.add_argument(
'name', nargs=1,
help="Name of the resulting extension"
"source_files", nargs="+", help="List of source files to compile"
)
parser.add_argument("name", nargs=1, help="Name of the resulting extension")
return parser
def main():
args = get_parser().parse_args()
print("Building {}...".format(args.name[0]))
ext = Extension(args.name[0], args.source_files)
setup(
script_args=['build_ext', '--inplace'],
ext_modules=[ext],
script_args=["build_ext", "--inplace"], ext_modules=[ext],
)
if __name__ == '__main__':
if __name__ == "__main__":
main()

17
hscommon/conflict.py
View File

@ -19,7 +19,8 @@ from .path import Path, pathify
# This matches [123], but not [12] (3 digits being the minimum).
# It also matches [1234] [12345] etc..
# And only at the start of the string
re_conflict = re.compile(r'^\[\d{3}\d*\] ')
re_conflict = re.compile(r"^\[\d{3}\d*\] ")
def get_conflicted_name(other_names, name):
"""Returns name with a ``[000]`` number in front of it.
@ -32,23 +33,26 @@ def get_conflicted_name(other_names, name):
return name
i = 0
while True:
newname = '[%03d] %s' % (i, name)
newname = "[%03d] %s" % (i, name)
if newname not in other_names:
return newname
i += 1
def get_unconflicted_name(name):
"""Returns ``name`` without ``[]`` brackets.
Brackets which, of course, might have been added by func:`get_conflicted_name`.
"""
return re_conflict.sub('',name,1)
return re_conflict.sub("", name, 1)
def is_conflicted(name):
"""Returns whether ``name`` is prepended with a bracketed number.
"""
return re_conflict.match(name) is not None
@pathify
def _smart_move_or_copy(operation, source_path: Path, dest_path: Path):
"""Use move() or copy() to move and copy file with the conflict management.
@ -62,18 +66,23 @@ def _smart_move_or_copy(operation, source_path: Path, dest_path: Path):
dest_path = dest_dir_path[newname]
operation(str(source_path), str(dest_path))
def smart_move(source_path, dest_path):
"""Same as :func:`smart_copy`, but it moves files instead.
"""
_smart_move_or_copy(shutil.move, source_path, dest_path)
def smart_copy(source_path, dest_path):
"""Copies ``source_path`` to ``dest_path``, recursively and with conflict resolution.
"""
try:
_smart_move_or_copy(shutil.copy, source_path, dest_path)
except IOError as e:
if e.errno in {21, 13}: # it's a directory, code is 21 on OS X / Linux and 13 on Windows
if e.errno in {
21,
13,
}: # it's a directory, code is 21 on OS X / Linux and 13 on Windows
_smart_move_or_copy(shutil.copytree, source_path, dest_path)
else:
raise

1
hscommon/debug.py
View File

@ -9,6 +9,7 @@
import sys
import traceback
# Taken from http://bzimmer.ziclix.com/2008/12/17/python-thread-dumps/
def stacktraces():
code = []

17
hscommon/desktop.py
View File

@ -9,25 +9,30 @@
import os.path as op
import logging
class SpecialFolder:
AppData = 1
Cache = 2
def open_url(url):
"""Open ``url`` with the default browser.
"""
_open_url(url)
def open_path(path):
"""Open ``path`` with its associated application.
"""
_open_path(str(path))
def reveal_path(path):
"""Open the folder containing ``path`` with the default file browser.
"""
_reveal_path(str(path))
def special_folder_path(special_folder, appname=None):
"""Returns the path of ``special_folder``.
@ -38,12 +43,14 @@ def special_folder_path(special_folder, appname=None):
"""
return _special_folder_path(special_folder, appname)
try:
# Normally, we would simply do "from cocoa import proxy", but due to a bug in pytest (currently
# at v2.4.2), our test suite is broken when we do that. This below is a workaround until that
# bug is fixed.
import cocoa
if not hasattr(cocoa, 'proxy'):
if not hasattr(cocoa, "proxy"):
raise ImportError()
proxy = cocoa.proxy
_open_url = proxy.openURL_
@ -56,13 +63,15 @@ try:
else:
base = proxy.getAppdataPath()
if not appname:
appname = proxy.bundleInfo_('CFBundleName')
appname = proxy.bundleInfo_("CFBundleName")
return op.join(base, appname)
except ImportError:
try:
from PyQt5.QtCore import QUrl, QStandardPaths
from PyQt5.QtGui import QDesktopServices
def _open_url(url):
QDesktopServices.openUrl(QUrl(url))
@ -79,10 +88,12 @@ except ImportError:
else:
qtfolder = QStandardPaths.DataLocation
return QStandardPaths.standardLocations(qtfolder)[0]
except ImportError:
# We're either running tests, and these functions don't matter much or we're in a really
# weird situation. Let's just have dummy fallbacks.
logging.warning("Can't setup desktop functions!")
def _open_path(path):
pass
@ -90,4 +101,4 @@ except ImportError:
pass
def _special_folder_path(special_folder, appname=None):
return '/tmp'
return "/tmp"

12
hscommon/geometry.py
View File

@ -11,13 +11,14 @@ from math import sqrt
VERY_SMALL = 0.0000001
class Point:
def __init__(self, x, y):
self.x = x
self.y = y
def __repr__(self):
return '<Point {:2.2f}, {:2.2f}>'.format(*self)
return "<Point {:2.2f}, {:2.2f}>".format(*self)
def __iter__(self):
yield self.x
@ -33,7 +34,7 @@ class Line:
self.p2 = p2
def __repr__(self):
return '<Line {}, {}>'.format(*self)
return "<Line {}, {}>".format(*self)
def __iter__(self):
yield self.p1
@ -69,8 +70,8 @@ class Line:
numera = (D.x - C.x) * (A.y - C.y) - (D.y - C.y) * (A.x - C.x)
numerb = (B.x - A.x) * (A.y - C.y) - (B.y - A.y) * (A.x - C.x)
mua = numera / denom;
mub = numerb / denom;
mua = numera / denom
mub = numerb / denom
if (0 <= mua <= 1) and (0 <= mub <= 1):
x = A.x + mua * (B.x - A.x)
y = A.y + mua * (B.y - A.y)
@ -93,7 +94,7 @@ class Rect:
yield self.h
def __repr__(self):
return '<Rect {:2.2f}, {:2.2f}, {:2.2f}, {:2.2f}>'.format(*self)
return "<Rect {:2.2f}, {:2.2f}, {:2.2f}, {:2.2f}>".format(*self)
@classmethod
def from_center(cls, center, width, height):
@ -215,4 +216,3 @@ class Rect:
@height.setter
def height(self, value):
self.h = value

5
hscommon/gui/base.py
View File

@ -4,13 +4,16 @@
# which should be included with this package. The terms are also available at
# http://www.gnu.org/licenses/gpl-3.0.html
def noop(*args, **kwargs):
pass
class NoopGUI:
def __getattr__(self, func_name):
return noop
class GUIObject:
"""Cross-toolkit "model" representation of a GUI layer object.
@ -32,6 +35,7 @@ class GUIObject:
However, sometimes you want to be able to re-bind another view. In this case, set the
``multibind`` flag to ``True`` and the safeguard will be disabled.
"""
def __init__(self, multibind=False):
self._view = None
self._multibind = multibind
@ -77,4 +81,3 @@ class GUIObject:
# Instead of None, we put a NoopGUI() there to avoid rogue view callback raising an
# exception.
self._view = NoopGUI()

53
hscommon/gui/column.py
View File

@ -10,12 +10,14 @@ import copy
from .base import GUIObject
class Column:
"""Holds column attributes such as its name, width, visibility, etc.
These attributes are then used to correctly configure the column on the "view" side.
"""
def __init__(self, name, display='', visible=True, optional=False):
def __init__(self, name, display="", visible=True, optional=False):
#: "programmatical" (not for display) name. Used as a reference in a couple of place, such
#: as :meth:`Columns.column_by_name`.
self.name = name
@ -40,6 +42,7 @@ class Column:
#: Whether the column can have :attr:`visible` set to false.
self.optional = optional
class ColumnsView:
"""Expected interface for :class:`Columns`'s view.
@ -48,6 +51,7 @@ class ColumnsView:
Our view, the columns controller of a table or outline, is expected to properly respond to
callbacks.
"""
def restore_columns(self):
"""Update all columns according to the model.
@ -62,11 +66,13 @@ class ColumnsView:
``colname``'s visibility status to ``visible``.
"""
class PrefAccessInterface:
"""Expected interface for :class:`Columns`'s prefaccess.
*Not actually used in the code. For documentation purposes only.*
"""
def get_default(self, key, fallback_value):
"""Retrieve the value for ``key`` in the currently running app's preference store.
@ -77,6 +83,7 @@ class PrefAccessInterface:
"""Set the value ``value`` for ``key`` in the currently running app's preference store.
"""
class Columns(GUIObject):
"""Cross-toolkit GUI-enabled column set for tables or outlines.
@ -97,6 +104,7 @@ class Columns(GUIObject):
a prefix. Preferences are saved under more than one name, but they will all
have that same prefix.
"""
def __init__(self, table, prefaccess=None, savename=None):
GUIObject.__init__(self)
self.table = table
@ -149,17 +157,17 @@ class Columns(GUIObject):
def column_display(self, colname):
"""Returns display name for column named ``colname``, or ``''`` if there's none.
"""
return self._get_colname_attr(colname, 'display', '')
return self._get_colname_attr(colname, "display", "")
def column_is_visible(self, colname):
"""Returns visibility for column named ``colname``, or ``True`` if there's none.
"""
return self._get_colname_attr(colname, 'visible', True)
return self._get_colname_attr(colname, "visible", True)
def column_width(self, colname):
"""Returns width for column named ``colname``, or ``0`` if there's none.
"""
return self._get_colname_attr(colname, 'width', 0)
return self._get_colname_attr(colname, "width", 0)
def columns_to_right(self, colname):
"""Returns the list of all columns to the right of ``colname``.
@ -169,7 +177,11 @@ class Columns(GUIObject):
"""
column = self.coldata[colname]
index = column.ordered_index
return [col.name for col in self.column_list if (col.visible and col.ordered_index > index)]
return [
col.name
for col in self.column_list
if (col.visible and col.ordered_index > index)
]
def menu_items(self):
"""Returns a list of items convenient for quick visibility menu generation.
@ -206,7 +218,7 @@ class Columns(GUIObject):
def resize_column(self, colname, newwidth):
"""Set column ``colname``'s width to ``newwidth``.
"""
self._set_colname_attr(colname, 'width', newwidth)
self._set_colname_attr(colname, "width", newwidth)
def restore_columns(self):
"""Restore's column persistent attributes from the last :meth:`save_columns`.
@ -218,14 +230,14 @@ class Columns(GUIObject):
self.view.restore_columns()
return
for col in self.column_list:
pref_name = '{}.Columns.{}'.format(self.savename, col.name)
pref_name = "{}.Columns.{}".format(self.savename, col.name)
coldata = self.prefaccess.get_default(pref_name, fallback_value={})
if 'index' in coldata:
col.ordered_index = coldata['index']
if 'width' in coldata:
col.width = coldata['width']
if col.optional and 'visible' in coldata:
col.visible = coldata['visible']
if "index" in coldata:
col.ordered_index = coldata["index"]
if "width" in coldata:
col.width = coldata["width"]
if col.optional and "visible" in coldata:
col.visible = coldata["visible"]
self.view.restore_columns()
def save_columns(self):
@ -234,10 +246,10 @@ class Columns(GUIObject):
if not (self.prefaccess and self.savename and self.coldata):
return
for col in self.column_list:
pref_name = '{}.Columns.{}'.format(self.savename, col.name)
coldata = {'index': col.ordered_index, 'width': col.width}
pref_name = "{}.Columns.{}".format(self.savename, col.name)
coldata = {"index": col.ordered_index, "width": col.width}
if col.optional:
coldata['visible'] = col.visible
coldata["visible"] = col.visible
self.prefaccess.set_default(pref_name, coldata)
def set_column_order(self, colnames):
@ -254,13 +266,13 @@ class Columns(GUIObject):
"""Set the visibility of column ``colname``.
"""
self.table.save_edits() # the table on the GUI side will stop editing when the columns change
self._set_colname_attr(colname, 'visible', visible)
self._set_colname_attr(colname, "visible", visible)
self.view.set_column_visible(colname, visible)
def set_default_width(self, colname, width):
"""Set the default width or column ``colname``.
"""
self._set_colname_attr(colname, 'default_width', width)
self._set_colname_attr(colname, "default_width", width)
def toggle_menu_item(self, index):
"""Toggles the visibility of an optional column.
@ -279,11 +291,12 @@ class Columns(GUIObject):
def ordered_columns(self):
"""List of :class:`Column` in visible order.
"""
return [col for col in sorted(self.column_list, key=lambda col: col.ordered_index)]
return [
col for col in sorted(self.column_list, key=lambda col: col.ordered_index)
]
@property
def colnames(self):
"""List of column names in visible order.
"""
return [col.name for col in self.ordered_columns]

7
hscommon/gui/progress_window.py
View File

@ -8,6 +8,7 @@ from ..jobprogress.performer import ThreadedJobPerformer
from .base import GUIObject
from .text_field import TextField
class ProgressWindowView:
"""Expected interface for :class:`ProgressWindow`'s view.
@ -18,6 +19,7 @@ class ProgressWindowView:
It's also expected to call :meth:`ProgressWindow.cancel` when the cancel button is clicked.
"""
def show(self):
"""Show the dialog.
"""
@ -36,6 +38,7 @@ class ProgressWindowView:
:param int progress: a value between ``0`` and ``100``.
"""
class ProgressWindow(GUIObject, ThreadedJobPerformer):
"""Cross-toolkit GUI-enabled progress window.
@ -58,6 +61,7 @@ class ProgressWindow(GUIObject, ThreadedJobPerformer):
if you want to. If the function returns ``True``, ``finish_func()`` will be
called as if the job terminated normally.
"""
def __init__(self, finish_func, error_func=None):
# finish_func(jobid) is the function that is called when a job is completed.
GUIObject.__init__(self)
@ -124,10 +128,9 @@ class ProgressWindow(GUIObject, ThreadedJobPerformer):
# target is a function with its first argument being a Job. It can then be followed by other
# arguments which are passed as `args`.
self.jobid = jobid
self.progressdesc_textfield.text = ''
self.progressdesc_textfield.text = ""
j = self.create_job()
args = tuple([j] + list(args))
self.run_threaded(target, args)
self.jobdesc_textfield.text = title
self.view.show()

10
hscommon/gui/selectable_list.py
View File

@ -10,12 +10,14 @@ from collections import Sequence, MutableSequence
from .base import GUIObject
class Selectable(Sequence):
"""Mix-in for a ``Sequence`` that manages its selection status.
When mixed in with a ``Sequence``, we enable it to manage its selection status. The selection
is held as a list of ``int`` indexes. Multiple selection is supported.
"""
def __init__(self):
self._selected_indexes = []
@ -25,7 +27,9 @@ class Selectable(Sequence):
self._selected_indexes = []
if not self._selected_indexes:
return
self._selected_indexes = [index for index in self._selected_indexes if index < len(self)]
self._selected_indexes = [
index for index in self._selected_indexes if index < len(self)
]
if not self._selected_indexes:
self._selected_indexes = [len(self) - 1]
@ -99,6 +103,7 @@ class SelectableList(MutableSequence, Selectable):
Subclasses :class:`Selectable`. Behaves like a ``list``.
"""
def __init__(self, items=None):
Selectable.__init__(self)
if items:
@ -160,6 +165,7 @@ class GUISelectableListView:
Our view, some kind of list view or combobox, is expected to sync with the list's contents by
appropriately behave to all callbacks in this interface.
"""
def refresh(self):
"""Refreshes the contents of the list widget.
@ -172,6 +178,7 @@ class GUISelectableListView:
Ensures that the list widget's selection is in sync with the model.
"""
class GUISelectableList(SelectableList, GUIObject):
"""Cross-toolkit GUI-enabled list view.
@ -182,6 +189,7 @@ class GUISelectableList(SelectableList, GUIObject):
:param iterable items: If specified, items to fill the list with initially.
"""
def __init__(self, items=None):
SelectableList.__init__(self, items)
GUIObject.__init__(self)

30
hscommon/gui/table.py
View File

@ -11,6 +11,7 @@ from collections import MutableSequence, namedtuple
from .base import GUIObject
from .selectable_list import Selectable
# We used to directly subclass list, but it caused problems at some point with deepcopy
class Table(MutableSequence, Selectable):
"""Sortable and selectable sequence of :class:`Row`.
@ -24,6 +25,7 @@ class Table(MutableSequence, Selectable):
Subclasses :class:`.Selectable`.
"""
def __init__(self):
Selectable.__init__(self)
self._rows = []
@ -216,6 +218,7 @@ class GUITableView:
Whenever the user changes the selection, we expect the view to call :meth:`Table.select`.
"""
def refresh(self):
"""Refreshes the contents of the table widget.
@ -238,7 +241,9 @@ class GUITableView:
"""
SortDescriptor = namedtuple('SortDescriptor', 'column desc')
SortDescriptor = namedtuple("SortDescriptor", "column desc")
class GUITable(Table, GUIObject):
"""Cross-toolkit GUI-enabled table view.
@ -254,6 +259,7 @@ class GUITable(Table, GUIObject):
Subclasses :class:`Table` and :class:`.GUIObject`. Expected view:
:class:`GUITableView`.
"""
def __init__(self):
GUIObject.__init__(self)
Table.__init__(self)
@ -444,6 +450,7 @@ class Row:
Of course, this is only default behavior. This can be overriden.
"""
def __init__(self, table):
super(Row, self).__init__()
self.table = table
@ -489,7 +496,7 @@ class Row:
there's none, raises ``AttributeError``.
"""
try:
return getattr(self, '_' + column_name)
return getattr(self, "_" + column_name)
except AttributeError:
return getattr(self, column_name)
@ -511,18 +518,18 @@ class Row:
return False
# '_' is in case column is a python keyword
if not hasattr(self, column_name):
if hasattr(self, column_name + '_'):
column_name = column_name + '_'
if hasattr(self, column_name + "_"):
column_name = column_name + "_"
else:
return False
if hasattr(self, 'can_edit_' + column_name):
return getattr(self, 'can_edit_' + column_name)
if hasattr(self, "can_edit_" + column_name):
return getattr(self, "can_edit_" + column_name)
# If the row has a settable property, we can edit the cell
rowclass = self.__class__
prop = getattr(rowclass, column_name, None)
if prop is None:
return False
return bool(getattr(prop, 'fset', None))
return bool(getattr(prop, "fset", None))
def get_cell_value(self, attrname):
"""Get cell value for ``attrname``.
@ -530,8 +537,8 @@ class Row:
By default, does a simple ``getattr()``, but it is used to allow subclasses to have
alternative value storage mechanisms.
"""
if attrname == 'from':
attrname = 'from_'
if attrname == "from":
attrname = "from_"
return getattr(self, attrname)
def set_cell_value(self, attrname, value):
@ -540,7 +547,6 @@ class Row:
By default, does a simple ``setattr()``, but it is used to allow subclasses to have
alternative value storage mechanisms.
"""
if attrname == 'from':
attrname = 'from_'
if attrname == "from":
attrname = "from_"
setattr(self, attrname, value)

9
hscommon/gui/text_field.py
View File

@ -8,6 +8,7 @@
from .base import GUIObject
from ..util import nonone
class TextFieldView:
"""Expected interface for :class:`TextField`'s view.
@ -17,12 +18,14 @@ class TextFieldView:
model's text when the user types something, but also update the text field when :meth:`refresh`
is called.
"""
def refresh(self):
"""Refreshes the contents of the input widget.
Ensures that the contents of the input widget is actually :attr:`TextField.text`.
"""
class TextField(GUIObject):
"""Cross-toolkit text field.
@ -34,9 +37,10 @@ class TextField(GUIObject):
Subclasses :class:`.GUIObject`. Expected view: :class:`TextFieldView`.
"""
def __init__(self):
GUIObject.__init__(self)
self._text = ''
self._text = ""
self._value = None
# --- Virtual
@ -84,7 +88,7 @@ class TextField(GUIObject):
@text.setter
def text(self, newtext):
self.value = self._parse(nonone(newtext, ''))
self.value = self._parse(nonone(newtext, ""))
@property
def value(self):
@ -105,4 +109,3 @@ class TextField(GUIObject):
self._text = self._format(newvalue)
self._update(self._value)
self.refresh()

8
hscommon/gui/tree.py
View File

@ -8,6 +8,7 @@ from collections import MutableSequence
from .base import GUIObject
class Node(MutableSequence):
"""Pretty bland node implementation to be used in a :class:`Tree`.
@ -19,6 +20,7 @@ class Node(MutableSequence):
Nodes are designed to be subclassed and given meaningful attributes (those you'll want to
display in your tree view), but they all have a :attr:`name`, which is given on initialization.
"""
def __init__(self, name):
self._name = name
self._parent = None
@ -26,7 +28,7 @@ class Node(MutableSequence):
self._children = []
def __repr__(self):
return '<Node %r>' % self.name
return "<Node %r>" % self.name
# --- MutableSequence overrides
def __delitem__(self, key):
@ -157,8 +159,9 @@ class Tree(Node, GUIObject):
Subclasses :class:`Node` (it is the root node of all its children) and :class:`.GUIObject`.
"""
def __init__(self):
Node.__init__(self, '')
Node.__init__(self, "")
GUIObject.__init__(self)
#: Where we store selected nodes (as a list of :class:`Node`)
self._selected_nodes = []
@ -248,4 +251,3 @@ class Tree(Node, GUIObject):
except IndexError:
pass
self._select_nodes(nodes)

21
hscommon/jobprogress/job.py
View File

@ -6,15 +6,19 @@
# which should be included with this package. The terms are also available at
# http://www.gnu.org/licenses/gpl-3.0.html
class JobCancelled(Exception):
"The user has cancelled the job"
class JobInProgressError(Exception):
"A job is already being performed, you can't perform more than one at the same time."
class JobCountError(Exception):
"The number of jobs started have exceeded the number of jobs allowed"
class Job:
"""Manages a job's progression and return it's progression through a callback.
@ -30,6 +34,7 @@ class Job:
Another one is that nothing stops you from calling add_progress right after
SkipJob.
"""
# ---Magic functions
def __init__(self, job_proportions, callback):
"""Initialize the Job with 'jobcount' jobs. Start every job with
@ -37,7 +42,7 @@ class Job:
'callback' takes a 'progress' int param, and a optional 'desc'
parameter. Callback must return false if the job must be cancelled.
"""
if not hasattr(callback, '__call__'):
if not hasattr(callback, "__call__"):
raise TypeError("'callback' MUST be set when creating a Job")
if isinstance(job_proportions, int):
job_proportions = [1] * job_proportions
@ -50,7 +55,7 @@ class Job:
self._currmax = 1
# ---Private
def _subjob_callback(self, progress, desc=''):
def _subjob_callback(self, progress, desc=""):
"""This is the callback passed to children jobs.
"""
self.set_progress(progress, desc)
@ -74,11 +79,11 @@ class Job:
raise JobCancelled()
# ---Public
def add_progress(self, progress=1, desc=''):
def add_progress(self, progress=1, desc=""):
self.set_progress(self._progress + progress, desc)
def check_if_cancelled(self):
self._do_update('')
self._do_update("")
def iter_with_progress(self, iterable, desc_format=None, every=1, count=None):
"""Iterate through ``iterable`` while automatically adding progress.
@ -89,7 +94,7 @@ class Job:
"""
if count is None:
count = len(iterable)
desc = ''
desc = ""
if desc_format:
desc = desc_format % (0, count)
self.start_job(count, desc)
@ -103,7 +108,7 @@ class Job:
desc = desc_format % (count, count)
self.set_progress(100, desc)
def start_job(self, max_progress=100, desc=''):
def start_job(self, max_progress=100, desc=""):
"""Begin work on the next job. You must not call start_job more than
'jobcount' (in __init__) times.
'max' is the job units you are to perform.
@ -118,7 +123,7 @@ class Job:
self._currmax = max(1, max_progress)
self._do_update(desc)
def start_subjob(self, job_proportions, desc=''):
def start_subjob(self, job_proportions, desc=""):
"""Starts a sub job. Use this when you want to split a job into
multiple smaller jobs. Pretty handy when starting a process where you
know how many subjobs you will have, but don't know the work unit count
@ -128,7 +133,7 @@ class Job:
self.start_job(100, desc)
return Job(job_proportions, self._subjob_callback)
def set_progress(self, progress, desc=''):
def set_progress(self, progress, desc=""):
"""Sets the progress of the current job to 'progress', and call the
callback
"""

7
hscommon/jobprogress/performer.py
View File

@ -11,6 +11,7 @@ import sys
from .job import Job, JobInProgressError, JobCancelled
class ThreadedJobPerformer:
"""Run threaded jobs and track progress.
@ -22,6 +23,7 @@ class ThreadedJobPerformer:
j = self._create_job()
self._run_threaded(self.some_work_func, (arg1, arg2, j))
"""
_job_running = False
last_error = None
@ -30,7 +32,7 @@ class ThreadedJobPerformer:
if self._job_running:
raise JobInProgressError()
self.last_progress = -1
self.last_desc = ''
self.last_desc = ""
self.job_cancelled = False
return Job(1, self._update_progress)
@ -58,7 +60,7 @@ class ThreadedJobPerformer:
if self.last_error is not None:
raise self.last_error.with_traceback(self.last_traceback)
def _update_progress(self, newprogress, newdesc=''):
def _update_progress(self, newprogress, newdesc=""):
self.last_progress = newprogress
if newdesc:
self.last_desc = newdesc
@ -69,4 +71,3 @@ class ThreadedJobPerformer:
raise JobInProgressError()
args = (target,) + args
Thread(target=self._async_run, args=args).start()

10
hscommon/jobprogress/qt.py
View File

@ -11,17 +11,18 @@ from PyQt5.QtWidgets import QProgressDialog
from . import performer
class Progress(QProgressDialog, performer.ThreadedJobPerformer):
finished = pyqtSignal(['QString'])
finished = pyqtSignal(["QString"])
def __init__(self, parent):
flags = Qt.CustomizeWindowHint | Qt.WindowTitleHint | Qt.WindowSystemMenuHint
QProgressDialog.__init__(self, '', "Cancel", 0, 100, parent, flags)
QProgressDialog.__init__(self, "", "Cancel", 0, 100, parent, flags)
self.setModal(True)
self.setAutoReset(False)
self.setAutoClose(False)
self._timer = QTimer()
self._jobid = ''
self._jobid = ""
self._timer.timeout.connect(self.updateProgress)
def updateProgress(self):
@ -44,9 +45,8 @@ class Progress(QProgressDialog, performer.ThreadedJobPerformer):
def run(self, jobid, title, target, args=()):
self._jobid = jobid
self.reset()
self.setLabelText('')
self.setLabelText("")
self.run_threaded(target, args)
self.setWindowTitle(title)
self.show()
self._timer.start(500)

104
hscommon/loc.py
View File

@ -7,26 +7,29 @@ import tempfile
import polib
from . import pygettext
from .util import modified_after, dedupe, ensure_folder, ensure_file
from .build import print_and_do, ensure_empty_folder, copy
from .util import modified_after, dedupe, ensure_folder
from .build import print_and_do, ensure_empty_folder
LC_MESSAGES = 'LC_MESSAGES'
LC_MESSAGES = "LC_MESSAGES"
# There isn't a 1-on-1 exact fit between .po language codes and cocoa ones
PO2COCOA = {
'pl_PL': 'pl',
'pt_BR': 'pt-BR',
'zh_CN': 'zh-Hans',
"pl_PL": "pl",
"pt_BR": "pt-BR",
"zh_CN": "zh-Hans",
}
COCOA2PO = {v: k for k, v in PO2COCOA.items()}
def get_langs(folder):
return [name for name in os.listdir(folder) if op.isdir(op.join(folder, name))]
def files_with_ext(folder, ext):
return [op.join(folder, fn) for fn in os.listdir(folder) if fn.endswith(ext)]
def generate_pot(folders, outpath, keywords, merge=False):
if merge and not op.exists(outpath):
merge = False
@ -37,21 +40,23 @@ def generate_pot(folders, outpath, keywords, merge=False):
pyfiles = []
for folder in folders:
for root, dirs, filenames in os.walk(folder):
keep = [fn for fn in filenames if fn.endswith('.py')]
keep = [fn for fn in filenames if fn.endswith(".py")]
pyfiles += [op.join(root, fn) for fn in keep]
pygettext.main(pyfiles, outpath=genpath, keywords=keywords)
if merge:
merge_po_and_preserve(genpath, outpath)
os.remove(genpath)
def compile_all_po(base_folder):
langs = get_langs(base_folder)
for lang in langs:
pofolder = op.join(base_folder, lang, LC_MESSAGES)
pofiles = files_with_ext(pofolder, '.po')
pofiles = files_with_ext(pofolder, ".po")
for pofile in pofiles:
p = polib.pofile(pofile)
p.save_as_mofile(pofile[:-3] + '.mo')
p.save_as_mofile(pofile[:-3] + ".mo")
def merge_locale_dir(target, mergeinto):
langs = get_langs(target)
@ -59,22 +64,24 @@ def merge_locale_dir(target, mergeinto):
if not op.exists(op.join(mergeinto, lang)):
continue
mofolder = op.join(target, lang, LC_MESSAGES)
mofiles = files_with_ext(mofolder, '.mo')
mofiles = files_with_ext(mofolder, ".mo")
for mofile in mofiles:
shutil.copy(mofile, op.join(mergeinto, lang, LC_MESSAGES))
def merge_pots_into_pos(folder):
# We're going to take all pot files in `folder` and for each lang, merge it with the po file
# with the same name.
potfiles = files_with_ext(folder, '.pot')
potfiles = files_with_ext(folder, ".pot")
for potfile in potfiles:
refpot = polib.pofile(potfile)
refname = op.splitext(op.basename(potfile))[0]
for lang in get_langs(folder):
po = polib.pofile(op.join(folder, lang, LC_MESSAGES, refname + '.po'))
po = polib.pofile(op.join(folder, lang, LC_MESSAGES, refname + ".po"))
po.merge(refpot)
po.save()
def merge_po_and_preserve(source, dest):
# Merges source entries into dest, but keep old entries intact
sourcepo = polib.pofile(source)
@ -86,6 +93,7 @@ def merge_po_and_preserve(source, dest):
destpo.append(entry)
destpo.save()
def normalize_all_pos(base_folder):
"""Normalize the format of .po files in base_folder.
@ -99,23 +107,27 @@ def normalize_all_pos(base_folder):
langs = get_langs(base_folder)
for lang in langs:
pofolder = op.join(base_folder, lang, LC_MESSAGES)
pofiles = files_with_ext(pofolder, '.po')
pofiles = files_with_ext(pofolder, ".po")
for pofile in pofiles:
p = polib.pofile(pofile)
p.save()
# --- Cocoa
def all_lproj_paths(folder):
return files_with_ext(folder, '.lproj')
return files_with_ext(folder, ".lproj")
def escape_cocoa_strings(s):
return s.replace('\\', '\\\\').replace('"', '\\"').replace('\n', '\\n')
return s.replace("\\", "\\\\").replace('"', '\\"').replace("\n", "\\n")
def unescape_cocoa_strings(s):
return s.replace('\\\\', '\\').replace('\\"', '"').replace('\\n', '\n')
return s.replace("\\\\", "\\").replace('\\"', '"').replace("\\n", "\n")
def strings2pot(target, dest):
with open(target, 'rt', encoding='utf-8') as fp:
with open(target, "rt", encoding="utf-8") as fp:
contents = fp.read()
# We're reading an en.lproj file. We only care about the righthand part of the translation.
re_trans = re.compile(r'".*" = "(.*)";')
@ -131,17 +143,21 @@ def strings2pot(target, dest):
entry = polib.POEntry(msgid=s)
po.append(entry)
# we don't know or care about a line number so we put 0
entry.occurrences.append((target, '0'))
entry.occurrences.append((target, "0"))
entry.occurrences = dedupe(entry.occurrences)
po.save(dest)
def allstrings2pot(lprojpath, dest, excludes=None):
allstrings = files_with_ext(lprojpath, '.strings')
allstrings = files_with_ext(lprojpath, ".strings")
if excludes:
allstrings = [p for p in allstrings if op.splitext(op.basename(p))[0] not in excludes]
allstrings = [
p for p in allstrings if op.splitext(op.basename(p))[0] not in excludes
]
for strings_path in allstrings:
strings2pot(strings_path, dest)
def po2strings(pofile, en_strings, dest):
# Takes en_strings and replace all righthand parts of "foo" = "bar"; entries with translations
# in pofile, then puts the result in dest.
@ -150,9 +166,10 @@ def po2strings(pofile, en_strings, dest):
return
ensure_folder(op.dirname(dest))
print("Creating {} from {}".format(dest, pofile))
with open(en_strings, 'rt', encoding='utf-8') as fp:
with open(en_strings, "rt", encoding="utf-8") as fp:
contents = fp.read()
re_trans = re.compile(r'(?<= = ").*(?=";\n)')
def repl(match):
s = match.group(0)
unescaped = unescape_cocoa_strings(s)
@ -162,10 +179,12 @@ def po2strings(pofile, en_strings, dest):
return s
trans = entry.msgstr
return escape_cocoa_strings(trans) if trans else s
contents = re_trans.sub(repl, contents)
with open(dest, 'wt', encoding='utf-8') as fp:
with open(dest, "wt", encoding="utf-8") as fp:
fp.write(contents)
def generate_cocoa_strings_from_code(code_folder, dest_folder):
# Uses the "genstrings" command to generate strings file from all .m files in "code_folder".
# The strings file (their name depends on the localization table used in the source) will be
@ -173,36 +192,49 @@ def generate_cocoa_strings_from_code(code_folder, dest_folder):
# genstrings produces utf-16 files with comments. After having generated the files, we convert
# them to utf-8 and remove the comments.
ensure_empty_folder(dest_folder)
print_and_do('genstrings -o "{}" `find "{}" -name *.m | xargs`'.format(dest_folder, code_folder))
print_and_do(
'genstrings -o "{}" `find "{}" -name *.m | xargs`'.format(
dest_folder, code_folder
)
)
for stringsfile in os.listdir(dest_folder):
stringspath = op.join(dest_folder, stringsfile)
with open(stringspath, 'rt', encoding='utf-16') as fp:
with open(stringspath, "rt", encoding="utf-16") as fp:
content = fp.read()
content = re.sub('/\*.*?\*/', '', content)
content = re.sub('\n{2,}', '\n', content)
content = re.sub(r"/\*.*?\*/", "", content)
content = re.sub(r"\n{2,}", "\n", content)
# I have no idea why, but genstrings seems to have problems with "%" character in strings
# and inserts (number)$ after it. Find these bogus inserts and remove them.
content = re.sub('%\d\$', '%', content)
with open(stringspath, 'wt', encoding='utf-8') as fp:
content = re.sub(r"%\d\$", "%", content)
with open(stringspath, "wt", encoding="utf-8") as fp:
fp.write(content)
def generate_cocoa_strings_from_xib(xib_folder):
xibs = [op.join(xib_folder, fn) for fn in os.listdir(xib_folder) if fn.endswith('.xib')]
xibs = [
op.join(xib_folder, fn) for fn in os.listdir(xib_folder) if fn.endswith(".xib")
]
for xib in xibs:
dest = xib.replace('.xib', '.strings')
print_and_do('ibtool {} --generate-strings-file {}'.format(xib, dest))
print_and_do('iconv -f utf-16 -t utf-8 {0} | tee {0}'.format(dest))
dest = xib.replace(".xib", ".strings")
print_and_do("ibtool {} --generate-strings-file {}".format(xib, dest))
print_and_do("iconv -f utf-16 -t utf-8 {0} | tee {0}".format(dest))
def localize_stringsfile(stringsfile, dest_root_folder):
stringsfile_name = op.basename(stringsfile)
for lang in get_langs('locale'):
pofile = op.join('locale', lang, 'LC_MESSAGES', 'ui.po')
for lang in get_langs("locale"):
pofile = op.join("locale", lang, "LC_MESSAGES", "ui.po")
cocoa_lang = PO2COCOA.get(lang, lang)
dest_lproj = op.join(dest_root_folder, cocoa_lang + '.lproj')
dest_lproj = op.join(dest_root_folder, cocoa_lang + ".lproj")
ensure_folder(dest_lproj)
po2strings(pofile, stringsfile, op.join(dest_lproj, stringsfile_name))
def localize_all_stringsfiles(src_folder, dest_root_folder):
stringsfiles = [op.join(src_folder, fn) for fn in os.listdir(src_folder) if fn.endswith('.strings')]
stringsfiles = [
op.join(src_folder, fn)
for fn in os.listdir(src_folder)
if fn.endswith(".strings")
]
for path in stringsfiles:
localize_stringsfile(path, dest_root_folder)

4
hscommon/notify.py
View File

@ -14,9 +14,11 @@ the method with the same name as the broadcasted message is called on the listen
from collections import defaultdict
class Broadcaster:
"""Broadcasts messages that are received by all listeners.
"""
def __init__(self):
self.listeners = set()
@ -39,6 +41,7 @@ class Broadcaster:
class Listener:
"""A listener is initialized with the broadcaster it's going to listen to. Initially, it is not connected.
"""
def __init__(self, broadcaster):
self.broadcaster = broadcaster
self._bound_notifications = defaultdict(list)
@ -86,4 +89,3 @@ class Repeater(Broadcaster, Listener):
def dispatch(self, msg):
Listener.dispatch(self, msg)
self._repeat_message(msg)

41
hscommon/path.py
View File

@ -15,6 +15,7 @@ from itertools import takewhile
from functools import wraps
from inspect import signature
class Path(tuple):
"""A handy class to work with paths.
@ -25,6 +26,7 @@ class Path(tuple):
* ``Path('/foo/bar')['baz']`` --> ``Path('/foo/bar/baz')``
* ``str(Path('/foo/bar/baz'))`` --> ``'/foo/bar/baz'``
"""
# Saves a little bit of memory usage
__slots__ = ()
@ -47,8 +49,8 @@ class Path(tuple):
value = unicode_if_needed(value)
if isinstance(value, str):
if value:
if (separator not in value) and ('/' in value):
separator = '/'
if (separator not in value) and ("/" in value):
separator = "/"
value = value.split(separator)
else:
value = ()
@ -60,7 +62,9 @@ class Path(tuple):
# We have a component with a separator in it. Let's rejoin it, and generate another path.
return Path(separator.join(value), separator)
if (len(value) > 1) and (not value[-1]):
value = value[:-1] #We never want a path to end with a '' (because Path() can be called with a trailing slash ending path)
value = value[
:-1
] # We never want a path to end with a '' (because Path() can be called with a trailing slash ending path)
return tuple.__new__(cls, value)
def __add__(self, other):
@ -81,10 +85,17 @@ class Path(tuple):
def __getitem__(self, key):
if isinstance(key, slice):
if isinstance(key.start, Path):
equal_elems = list(takewhile(lambda pair: pair[0] == pair[1], zip(self, key.start)))
equal_elems = list(
takewhile(lambda pair: pair[0] == pair[1], zip(self, key.start))
)
key = slice(len(equal_elems), key.stop, key.step)
if isinstance(key.stop, Path):
equal_elems = list(takewhile(lambda pair: pair[0] == pair[1], zip(reversed(self), reversed(key.stop))))
equal_elems = list(
takewhile(
lambda pair: pair[0] == pair[1],
zip(reversed(self), reversed(key.stop)),
)
)
stop = -len(equal_elems) if equal_elems else None
key = slice(key.start, stop, key.step)
return Path(tuple.__getitem__(self, key))
@ -105,17 +116,17 @@ class Path(tuple):
def __str__(self):
if len(self) == 1:
first = self[0]
if (len(first) == 2) and (first[1] == ':'): #Windows drive letter
return first + '\\'
if (len(first) == 2) and (first[1] == ":"): # Windows drive letter
return first + "\\"
elif not len(first): # root directory
return '/'
return "/"
return os.sep.join(self)
def has_drive_letter(self):
if not self:
return False
first = self[0]
return (len(first) == 2) and (first[1] == ':')
return (len(first) == 2) and (first[1] == ":")
def is_parent_of(self, other):
"""Whether ``other`` is a subpath of ``self``.
@ -201,6 +212,7 @@ class Path(tuple):
def stat(self):
return os.stat(str(self))
def pathify(f):
"""Ensure that every annotated :class:`Path` arguments are actually paths.
@ -214,22 +226,29 @@ def pathify(f):
Calling ``foo('/bar', 0)`` will convert ``'/bar'`` to ``Path('/bar')``.
"""
sig = signature(f)
pindexes = {i for i, p in enumerate(sig.parameters.values()) if p.annotation is Path}
pindexes = {
i for i, p in enumerate(sig.parameters.values()) if p.annotation is Path
}
pkeys = {k: v for k, v in sig.parameters.items() if v.annotation is Path}
def path_or_none(p):
return None if p is None else Path(p)
@wraps(f)
def wrapped(*args, **kwargs):
args = tuple((path_or_none(a) if i in pindexes else a) for i, a in enumerate(args))
args = tuple(
(path_or_none(a) if i in pindexes else a) for i, a in enumerate(args)
)
kwargs = {k: (path_or_none(v) if k in pkeys else v) for k, v in kwargs.items()}
return f(*args, **kwargs)
return wrapped
def log_io_error(func):
""" Catches OSError, IOError and WindowsError and log them
"""
@wraps(func)
def wrapper(path, *args, **kwargs):
try:

6
hscommon/plat.py
View File

@ -11,6 +11,6 @@
import sys
ISWINDOWS = sys.platform == 'win32'
ISOSX = sys.platform == 'darwin'
ISLINUX = sys.platform.startswith('linux')
ISWINDOWS = sys.platform == "win32"
ISOSX = sys.platform == "darwin"
ISLINUX = sys.platform.startswith("linux")

146
hscommon/pygettext.py
View File

@ -18,20 +18,17 @@ import os
import imp
import sys
import glob
import time
import token
import tokenize
import operator
__version__ = '1.5'
__version__ = "1.5"
default_keywords = ['_']
DEFAULTKEYWORDS = ', '.join(default_keywords)
default_keywords = ["_"]
DEFAULTKEYWORDS = ", ".join(default_keywords)
EMPTYSTRING = ''
EMPTYSTRING = ""
# The normal pot-file header. msgmerge and Emacs's po-mode work better if it's
# there.
pot_header = """
@ -41,17 +38,17 @@ msgstr ""
"Content-Transfer-Encoding: utf-8\\n"
"""
def usage(code, msg=''):
def usage(code, msg=""):
print(__doc__ % globals(), file=sys.stderr)
if msg:
print(msg, file=sys.stderr)
sys.exit(code)
escapes = []
def make_escapes(pass_iso8859):
global escapes
if pass_iso8859:
@ -66,11 +63,11 @@ def make_escapes(pass_iso8859):
escapes.append(chr(i))
else:
escapes.append("\\%03o" % i)
escapes[ord('\\')] = '\\\\'
escapes[ord('\t')] = '\\t'
escapes[ord('\r')] = '\\r'
escapes[ord('\n')] = '\\n'
escapes[ord('\"')] = '\\"'
escapes[ord("\\")] = "\\\\"
escapes[ord("\t")] = "\\t"
escapes[ord("\r")] = "\\r"
escapes[ord("\n")] = "\\n"
escapes[ord('"')] = '\\"'
def escape(s):
@ -83,26 +80,26 @@ def escape(s):
def safe_eval(s):
# unwrap quotes, safely
return eval(s, {'__builtins__':{}}, {})
return eval(s, {"__builtins__": {}}, {})
def normalize(s):
# This converts the various Python string types into a format that is
# appropriate for .po files, namely much closer to C style.
lines = s.split('\n')
lines = s.split("\n")
if len(lines) == 1:
s = '"' + escape(s) + '"'
else:
if not lines[-1]:
del lines[-1]
lines[-1] = lines[-1] + '\n'
lines[-1] = lines[-1] + "\n"
for i in range(len(lines)):
lines[i] = escape(lines[i])
lineterm = '\\n"\n"'
s = '""\n"' + lineterm.join(lines) + '"'
return s
def containsAny(str, set):
"""Check whether 'str' contains ANY of the chars in 'set'"""
return 1 in [c in str for c in set]
@ -111,19 +108,23 @@ def containsAny(str, set):
def _visit_pyfiles(list, dirname, names):
"""Helper for getFilesForName()."""
# get extension for python source files
if '_py_ext' not in globals():
if "_py_ext" not in globals():
global _py_ext
_py_ext = [triple[0] for triple in imp.get_suffixes()
if triple[2] == imp.PY_SOURCE][0]
_py_ext = [
triple[0] for triple in imp.get_suffixes() if triple[2] == imp.PY_SOURCE
][0]
# don't recurse into CVS directories
if 'CVS' in names:
names.remove('CVS')
if "CVS" in names:
names.remove("CVS")
# add all *.py files to list
list.extend(
[os.path.join(dirname, file) for file in names
if os.path.splitext(file)[1] == _py_ext]
[
os.path.join(dirname, file)
for file in names
if os.path.splitext(file)[1] == _py_ext
]
)
@ -135,13 +136,14 @@ def _get_modpkg_path(dotted_name, pathlist=None):
extension module.
"""
# split off top-most name
parts = dotted_name.split('.', 1)
parts = dotted_name.split(".", 1)
if len(parts) > 1:
# we have a dotted path, import top-level package
try:
file, pathname, description = imp.find_module(parts[0], pathlist)
if file: file.close()
if file:
file.close()
except ImportError:
return None
@ -154,8 +156,7 @@ def _get_modpkg_path(dotted_name, pathlist=None):
else:
# plain name
try:
file, pathname, description = imp.find_module(
dotted_name, pathlist)
file, pathname, description = imp.find_module(dotted_name, pathlist)
if file:
file.close()
if description[2] not in [imp.PY_SOURCE, imp.PKG_DIRECTORY]:
@ -195,7 +196,7 @@ def getFilesForName(name):
return []
class TokenEater:
def __init__(self, options):
self.__options = options
@ -208,9 +209,9 @@ class TokenEater:
def __call__(self, ttype, tstring, stup, etup, line):
# dispatch
## import token
## print >> sys.stderr, 'ttype:', token.tok_name[ttype], \
## 'tstring:', tstring
# import token
# print >> sys.stderr, 'ttype:', token.tok_name[ttype], \
# 'tstring:', tstring
self.__state(ttype, tstring, stup[0])
def __waiting(self, ttype, tstring, lineno):
@ -226,7 +227,7 @@ class TokenEater:
self.__freshmodule = 0
return
# class docstring?
if ttype == tokenize.NAME and tstring in ('class', 'def'):
if ttype == tokenize.NAME and tstring in ("class", "def"):
self.__state = self.__suiteseen
return
if ttype == tokenize.NAME and tstring in opts.keywords:
@ -234,7 +235,7 @@ class TokenEater:
def __suiteseen(self, ttype, tstring, lineno):
# ignore anything until we see the colon
if ttype == tokenize.OP and tstring == ':':
if ttype == tokenize.OP and tstring == ":":
self.__state = self.__suitedocstring
def __suitedocstring(self, ttype, tstring, lineno):
@ -242,13 +243,12 @@ class TokenEater:
if ttype == tokenize.STRING:
self.__addentry(safe_eval(tstring), lineno, isdocstring=1)
self.__state = self.__waiting
elif ttype not in (tokenize.NEWLINE, tokenize.INDENT,
tokenize.COMMENT):
elif ttype not in (tokenize.NEWLINE, tokenize.INDENT, tokenize.COMMENT):
# there was no class docstring
self.__state = self.__waiting
def __keywordseen(self, ttype, tstring, lineno):
if ttype == tokenize.OP and tstring == '(':
if ttype == tokenize.OP and tstring == "(":
self.__data = []
self.__lineno = lineno
self.__state = self.__openseen
@ -256,7 +256,7 @@ class TokenEater:
self.__state = self.__waiting
def __openseen(self, ttype, tstring, lineno):
if ttype == tokenize.OP and tstring == ')':
if ttype == tokenize.OP and tstring == ")":
# We've seen the last of the translatable strings. Record the
# line number of the first line of the strings and update the list
# of messages seen. Reset state for the next batch. If there
@ -266,20 +266,25 @@ class TokenEater:
self.__state = self.__waiting
elif ttype == tokenize.STRING:
self.__data.append(safe_eval(tstring))
elif ttype not in [tokenize.COMMENT, token.INDENT, token.DEDENT,
token.NEWLINE, tokenize.NL]:
elif ttype not in [
tokenize.COMMENT,
token.INDENT,
token.DEDENT,
token.NEWLINE,
tokenize.NL,
]:
# warn if we see anything else than STRING or whitespace
print('*** %(file)s:%(lineno)s: Seen unexpected token "%(token)s"' % {
'token': tstring,
'file': self.__curfile,
'lineno': self.__lineno
}, file=sys.stderr)
print(
'*** %(file)s:%(lineno)s: Seen unexpected token "%(token)s"'
% {"token": tstring, "file": self.__curfile, "lineno": self.__lineno},
file=sys.stderr,
)
self.__state = self.__waiting
def __addentry(self, msg, lineno=None, isdocstring=0):
if lineno is None:
lineno = self.__lineno
if not msg in self.__options.toexclude:
if msg not in self.__options.toexclude:
entry = (self.__curfile, lineno)
self.__messages.setdefault(msg, {})[entry] = isdocstring
@ -289,7 +294,6 @@ class TokenEater:
def write(self, fp):
options = self.__options
timestamp = time.strftime('%Y-%m-%d %H:%M+%Z')
# The time stamp in the header doesn't have the same format as that
# generated by xgettext...
print(pot_header, file=fp)
@ -317,15 +321,15 @@ class TokenEater:
# location comments are different b/w Solaris and GNU:
elif options.locationstyle == options.SOLARIS:
for filename, lineno in v:
d = {'filename': filename, 'lineno': lineno}
print('# File: %(filename)s, line: %(lineno)d' % d, file=fp)
d = {"filename": filename, "lineno": lineno}
print("# File: %(filename)s, line: %(lineno)d" % d, file=fp)
elif options.locationstyle == options.GNU:
# fit as many locations on one line, as long as the
# resulting line length doesn't exceeds 'options.width'
locline = '#:'
locline = "#:"
for filename, lineno in v:
d = {'filename': filename, 'lineno': lineno}
s = ' %(filename)s:%(lineno)d' % d
d = {"filename": filename, "lineno": lineno}
s = " %(filename)s:%(lineno)d" % d
if len(locline) + len(s) <= options.width:
locline = locline + s
else:
@ -334,14 +338,14 @@ class TokenEater:
if len(locline) > 2:
print(locline, file=fp)
if isdocstring:
print('#, docstring', file=fp)
print('msgid', normalize(k), file=fp)
print("#, docstring", file=fp)
print("msgid", normalize(k), file=fp)
print('msgstr ""\n', file=fp)
def main(source_files, outpath, keywords=None):
global default_keywords
# for holding option values
class Options:
# constants
@ -351,19 +355,16 @@ def main(source_files, outpath, keywords=None):
extractall = 0 # FIXME: currently this option has no effect at all.
escape = 0
keywords = []
outfile = 'messages.pot'
outfile = "messages.pot"
writelocations = 1
locationstyle = GNU
verbose = 0
width = 78
excludefilename = ''
excludefilename = ""
docstrings = 0
nodocstrings = {}
options = Options()
locations = {'gnu' : options.GNU,
'solaris' : options.SOLARIS,
}
options.outfile = outpath
if keywords:
@ -378,11 +379,14 @@ def main(source_files, outpath, keywords=None):
# initialize list of strings to exclude
if options.excludefilename:
try:
fp = open(options.excludefilename, encoding='utf-8')
fp = open(options.excludefilename, encoding="utf-8")
options.toexclude = fp.readlines()
fp.close()
except IOError:
print("Can't read --exclude-file: %s" % options.excludefilename, file=sys.stderr)
print(
"Can't read --exclude-file: %s" % options.excludefilename,
file=sys.stderr,
)
sys.exit(1)
else:
options.toexclude = []
@ -391,8 +395,8 @@ def main(source_files, outpath, keywords=None):
eater = TokenEater(options)
for filename in source_files:
if options.verbose:
print('Working on %s' % filename)
fp = open(filename, encoding='utf-8')
print("Working on %s" % filename)
fp = open(filename, encoding="utf-8")
closep = 1
try:
eater.set_filename(filename)
@ -401,14 +405,16 @@ def main(source_files, outpath, keywords=None):
for _token in tokens:
eater(*_token)
except tokenize.TokenError as e:
print('%s: %s, line %d, column %d' % (
e.args[0], filename, e.args[1][0], e.args[1][1]),
file=sys.stderr)
print(
"%s: %s, line %d, column %d"
% (e.args[0], filename, e.args[1][0], e.args[1][1]),
file=sys.stderr,
)
finally:
if closep:
fp.close()
fp = open(options.outfile, 'w', encoding='utf-8')
fp = open(options.outfile, "w", encoding="utf-8")
closep = 1
try:
eater.write(fp)

54
hscommon/sphinxgen.py
View File

@ -19,16 +19,28 @@ CHANGELOG_FORMAT = """
{description}
"""
def tixgen(tixurl):
"""This is a filter *generator*. tixurl is a url pattern for the tix with a {0} placeholder
for the tix #
"""
urlpattern = tixurl.format('\\1') # will be replaced buy the content of the first group in re
R = re.compile(r'#(\d+)')
repl = '`#\\1 <{}>`__'.format(urlpattern)
urlpattern = tixurl.format(
"\\1"
) # will be replaced buy the content of the first group in re
R = re.compile(r"#(\d+)")
repl = "`#\\1 <{}>`__".format(urlpattern)
return lambda text: R.sub(repl, text)
def gen(basepath, destpath, changelogpath, tixurl, confrepl=None, confpath=None, changelogtmpl=None):
def gen(
basepath,
destpath,
changelogpath,
tixurl,
confrepl=None,
confpath=None,
changelogtmpl=None,
):
"""Generate sphinx docs with all bells and whistles.
basepath: The base sphinx source path.
@ -40,41 +52,47 @@ def gen(basepath, destpath, changelogpath, tixurl, confrepl=None, confpath=None,
if confrepl is None:
confrepl = {}
if confpath is None:
confpath = op.join(basepath, 'conf.tmpl')
confpath = op.join(basepath, "conf.tmpl")
if changelogtmpl is None:
changelogtmpl = op.join(basepath, 'changelog.tmpl')
changelogtmpl = op.join(basepath, "changelog.tmpl")
changelog = read_changelog_file(changelogpath)
tix = tixgen(tixurl)
rendered_logs = []
for log in changelog:
description = tix(log['description'])
description = tix(log["description"])
# The format of the changelog descriptions is in markdown, but since we only use bulled list
# and links, it's not worth depending on the markdown package. A simple regexp suffice.
description = re.sub(r'\[(.*?)\]\((.*?)\)', '`\\1 <\\2>`__', description)
rendered = CHANGELOG_FORMAT.format(version=log['version'], date=log['date_str'],
description=description)
description = re.sub(r"\[(.*?)\]\((.*?)\)", "`\\1 <\\2>`__", description)
rendered = CHANGELOG_FORMAT.format(
version=log["version"], date=log["date_str"], description=description
)
rendered_logs.append(rendered)
confrepl['version'] = changelog[0]['version']
changelog_out = op.join(basepath, 'changelog.rst')
filereplace(changelogtmpl, changelog_out, changelog='\n'.join(rendered_logs))
confrepl["version"] = changelog[0]["version"]
changelog_out = op.join(basepath, "changelog.rst")
filereplace(changelogtmpl, changelog_out, changelog="\n".join(rendered_logs))
if op.exists(confpath):
conf_out = op.join(basepath, 'conf.py')
conf_out = op.join(basepath, "conf.py")
filereplace(confpath, conf_out, **confrepl)
if LooseVersion(get_distribution("sphinx").version) >= LooseVersion("1.7.0"):
from sphinx.cmd.build import build_main as sphinx_build
# Call the sphinx_build function, which is the same as doing sphinx-build from cli
try:
sphinx_build([basepath, destpath])
except SystemExit:
print("Sphinx called sys.exit(), but we're cancelling it because we don't actually want to exit")
print(
"Sphinx called sys.exit(), but we're cancelling it because we don't actually want to exit"
)
else:
# We used to call sphinx-build with print_and_do(), but the problem was that the virtualenv
# of the calling python wasn't correctly considered and caused problems with documentation
# relying on autodoc (which tries to import the module to auto-document, but fail because of
# missing dependencies which are in the virtualenv). Here, we do exactly what is done when
# calling the command from bash.
cmd = load_entry_point('Sphinx', 'console_scripts', 'sphinx-build')
cmd = load_entry_point("Sphinx", "console_scripts", "sphinx-build")
try:
cmd(['sphinx-build', basepath, destpath])
cmd(["sphinx-build", basepath, destpath])
except SystemExit:
print("Sphinx called sys.exit(), but we're cancelling it because we don't actually want to exit")
print(
"Sphinx called sys.exit(), but we're cancelling it because we don't actually want to exit"
)

10
hscommon/sqlite.py
View File

@ -6,18 +6,17 @@
# which should be included with this package. The terms are also available at
# http://www.gnu.org/licenses/gpl-3.0.html
import sys
import os
import os.path as op
import threading
from queue import Queue
import time
import sqlite3 as sqlite
STOP = object()
COMMIT = object()
ROLLBACK = object()
class FakeCursor(list):
# It's not possible to use sqlite cursors on another thread than the connection. Thus,
# we can't directly return the cursor. We have to fatch all results, and support its interface.
@ -32,9 +31,10 @@ class FakeCursor(list):
class _ActualThread(threading.Thread):
''' We can't use this class directly because thread object are not automatically freed when
""" We can't use this class directly because thread object are not automatically freed when
nothing refers to it, making it hang the application if not explicitely closed.
'''
"""
def __init__(self, dbname, autocommit):
threading.Thread.__init__(self)
self._queries = Queue()
@ -118,6 +118,7 @@ class ThreadedConn:
connection in its own thread and sends it queries through a queue, making it suitable in
multi-threaded environment.
"""
def __init__(self, dbname, autocommit):
self._t = _ActualThread(dbname, autocommit)
self.lastrowid = -1
@ -138,4 +139,3 @@ class ThreadedConn:
def rollback(self):
self._t.rollback()

110
hscommon/tests/conflict_test.py
View File

@ -10,95 +10,97 @@ from ..conflict import *
from ..path import Path
from ..testutil import eq_
class TestCase_GetConflictedName:
def test_simple(self):
name = get_conflicted_name(['bar'], 'bar')
eq_('[000] bar', name)
name = get_conflicted_name(['bar', '[000] bar'], 'bar')
eq_('[001] bar', name)
name = get_conflicted_name(["bar"], "bar")
eq_("[000] bar", name)
name = get_conflicted_name(["bar", "[000] bar"], "bar")
eq_("[001] bar", name)
def test_no_conflict(self):
name = get_conflicted_name(['bar'], 'foobar')
eq_('foobar', name)
name = get_conflicted_name(["bar"], "foobar")
eq_("foobar", name)
def test_fourth_digit(self):
# This test is long because every time we have to add a conflicted name,
# a test must be made for every other conflicted name existing...
# Anyway, this has very few chances to happen.
names = ['bar'] + ['[%03d] bar' % i for i in range(1000)]
name = get_conflicted_name(names, 'bar')
eq_('[1000] bar', name)
names = ["bar"] + ["[%03d] bar" % i for i in range(1000)]
name = get_conflicted_name(names, "bar")
eq_("[1000] bar", name)
def test_auto_unconflict(self):
# Automatically unconflict the name if it's already conflicted.
name = get_conflicted_name([], '[000] foobar')
eq_('foobar', name)
name = get_conflicted_name(['bar'], '[001] bar')
eq_('[000] bar', name)
name = get_conflicted_name([], "[000] foobar")
eq_("foobar", name)
name = get_conflicted_name(["bar"], "[001] bar")
eq_("[000] bar", name)
class TestCase_GetUnconflictedName:
def test_main(self):
eq_('foobar',get_unconflicted_name('[000] foobar'))
eq_('foobar',get_unconflicted_name('[9999] foobar'))
eq_('[000]foobar',get_unconflicted_name('[000]foobar'))
eq_('[000a] foobar',get_unconflicted_name('[000a] foobar'))
eq_('foobar',get_unconflicted_name('foobar'))
eq_('foo [000] bar',get_unconflicted_name('foo [000] bar'))
eq_("foobar", get_unconflicted_name("[000] foobar"))
eq_("foobar", get_unconflicted_name("[9999] foobar"))
eq_("[000]foobar", get_unconflicted_name("[000]foobar"))
eq_("[000a] foobar", get_unconflicted_name("[000a] foobar"))
eq_("foobar", get_unconflicted_name("foobar"))
eq_("foo [000] bar", get_unconflicted_name("foo [000] bar"))
class TestCase_IsConflicted:
def test_main(self):
assert is_conflicted('[000] foobar')
assert is_conflicted('[9999] foobar')
assert not is_conflicted('[000]foobar')
assert not is_conflicted('[000a] foobar')
assert not is_conflicted('foobar')
assert not is_conflicted('foo [000] bar')
assert is_conflicted("[000] foobar")
assert is_conflicted("[9999] foobar")
assert not is_conflicted("[000]foobar")
assert not is_conflicted("[000a] foobar")
assert not is_conflicted("foobar")
assert not is_conflicted("foo [000] bar")
class TestCase_move_copy:
def pytest_funcarg__do_setup(self, request):
tmpdir = request.getfuncargvalue('tmpdir')
tmpdir = request.getfuncargvalue("tmpdir")
self.path = Path(str(tmpdir))
self.path['foo'].open('w').close()
self.path['bar'].open('w').close()
self.path['dir'].mkdir()
self.path["foo"].open("w").close()
self.path["bar"].open("w").close()
self.path["dir"].mkdir()
def test_move_no_conflict(self, do_setup):
smart_move(self.path + 'foo', self.path + 'baz')
assert self.path['baz'].exists()
assert not self.path['foo'].exists()
smart_move(self.path + "foo", self.path + "baz")
assert self.path["baz"].exists()
assert not self.path["foo"].exists()
def test_copy_no_conflict(self, do_setup): # No need to duplicate the rest of the tests... Let's just test on move
smart_copy(self.path + 'foo', self.path + 'baz')
assert self.path['baz'].exists()
assert self.path['foo'].exists()
def test_copy_no_conflict(
self, do_setup
): # No need to duplicate the rest of the tests... Let's just test on move
smart_copy(self.path + "foo", self.path + "baz")
assert self.path["baz"].exists()
assert self.path["foo"].exists()
def test_move_no_conflict_dest_is_dir(self, do_setup):
smart_move(self.path + 'foo', self.path + 'dir')
assert self.path['dir']['foo'].exists()
assert not self.path['foo'].exists()
smart_move(self.path + "foo", self.path + "dir")
assert self.path["dir"]["foo"].exists()
assert not self.path["foo"].exists()
def test_move_conflict(self, do_setup):
smart_move(self.path + 'foo', self.path + 'bar')
assert self.path['[000] bar'].exists()
assert not self.path['foo'].exists()
smart_move(self.path + "foo", self.path + "bar")
assert self.path["[000] bar"].exists()
assert not self.path["foo"].exists()
def test_move_conflict_dest_is_dir(self, do_setup):
smart_move(self.path['foo'], self.path['dir'])
smart_move(self.path['bar'], self.path['foo'])
smart_move(self.path['foo'], self.path['dir'])
assert self.path['dir']['foo'].exists()
assert self.path['dir']['[000] foo'].exists()
assert not self.path['foo'].exists()
assert not self.path['bar'].exists()
smart_move(self.path["foo"], self.path["dir"])
smart_move(self.path["bar"], self.path["foo"])
smart_move(self.path["foo"], self.path["dir"])
assert self.path["dir"]["foo"].exists()
assert self.path["dir"]["[000] foo"].exists()
assert not self.path["foo"].exists()
assert not self.path["bar"].exists()
def test_copy_folder(self, tmpdir):
# smart_copy also works on folders
path = Path(str(tmpdir))
path['foo'].mkdir()
path['bar'].mkdir()
smart_copy(path['foo'], path['bar']) # no crash
assert path['[000] bar'].exists()
path["foo"].mkdir()
path["bar"].mkdir()
smart_copy(path["foo"], path["bar"]) # no crash
assert path["[000] bar"].exists()

45
hscommon/tests/notify_test.py
View File

@ -7,6 +7,7 @@
from ..testutil import eq_
from ..notify import Broadcaster, Listener, Repeater
class HelloListener(Listener):
def __init__(self, broadcaster):
Listener.__init__(self, broadcaster)
@ -15,6 +16,7 @@ class HelloListener(Listener):
def hello(self):
self.hello_count += 1
class HelloRepeater(Repeater):
def __init__(self, broadcaster):
Repeater.__init__(self, broadcaster)
@ -23,11 +25,13 @@ class HelloRepeater(Repeater):
def hello(self):
self.hello_count += 1
def create_pair():
b = Broadcaster()
l = HelloListener(b)
return b, l
def test_disconnect_during_notification():
# When a listener disconnects another listener the other listener will not receive a
# notification.
@ -49,63 +53,72 @@ def test_disconnect_during_notification():
first.other, second.other = second, first
first.connect()
second.connect()
broadcaster.notify('hello')
broadcaster.notify("hello")
# only one of them was notified
eq_(first.hello_count + second.hello_count, 1)
def test_disconnect():
# After a disconnect, the listener doesn't hear anything.
b, l = create_pair()
l.connect()
l.disconnect()
b.notify('hello')
b.notify("hello")
eq_(l.hello_count, 0)
def test_disconnect_when_not_connected():
# When disconnecting an already disconnected listener, nothing happens.
b, l = create_pair()
l.disconnect()
def test_not_connected_on_init():
# A listener is not initialized connected.
b, l = create_pair()
b.notify('hello')
b.notify("hello")
eq_(l.hello_count, 0)
def test_notify():
# The listener listens to the broadcaster.
b, l = create_pair()
l.connect()
b.notify('hello')
b.notify("hello")
eq_(l.hello_count, 1)
def test_reconnect():
# It's possible to reconnect a listener after disconnection.
b, l = create_pair()
l.connect()
l.disconnect()
l.connect()
b.notify('hello')
b.notify("hello")
eq_(l.hello_count, 1)
def test_repeater():
b = Broadcaster()
r = HelloRepeater(b)
l = HelloListener(r)
r.connect()
l.connect()
b.notify('hello')
b.notify("hello")
eq_(r.hello_count, 1)
eq_(l.hello_count, 1)
def test_repeater_with_repeated_notifications():
# If REPEATED_NOTIFICATIONS is not empty, only notifs in this set are repeated (but they're
# still dispatched locally).
class MyRepeater(HelloRepeater):
REPEATED_NOTIFICATIONS = set(['hello'])
REPEATED_NOTIFICATIONS = set(["hello"])
def __init__(self, broadcaster):
HelloRepeater.__init__(self, broadcaster)
self.foo_count = 0
def foo(self):
self.foo_count += 1
@ -114,12 +127,15 @@ def test_repeater_with_repeated_notifications():
l = HelloListener(r)
r.connect()
l.connect()
b.notify('hello')
b.notify('foo') # if the repeater repeated this notif, we'd get a crash on HelloListener
b.notify("hello")
b.notify(
"foo"
) # if the repeater repeated this notif, we'd get a crash on HelloListener
eq_(r.hello_count, 1)
eq_(l.hello_count, 1)
eq_(r.foo_count, 1)
def test_repeater_doesnt_try_to_dispatch_to_self_if_it_cant():
# if a repeater doesn't handle a particular message, it doesn't crash and simply repeats it.
b = Broadcaster()
@ -127,14 +143,15 @@ def test_repeater_doesnt_try_to_dispatch_to_self_if_it_cant():
l = HelloListener(r)
r.connect()
l.connect()
b.notify('hello') # no crash
b.notify("hello") # no crash
eq_(l.hello_count, 1)
def test_bind_messages():
b, l = create_pair()
l.bind_messages({'foo', 'bar'}, l.hello)
l.bind_messages({"foo", "bar"}, l.hello)
l.connect()
b.notify('foo')
b.notify('bar')
b.notify('hello') # Normal dispatching still work
b.notify("foo")
b.notify("bar")
b.notify("hello") # Normal dispatching still work
eq_(l.hello_count, 3)

268
hscommon/tests/path_test.py
View File

@ -14,33 +14,39 @@ from pytest import raises, mark
from ..path import Path, pathify
from ..testutil import eq_
def pytest_funcarg__force_ossep(request):
monkeypatch = request.getfuncargvalue('monkeypatch')
monkeypatch.setattr(os, 'sep', '/')
monkeypatch = request.getfuncargvalue("monkeypatch")
monkeypatch.setattr(os, "sep", "/")
def test_empty(force_ossep):
path = Path('')
eq_('',str(path))
path = Path("")
eq_("", str(path))
eq_(0, len(path))
path = Path(())
eq_('',str(path))
eq_("", str(path))
eq_(0, len(path))
def test_single(force_ossep):
path = Path('foobar')
eq_('foobar',path)
path = Path("foobar")
eq_("foobar", path)
eq_(1, len(path))
def test_multiple(force_ossep):
path = Path('foo/bar')
eq_('foo/bar',path)
path = Path("foo/bar")
eq_("foo/bar", path)
eq_(2, len(path))
def test_init_with_tuple_and_list(force_ossep):
path = Path(('foo','bar'))
eq_('foo/bar',path)
path = Path(['foo','bar'])
eq_('foo/bar',path)
path = Path(("foo", "bar"))
eq_("foo/bar", path)
path = Path(["foo", "bar"])
eq_("foo/bar", path)
def test_init_with_invalid_value(force_ossep):
try:
@ -49,202 +55,230 @@ def test_init_with_invalid_value(force_ossep):
except TypeError:
pass
def test_access(force_ossep):
path = Path('foo/bar/bleh')
eq_('foo',path[0])
eq_('foo',path[-3])
eq_('bar',path[1])
eq_('bar',path[-2])
eq_('bleh',path[2])
eq_('bleh',path[-1])
path = Path("foo/bar/bleh")
eq_("foo", path[0])
eq_("foo", path[-3])
eq_("bar", path[1])
eq_("bar", path[-2])
eq_("bleh", path[2])
eq_("bleh", path[-1])
def test_slicing(force_ossep):
path = Path('foo/bar/bleh')
path = Path("foo/bar/bleh")
subpath = path[:2]
eq_('foo/bar',subpath)
eq_("foo/bar", subpath)
assert isinstance(subpath, Path)
def test_parent(force_ossep):
path = Path('foo/bar/bleh')
path = Path("foo/bar/bleh")
subpath = path.parent()
eq_('foo/bar', subpath)
eq_("foo/bar", subpath)
assert isinstance(subpath, Path)
def test_filename(force_ossep):
path = Path('foo/bar/bleh.ext')
eq_(path.name, 'bleh.ext')
path = Path("foo/bar/bleh.ext")
eq_(path.name, "bleh.ext")
def test_deal_with_empty_components(force_ossep):
"""Keep ONLY a leading space, which means we want a leading slash.
"""
eq_('foo//bar',str(Path(('foo','','bar'))))
eq_('/foo/bar',str(Path(('','foo','bar'))))
eq_('foo/bar',str(Path('foo/bar/')))
eq_("foo//bar", str(Path(("foo", "", "bar"))))
eq_("/foo/bar", str(Path(("", "foo", "bar"))))
eq_("foo/bar", str(Path("foo/bar/")))
def test_old_compare_paths(force_ossep):
eq_(Path('foobar'),Path('foobar'))
eq_(Path('foobar/'),Path('foobar\\','\\'))
eq_(Path('/foobar/'),Path('\\foobar\\','\\'))
eq_(Path('/foo/bar'),Path('\\foo\\bar','\\'))
eq_(Path('/foo/bar'),Path('\\foo\\bar\\','\\'))
assert Path('/foo/bar') != Path('\\foo\\foo','\\')
eq_(Path("foobar"), Path("foobar"))
eq_(Path("foobar/"), Path("foobar\\", "\\"))
eq_(Path("/foobar/"), Path("\\foobar\\", "\\"))
eq_(Path("/foo/bar"), Path("\\foo\\bar", "\\"))
eq_(Path("/foo/bar"), Path("\\foo\\bar\\", "\\"))
assert Path("/foo/bar") != Path("\\foo\\foo", "\\")
# We also have to test __ne__
assert not (Path('foobar') != Path('foobar'))
assert Path('/a/b/c.x') != Path('/a/b/c.y')
assert not (Path("foobar") != Path("foobar"))
assert Path("/a/b/c.x") != Path("/a/b/c.y")
def test_old_split_path(force_ossep):
eq_(Path('foobar'),('foobar',))
eq_(Path('foo/bar'),('foo','bar'))
eq_(Path('/foo/bar/'),('','foo','bar'))
eq_(Path('\\foo\\bar','\\'),('','foo','bar'))
eq_(Path("foobar"), ("foobar",))
eq_(Path("foo/bar"), ("foo", "bar"))
eq_(Path("/foo/bar/"), ("", "foo", "bar"))
eq_(Path("\\foo\\bar", "\\"), ("", "foo", "bar"))
def test_representation(force_ossep):
eq_("('foo', 'bar')",repr(Path(('foo','bar'))))
eq_("('foo', 'bar')", repr(Path(("foo", "bar"))))
def test_add(force_ossep):
eq_('foo/bar/bar/foo',Path(('foo','bar')) + Path('bar/foo'))
eq_('foo/bar/bar/foo',Path('foo/bar') + 'bar/foo')
eq_('foo/bar/bar/foo',Path('foo/bar') + ('bar','foo'))
eq_('foo/bar/bar/foo',('foo','bar') + Path('bar/foo'))
eq_('foo/bar/bar/foo','foo/bar' + Path('bar/foo'))
eq_("foo/bar/bar/foo", Path(("foo", "bar")) + Path("bar/foo"))
eq_("foo/bar/bar/foo", Path("foo/bar") + "bar/foo")
eq_("foo/bar/bar/foo", Path("foo/bar") + ("bar", "foo"))
eq_("foo/bar/bar/foo", ("foo", "bar") + Path("bar/foo"))
eq_("foo/bar/bar/foo", "foo/bar" + Path("bar/foo"))
# Invalid concatenation
try:
Path(('foo','bar')) + 1
Path(("foo", "bar")) + 1
assert False
except TypeError:
pass
def test_path_slice(force_ossep):
foo = Path('foo')
bar = Path('bar')
foobar = Path('foo/bar')
eq_('bar',foobar[foo:])
eq_('foo',foobar[:bar])
eq_('foo/bar',foobar[bar:])
eq_('foo/bar',foobar[:foo])
foo = Path("foo")
bar = Path("bar")
foobar = Path("foo/bar")
eq_("bar", foobar[foo:])
eq_("foo", foobar[:bar])
eq_("foo/bar", foobar[bar:])
eq_("foo/bar", foobar[:foo])
eq_((), foobar[foobar:])
eq_((), foobar[:foobar])
abcd = Path('a/b/c/d')
a = Path('a')
b = Path('b')
c = Path('c')
d = Path('d')
z = Path('z')
eq_('b/c',abcd[a:d])
eq_('b/c/d',abcd[a:d+z])
eq_('b/c',abcd[a:z+d])
eq_('a/b/c/d',abcd[:z])
abcd = Path("a/b/c/d")
a = Path("a")
b = Path("b")
c = Path("c")
d = Path("d")
z = Path("z")
eq_("b/c", abcd[a:d])
eq_("b/c/d", abcd[a : d + z])
eq_("b/c", abcd[a : z + d])
eq_("a/b/c/d", abcd[:z])
def test_add_with_root_path(force_ossep):
"""if I perform /a/b/c + /d/e/f, I want /a/b/c/d/e/f, not /a/b/c//d/e/f
"""
eq_('/foo/bar',str(Path('/foo') + Path('/bar')))
eq_("/foo/bar", str(Path("/foo") + Path("/bar")))
def test_create_with_tuple_that_have_slash_inside(force_ossep, monkeypatch):
eq_(('','foo','bar'), Path(('/foo','bar')))
monkeypatch.setattr(os, 'sep', '\\')
eq_(('','foo','bar'), Path(('\\foo','bar')))
eq_(("", "foo", "bar"), Path(("/foo", "bar")))
monkeypatch.setattr(os, "sep", "\\")
eq_(("", "foo", "bar"), Path(("\\foo", "bar")))
def test_auto_decode_os_sep(force_ossep, monkeypatch):
"""Path should decode any either / or os.sep, but always encode in os.sep.
"""
eq_(('foo\\bar','bleh'),Path('foo\\bar/bleh'))
monkeypatch.setattr(os, 'sep', '\\')
eq_(('foo','bar/bleh'),Path('foo\\bar/bleh'))
path = Path('foo/bar')
eq_(('foo','bar'),path)
eq_('foo\\bar',str(path))
eq_(("foo\\bar", "bleh"), Path("foo\\bar/bleh"))
monkeypatch.setattr(os, "sep", "\\")
eq_(("foo", "bar/bleh"), Path("foo\\bar/bleh"))
path = Path("foo/bar")
eq_(("foo", "bar"), path)
eq_("foo\\bar", str(path))
def test_contains(force_ossep):
p = Path(('foo','bar'))
assert Path(('foo','bar','bleh')) in p
assert Path(('foo','bar')) in p
assert 'foo' in p
assert 'bleh' not in p
assert Path('foo') not in p
p = Path(("foo", "bar"))
assert Path(("foo", "bar", "bleh")) in p
assert Path(("foo", "bar")) in p
assert "foo" in p
assert "bleh" not in p
assert Path("foo") not in p
def test_is_parent_of(force_ossep):
assert Path(('foo','bar')).is_parent_of(Path(('foo','bar','bleh')))
assert not Path(('foo','bar')).is_parent_of(Path(('foo','baz')))
assert not Path(('foo','bar')).is_parent_of(Path(('foo','bar')))
assert Path(("foo", "bar")).is_parent_of(Path(("foo", "bar", "bleh")))
assert not Path(("foo", "bar")).is_parent_of(Path(("foo", "baz")))
assert not Path(("foo", "bar")).is_parent_of(Path(("foo", "bar")))
def test_windows_drive_letter(force_ossep):
p = Path(('c:',))
eq_('c:\\',str(p))
p = Path(("c:",))
eq_("c:\\", str(p))
def test_root_path(force_ossep):
p = Path('/')
eq_('/',str(p))
p = Path("/")
eq_("/", str(p))
def test_str_encodes_unicode_to_getfilesystemencoding(force_ossep):
p = Path(('foo','bar\u00e9'))
eq_('foo/bar\u00e9'.encode(sys.getfilesystemencoding()), p.tobytes())
p = Path(("foo", "bar\u00e9"))
eq_("foo/bar\u00e9".encode(sys.getfilesystemencoding()), p.tobytes())
def test_unicode(force_ossep):
p = Path(('foo','bar\u00e9'))
eq_('foo/bar\u00e9',str(p))
p = Path(("foo", "bar\u00e9"))
eq_("foo/bar\u00e9", str(p))
def test_str_repr_of_mix_between_non_ascii_str_and_unicode(force_ossep):
u = 'foo\u00e9'
u = "foo\u00e9"
encoded = u.encode(sys.getfilesystemencoding())
p = Path((encoded,'bar'))
p = Path((encoded, "bar"))
print(repr(tuple(p)))
eq_('foo\u00e9/bar'.encode(sys.getfilesystemencoding()), p.tobytes())
eq_("foo\u00e9/bar".encode(sys.getfilesystemencoding()), p.tobytes())
def test_Path_of_a_Path_returns_self(force_ossep):
# if Path() is called with a path as value, just return value.
p = Path('foo/bar')
p = Path("foo/bar")
assert Path(p) is p
def test_getitem_str(force_ossep):
# path['something'] returns the child path corresponding to the name
p = Path('/foo/bar')
eq_(p['baz'], Path('/foo/bar/baz'))
p = Path("/foo/bar")
eq_(p["baz"], Path("/foo/bar/baz"))
def test_getitem_path(force_ossep):
# path[Path('something')] returns the child path corresponding to the name (or subpath)
p = Path('/foo/bar')
eq_(p[Path('baz/bleh')], Path('/foo/bar/baz/bleh'))
p = Path("/foo/bar")
eq_(p[Path("baz/bleh")], Path("/foo/bar/baz/bleh"))
@mark.xfail(reason="pytest's capture mechanism is flaky, I have to investigate")
def test_log_unicode_errors(force_ossep, monkeypatch, capsys):
# When an there's a UnicodeDecodeError on path creation, log it so it can be possible
# to debug the cause of it.
monkeypatch.setattr(sys, 'getfilesystemencoding', lambda: 'ascii')
monkeypatch.setattr(sys, "getfilesystemencoding", lambda: "ascii")
with raises(UnicodeDecodeError):
Path(['', b'foo\xe9'])
Path(["", b"foo\xe9"])
out, err = capsys.readouterr()
assert repr(b'foo\xe9') in err
assert repr(b"foo\xe9") in err
def test_has_drive_letter(monkeypatch):
monkeypatch.setattr(os, 'sep', '\\')
p = Path('foo\\bar')
monkeypatch.setattr(os, "sep", "\\")
p = Path("foo\\bar")
assert not p.has_drive_letter()
p = Path('C:\\')
p = Path("C:\\")
assert p.has_drive_letter()
p = Path('z:\\foo')
p = Path("z:\\foo")
assert p.has_drive_letter()
def test_remove_drive_letter(monkeypatch):
monkeypatch.setattr(os, 'sep', '\\')
p = Path('foo\\bar')
eq_(p.remove_drive_letter(), Path('foo\\bar'))
p = Path('C:\\')
eq_(p.remove_drive_letter(), Path(''))
p = Path('z:\\foo')
eq_(p.remove_drive_letter(), Path('foo'))
monkeypatch.setattr(os, "sep", "\\")
p = Path("foo\\bar")
eq_(p.remove_drive_letter(), Path("foo\\bar"))
p = Path("C:\\")
eq_(p.remove_drive_letter(), Path(""))
p = Path("z:\\foo")
eq_(p.remove_drive_letter(), Path("foo"))
def test_pathify():
@pathify
def foo(a: Path, b, c: Path):
return a, b, c
a, b, c = foo('foo', 0, c=Path('bar'))
a, b, c = foo("foo", 0, c=Path("bar"))
assert isinstance(a, Path)
assert a == Path('foo')
assert a == Path("foo")
assert b == 0
assert isinstance(c, Path)
assert c == Path('bar')
assert c == Path("bar")
def test_pathify_preserve_none():
# @pathify preserves None value and doesn't try to return a Path

45
hscommon/tests/selectable_list_test.py
View File

@ -9,6 +9,7 @@
from ..testutil import eq_, callcounter, CallLogger
from ..gui.selectable_list import SelectableList, GUISelectableList
def test_in():
# When a SelectableList is in a list, doing "in list" with another instance returns false, even
# if they're the same as lists.
@ -16,50 +17,56 @@ def test_in():
some_list = [sl]
assert SelectableList() not in some_list
def test_selection_range():
# selection is correctly adjusted on deletion
sl = SelectableList(['foo', 'bar', 'baz'])
sl = SelectableList(["foo", "bar", "baz"])
sl.selected_index = 3
eq_(sl.selected_index, 2)
del sl[2]
eq_(sl.selected_index, 1)
def test_update_selection_called():
# _update_selection_is called after a change in selection. However, we only do so on select()
# calls. I follow the old behavior of the Table class. At the moment, I don't quite remember
# why there was a specific select() method for triggering _update_selection(), but I think I
# remember there was a reason, so I keep it that way.
sl = SelectableList(['foo', 'bar'])
sl = SelectableList(["foo", "bar"])
sl._update_selection = callcounter()
sl.select(1)
eq_(sl._update_selection.callcount, 1)
sl.selected_index = 0
eq_(sl._update_selection.callcount, 1) # no call
def test_guicalls():
# A GUISelectableList appropriately calls its view.
sl = GUISelectableList(['foo', 'bar'])
sl = GUISelectableList(["foo", "bar"])
sl.view = CallLogger()
sl.view.check_gui_calls(['refresh']) # Upon setting the view, we get a call to refresh()
sl[1] = 'baz'
sl.view.check_gui_calls(['refresh'])
sl.append('foo')
sl.view.check_gui_calls(['refresh'])
sl.view.check_gui_calls(
["refresh"]
) # Upon setting the view, we get a call to refresh()
sl[1] = "baz"
sl.view.check_gui_calls(["refresh"])
sl.append("foo")
sl.view.check_gui_calls(["refresh"])
del sl[2]
sl.view.check_gui_calls(['refresh'])
sl.remove('baz')
sl.view.check_gui_calls(['refresh'])
sl.insert(0, 'foo')
sl.view.check_gui_calls(['refresh'])
sl.view.check_gui_calls(["refresh"])
sl.remove("baz")
sl.view.check_gui_calls(["refresh"])
sl.insert(0, "foo")
sl.view.check_gui_calls(["refresh"])
sl.select(1)
sl.view.check_gui_calls(['update_selection'])
sl.view.check_gui_calls(["update_selection"])
# XXX We have to give up on this for now because of a breakage it causes in the tables.
# sl.select(1) # don't update when selection stays the same
# gui.check_gui_calls([])
def test_search_by_prefix():
sl = SelectableList(['foo', 'bAr', 'baZ'])
eq_(sl.search_by_prefix('b'), 1)
eq_(sl.search_by_prefix('BA'), 1)
eq_(sl.search_by_prefix('BAZ'), 2)
eq_(sl.search_by_prefix('BAZZ'), -1)
sl = SelectableList(["foo", "bAr", "baZ"])
eq_(sl.search_by_prefix("b"), 1)
eq_(sl.search_by_prefix("BA"), 1)
eq_(sl.search_by_prefix("BAZ"), 2)
eq_(sl.search_by_prefix("BAZZ"), -1)

91
hscommon/tests/sqlite_test.py
View File

@ -19,69 +19,75 @@ from ..sqlite import ThreadedConn
# Threading is hard to test. In a lot of those tests, a failure means that the test run will
# hang forever. Well... I don't know a better alternative.
def test_can_access_from_multiple_threads():
def run():
con.execute('insert into foo(bar) values(\'baz\')')
con.execute("insert into foo(bar) values('baz')")
con = ThreadedConn(':memory:', True)
con.execute('create table foo(bar TEXT)')
con = ThreadedConn(":memory:", True)
con.execute("create table foo(bar TEXT)")
t = threading.Thread(target=run)
t.start()
t.join()
result = con.execute('select * from foo')
result = con.execute("select * from foo")
eq_(1, len(result))
eq_('baz', result[0][0])
eq_("baz", result[0][0])
def test_exception_during_query():
con = ThreadedConn(':memory:', True)
con.execute('create table foo(bar TEXT)')
con = ThreadedConn(":memory:", True)
con.execute("create table foo(bar TEXT)")
with raises(sqlite.OperationalError):
con.execute('select * from bleh')
con.execute("select * from bleh")
def test_not_autocommit(tmpdir):
dbpath = str(tmpdir.join('foo.db'))
dbpath = str(tmpdir.join("foo.db"))
con = ThreadedConn(dbpath, False)
con.execute('create table foo(bar TEXT)')
con.execute('insert into foo(bar) values(\'baz\')')
con.execute("create table foo(bar TEXT)")
con.execute("insert into foo(bar) values('baz')")
del con
# The data shouldn't have been inserted
con = ThreadedConn(dbpath, False)
result = con.execute('select * from foo')
result = con.execute("select * from foo")
eq_(0, len(result))
con.execute('insert into foo(bar) values(\'baz\')')
con.execute("insert into foo(bar) values('baz')")
con.commit()
del con
# Now the data should be there
con = ThreadedConn(dbpath, False)
result = con.execute('select * from foo')
result = con.execute("select * from foo")
eq_(1, len(result))
def test_rollback():
con = ThreadedConn(':memory:', False)
con.execute('create table foo(bar TEXT)')
con.execute('insert into foo(bar) values(\'baz\')')
con = ThreadedConn(":memory:", False)
con.execute("create table foo(bar TEXT)")
con.execute("insert into foo(bar) values('baz')")
con.rollback()
result = con.execute('select * from foo')
result = con.execute("select * from foo")
eq_(0, len(result))
def test_query_palceholders():
con = ThreadedConn(':memory:', True)
con.execute('create table foo(bar TEXT)')
con.execute('insert into foo(bar) values(?)', ['baz'])
result = con.execute('select * from foo')
con = ThreadedConn(":memory:", True)
con.execute("create table foo(bar TEXT)")
con.execute("insert into foo(bar) values(?)", ["baz"])
result = con.execute("select * from foo")
eq_(1, len(result))
eq_('baz', result[0][0])
eq_("baz", result[0][0])
def test_make_sure_theres_no_messup_between_queries():
def run(expected_rowid):
time.sleep(0.1)
result = con.execute('select rowid from foo where rowid = ?', [expected_rowid])
result = con.execute("select rowid from foo where rowid = ?", [expected_rowid])
assert expected_rowid == result[0][0]
con = ThreadedConn(':memory:', True)
con.execute('create table foo(bar TEXT)')
con = ThreadedConn(":memory:", True)
con.execute("create table foo(bar TEXT)")
for i in range(100):
con.execute('insert into foo(bar) values(\'baz\')')
con.execute("insert into foo(bar) values('baz')")
threads = []
for i in range(1, 101):
t = threading.Thread(target=run, args=(i,))
@ -91,36 +97,41 @@ def test_make_sure_theres_no_messup_between_queries():
time.sleep(0.1)
threads = [t for t in threads if t.isAlive()]
def test_query_after_close():
con = ThreadedConn(':memory:', True)
con = ThreadedConn(":memory:", True)
con.close()
con.execute('select 1')
con.execute("select 1")
def test_lastrowid():
# It's not possible to return a cursor because of the threading, but lastrowid should be
# fetchable from the connection itself
con = ThreadedConn(':memory:', True)
con.execute('create table foo(bar TEXT)')
con.execute('insert into foo(bar) values(\'baz\')')
con = ThreadedConn(":memory:", True)
con.execute("create table foo(bar TEXT)")
con.execute("insert into foo(bar) values('baz')")
eq_(1, con.lastrowid)
def test_add_fetchone_fetchall_interface_to_results():
con = ThreadedConn(':memory:', True)
con.execute('create table foo(bar TEXT)')
con.execute('insert into foo(bar) values(\'baz1\')')
con.execute('insert into foo(bar) values(\'baz2\')')
result = con.execute('select * from foo')
con = ThreadedConn(":memory:", True)
con.execute("create table foo(bar TEXT)")
con.execute("insert into foo(bar) values('baz1')")
con.execute("insert into foo(bar) values('baz2')")
result = con.execute("select * from foo")
ref = result[:]
eq_(ref, result.fetchall())
eq_(ref[0], result.fetchone())
eq_(ref[1], result.fetchone())
assert result.fetchone() is None
def test_non_ascii_dbname(tmpdir):
ThreadedConn(str(tmpdir.join('foo\u00e9.db')), True)
ThreadedConn(str(tmpdir.join("foo\u00e9.db")), True)
def test_non_ascii_dbdir(tmpdir):
# when this test fails, it doesn't fail gracefully, it brings the whole test suite with it.
dbdir = tmpdir.join('foo\u00e9')
dbdir = tmpdir.join("foo\u00e9")
os.mkdir(str(dbdir))
ThreadedConn(str(dbdir.join('foo.db')), True)
ThreadedConn(str(dbdir.join("foo.db")), True)

78
hscommon/tests/table_test.py
View File

@ -9,6 +9,7 @@
from ..testutil import CallLogger, eq_
from ..gui.table import Table, GUITable, Row
class TestRow(Row):
def __init__(self, table, index, is_new=False):
Row.__init__(self, table)
@ -55,6 +56,7 @@ def table_with_footer():
table.footer = footer
return table, footer
def table_with_header():
table = Table()
table.append(TestRow(table, 1))
@ -62,6 +64,7 @@ def table_with_header():
table.header = header
return table, header
# --- Tests
def test_allow_edit_when_attr_is_property_with_fset():
# When a row has a property that has a fset, by default, make that cell editable.
@ -69,17 +72,20 @@ def test_allow_edit_when_attr_is_property_with_fset():
@property
def foo(self):
pass
@property
def bar(self):
pass
@bar.setter
def bar(self, value):
pass
row = TestRow(Table())
assert row.can_edit_cell('bar')
assert not row.can_edit_cell('foo')
assert not row.can_edit_cell('baz') # doesn't exist, can't edit
assert row.can_edit_cell("bar")
assert not row.can_edit_cell("foo")
assert not row.can_edit_cell("baz") # doesn't exist, can't edit
def test_can_edit_prop_has_priority_over_fset_checks():
# When a row has a cen_edit_* property, it's the result of that property that is used, not the
@ -88,13 +94,16 @@ def test_can_edit_prop_has_priority_over_fset_checks():
@property
def bar(self):
pass
@bar.setter
def bar(self, value):
pass
can_edit_bar = False
row = TestRow(Table())
assert not row.can_edit_cell('bar')
assert not row.can_edit_cell("bar")
def test_in():
# When a table is in a list, doing "in list" with another instance returns false, even if
@ -103,12 +112,14 @@ def test_in():
some_list = [table]
assert Table() not in some_list
def test_footer_del_all():
# Removing all rows doesn't crash when doing the footer check.
table, footer = table_with_footer()
del table[:]
assert table.footer is None
def test_footer_del_row():
# Removing the footer row sets it to None
table, footer = table_with_footer()
@ -116,18 +127,21 @@ def test_footer_del_row():
assert table.footer is None
eq_(len(table), 1)
def test_footer_is_appened_to_table():
# A footer is appended at the table's bottom
table, footer = table_with_footer()
eq_(len(table), 2)
assert table[1] is footer
def test_footer_remove():
# remove() on footer sets it to None
table, footer = table_with_footer()
table.remove(footer)
assert table.footer is None
def test_footer_replaces_old_footer():
table, footer = table_with_footer()
other = Row(table)
@ -136,18 +150,21 @@ def test_footer_replaces_old_footer():
eq_(len(table), 2)
assert table[1] is other
def test_footer_rows_and_row_count():
# rows() and row_count() ignore footer.
table, footer = table_with_footer()
eq_(table.row_count, 1)
eq_(table.rows, table[:-1])
def test_footer_setting_to_none_removes_old_one():
table, footer = table_with_footer()
table.footer = None
assert table.footer is None
eq_(len(table), 1)
def test_footer_stays_there_on_append():
# Appending another row puts it above the footer
table, footer = table_with_footer()
@ -155,6 +172,7 @@ def test_footer_stays_there_on_append():
eq_(len(table), 3)
assert table[2] is footer
def test_footer_stays_there_on_insert():
# Inserting another row puts it above the footer
table, footer = table_with_footer()
@ -162,12 +180,14 @@ def test_footer_stays_there_on_insert():
eq_(len(table), 3)
assert table[2] is footer
def test_header_del_all():
# Removing all rows doesn't crash when doing the header check.
table, header = table_with_header()
del table[:]
assert table.header is None
def test_header_del_row():
# Removing the header row sets it to None
table, header = table_with_header()
@ -175,18 +195,21 @@ def test_header_del_row():
assert table.header is None
eq_(len(table), 1)
def test_header_is_inserted_in_table():
# A header is inserted at the table's top
table, header = table_with_header()
eq_(len(table), 2)
assert table[0] is header
def test_header_remove():
# remove() on header sets it to None
table, header = table_with_header()
table.remove(header)
assert table.header is None
def test_header_replaces_old_header():
table, header = table_with_header()
other = Row(table)
@ -195,18 +218,21 @@ def test_header_replaces_old_header():
eq_(len(table), 2)
assert table[0] is other
def test_header_rows_and_row_count():
# rows() and row_count() ignore header.
table, header = table_with_header()
eq_(table.row_count, 1)
eq_(table.rows, table[1:])
def test_header_setting_to_none_removes_old_one():
table, header = table_with_header()
table.header = None
assert table.header is None
eq_(len(table), 1)
def test_header_stays_there_on_insert():
# Inserting another row at the top puts it below the header
table, header = table_with_header()
@ -214,21 +240,24 @@ def test_header_stays_there_on_insert():
eq_(len(table), 3)
assert table[0] is header
def test_refresh_view_on_refresh():
# If refresh_view is not False, we refresh the table's view on refresh()
table = TestGUITable(1)
table.refresh()
table.view.check_gui_calls(['refresh'])
table.view.check_gui_calls(["refresh"])
table.view.clear_calls()
table.refresh(refresh_view=False)
table.view.check_gui_calls([])
def test_restore_selection():
# By default, after a refresh, selection goes on the last row
table = TestGUITable(10)
table.refresh()
eq_(table.selected_indexes, [9])
def test_restore_selection_after_cancel_edits():
# _restore_selection() is called after cancel_edits(). Previously, only _update_selection would
# be called.
@ -242,6 +271,7 @@ def test_restore_selection_after_cancel_edits():
table.cancel_edits()
eq_(table.selected_indexes, [6])
def test_restore_selection_with_previous_selection():
# By default, we try to restore the selection that was there before a refresh
table = TestGUITable(10)
@ -250,6 +280,7 @@ def test_restore_selection_with_previous_selection():
table.refresh()
eq_(table.selected_indexes, [2, 4])
def test_restore_selection_custom():
# After a _fill() called, the virtual _restore_selection() is called so that it's possible for a
# GUITable subclass to customize its post-refresh selection behavior.
@ -261,58 +292,64 @@ def test_restore_selection_custom():
table.refresh()
eq_(table.selected_indexes, [6])
def test_row_cell_value():
# *_cell_value() correctly mangles attrnames that are Python reserved words.
row = Row(Table())
row.from_ = 'foo'
eq_(row.get_cell_value('from'), 'foo')
row.set_cell_value('from', 'bar')
eq_(row.get_cell_value('from'), 'bar')
row.from_ = "foo"
eq_(row.get_cell_value("from"), "foo")
row.set_cell_value("from", "bar")
eq_(row.get_cell_value("from"), "bar")
def test_sort_table_also_tries_attributes_without_underscores():
# When determining a sort key, after having unsuccessfully tried the attribute with the,
# underscore, try the one without one.
table = Table()
row1 = Row(table)
row1._foo = 'a' # underscored attr must be checked first
row1.foo = 'b'
row1.bar = 'c'
row1._foo = "a" # underscored attr must be checked first
row1.foo = "b"
row1.bar = "c"
row2 = Row(table)
row2._foo = 'b'
row2.foo = 'a'
row2.bar = 'b'
row2._foo = "b"
row2.foo = "a"
row2.bar = "b"
table.append(row1)
table.append(row2)
table.sort_by('foo')
table.sort_by("foo")
assert table[0] is row1
assert table[1] is row2
table.sort_by('bar')
table.sort_by("bar")
assert table[0] is row2
assert table[1] is row1
def test_sort_table_updates_selection():
table = TestGUITable(10)
table.refresh()
table.select([2, 4])
table.sort_by('index', desc=True)
table.sort_by("index", desc=True)
# Now, the updated rows should be 7 and 5
eq_(len(table.updated_rows), 2)
r1, r2 = table.updated_rows
eq_(r1.index, 7)
eq_(r2.index, 5)
def test_sort_table_with_footer():
# Sorting a table with a footer keeps it at the bottom
table, footer = table_with_footer()
table.sort_by('index', desc=True)
table.sort_by("index", desc=True)
assert table[-1] is footer
def test_sort_table_with_header():
# Sorting a table with a header keeps it at the top
table, header = table_with_header()
table.sort_by('index', desc=True)
table.sort_by("index", desc=True)
assert table[0] is header
def test_add_with_view_that_saves_during_refresh():
# Calling save_edits during refresh() called by add() is ignored.
class TableView(CallLogger):
@ -322,4 +359,3 @@ def test_add_with_view_that_saves_during_refresh():
table = TestGUITable(10, viewclass=TableView)
table.add()
assert table.edited is not None # still in edit mode

34
hscommon/tests/tree_test.py
View File

@ -9,15 +9,17 @@
from ..testutil import eq_
from ..gui.tree import Tree, Node
def tree_with_some_nodes():
t = Tree()
t.append(Node('foo'))
t.append(Node('bar'))
t.append(Node('baz'))
t[0].append(Node('sub1'))
t[0].append(Node('sub2'))
t.append(Node("foo"))
t.append(Node("bar"))
t.append(Node("baz"))
t[0].append(Node("sub1"))
t[0].append(Node("sub2"))
return t
def test_selection():
t = tree_with_some_nodes()
assert t.selected_node is None
@ -25,6 +27,7 @@ def test_selection():
assert t.selected_path is None
eq_(t.selected_paths, [])
def test_select_one_node():
t = tree_with_some_nodes()
t.selected_node = t[0][0]
@ -33,33 +36,39 @@ def test_select_one_node():
eq_(t.selected_path, [0, 0])
eq_(t.selected_paths, [[0, 0]])
def test_select_one_path():
t = tree_with_some_nodes()
t.selected_path = [0, 1]
assert t.selected_node is t[0][1]
def test_select_multiple_nodes():
t = tree_with_some_nodes()
t.selected_nodes = [t[0], t[1]]
eq_(t.selected_paths, [[0], [1]])
def test_select_multiple_paths():
t = tree_with_some_nodes()
t.selected_paths = [[0], [1]]
eq_(t.selected_nodes, [t[0], t[1]])
def test_select_none_path():
# setting selected_path to None clears the selection
t = Tree()
t.selected_path = None
assert t.selected_path is None
def test_select_none_node():
# setting selected_node to None clears the selection
t = Tree()
t.selected_node = None
eq_(t.selected_nodes, [])
def test_clear_removes_selection():
# When clearing a tree, we want to clear the selection as well or else we end up with a crash
# when calling selected_paths.
@ -68,15 +77,16 @@ def test_clear_removes_selection():
t.clear()
assert t.selected_node is None
def test_selection_override():
# All selection changed pass through the _select_node() method so it's easy for subclasses to
# customize the tree's behavior.
class MyTree(Tree):
called = False
def _select_nodes(self, nodes):
self.called = True
t = MyTree()
t.selected_paths = []
assert t.called
@ -84,25 +94,31 @@ def test_selection_override():
t.selected_node = None
assert t.called
def test_findall():
t = tree_with_some_nodes()
r = t.findall(lambda n: n.name.startswith('sub'))
r = t.findall(lambda n: n.name.startswith("sub"))
eq_(set(r), set([t[0][0], t[0][1]]))
def test_findall_dont_include_self():
# When calling findall with include_self=False, the node itself is never evaluated.
t = tree_with_some_nodes()
del t._name # so that if the predicate is called on `t`, we crash
r = t.findall(lambda n: not n.name.startswith('sub'), include_self=False) # no crash
r = t.findall(
lambda n: not n.name.startswith("sub"), include_self=False
) # no crash
eq_(set(r), set([t[0], t[1], t[2]]))
def test_find_dont_include_self():
# When calling find with include_self=False, the node itself is never evaluated.
t = tree_with_some_nodes()
del t._name # so that if the predicate is called on `t`, we crash
r = t.find(lambda n: not n.name.startswith('sub'), include_self=False) # no crash
r = t.find(lambda n: not n.name.startswith("sub"), include_self=False) # no crash
assert r is t[0]
def test_find_none():
# when find() yields no result, return None
t = Tree()

311
hscommon/tests/util_test.py
View File

@ -14,43 +14,53 @@ from ..testutil import eq_
from ..path import Path
from ..util import *
def test_nonone():
eq_('foo', nonone('foo', 'bar'))
eq_('bar', nonone(None, 'bar'))
eq_("foo", nonone("foo", "bar"))
eq_("bar", nonone(None, "bar"))
def test_tryint():
eq_(42,tryint('42'))
eq_(0,tryint('abc'))
eq_(42, tryint("42"))
eq_(0, tryint("abc"))
eq_(0, tryint(None))
eq_(42, tryint(None, 42))
def test_minmax():
eq_(minmax(2, 1, 3), 2)
eq_(minmax(0, 1, 3), 1)
eq_(minmax(4, 1, 3), 3)
# --- Sequence
def test_first():
eq_(first([3, 2, 1]), 3)
eq_(first(i for i in [3, 2, 1] if i < 3), 2)
def test_flatten():
eq_([1, 2, 3, 4], flatten([[1, 2], [3, 4]]))
eq_([], flatten([]))
def test_dedupe():
reflist = [0, 7, 1, 2, 3, 4, 4, 5, 6, 7, 1, 2, 3]
eq_(dedupe(reflist), [0, 7, 1, 2, 3, 4, 5, 6])
def test_stripfalse():
eq_([1, 2, 3], stripfalse([None, 0, 1, 2, 3, None]))
def test_extract():
wheat, shaft = extract(lambda n: n % 2 == 0, list(range(10)))
eq_(wheat, [0, 2, 4, 6, 8])
eq_(shaft, [1, 3, 5, 7, 9])
def test_allsame():
assert allsame([42, 42, 42])
assert not allsame([42, 43, 42])
@ -58,25 +68,32 @@ def test_allsame():
# Works on non-sequence as well
assert allsame(iter([42, 42, 42]))
def test_trailiter():
eq_(list(trailiter([])), [])
eq_(list(trailiter(['foo'])), [(None, 'foo')])
eq_(list(trailiter(['foo', 'bar'])), [(None, 'foo'), ('foo', 'bar')])
eq_(list(trailiter(['foo', 'bar'], skipfirst=True)), [('foo', 'bar')])
eq_(list(trailiter(["foo"])), [(None, "foo")])
eq_(list(trailiter(["foo", "bar"])), [(None, "foo"), ("foo", "bar")])
eq_(list(trailiter(["foo", "bar"], skipfirst=True)), [("foo", "bar")])
eq_(list(trailiter([], skipfirst=True)), []) # no crash
def test_iterconsume():
# We just want to make sure that we return *all* items and that we're not mistakenly skipping
# one.
eq_(list(range(2500)), list(iterconsume(list(range(2500)))))
eq_(list(reversed(range(2500))), list(iterconsume(list(range(2500)), reverse=False)))
eq_(
list(reversed(range(2500))), list(iterconsume(list(range(2500)), reverse=False))
)
# --- String
def test_escape():
eq_('f\\o\\ob\\ar', escape('foobar', 'oa'))
eq_('f*o*ob*ar', escape('foobar', 'oa', '*'))
eq_('f*o*ob*ar', escape('foobar', set('oa'), '*'))
eq_("f\\o\\ob\\ar", escape("foobar", "oa"))
eq_("f*o*ob*ar", escape("foobar", "oa", "*"))
eq_("f*o*ob*ar", escape("foobar", set("oa"), "*"))
def test_get_file_ext():
eq_(get_file_ext("foobar"), "")
@ -84,146 +101,155 @@ def test_get_file_ext():
eq_(get_file_ext("foobar."), "")
eq_(get_file_ext(".foobar"), "foobar")
def test_rem_file_ext():
eq_(rem_file_ext("foobar"), "foobar")
eq_(rem_file_ext("foo.bar"), "foo")
eq_(rem_file_ext("foobar."), "foobar")
eq_(rem_file_ext(".foobar"), "")
def test_pluralize():
eq_('0 song', pluralize(0,'song'))
eq_('1 song', pluralize(1,'song'))
eq_('2 songs', pluralize(2,'song'))
eq_('1 song', pluralize(1.1,'song'))
eq_('2 songs', pluralize(1.5,'song'))
eq_('1.1 songs', pluralize(1.1,'song',1))
eq_('1.5 songs', pluralize(1.5,'song',1))
eq_('2 entries', pluralize(2,'entry', plural_word='entries'))
eq_("0 song", pluralize(0, "song"))
eq_("1 song", pluralize(1, "song"))
eq_("2 songs", pluralize(2, "song"))
eq_("1 song", pluralize(1.1, "song"))
eq_("2 songs", pluralize(1.5, "song"))
eq_("1.1 songs", pluralize(1.1, "song", 1))
eq_("1.5 songs", pluralize(1.5, "song", 1))
eq_("2 entries", pluralize(2, "entry", plural_word="entries"))
def test_format_time():
eq_(format_time(0),'00:00:00')
eq_(format_time(1),'00:00:01')
eq_(format_time(23),'00:00:23')
eq_(format_time(60),'00:01:00')
eq_(format_time(101),'00:01:41')
eq_(format_time(683),'00:11:23')
eq_(format_time(3600),'01:00:00')
eq_(format_time(3754),'01:02:34')
eq_(format_time(36000),'10:00:00')
eq_(format_time(366666),'101:51:06')
eq_(format_time(0, with_hours=False),'00:00')
eq_(format_time(1, with_hours=False),'00:01')
eq_(format_time(23, with_hours=False),'00:23')
eq_(format_time(60, with_hours=False),'01:00')
eq_(format_time(101, with_hours=False),'01:41')
eq_(format_time(683, with_hours=False),'11:23')
eq_(format_time(3600, with_hours=False),'60:00')
eq_(format_time(6036, with_hours=False),'100:36')
eq_(format_time(60360, with_hours=False),'1006:00')
eq_(format_time(0), "00:00:00")
eq_(format_time(1), "00:00:01")
eq_(format_time(23), "00:00:23")
eq_(format_time(60), "00:01:00")
eq_(format_time(101), "00:01:41")
eq_(format_time(683), "00:11:23")
eq_(format_time(3600), "01:00:00")
eq_(format_time(3754), "01:02:34")
eq_(format_time(36000), "10:00:00")
eq_(format_time(366666), "101:51:06")
eq_(format_time(0, with_hours=False), "00:00")
eq_(format_time(1, with_hours=False), "00:01")
eq_(format_time(23, with_hours=False), "00:23")
eq_(format_time(60, with_hours=False), "01:00")
eq_(format_time(101, with_hours=False), "01:41")
eq_(format_time(683, with_hours=False), "11:23")
eq_(format_time(3600, with_hours=False), "60:00")
eq_(format_time(6036, with_hours=False), "100:36")
eq_(format_time(60360, with_hours=False), "1006:00")
def test_format_time_decimal():
eq_(format_time_decimal(0), '0.0 second')
eq_(format_time_decimal(1), '1.0 second')
eq_(format_time_decimal(23), '23.0 seconds')
eq_(format_time_decimal(60), '1.0 minute')
eq_(format_time_decimal(101), '1.7 minutes')
eq_(format_time_decimal(683), '11.4 minutes')
eq_(format_time_decimal(3600), '1.0 hour')
eq_(format_time_decimal(6036), '1.7 hours')
eq_(format_time_decimal(86400), '1.0 day')
eq_(format_time_decimal(160360), '1.9 days')
eq_(format_time_decimal(0), "0.0 second")
eq_(format_time_decimal(1), "1.0 second")
eq_(format_time_decimal(23), "23.0 seconds")
eq_(format_time_decimal(60), "1.0 minute")
eq_(format_time_decimal(101), "1.7 minutes")
eq_(format_time_decimal(683), "11.4 minutes")
eq_(format_time_decimal(3600), "1.0 hour")
eq_(format_time_decimal(6036), "1.7 hours")
eq_(format_time_decimal(86400), "1.0 day")
eq_(format_time_decimal(160360), "1.9 days")
def test_format_size():
eq_(format_size(1024), '1 KB')
eq_(format_size(1024,2), '1.00 KB')
eq_(format_size(1024,0,2), '1 MB')
eq_(format_size(1024,2,2), '0.01 MB')
eq_(format_size(1024,3,2), '0.001 MB')
eq_(format_size(1024,3,2,False), '0.001')
eq_(format_size(1023), '1023 B')
eq_(format_size(1023,0,1), '1 KB')
eq_(format_size(511,0,1), '1 KB')
eq_(format_size(9), '9 B')
eq_(format_size(99), '99 B')
eq_(format_size(999), '999 B')
eq_(format_size(9999), '10 KB')
eq_(format_size(99999), '98 KB')
eq_(format_size(999999), '977 KB')
eq_(format_size(9999999), '10 MB')
eq_(format_size(99999999), '96 MB')
eq_(format_size(999999999), '954 MB')
eq_(format_size(9999999999), '10 GB')
eq_(format_size(99999999999), '94 GB')
eq_(format_size(999999999999), '932 GB')
eq_(format_size(9999999999999), '10 TB')
eq_(format_size(99999999999999), '91 TB')
eq_(format_size(999999999999999), '910 TB')
eq_(format_size(9999999999999999), '9 PB')
eq_(format_size(99999999999999999), '89 PB')
eq_(format_size(999999999999999999), '889 PB')
eq_(format_size(9999999999999999999), '9 EB')
eq_(format_size(99999999999999999999), '87 EB')
eq_(format_size(999999999999999999999), '868 EB')
eq_(format_size(9999999999999999999999), '9 ZB')
eq_(format_size(99999999999999999999999), '85 ZB')
eq_(format_size(999999999999999999999999), '848 ZB')
eq_(format_size(1024), "1 KB")
eq_(format_size(1024, 2), "1.00 KB")
eq_(format_size(1024, 0, 2), "1 MB")
eq_(format_size(1024, 2, 2), "0.01 MB")
eq_(format_size(1024, 3, 2), "0.001 MB")
eq_(format_size(1024, 3, 2, False), "0.001")
eq_(format_size(1023), "1023 B")
eq_(format_size(1023, 0, 1), "1 KB")
eq_(format_size(511, 0, 1), "1 KB")
eq_(format_size(9), "9 B")
eq_(format_size(99), "99 B")
eq_(format_size(999), "999 B")
eq_(format_size(9999), "10 KB")
eq_(format_size(99999), "98 KB")
eq_(format_size(999999), "977 KB")
eq_(format_size(9999999), "10 MB")
eq_(format_size(99999999), "96 MB")
eq_(format_size(999999999), "954 MB")
eq_(format_size(9999999999), "10 GB")
eq_(format_size(99999999999), "94 GB")
eq_(format_size(999999999999), "932 GB")
eq_(format_size(9999999999999), "10 TB")
eq_(format_size(99999999999999), "91 TB")
eq_(format_size(999999999999999), "910 TB")
eq_(format_size(9999999999999999), "9 PB")
eq_(format_size(99999999999999999), "89 PB")
eq_(format_size(999999999999999999), "889 PB")
eq_(format_size(9999999999999999999), "9 EB")
eq_(format_size(99999999999999999999), "87 EB")
eq_(format_size(999999999999999999999), "868 EB")
eq_(format_size(9999999999999999999999), "9 ZB")
eq_(format_size(99999999999999999999999), "85 ZB")
eq_(format_size(999999999999999999999999), "848 ZB")
def test_remove_invalid_xml():
eq_(remove_invalid_xml('foo\0bar\x0bbaz'), 'foo bar baz')
eq_(remove_invalid_xml("foo\0bar\x0bbaz"), "foo bar baz")
# surrogate blocks have to be replaced, but not the rest
eq_(remove_invalid_xml('foo\ud800bar\udfffbaz\ue000'), 'foo bar baz\ue000')
eq_(remove_invalid_xml("foo\ud800bar\udfffbaz\ue000"), "foo bar baz\ue000")
# replace with something else
eq_(remove_invalid_xml('foo\0baz', replace_with='bar'), 'foobarbaz')
eq_(remove_invalid_xml("foo\0baz", replace_with="bar"), "foobarbaz")
def test_multi_replace():
eq_('136',multi_replace('123456',('2','45')))
eq_('1 3 6',multi_replace('123456',('2','45'),' '))
eq_('1 3 6',multi_replace('123456','245',' '))
eq_('173896',multi_replace('123456','245','789'))
eq_('173896',multi_replace('123456','245',('7','8','9')))
eq_('17386',multi_replace('123456',('2','45'),'78'))
eq_('17386',multi_replace('123456',('2','45'),('7','8')))
eq_("136", multi_replace("123456", ("2", "45")))
eq_("1 3 6", multi_replace("123456", ("2", "45"), " "))
eq_("1 3 6", multi_replace("123456", "245", " "))
eq_("173896", multi_replace("123456", "245", "789"))
eq_("173896", multi_replace("123456", "245", ("7", "8", "9")))
eq_("17386", multi_replace("123456", ("2", "45"), "78"))
eq_("17386", multi_replace("123456", ("2", "45"), ("7", "8")))
with raises(ValueError):
multi_replace('123456',('2','45'),('7','8','9'))
eq_('17346',multi_replace('12346',('2','45'),'78'))
multi_replace("123456", ("2", "45"), ("7", "8", "9"))
eq_("17346", multi_replace("12346", ("2", "45"), "78"))
# --- Files
class TestCase_modified_after:
def test_first_is_modified_after(self, monkeyplus):
monkeyplus.patch_osstat('first', st_mtime=42)
monkeyplus.patch_osstat('second', st_mtime=41)
assert modified_after('first', 'second')
monkeyplus.patch_osstat("first", st_mtime=42)
monkeyplus.patch_osstat("second", st_mtime=41)
assert modified_after("first", "second")
def test_second_is_modified_after(self, monkeyplus):
monkeyplus.patch_osstat('first', st_mtime=42)
monkeyplus.patch_osstat('second', st_mtime=43)
assert not modified_after('first', 'second')
monkeyplus.patch_osstat("first", st_mtime=42)
monkeyplus.patch_osstat("second", st_mtime=43)
assert not modified_after("first", "second")
def test_same_mtime(self, monkeyplus):
monkeyplus.patch_osstat('first', st_mtime=42)
monkeyplus.patch_osstat('second', st_mtime=42)
assert not modified_after('first', 'second')
monkeyplus.patch_osstat("first", st_mtime=42)
monkeyplus.patch_osstat("second", st_mtime=42)
assert not modified_after("first", "second")
def test_first_file_does_not_exist(self, monkeyplus):
# when the first file doesn't exist, we return False
monkeyplus.patch_osstat('second', st_mtime=42)
assert not modified_after('does_not_exist', 'second') # no crash
monkeyplus.patch_osstat("second", st_mtime=42)
assert not modified_after("does_not_exist", "second") # no crash
def test_second_file_does_not_exist(self, monkeyplus):
# when the second file doesn't exist, we return True
monkeyplus.patch_osstat('first', st_mtime=42)
assert modified_after('first', 'does_not_exist') # no crash
monkeyplus.patch_osstat("first", st_mtime=42)
assert modified_after("first", "does_not_exist") # no crash
def test_first_file_is_none(self, monkeyplus):
# when the first file is None, we return False
monkeyplus.patch_osstat('second', st_mtime=42)
assert not modified_after(None, 'second') # no crash
monkeyplus.patch_osstat("second", st_mtime=42)
assert not modified_after(None, "second") # no crash
def test_second_file_is_none(self, monkeyplus):
# when the second file is None, we return True
monkeyplus.patch_osstat('first', st_mtime=42)
assert modified_after('first', None) # no crash
monkeyplus.patch_osstat("first", st_mtime=42)
assert modified_after("first", None) # no crash
class TestCase_delete_if_empty:
@ -234,39 +260,39 @@ class TestCase_delete_if_empty:
def test_not_empty(self, tmpdir):
testpath = Path(str(tmpdir))
testpath['foo'].mkdir()
testpath["foo"].mkdir()
assert not delete_if_empty(testpath)
assert testpath.exists()
def test_with_files_to_delete(self, tmpdir):
testpath = Path(str(tmpdir))
testpath['foo'].open('w')
testpath['bar'].open('w')
assert delete_if_empty(testpath, ['foo', 'bar'])
testpath["foo"].open("w")
testpath["bar"].open("w")
assert delete_if_empty(testpath, ["foo", "bar"])
assert not testpath.exists()
def test_directory_in_files_to_delete(self, tmpdir):
testpath = Path(str(tmpdir))
testpath['foo'].mkdir()
assert not delete_if_empty(testpath, ['foo'])
testpath["foo"].mkdir()
assert not delete_if_empty(testpath, ["foo"])
assert testpath.exists()
def test_delete_files_to_delete_only_if_dir_is_empty(self, tmpdir):
testpath = Path(str(tmpdir))
testpath['foo'].open('w')
testpath['bar'].open('w')
assert not delete_if_empty(testpath, ['foo'])
testpath["foo"].open("w")
testpath["bar"].open("w")
assert not delete_if_empty(testpath, ["foo"])
assert testpath.exists()
assert testpath['foo'].exists()
assert testpath["foo"].exists()
def test_doesnt_exist(self):
# When the 'path' doesn't exist, just do nothing.
delete_if_empty(Path('does_not_exist')) # no crash
delete_if_empty(Path("does_not_exist")) # no crash
def test_is_file(self, tmpdir):
# When 'path' is a file, do nothing.
p = Path(str(tmpdir)) + 'filename'
p.open('w').close()
p = Path(str(tmpdir)) + "filename"
p.open("w").close()
delete_if_empty(p) # no crash
def test_ioerror(self, tmpdir, monkeypatch):
@ -274,52 +300,51 @@ class TestCase_delete_if_empty:
def do_raise(*args, **kw):
raise OSError()
monkeypatch.setattr(Path, 'rmdir', do_raise)
monkeypatch.setattr(Path, "rmdir", do_raise)
delete_if_empty(Path(str(tmpdir))) # no crash
class TestCase_open_if_filename:
def test_file_name(self, tmpdir):
filepath = str(tmpdir.join('test.txt'))
open(filepath, 'wb').write(b'test_data')
filepath = str(tmpdir.join("test.txt"))
open(filepath, "wb").write(b"test_data")
file, close = open_if_filename(filepath)
assert close
eq_(b'test_data', file.read())
eq_(b"test_data", file.read())
file.close()
def test_opened_file(self):
sio = StringIO()
sio.write('test_data')
sio.write("test_data")
sio.seek(0)
file, close = open_if_filename(sio)
assert not close
eq_('test_data', file.read())
eq_("test_data", file.read())
def test_mode_is_passed_to_open(self, tmpdir):
filepath = str(tmpdir.join('test.txt'))
open(filepath, 'w').close()
file, close = open_if_filename(filepath, 'a')
eq_('a', file.mode)
filepath = str(tmpdir.join("test.txt"))
open(filepath, "w").close()
file, close = open_if_filename(filepath, "a")
eq_("a", file.mode)
file.close()
class TestCase_FileOrPath:
def test_path(self, tmpdir):
filepath = str(tmpdir.join('test.txt'))
open(filepath, 'wb').write(b'test_data')
filepath = str(tmpdir.join("test.txt"))
open(filepath, "wb").write(b"test_data")
with FileOrPath(filepath) as fp:
eq_(b'test_data', fp.read())
eq_(b"test_data", fp.read())
def test_opened_file(self):
sio = StringIO()
sio.write('test_data')
sio.write("test_data")
sio.seek(0)
with FileOrPath(sio) as fp:
eq_('test_data', fp.read())
eq_("test_data", fp.read())
def test_mode_is_passed_to_open(self, tmpdir):
filepath = str(tmpdir.join('test.txt'))
open(filepath, 'w').close()
with FileOrPath(filepath, 'a') as fp:
eq_('a', fp.mode)
filepath = str(tmpdir.join("test.txt"))
open(filepath, "w").close()
with FileOrPath(filepath, "a") as fp:
eq_("a", fp.mode)

62
hscommon/testutil.py
View File

@ -9,10 +9,12 @@
import threading
import py.path
def eq_(a, b, msg=None):
__tracebackhide__ = True
assert a == b, msg or "%r != %r" % (a, b)
def eq_sorted(a, b, msg=None):
"""If both a and b are iterable sort them and compare using eq_, otherwise just pass them through to eq_ anyway."""
try:
@ -20,10 +22,12 @@ def eq_sorted(a, b, msg=None):
except TypeError:
eq_(a, b, msg)
def assert_almost_equal(a, b, places=7):
__tracebackhide__ = True
assert round(a, ndigits=places) == round(b, ndigits=places)
def callcounter():
def f(*args, **kwargs):
f.callcount += 1
@ -31,6 +35,7 @@ def callcounter():
f.callcount = 0
return f
class TestData:
def __init__(self, datadirpath):
self.datadirpath = py.path.local(datadirpath)
@ -53,12 +58,14 @@ class CallLogger:
It is used to simulate the GUI layer.
"""
def __init__(self):
self.calls = []
def __getattr__(self, func_name):
def func(*args, **kw):
self.calls.append(func_name)
return func
def clear_calls(self):
@ -77,7 +84,9 @@ class CallLogger:
eq_(set(self.calls), set(expected))
self.clear_calls()
def check_gui_calls_partial(self, expected=None, not_expected=None, verify_order=False):
def check_gui_calls_partial(
self, expected=None, not_expected=None, verify_order=False
):
"""Checks that the expected calls have been made to 'self', then clears the log.
`expected` is an iterable of strings representing method names. Order doesn't matter.
@ -88,17 +97,25 @@ class CallLogger:
__tracebackhide__ = True
if expected is not None:
not_called = set(expected) - set(self.calls)
assert not not_called, "These calls haven't been made: {0}".format(not_called)
assert not not_called, "These calls haven't been made: {0}".format(
not_called
)
if verify_order:
max_index = 0
for call in expected:
index = self.calls.index(call)
if index < max_index:
raise AssertionError("The call {0} hasn't been made in the correct order".format(call))
raise AssertionError(
"The call {0} hasn't been made in the correct order".format(
call
)
)
max_index = index
if not_expected is not None:
called = set(not_expected) & set(self.calls)
assert not called, "These calls shouldn't have been made: {0}".format(called)
assert not called, "These calls shouldn't have been made: {0}".format(
called
)
self.clear_calls()
@ -124,7 +141,7 @@ class TestApp:
parent = self.default_parent
if holder is None:
holder = self
setattr(holder, '{0}_gui'.format(name), view)
setattr(holder, "{0}_gui".format(name), view)
gui = class_(parent)
gui.view = view
setattr(holder, name, gui)
@ -136,38 +153,44 @@ def with_app(setupfunc):
def decorator(func):
func.setupfunc = setupfunc
return func
return decorator
def pytest_funcarg__app(request):
setupfunc = request.function.setupfunc
if hasattr(setupfunc, '__code__'):
if hasattr(setupfunc, "__code__"):
argnames = setupfunc.__code__.co_varnames[: setupfunc.__code__.co_argcount]
def getarg(name):
if name == 'self':
if name == "self":
return request.function.__self__
else:
return request.getfixturevalue(name)
args = [getarg(argname) for argname in argnames]
else:
args = []
app = setupfunc(*args)
return app
def jointhreads():
"""Join all threads to the main thread"""
for thread in threading.enumerate():
if hasattr(thread, 'BUGGY'):
if hasattr(thread, "BUGGY"):
continue
if thread.getName() != 'MainThread' and thread.isAlive():
if hasattr(thread, 'close'):
if thread.getName() != "MainThread" and thread.isAlive():
if hasattr(thread, "close"):
thread.close()
thread.join(1)
if thread.isAlive():
print("Thread problem. Some thread doesn't want to stop.")
thread.BUGGY = True
def _unify_args(func, args, kwargs, args_to_ignore=None):
''' Unify args and kwargs in the same dictionary.
""" Unify args and kwargs in the same dictionary.
The result is kwargs with args added to it. func.func_code.co_varnames is used to determine
under what key each elements of arg will be mapped in kwargs.
@ -181,11 +204,13 @@ def _unify_args(func, args, kwargs, args_to_ignore=None):
def foo(bar, baz)
_unifyArgs(foo, (42,), {'baz': 23}) --> {'bar': 42, 'baz': 23}
_unifyArgs(foo, (42,), {'baz': 23}, ['bar']) --> {'baz': 23}
'''
"""
result = kwargs.copy()
if hasattr(func, '__code__'): # built-in functions don't have func_code
if hasattr(func, "__code__"): # built-in functions don't have func_code
args = list(args)
if getattr(func, '__self__', None) is not None: # bound method, we have to add self to args list
if (
getattr(func, "__self__", None) is not None
): # bound method, we have to add self to args list
args = [func.__self__] + args
defaults = list(func.__defaults__) if func.__defaults__ is not None else []
arg_count = func.__code__.co_argcount
@ -198,19 +223,21 @@ def _unify_args(func, args, kwargs, args_to_ignore=None):
result.setdefault(arg_name, arg)
else:
# 'func' has a *args argument
result['args'] = args
result["args"] = args
if args_to_ignore:
for kw in args_to_ignore:
del result[kw]
return result
def log_calls(func):
''' Logs all func calls' arguments under func.calls.
""" Logs all func calls' arguments under func.calls.
func.calls is a list of _unify_args() result (dict).
Mostly used for unit testing.
'''
"""
def wrapper(*args, **kwargs):
unifiedArgs = _unify_args(func, args, kwargs)
wrapper.calls.append(unifiedArgs)
@ -218,4 +245,3 @@ def log_calls(func):
wrapper.calls = []
return wrapper

90
hscommon/trans.py
View File

@ -19,6 +19,7 @@ _trfunc = None
_trget = None
installed_lang = None
def tr(s, context=None):
if _trfunc is None:
return s
@ -28,6 +29,7 @@ def tr(s, context=None):
else:
return _trfunc(s)
def trget(domain):
# Returns a tr() function for the specified domain.
if _trget is None:
@ -35,57 +37,61 @@ def trget(domain):
else:
return _trget(domain)
def set_tr(new_tr, new_trget=None):
global _trfunc, _trget
_trfunc = new_tr
if new_trget is not None:
_trget = new_trget
def get_locale_name(lang):
if ISWINDOWS:
# http://msdn.microsoft.com/en-us/library/39cwe7zf(vs.71).aspx
LANG2LOCALENAME = {
'cs': 'czy',
'de': 'deu',
'el': 'grc',
'es': 'esn',
'fr': 'fra',
'it': 'ita',
'ko': 'korean',
'nl': 'nld',
'pl_PL': 'polish_poland',
'pt_BR': 'ptb',
'ru': 'rus',
'zh_CN': 'chs',
"cs": "czy",
"de": "deu",
"el": "grc",
"es": "esn",
"fr": "fra",
"it": "ita",
"ko": "korean",
"nl": "nld",
"pl_PL": "polish_poland",
"pt_BR": "ptb",
"ru": "rus",
"zh_CN": "chs",
}
else:
LANG2LOCALENAME = {
'cs': 'cs_CZ',
'de': 'de_DE',
'el': 'el_GR',
'es': 'es_ES',
'fr': 'fr_FR',
'it': 'it_IT',
'nl': 'nl_NL',
'hy': 'hy_AM',
'ko': 'ko_KR',
'pl_PL': 'pl_PL',
'pt_BR': 'pt_BR',
'ru': 'ru_RU',
'uk': 'uk_UA',
'vi': 'vi_VN',
'zh_CN': 'zh_CN',
"cs": "cs_CZ",
"de": "de_DE",
"el": "el_GR",
"es": "es_ES",
"fr": "fr_FR",
"it": "it_IT",
"nl": "nl_NL",
"hy": "hy_AM",
"ko": "ko_KR",
"pl_PL": "pl_PL",
"pt_BR": "pt_BR",
"ru": "ru_RU",
"uk": "uk_UA",
"vi": "vi_VN",
"zh_CN": "zh_CN",
}
if lang not in LANG2LOCALENAME:
return None
result = LANG2LOCALENAME[lang]
if ISLINUX:
result += '.UTF-8'
result += ".UTF-8"
return result
# --- Qt
def install_qt_trans(lang=None):
from PyQt5.QtCore import QCoreApplication, QTranslator, QLocale
if not lang:
lang = str(QLocale.system().name())[:2]
localename = get_locale_name(lang)
@ -95,54 +101,66 @@ def install_qt_trans(lang=None):
except locale.Error:
logging.warning("Couldn't set locale %s", localename)
else:
lang = 'en'
lang = "en"
qtr1 = QTranslator(QCoreApplication.instance())
qtr1.load(':/qt_%s' % lang)
qtr1.load(":/qt_%s" % lang)
QCoreApplication.installTranslator(qtr1)
qtr2 = QTranslator(QCoreApplication.instance())
qtr2.load(':/%s' % lang)
qtr2.load(":/%s" % lang)
QCoreApplication.installTranslator(qtr2)
def qt_tr(s, context='core'):
def qt_tr(s, context="core"):
return str(QCoreApplication.translate(context, s, None))
set_tr(qt_tr)
# --- gettext
def install_gettext_trans(base_folder, lang):
import gettext
def gettext_trget(domain):
if not lang:
return lambda s: s
try:
return gettext.translation(domain, localedir=base_folder, languages=[lang]).gettext
return gettext.translation(
domain, localedir=base_folder, languages=[lang]
).gettext
except IOError:
return lambda s: s
default_gettext = gettext_trget('core')
default_gettext = gettext_trget("core")
def gettext_tr(s, context=None):
if not context:
return default_gettext(s)
else:
trfunc = gettext_trget(context)
return trfunc(s)
set_tr(gettext_tr, gettext_trget)
global installed_lang
installed_lang = lang
def install_gettext_trans_under_cocoa():
from cocoa import proxy
resFolder = proxy.getResourcePath()
baseFolder = op.join(resFolder, 'locale')
baseFolder = op.join(resFolder, "locale")
currentLang = proxy.systemLang()
install_gettext_trans(baseFolder, currentLang)
localename = get_locale_name(currentLang)
if localename is not None:
locale.setlocale(locale.LC_ALL, localename)
def install_gettext_trans_under_qt(base_folder, lang=None):
# So, we install the gettext locale, great, but we also should try to install qt_*.qm if
# available so that strings that are inside Qt itself over which I have no control are in the
# right language.
from PyQt5.QtCore import QCoreApplication, QTranslator, QLocale, QLibraryInfo
if not lang:
lang = str(QLocale.system().name())[:2]
localename = get_locale_name(lang)
@ -151,7 +169,7 @@ def install_gettext_trans_under_qt(base_folder, lang=None):
locale.setlocale(locale.LC_ALL, localename)
except locale.Error:
logging.warning("Couldn't set locale %s", localename)
qmname = 'qt_%s' % lang
qmname = "qt_%s" % lang
if ISLINUX:
# Under linux, a full Qt installation is already available in the system, we didn't bundle
# up the qm files in our package, so we have to load translations from the system.

97
hscommon/util.py
View File

@ -17,6 +17,7 @@ from datetime import timedelta
from .path import Path, pathify, log_io_error
def nonone(value, replace_value):
"""Returns ``value`` if ``value`` is not ``None``. Returns ``replace_value`` otherwise.
"""
@ -25,6 +26,7 @@ def nonone(value, replace_value):
else:
return value
def tryint(value, default=0):
"""Tries to convert ``value`` to in ``int`` and returns ``default`` if it fails.
"""
@ -33,13 +35,16 @@ def tryint(value, default=0):
except (TypeError, ValueError):
return default
def minmax(value, min_value, max_value):
"""Returns `value` or one of the min/max bounds if `value` is not between them.
"""
return min(max(value, min_value), max_value)
# --- Sequence related
def dedupe(iterable):
"""Returns a list of elements in ``iterable`` with all dupes removed.
@ -54,6 +59,7 @@ def dedupe(iterable):
result.append(item)
return result
def flatten(iterables, start_with=None):
"""Takes a list of lists ``iterables`` and returns a list containing elements of every list.
@ -67,6 +73,7 @@ def flatten(iterables, start_with=None):
result.extend(iterable)
return result
def first(iterable):
"""Returns the first item of ``iterable``.
"""
@ -75,11 +82,13 @@ def first(iterable):
except StopIteration:
return None
def stripfalse(seq):
"""Returns a sequence with all false elements stripped out of seq.
"""
return [x for x in seq if x]
def extract(predicate, iterable):
"""Separates the wheat from the shaft (`predicate` defines what's the wheat), and returns both.
"""
@ -92,6 +101,7 @@ def extract(predicate, iterable):
shaft.append(item)
return wheat, shaft
def allsame(iterable):
"""Returns whether all elements of 'iterable' are the same.
"""
@ -102,6 +112,7 @@ def allsame(iterable):
raise ValueError("iterable cannot be empty")
return all(element == first_item for element in it)
def trailiter(iterable, skipfirst=False):
"""Yields (prev_element, element), starting with (None, first_element).
@ -120,6 +131,7 @@ def trailiter(iterable, skipfirst=False):
yield prev, item
prev = item
def iterconsume(seq, reverse=True):
"""Iterate over ``seq`` and pops yielded objects.
@ -135,31 +147,36 @@ def iterconsume(seq, reverse=True):
while seq:
yield seq.pop()
# --- String related
def escape(s, to_escape, escape_with='\\'):
def escape(s, to_escape, escape_with="\\"):
"""Returns ``s`` with characters in ``to_escape`` all prepended with ``escape_with``.
"""
return ''.join((escape_with + c if c in to_escape else c) for c in s)
return "".join((escape_with + c if c in to_escape else c) for c in s)
def get_file_ext(filename):
"""Returns the lowercase extension part of filename, without the dot.
"""
pos = filename.rfind('.')
pos = filename.rfind(".")
if pos > -1:
return filename[pos + 1 :].lower()
else:
return ''
return ""
def rem_file_ext(filename):
"""Returns the filename without extension.
"""
pos = filename.rfind('.')
pos = filename.rfind(".")
if pos > -1:
return filename[:pos]
else:
return filename
def pluralize(number, word, decimals=0, plural_word=None):
"""Returns a pluralized string with ``number`` in front of ``word``.
@ -173,11 +190,12 @@ def pluralize(number, word, decimals=0, plural_word=None):
format = "%%1.%df %%s" % decimals
if number > 1:
if plural_word is None:
word += 's'
word += "s"
else:
word = plural_word
return format % (number, word)
def format_time(seconds, with_hours=True):
"""Transforms seconds in a hh:mm:ss string.
@ -189,14 +207,15 @@ def format_time(seconds, with_hours=True):
m, s = divmod(seconds, 60)
if with_hours:
h, m = divmod(m, 60)
r = '%02d:%02d:%02d' % (h, m, s)
r = "%02d:%02d:%02d" % (h, m, s)
else:
r = '%02d:%02d' % (m,s)
r = "%02d:%02d" % (m, s)
if minus:
return '-' + r
return "-" + r
else:
return r
def format_time_decimal(seconds):
"""Transforms seconds in a strings like '3.4 minutes'.
"""
@ -204,20 +223,23 @@ def format_time_decimal(seconds):
if minus:
seconds *= -1
if seconds < 60:
r = pluralize(seconds, 'second', 1)
r = pluralize(seconds, "second", 1)
elif seconds < 3600:
r = pluralize(seconds / 60.0, 'minute', 1)
r = pluralize(seconds / 60.0, "minute", 1)
elif seconds < 86400:
r = pluralize(seconds / 3600.0, 'hour', 1)
r = pluralize(seconds / 3600.0, "hour", 1)
else:
r = pluralize(seconds / 86400.0, 'day', 1)
r = pluralize(seconds / 86400.0, "day", 1)
if minus:
return '-' + r
return "-" + r
else:
return r
SIZE_DESC = ('B','KB','MB','GB','TB','PB','EB','ZB','YB')
SIZE_DESC = ("B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB")
SIZE_VALS = tuple(1024 ** i for i in range(1, 9))
def format_size(size, decimal=0, forcepower=-1, showdesc=True):
"""Transform a byte count in a formatted string (KB, MB etc..).
@ -241,9 +263,9 @@ def format_size(size, decimal=0, forcepower=-1, showdesc=True):
div = SIZE_VALS[i - 1]
else:
div = 1
format = '%%%d.%df' % (decimal,decimal)
format = "%%%d.%df" % (decimal, decimal)
negative = size < 0
divided_size = ((0.0 + abs(size)) / div)
divided_size = (0.0 + abs(size)) / div
if decimal == 0:
divided_size = ceil(divided_size)
else:
@ -252,18 +274,21 @@ def format_size(size, decimal=0, forcepower=-1, showdesc=True):
divided_size *= -1
result = format % divided_size
if showdesc:
result += ' ' + SIZE_DESC[i]
result += " " + SIZE_DESC[i]
return result
_valid_xml_range = '\x09\x0A\x0D\x20-\uD7FF\uE000-\uFFFD'
if sys.maxunicode > 0x10000:
_valid_xml_range += '%s-%s' % (chr(0x10000), chr(min(sys.maxunicode, 0x10FFFF)))
RE_INVALID_XML_SUB = re.compile('[^%s]' % _valid_xml_range, re.U).sub
def remove_invalid_xml(s, replace_with=' '):
_valid_xml_range = "\x09\x0A\x0D\x20-\uD7FF\uE000-\uFFFD"
if sys.maxunicode > 0x10000:
_valid_xml_range += "%s-%s" % (chr(0x10000), chr(min(sys.maxunicode, 0x10FFFF)))
RE_INVALID_XML_SUB = re.compile("[^%s]" % _valid_xml_range, re.U).sub
def remove_invalid_xml(s, replace_with=" "):
return RE_INVALID_XML_SUB(replace_with, s)
def multi_replace(s, replace_from, replace_to=''):
def multi_replace(s, replace_from, replace_to=""):
"""A function like str.replace() with multiple replacements.
``replace_from`` is a list of things you want to replace. Ex: ['a','bc','d']
@ -280,17 +305,20 @@ def multi_replace(s, replace_from, replace_to=''):
if isinstance(replace_to, str) and (len(replace_from) != len(replace_to)):
replace_to = [replace_to for r in replace_from]
if len(replace_from) != len(replace_to):
raise ValueError('len(replace_from) must be equal to len(replace_to)')
raise ValueError("len(replace_from) must be equal to len(replace_to)")
replace = list(zip(replace_from, replace_to))
for r_from, r_to in [r for r in replace if r[0] in s]:
s = s.replace(r_from, r_to)
return s
# --- Date related
# It might seem like needless namespace pollution, but the speedup gained by this constant is
# significant, so it stays.
ONE_DAY = timedelta(1)
def iterdaterange(start, end):
"""Yields every day between ``start`` and ``end``.
"""
@ -299,8 +327,10 @@ def iterdaterange(start, end):
yield date
date += ONE_DAY
# --- Files related
@pathify
def modified_after(first_path: Path, second_path: Path):
"""Returns ``True`` if first_path's mtime is higher than second_path's mtime.
@ -317,12 +347,13 @@ def modified_after(first_path: Path, second_path: Path):
return True
return first_mtime > second_mtime
def find_in_path(name, paths=None):
"""Search for `name` in all directories of `paths` and return the absolute path of the first
occurrence. If `paths` is None, $PATH is used.
"""
if paths is None:
paths = os.environ['PATH']
paths = os.environ["PATH"]
if isinstance(paths, str): # if it's not a string, it's already a list
paths = paths.split(os.pathsep)
for path in paths:
@ -330,6 +361,7 @@ def find_in_path(name, paths=None):
return op.join(path, name)
return None
@log_io_error
@pathify
def delete_if_empty(path: Path, files_to_delete=[]):
@ -345,7 +377,8 @@ def delete_if_empty(path: Path, files_to_delete=[]):
path.rmdir()
return True
def open_if_filename(infile, mode='rb'):
def open_if_filename(infile, mode="rb"):
"""If ``infile`` is a string, it opens and returns it. If it's already a file object, it simply returns it.
This function returns ``(file, should_close_flag)``. The should_close_flag is True is a file has
@ -364,15 +397,18 @@ def open_if_filename(infile, mode='rb'):
else:
return (infile, False)
def ensure_folder(path):
"Create `path` as a folder if it doesn't exist."
if not op.exists(path):
os.makedirs(path)
def ensure_file(path):
"Create `path` as an empty file if it doesn't exist."
if not op.exists(path):
open(path, 'w').close()
open(path, "w").close()
def delete_files_with_pattern(folder_path, pattern, recursive=True):
"""Delete all files (or folders) in `folder_path` that match the glob `pattern`.
@ -389,6 +425,7 @@ def delete_files_with_pattern(folder_path, pattern, recursive=True):
for p in subfolders:
delete_files_with_pattern(p, pattern, True)
class FileOrPath:
"""Does the same as :func:`open_if_filename`, but it can be used with a ``with`` statement.
@ -397,7 +434,8 @@ class FileOrPath:
with FileOrPath(infile):
dostuff()
"""
def __init__(self, file_or_path, mode='rb'):
def __init__(self, file_or_path, mode="rb"):
self.file_or_path = file_or_path
self.mode = mode
self.mustclose = False
@ -410,4 +448,3 @@ class FileOrPath:
def __exit__(self, exc_type, exc_value, traceback):
if self.fp and self.mustclose:
self.fp.close()

178
package.py
View File

@ -15,16 +15,23 @@ import platform
import re
from hscommon.build import (
print_and_do, copy_packages, build_debian_changelog,
get_module_version, filereplace, copy, setup_package_argparser,
copy_all
print_and_do,
copy_packages,
build_debian_changelog,
get_module_version,
filereplace,
copy,
setup_package_argparser,
copy_all,
)
def parse_args():
parser = ArgumentParser()
setup_package_argparser(parser)
return parser.parse_args()
def copy_files_to_package(destpath, packages, with_so):
# when with_so is true, we keep .so files in the package, and otherwise, we don't. We need this
# flag because when building debian src pkg, we *don't* want .so files (they're compiled later)
@ -32,102 +39,129 @@ def copy_files_to_package(destpath, packages, with_so):
if op.exists(destpath):
shutil.rmtree(destpath)
os.makedirs(destpath)
shutil.copy('run.py', op.join(destpath, 'run.py'))
extra_ignores = ['*.so'] if not with_so else None
shutil.copy("run.py", op.join(destpath, "run.py"))
extra_ignores = ["*.so"] if not with_so else None
copy_packages(packages, destpath, extra_ignores=extra_ignores)
shutil.copytree(op.join('build', 'help'), op.join(destpath, 'help'))
shutil.copytree(op.join('build', 'locale'), op.join(destpath, 'locale'))
shutil.copytree(op.join("build", "help"), op.join(destpath, "help"))
shutil.copytree(op.join("build", "locale"), op.join(destpath, "locale"))
compileall.compile_dir(destpath)
def package_debian_distribution(distribution):
app_version = get_module_version('core')
version = '{}~{}'.format(app_version, distribution)
destpath = op.join('build', 'dupeguru-{}'.format(version))
srcpath = op.join(destpath, 'src')
packages = [
'hscommon', 'core', 'qtlib', 'qt', 'send2trash', 'hsaudiotag'
]
app_version = get_module_version("core")
version = "{}~{}".format(app_version, distribution)
destpath = op.join("build", "dupeguru-{}".format(version))
srcpath = op.join(destpath, "src")
packages = ["hscommon", "core", "qtlib", "qt", "send2trash", "hsaudiotag"]
copy_files_to_package(srcpath, packages, with_so=False)
os.mkdir(op.join(destpath, 'modules'))
copy_all(op.join('core', 'pe', 'modules', '*.*'), op.join(destpath, 'modules'))
copy(op.join('qt', 'pe', 'modules', 'block.c'), op.join(destpath, 'modules', 'block_qt.c'))
copy(op.join('pkg', 'debian', 'build_pe_modules.py'), op.join(destpath, 'build_pe_modules.py'))
debdest = op.join(destpath, 'debian')
debskel = op.join('pkg', 'debian')
os.makedirs(debdest)
debopts = json.load(open(op.join(debskel, 'dupeguru.json')))
for fn in ['compat', 'copyright', 'dirs', 'rules', 'source']:
copy(op.join(debskel, fn), op.join(debdest, fn))
filereplace(op.join(debskel, 'control'), op.join(debdest, 'control'), **debopts)
filereplace(op.join(debskel, 'Makefile'), op.join(destpath, 'Makefile'), **debopts)
filereplace(op.join(debskel, 'dupeguru.desktop'), op.join(debdest, 'dupeguru.desktop'), **debopts)
changelogpath = op.join('help', 'changelog')
changelog_dest = op.join(debdest, 'changelog')
project_name = debopts['pkgname']
from_version = '2.9.2'
build_debian_changelog(
changelogpath, changelog_dest, project_name, from_version=from_version,
distribution=distribution
os.mkdir(op.join(destpath, "modules"))
copy_all(op.join("core", "pe", "modules", "*.*"), op.join(destpath, "modules"))
copy(
op.join("qt", "pe", "modules", "block.c"),
op.join(destpath, "modules", "block_qt.c"),
)
shutil.copy(op.join('images', 'dgse_logo_128.png'), srcpath)
copy(
op.join("pkg", "debian", "build_pe_modules.py"),
op.join(destpath, "build_pe_modules.py"),
)
debdest = op.join(destpath, "debian")
debskel = op.join("pkg", "debian")
os.makedirs(debdest)
debopts = json.load(open(op.join(debskel, "dupeguru.json")))
for fn in ["compat", "copyright", "dirs", "rules", "source"]:
copy(op.join(debskel, fn), op.join(debdest, fn))
filereplace(op.join(debskel, "control"), op.join(debdest, "control"), **debopts)
filereplace(op.join(debskel, "Makefile"), op.join(destpath, "Makefile"), **debopts)
filereplace(
op.join(debskel, "dupeguru.desktop"),
op.join(debdest, "dupeguru.desktop"),
**debopts
)
changelogpath = op.join("help", "changelog")
changelog_dest = op.join(debdest, "changelog")
project_name = debopts["pkgname"]
from_version = "2.9.2"
build_debian_changelog(
changelogpath,
changelog_dest,
project_name,
from_version=from_version,
distribution=distribution,
)
shutil.copy(op.join("images", "dgse_logo_128.png"), srcpath)
os.chdir(destpath)
cmd = "dpkg-buildpackage -S -us -uc"
os.system(cmd)
os.chdir('../..')
os.chdir("../..")
def package_debian():
print("Packaging for Debian/Ubuntu")
for distribution in ['unstable']:
for distribution in ["unstable"]:
package_debian_distribution(distribution)
def package_arch():
# For now, package_arch() will only copy the source files into build/. It copies less packages
# than package_debian because there are more python packages available in Arch (so we don't
# need to include them).
print("Packaging for Arch")
srcpath = op.join('build', 'dupeguru-arch')
srcpath = op.join("build", "dupeguru-arch")
packages = [
'hscommon', 'core', 'qtlib', 'qt', 'send2trash', 'hsaudiotag',
"hscommon",
"core",
"qtlib",
"qt",
"send2trash",
"hsaudiotag",
]
copy_files_to_package(srcpath, packages, with_so=True)
shutil.copy(op.join('images', 'dgse_logo_128.png'), srcpath)
debopts = json.load(open(op.join('pkg', 'arch', 'dupeguru.json')))
filereplace(op.join('pkg', 'arch', 'dupeguru.desktop'), op.join(srcpath, 'dupeguru.desktop'), **debopts)
shutil.copy(op.join("images", "dgse_logo_128.png"), srcpath)
debopts = json.load(open(op.join("pkg", "arch", "dupeguru.json")))
filereplace(
op.join("pkg", "arch", "dupeguru.desktop"),
op.join(srcpath, "dupeguru.desktop"),
**debopts
)
def package_source_txz():
print("Creating git archive")
app_version = get_module_version('core')
name = 'dupeguru-src-{}.tar'.format(app_version)
app_version = get_module_version("core")
name = "dupeguru-src-{}.tar".format(app_version)
base_path = os.getcwd()
build_path = op.join(base_path, 'build')
build_path = op.join(base_path, "build")
dest = op.join(build_path, name)
print_and_do('git archive -o {} HEAD'.format(dest))
print_and_do("git archive -o {} HEAD".format(dest))
# Now, we need to include submodules
SUBMODULES = ['hscommon', 'qtlib']
SUBMODULES = ["hscommon", "qtlib"]
for submodule in SUBMODULES:
print("Adding submodule {} to archive".format(submodule))
os.chdir(submodule)
archive_path = op.join(build_path, '{}.tar'.format(submodule))
print_and_do('git archive -o {} --prefix {}/ HEAD'.format(archive_path, submodule))
archive_path = op.join(build_path, "{}.tar".format(submodule))
print_and_do(
"git archive -o {} --prefix {}/ HEAD".format(archive_path, submodule)
)
os.chdir(base_path)
print_and_do('tar -A {} -f {}'.format(archive_path, dest))
print_and_do('xz {}'.format(dest))
print_and_do("tar -A {} -f {}".format(archive_path, dest))
print_and_do("xz {}".format(dest))
def package_windows():
app_version = get_module_version('core')
app_version = get_module_version("core")
arch = platform.architecture()[0]
# Information to pass to pyinstaller and NSIS
match = re.search('[0-9]+.[0-9]+.[0-9]+', app_version)
version_array = match.group(0).split('.')
match = re.search('[0-9]+', arch)
match = re.search("[0-9]+.[0-9]+.[0-9]+", app_version)
version_array = match.group(0).split(".")
match = re.search("[0-9]+", arch)
bits = match.group(0)
# include locale files if they are built otherwise exit as it will break
# the localization
if not op.exists('build/locale'):
if not op.exists("build/locale"):
print("Locale files not built, exiting...")
return
# include help files if they are built otherwise exit as they should be included?
if not op.exists('build/help'):
if not op.exists("build/help"):
print("Help files not built, exiting...")
return
# create version information file from template
@ -136,22 +170,31 @@ def package_windows():
version_info = version_template.read()
version_template.close()
version_info_file = open("win_version_info.txt", "w")
version_info_file.write(version_info.format(version_array[0], version_array[1], version_array[2], bits))
version_info_file.write(
version_info.format(
version_array[0], version_array[1], version_array[2], bits
)
)
version_info_file.close()
except Exception:
print("Error creating version info file, exiting...")
return
# run pyinstaller via command line
print_and_do('pyinstaller -w --name=dupeguru-win{0} --icon=images/dgse_logo.ico '
print_and_do(
"pyinstaller -w --name=dupeguru-win{0} --icon=images/dgse_logo.ico "
'--add-data "build/locale;locale" --add-data "build/help;help" '
'--version-file win_version_info.txt run.py'.format(bits))
"--version-file win_version_info.txt run.py".format(bits)
)
# remove version info file
os.remove('win_version_info.txt')
os.remove("win_version_info.txt")
# Call NSIS (TODO update to not use hardcoded path)
cmd = ('"C:\\Program Files (x86)\\NSIS\\Bin\\makensis.exe" '
'/DVERSIONMAJOR={0} /DVERSIONMINOR={1} /DVERSIONPATCH={2} /DBITS={3} setup.nsi')
cmd = (
'"C:\\Program Files (x86)\\NSIS\\Bin\\makensis.exe" '
"/DVERSIONMAJOR={0} /DVERSIONMINOR={1} /DVERSIONPATCH={2} /DBITS={3} setup.nsi"
)
print_and_do(cmd.format(version_array[0], version_array[1], version_array[2], bits))
def main():
args = parse_args()
if args.src_pkg:
@ -159,17 +202,18 @@ def main():
package_source_txz()
return
print("Packaging dupeGuru with UI qt")
if sys.platform == 'win32':
if sys.platform == "win32":
package_windows()
else:
if not args.arch_pkg:
distname, _, _ = platform.dist()
else:
distname = 'arch'
if distname == 'arch':
distname = "arch"
if distname == "arch":
package_arch()
else:
package_debian()
if __name__ == '__main__':
if __name__ == "__main__":
main()

135
qt/app.py
View File

@ -36,11 +36,12 @@ from .me.preferences_dialog import PreferencesDialog as PreferencesDialogMusic
from .pe.preferences_dialog import PreferencesDialog as PreferencesDialogPicture
from .pe.photo import File as PlatSpecificPhoto
tr = trget('ui')
tr = trget("ui")
class DupeGuru(QObject):
LOGO_NAME = 'logo_se'
NAME = 'dupeGuru'
LOGO_NAME = "logo_se"
NAME = "dupeGuru"
def __init__(self, **kwargs):
super().__init__(**kwargs)
@ -54,15 +55,23 @@ class DupeGuru(QObject):
core.pe.photo.PLAT_SPECIFIC_PHOTO_CLASS = PlatSpecificPhoto
self._setupActions()
self._update_options()
self.recentResults = Recent(self, 'recentResults')
self.recentResults = Recent(self, "recentResults")
self.recentResults.mustOpenItem.connect(self.model.load_from)
self.resultWindow = None
self.details_dialog = None
self.directories_dialog = DirectoriesDialog(self)
self.progress_window = ProgressWindow(self.directories_dialog, self.model.progress_window)
self.problemDialog = ProblemDialog(parent=self.directories_dialog, model=self.model.problem_dialog)
self.ignoreListDialog = IgnoreListDialog(parent=self.directories_dialog, model=self.model.ignore_list_dialog)
self.deletionOptions = DeletionOptions(parent=self.directories_dialog, model=self.model.deletion_options)
self.progress_window = ProgressWindow(
self.directories_dialog, self.model.progress_window
)
self.problemDialog = ProblemDialog(
parent=self.directories_dialog, model=self.model.problem_dialog
)
self.ignoreListDialog = IgnoreListDialog(
parent=self.directories_dialog, model=self.model.ignore_list_dialog
)
self.deletionOptions = DeletionOptions(
parent=self.directories_dialog, model=self.model.deletion_options
)
self.about_box = AboutBox(self.directories_dialog, self)
self.directories_dialog.show()
@ -80,44 +89,68 @@ class DupeGuru(QObject):
# Setup actions that are common to both the directory dialog and the results window.
# (name, shortcut, icon, desc, func)
ACTIONS = [
('actionQuit', 'Ctrl+Q', '', tr("Quit"), self.quitTriggered),
('actionPreferences', 'Ctrl+P', '', tr("Options"), self.preferencesTriggered),
('actionIgnoreList', '', '', tr("Ignore List"), self.ignoreListTriggered),
('actionClearPictureCache', 'Ctrl+Shift+P', '', tr("Clear Picture Cache"), self.clearPictureCacheTriggered),
('actionShowHelp', 'F1', '', tr("dupeGuru Help"), self.showHelpTriggered),
('actionAbout', '', '', tr("About dupeGuru"), self.showAboutBoxTriggered),
('actionOpenDebugLog', '', '', tr("Open Debug Log"), self.openDebugLogTriggered),
("actionQuit", "Ctrl+Q", "", tr("Quit"), self.quitTriggered),
(
"actionPreferences",
"Ctrl+P",
"",
tr("Options"),
self.preferencesTriggered,
),
("actionIgnoreList", "", "", tr("Ignore List"), self.ignoreListTriggered),
(
"actionClearPictureCache",
"Ctrl+Shift+P",
"",
tr("Clear Picture Cache"),
self.clearPictureCacheTriggered,
),
("actionShowHelp", "F1", "", tr("dupeGuru Help"), self.showHelpTriggered),
("actionAbout", "", "", tr("About dupeGuru"), self.showAboutBoxTriggered),
(
"actionOpenDebugLog",
"",
"",
tr("Open Debug Log"),
self.openDebugLogTriggered,
),
]
createActions(ACTIONS, self)
def _update_options(self):
self.model.options['mix_file_kind'] = self.prefs.mix_file_kind
self.model.options['escape_filter_regexp'] = not self.prefs.use_regexp
self.model.options['clean_empty_dirs'] = self.prefs.remove_empty_folders
self.model.options['ignore_hardlink_matches'] = self.prefs.ignore_hardlink_matches
self.model.options['copymove_dest_type'] = self.prefs.destination_type
self.model.options['scan_type'] = self.prefs.get_scan_type(self.model.app_mode)
self.model.options['min_match_percentage'] = self.prefs.filter_hardness
self.model.options['word_weighting'] = self.prefs.word_weighting
self.model.options['match_similar_words'] = self.prefs.match_similar
threshold = self.prefs.small_file_threshold if self.prefs.ignore_small_files else 0
self.model.options['size_threshold'] = threshold * 1024 # threshold is in KB. the scanner wants bytes
self.model.options["mix_file_kind"] = self.prefs.mix_file_kind
self.model.options["escape_filter_regexp"] = not self.prefs.use_regexp
self.model.options["clean_empty_dirs"] = self.prefs.remove_empty_folders
self.model.options[
"ignore_hardlink_matches"
] = self.prefs.ignore_hardlink_matches
self.model.options["copymove_dest_type"] = self.prefs.destination_type
self.model.options["scan_type"] = self.prefs.get_scan_type(self.model.app_mode)
self.model.options["min_match_percentage"] = self.prefs.filter_hardness
self.model.options["word_weighting"] = self.prefs.word_weighting
self.model.options["match_similar_words"] = self.prefs.match_similar
threshold = (
self.prefs.small_file_threshold if self.prefs.ignore_small_files else 0
)
self.model.options["size_threshold"] = (
threshold * 1024
) # threshold is in KB. the scanner wants bytes
scanned_tags = set()
if self.prefs.scan_tag_track:
scanned_tags.add('track')
scanned_tags.add("track")
if self.prefs.scan_tag_artist:
scanned_tags.add('artist')
scanned_tags.add("artist")
if self.prefs.scan_tag_album:
scanned_tags.add('album')
scanned_tags.add("album")
if self.prefs.scan_tag_title:
scanned_tags.add('title')
scanned_tags.add("title")
if self.prefs.scan_tag_genre:
scanned_tags.add('genre')
scanned_tags.add("genre")
if self.prefs.scan_tag_year:
scanned_tags.add('year')
self.model.options['scanned_tags'] = scanned_tags
self.model.options['match_scaled'] = self.prefs.match_scaled
self.model.options['picture_cache_type'] = self.prefs.picture_cache_type
scanned_tags.add("year")
self.model.options["scanned_tags"] = scanned_tags
self.model.options["match_scaled"] = self.prefs.match_scaled
self.model.options["picture_cache_type"] = self.prefs.picture_cache_type
# --- Private
def _get_details_dialog_class(self):
@ -172,11 +205,13 @@ class DupeGuru(QObject):
# --- Events
def finishedLaunching(self):
if sys.getfilesystemencoding() == 'ascii':
if sys.getfilesystemencoding() == "ascii":
# No need to localize this, it's a debugging message.
msg = "Something is wrong with the way your system locale is set. If the files you're "\
"scanning have accented letters, you'll probably get a crash. It is advised that "\
msg = (
"Something is wrong with the way your system locale is set. If the files you're "
"scanning have accented letters, you'll probably get a crash. It is advised that "
"you set your system locale properly."
)
QMessageBox.warning(self.directories_dialog, "Wrong Locale", msg)
def clearPictureCacheTriggered(self):
@ -191,11 +226,13 @@ class DupeGuru(QObject):
self.model.ignore_list_dialog.show()
def openDebugLogTriggered(self):
debugLogPath = op.join(self.model.appdata, 'debug.log')
debugLogPath = op.join(self.model.appdata, "debug.log")
desktop.open_path(debugLogPath)
def preferencesTriggered(self):
preferences_dialog = self._get_preferences_dialog_class()(self.directories_dialog, self)
preferences_dialog = self._get_preferences_dialog_class()(
self.directories_dialog, self
)
preferences_dialog.load()
result = preferences_dialog.exec()
if result == QDialog.Accepted:
@ -212,11 +249,11 @@ class DupeGuru(QObject):
def showHelpTriggered(self):
base_path = platform.HELP_PATH
help_path = op.abspath(op.join(base_path, 'index.html'))
help_path = op.abspath(op.join(base_path, "index.html"))
if op.exists(help_path):
url = QUrl.fromLocalFile(help_path)
else:
url = QUrl('https://www.hardcoded.net/dupeguru/help/en/')
url = QUrl("https://www.hardcoded.net/dupeguru/help/en/")
QDesktopServices.openUrl(url)
def handleSIGTERM(self):
@ -231,10 +268,10 @@ class DupeGuru(QObject):
def show_message(self, msg):
window = QApplication.activeWindow()
QMessageBox.information(window, '', msg)
QMessageBox.information(window, "", msg)
def ask_yes_no(self, prompt):
return self.confirm('', prompt)
return self.confirm("", prompt)
def create_results_window(self):
"""Creates resultWindow and details_dialog depending on the selected ``app_mode``.
@ -256,11 +293,13 @@ class DupeGuru(QObject):
def select_dest_folder(self, prompt):
flags = QFileDialog.ShowDirsOnly
return QFileDialog.getExistingDirectory(self.resultWindow, prompt, '', flags)
return QFileDialog.getExistingDirectory(self.resultWindow, prompt, "", flags)
def select_dest_file(self, prompt, extension):
files = tr("{} file (*.{})").format(extension.upper(), extension)
destination, chosen_filter = QFileDialog.getSaveFileName(self.resultWindow, prompt, '', files)
if not destination.endswith('.{}'.format(extension)):
destination = '{}.{}'.format(destination, extension)
destination, chosen_filter = QFileDialog.getSaveFileName(
self.resultWindow, prompt, "", files
)
if not destination.endswith(".{}".format(extension)):
destination = "{}.{}".format(destination, extension)
return destination

8
qt/deletion_options.py
View File

@ -12,7 +12,8 @@ from PyQt5.QtWidgets import QDialog, QVBoxLayout, QLabel, QCheckBox, QDialogButt
from hscommon.trans import trget
from qtlib.radio_box import RadioBox
tr = trget('ui')
tr = trget("ui")
class DeletionOptions(QDialog):
def __init__(self, parent, model, **kwargs):
@ -41,7 +42,9 @@ class DeletionOptions(QDialog):
self.linkMessageLabel = QLabel(text)
self.linkMessageLabel.setWordWrap(True)
self.verticalLayout.addWidget(self.linkMessageLabel)
self.linkTypeRadio = RadioBox(items=[tr("Symlink"), tr("Hardlink")], spread=False)
self.linkTypeRadio = RadioBox(
items=[tr("Symlink"), tr("Hardlink")], spread=False
)
self.verticalLayout.addWidget(self.linkTypeRadio)
if not self.model.supports_links():
self.linkCheckbox.setEnabled(False)
@ -80,4 +83,3 @@ class DeletionOptions(QDialog):
def set_hardlink_option_enabled(self, is_enabled: bool):
self.linkTypeRadio.setEnabled(is_enabled)

6
qt/details_dialog.py
View File

@ -11,6 +11,7 @@ from PyQt5.QtWidgets import QDialog
from .details_table import DetailsModel
class DetailsDialog(QDialog):
def __init__(self, parent, app, **kwargs):
super().__init__(parent, Qt.Tool, **kwargs)
@ -20,7 +21,7 @@ class DetailsDialog(QDialog):
# To avoid saving uninitialized geometry on appWillSavePrefs, we track whether our dialog
# has been shown. If it has, we know that our geometry should be saved.
self._shown_once = False
self.app.prefs.restoreGeometry('DetailsWindowRect', self)
self.app.prefs.restoreGeometry("DetailsWindowRect", self)
self.tableModel = DetailsModel(self.model)
# tableView is defined in subclasses
self.tableView.setModel(self.tableModel)
@ -38,10 +39,9 @@ class DetailsDialog(QDialog):
# --- Events
def appWillSavePrefs(self):
if self._shown_once:
self.app.prefs.saveGeometry('DetailsWindowRect', self)
self.app.prefs.saveGeometry("DetailsWindowRect", self)
# --- model --> view
def refresh(self):
self.tableModel.beginResetModel()
self.tableModel.endResetModel()

10
qt/details_table.py
View File

@ -11,10 +11,11 @@ from PyQt5.QtWidgets import QHeaderView, QTableView
from hscommon.trans import trget
tr = trget('ui')
tr = trget("ui")
HEADER = [tr("Attribute"), tr("Selected"), tr("Reference")]
class DetailsModel(QAbstractTableModel):
def __init__(self, model, **kwargs):
super().__init__(**kwargs)
@ -33,7 +34,11 @@ class DetailsModel(QAbstractTableModel):
return self.model.row(row)[column]
def headerData(self, section, orientation, role):
if orientation == Qt.Horizontal and role == Qt.DisplayRole and section < len(HEADER):
if (
orientation == Qt.Horizontal
and role == Qt.DisplayRole
and section < len(HEADER)
):
return HEADER[section]
return None
@ -61,4 +66,3 @@ class DetailsTable(QTableView):
vheader = self.verticalHeader()
vheader.setVisible(False)
vheader.setDefaultSectionSize(18)

79
qt/directories_dialog.py
View File

@ -6,9 +6,21 @@
from PyQt5.QtCore import QRect, Qt
from PyQt5.QtWidgets import (
QWidget, QFileDialog, QHeaderView, QVBoxLayout, QHBoxLayout, QTreeView,
QAbstractItemView, QSpacerItem, QSizePolicy, QPushButton, QMainWindow, QMenuBar, QMenu, QLabel,
QComboBox
QWidget,
QFileDialog,
QHeaderView,
QVBoxLayout,
QHBoxLayout,
QTreeView,
QAbstractItemView,
QSpacerItem,
QSizePolicy,
QPushButton,
QMainWindow,
QMenuBar,
QMenu,
QLabel,
QComboBox,
)
from PyQt5.QtGui import QPixmap, QIcon
@ -21,17 +33,20 @@ from qtlib.util import moveToScreenCenter, createActions
from . import platform
from .directories_model import DirectoriesModel, DirectoriesDelegate
tr = trget('ui')
tr = trget("ui")
class DirectoriesDialog(QMainWindow):
def __init__(self, app, **kwargs):
super().__init__(None, **kwargs)
self.app = app
self.lastAddedFolder = platform.INITIAL_FOLDER_IN_DIALOGS
self.recentFolders = Recent(self.app, 'recentFolders')
self.recentFolders = Recent(self.app, "recentFolders")
self._setupUi()
self._updateScanTypeList()
self.directoriesModel = DirectoriesModel(self.app.model.directory_tree, view=self.treeView)
self.directoriesModel = DirectoriesModel(
self.app.model.directory_tree, view=self.treeView
)
self.directoriesDelegate = DirectoriesDelegate()
self.treeView.setItemDelegate(self.directoriesDelegate)
self._setupColumns()
@ -61,9 +76,21 @@ class DirectoriesDialog(QMainWindow):
def _setupActions(self):
# (name, shortcut, icon, desc, func)
ACTIONS = [
('actionLoadResults', 'Ctrl+L', '', tr("Load Results..."), self.loadResultsTriggered),
('actionShowResultsWindow', '', '', tr("Results Window"), self.app.showResultsWindow),
('actionAddFolder', '', '', tr("Add Folder..."), self.addFolderTriggered),
(
"actionLoadResults",
"Ctrl+L",
"",
tr("Load Results..."),
self.loadResultsTriggered,
),
(
"actionShowResultsWindow",
"",
"",
tr("Results Window"),
self.app.showResultsWindow,
),
("actionAddFolder", "", "", tr("Add Folder..."), self.addFolderTriggered),
]
createActions(ACTIONS, self)
@ -117,9 +144,7 @@ class DirectoriesDialog(QMainWindow):
label.setSizePolicy(QSizePolicy.Fixed, QSizePolicy.Fixed)
hl.addWidget(label)
self.appModeRadioBox = RadioBox(
self,
items=[tr("Standard"), tr("Music"), tr("Picture")],
spread=False
self, items=[tr("Standard"), tr("Music"), tr("Picture")], spread=False
)
hl.addWidget(self.appModeRadioBox)
self.verticalLayout.addLayout(hl)
@ -129,21 +154,28 @@ class DirectoriesDialog(QMainWindow):
label.setSizePolicy(QSizePolicy.Fixed, QSizePolicy.Fixed)
hl.addWidget(label)
self.scanTypeComboBox = QComboBox(self)
self.scanTypeComboBox.setSizePolicy(QSizePolicy(QSizePolicy.Expanding, QSizePolicy.Fixed))
self.scanTypeComboBox.setSizePolicy(
QSizePolicy(QSizePolicy.Expanding, QSizePolicy.Fixed)
)
self.scanTypeComboBox.setMaximumWidth(400)
hl.addWidget(self.scanTypeComboBox)
self.showPreferencesButton = QPushButton(tr("More Options"), self.centralwidget)
self.showPreferencesButton.setSizePolicy(QSizePolicy.Fixed, QSizePolicy.Fixed)
hl.addWidget(self.showPreferencesButton)
self.verticalLayout.addLayout(hl)
self.promptLabel = QLabel(tr("Select folders to scan and press \"Scan\"."), self.centralwidget)
self.promptLabel = QLabel(
tr('Select folders to scan and press "Scan".'), self.centralwidget
)
self.verticalLayout.addWidget(self.promptLabel)
self.treeView = QTreeView(self.centralwidget)
self.treeView.setSelectionMode(QAbstractItemView.ExtendedSelection)
self.treeView.setSelectionBehavior(QAbstractItemView.SelectRows)
self.treeView.setAcceptDrops(True)
triggers = QAbstractItemView.DoubleClicked | QAbstractItemView.EditKeyPressed\
triggers = (
QAbstractItemView.DoubleClicked
| QAbstractItemView.EditKeyPressed
| QAbstractItemView.SelectedClicked
)
self.treeView.setEditTriggers(triggers)
self.treeView.setDragDropOverwriteMode(True)
self.treeView.setDragDropMode(QAbstractItemView.DropOnly)
@ -208,7 +240,9 @@ class DirectoriesDialog(QMainWindow):
def _updateScanTypeList(self):
try:
self.scanTypeComboBox.currentIndexChanged[int].disconnect(self.scanTypeChanged)
self.scanTypeComboBox.currentIndexChanged[int].disconnect(
self.scanTypeChanged
)
except TypeError:
# Not connected, ignore
pass
@ -238,7 +272,9 @@ class DirectoriesDialog(QMainWindow):
def addFolderTriggered(self):
title = tr("Select a folder to add to the scanning list")
flags = QFileDialog.ShowDirsOnly
dirpath = str(QFileDialog.getExistingDirectory(self, title, self.lastAddedFolder, flags))
dirpath = str(
QFileDialog.getExistingDirectory(self, title, self.lastAddedFolder, flags)
)
if not dirpath:
return
self.lastAddedFolder = dirpath
@ -264,8 +300,8 @@ class DirectoriesDialog(QMainWindow):
def loadResultsTriggered(self):
title = tr("Select a results file to load")
files = ';;'.join([tr("dupeGuru Results (*.dupeguru)"), tr("All Files (*.*)")])
destination = QFileDialog.getOpenFileName(self, title, '', files)[0]
files = ";;".join([tr("dupeGuru Results (*.dupeguru)"), tr("All Files (*.*)")])
destination = QFileDialog.getOpenFileName(self, title, "", files)[0]
if destination:
self.app.model.load_from(destination)
self.app.recentResults.insertItem(destination)
@ -283,9 +319,10 @@ class DirectoriesDialog(QMainWindow):
def scanTypeChanged(self, index):
scan_options = self.app.model.SCANNER_CLASS.get_scan_options()
self.app.prefs.set_scan_type(self.app.model.app_mode, scan_options[index].scan_type)
self.app.prefs.set_scan_type(
self.app.model.app_mode, scan_options[index].scan_type
)
self.app._update_options()
def selectionChanged(self, selected, deselected):
self._updateRemoveButton()

33
qt/directories_model.py
View File

@ -10,19 +10,24 @@ import urllib.parse
from PyQt5.QtCore import pyqtSignal, Qt, QRect, QUrl, QModelIndex, QItemSelection
from PyQt5.QtWidgets import (
QComboBox, QStyledItemDelegate, QStyle, QStyleOptionComboBox,
QStyleOptionViewItem, QApplication
QComboBox,
QStyledItemDelegate,
QStyle,
QStyleOptionComboBox,
QStyleOptionViewItem,
QApplication,
)
from PyQt5.QtGui import QBrush
from hscommon.trans import trget
from qtlib.tree_model import RefNode, TreeModel
tr = trget('ui')
tr = trget("ui")
HEADERS = [tr("Name"), tr("State")]
STATES = [tr("Normal"), tr("Reference"), tr("Excluded")]
class DirectoriesDelegate(QStyledItemDelegate):
def createEditor(self, parent, option, index):
editor = QComboBox(parent)
@ -39,7 +44,9 @@ class DirectoriesDelegate(QStyledItemDelegate):
# On OS X (with Qt4.6.0), adding State_Enabled to the flags causes the whole drawing to
# fail (draw nothing), but it's an OS X only glitch. On Windows, it works alright.
cboption.state |= QStyle.State_Enabled
QApplication.style().drawComplexControl(QStyle.CC_ComboBox, cboption, painter)
QApplication.style().drawComplexControl(
QStyle.CC_ComboBox, cboption, painter
)
painter.setBrush(option.palette.text())
rect = QRect(option.rect)
rect.setLeft(rect.left() + 4)
@ -68,7 +75,9 @@ class DirectoriesModel(TreeModel):
self.view = view
self.view.setModel(self)
self.view.selectionModel().selectionChanged[(QItemSelection, QItemSelection)].connect(self.selectionChanged)
self.view.selectionModel().selectionChanged[
(QItemSelection, QItemSelection)
].connect(self.selectionChanged)
def _createNode(self, ref, row):
return RefNode(self, None, ref, row)
@ -102,11 +111,11 @@ class DirectoriesModel(TreeModel):
def dropMimeData(self, mimeData, action, row, column, parentIndex):
# the data in mimeData is urlencoded **in utf-8**!!! What we do is to decode, the mime data
# with 'ascii', which works since it's urlencoded. Then, we pass that to urllib.
if not mimeData.hasFormat('text/uri-list'):
if not mimeData.hasFormat("text/uri-list"):
return False
data = bytes(mimeData.data('text/uri-list')).decode('ascii')
data = bytes(mimeData.data("text/uri-list")).decode("ascii")
unquoted = urllib.parse.unquote(data)
urls = unquoted.split('\r\n')
urls = unquoted.split("\r\n")
paths = [QUrl(url).toLocalFile() for url in urls if url]
for path in paths:
self.model.add_directory(path)
@ -129,7 +138,7 @@ class DirectoriesModel(TreeModel):
return None
def mimeTypes(self):
return ['text/uri-list']
return ["text/uri-list"]
def setData(self, index, value, role):
if not index.isValid() or role != Qt.EditRole or index.column() != 1:
@ -146,7 +155,10 @@ class DirectoriesModel(TreeModel):
# --- Events
def selectionChanged(self, selected, deselected):
newNodes = [modelIndex.internalPointer().ref for modelIndex in self.view.selectionModel().selectedRows()]
newNodes = [
modelIndex.internalPointer().ref
for modelIndex in self.view.selectionModel().selectedRows()
]
self.model.selected_nodes = newNodes
# --- Signals
@ -158,4 +170,3 @@ class DirectoriesModel(TreeModel):
def refresh_states(self):
self.refreshData()

19
qt/ignore_list_dialog.py
View File

@ -7,13 +7,20 @@
# http://www.gnu.org/licenses/gpl-3.0.html
from PyQt5.QtCore import Qt
from PyQt5.QtWidgets import QDialog, QVBoxLayout, QPushButton, QTableView, QAbstractItemView
from PyQt5.QtWidgets import (
QDialog,
QVBoxLayout,
QPushButton,
QTableView,
QAbstractItemView,
)
from hscommon.trans import trget
from qtlib.util import horizontalWrap
from .ignore_list_table import IgnoreListTable
tr = trget('ui')
tr = trget("ui")
class IgnoreListDialog(QDialog):
def __init__(self, parent, model, **kwargs):
@ -46,13 +53,11 @@ class IgnoreListDialog(QDialog):
self.clearButton = QPushButton(tr("Clear"))
self.closeButton = QPushButton(tr("Close"))
self.verticalLayout.addLayout(
horizontalWrap([
self.removeSelectedButton, self.clearButton,
None, self.closeButton
])
horizontalWrap(
[self.removeSelectedButton, self.clearButton, None, self.closeButton]
)
)
# --- model --> view
def show(self):
super().show()

5
qt/ignore_list_table.py
View File

@ -8,8 +8,9 @@
from qtlib.column import Column
from qtlib.table import Table
class IgnoreListTable(Table):
COLUMNS = [
Column('path1', defaultWidth=230),
Column('path2', defaultWidth=230),
Column("path1", defaultWidth=230),
Column("path2", defaultWidth=230),
]

Some files were not shown because too many files have changed in this diff Show More