arsenetar
/
dupeguru
mirror of https://github.com/arsenetar/dupeguru.git
1
0
Fork 0
Code Releases Wiki Activity

Integrated the jobprogress library into hscommon 

I have a fix to make in it and it's really silly to pretend that this
lib is of any use to anybody outside HS apps. Bringing it back here will
make things more simple.
This commit is contained in:
Virgil Dupras 2014-10-05 16:31:16 -04:00
parent 87c2fa2573
commit ac32305532
21 changed files with 775 additions and 487 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
build.py
View File

@ -1,9 +1,9 @@
# Created By: Virgil Dupras # Created By: Virgil Dupras
# Created On: 2009-12-30 # Created On: 2009-12-30
# Copyright 2014 Hardcoded Software (http://www.hardcoded.net) # Copyright 2014 Hardcoded Software (http://www.hardcoded.net)
# #
# This software is licensed under the "BSD" License as described in the "LICENSE" file, # This software is licensed under the "BSD" License as described in the "LICENSE" file,
# which should be included with this package. The terms are also available at # which should be included with this package. The terms are also available at
# http://www.hardcoded.net/licenses/bsd_license # http://www.hardcoded.net/licenses/bsd_license
import sys import sys
@ -110,8 +110,9 @@ def build_cocoa(edition, dev):
'me': ['core_me'] + appscript_pkgs + ['hsaudiotag'], 'me': ['core_me'] + appscript_pkgs + ['hsaudiotag'],
'pe': ['core_pe'] + appscript_pkgs, 'pe': ['core_pe'] + appscript_pkgs,
}[edition] }[edition]
tocopy = ['core', 'hscommon', 'cocoa/inter', 'cocoalib/cocoa', 'jobprogress', 'objp', tocopy = [
'send2trash'] + specific_packages 'core', 'hscommon', 'cocoa/inter', 'cocoalib/cocoa', 'objp', 'send2trash'
] + specific_packages
copy_packages(tocopy, pydep_folder, create_links=dev) copy_packages(tocopy, pydep_folder, create_links=dev)
sys.path.insert(0, 'build') sys.path.insert(0, 'build')
extra_deps = None extra_deps = None

11
cocoalib/cocoa/__init__.py
View File

@ -2,8 +2,8 @@
# Created On: 2007-10-06 # Created On: 2007-10-06
# Copyright 2014 Hardcoded Software (http://www.hardcoded.net) # Copyright 2014 Hardcoded Software (http://www.hardcoded.net)
# This software is licensed under the "BSD" License as described in the "LICENSE" file, # This software is licensed under the "BSD" License as described in the "LICENSE" file,
# which should be included with this package. The terms are also available at # which should be included with this package. The terms are also available at
# http://www.hardcoded.net/licenses/bsd_license # http://www.hardcoded.net/licenses/bsd_license
import logging import logging
@ -26,7 +26,7 @@ def autoreleasepool(func):
def as_fetch(as_list, as_type, step_size=1000): def as_fetch(as_list, as_type, step_size=1000):
"""When fetching items from a very big list through applescript, the connection with the app """When fetching items from a very big list through applescript, the connection with the app
will timeout. This function is to circumvent that. 'as_type' is the type of the items in the will timeout. This function is to circumvent that. 'as_type' is the type of the items in the
list (found in appscript.k). If we don't pass it to the 'each' arg of 'count()', it doesn't work. list (found in appscript.k). If we don't pass it to the 'each' arg of 'count()', it doesn't work.
applescript is rather stupid...""" applescript is rather stupid..."""
result = [] result = []
@ -66,7 +66,7 @@ def extract_tb_noline(tb):
def safe_format_exception(type, value, tb): def safe_format_exception(type, value, tb):
"""Format exception from type, value and tb and fallback if there's a problem. """Format exception from type, value and tb and fallback if there's a problem.
In some cases in threaded exceptions under Cocoa, I get tracebacks targeting pyc files instead In some cases in threaded exceptions under Cocoa, I get tracebacks targeting pyc files instead
of py files, which results in traceback.format_exception() trying to print lines from pyc files of py files, which results in traceback.format_exception() trying to print lines from pyc files
and then crashing when trying to interpret that binary data as utf-8. We want a fallback in and then crashing when trying to interpret that binary data as utf-8. We want a fallback in
@ -113,5 +113,6 @@ def patch_threaded_job_performer():
# _async_run, under cocoa, has to be run within an autorelease pool to prevent leaks. # _async_run, under cocoa, has to be run within an autorelease pool to prevent leaks.
# You only need this patch is you use one of CocoaProxy's function (which allocate objc # You only need this patch is you use one of CocoaProxy's function (which allocate objc
# structures) inside a threaded job. # structures) inside a threaded job.
from jobprogress.performer import ThreadedJobPerformer from hscommon.jobprogress.performer import ThreadedJobPerformer
ThreadedJobPerformer._async_run = autoreleasepool(ThreadedJobPerformer._async_run) ThreadedJobPerformer._async_run = autoreleasepool(ThreadedJobPerformer._async_run)

186
core/app.py
View File

@ -1,9 +1,9 @@
# Created By: Virgil Dupras # Created By: Virgil Dupras
# Created On: 2006/11/11 # Created On: 2006/11/11
# Copyright 2014 Hardcoded Software (http://www.hardcoded.net) # Copyright 2014 Hardcoded Software (http://www.hardcoded.net)
# #
# This software is licensed under the "BSD" License as described in the "LICENSE" file, # This software is licensed under the "BSD" License as described in the "LICENSE" file,
# which should be included with this package. The terms are also available at # which should be included with this package. The terms are also available at
# http://www.hardcoded.net/licenses/bsd_license # http://www.hardcoded.net/licenses/bsd_license
import os import os
@ -15,7 +15,7 @@ import time
import shutil import shutil
from send2trash import send2trash from send2trash import send2trash
from jobprogress import job from hscommon.jobprogress import job
from hscommon.notify import Broadcaster from hscommon.notify import Broadcaster
from hscommon.path import Path from hscommon.path import Path
from hscommon.conflict import smart_move, smart_copy from hscommon.conflict import smart_move, smart_copy
@ -78,7 +78,7 @@ def format_words(w):
return '(%s)' % ', '.join(do_format(item) for item in w) return '(%s)' % ', '.join(do_format(item) for item in w)
else: else:
return w.replace('\n', ' ') return w.replace('\n', ' ')
return ', '.join(do_format(item) for item in w) return ', '.join(do_format(item) for item in w)
def format_perc(p): def format_perc(p):
@ -110,33 +110,33 @@ def fix_surrogate_encoding(s, encoding='utf-8'):
class DupeGuru(Broadcaster): class DupeGuru(Broadcaster):
"""Holds everything together. """Holds everything together.
Instantiated once per running application, it holds a reference to every high-level object Instantiated once per running application, it holds a reference to every high-level object
whose reference needs to be held: :class:`~core.results.Results`, :class:`Scanner`, whose reference needs to be held: :class:`~core.results.Results`, :class:`Scanner`,
:class:`~core.directories.Directories`, :mod:`core.gui` instances, etc.. :class:`~core.directories.Directories`, :mod:`core.gui` instances, etc..
It also hosts high level methods and acts as a coordinator for all those elements. This is why It also hosts high level methods and acts as a coordinator for all those elements. This is why
some of its methods seem a bit shallow, like for example :meth:`mark_all` and some of its methods seem a bit shallow, like for example :meth:`mark_all` and
:meth:`remove_duplicates`. These methos are just proxies for a method in :attr:`results`, but :meth:`remove_duplicates`. These methos are just proxies for a method in :attr:`results`, but
they are also followed by a notification call which is very important if we want GUI elements they are also followed by a notification call which is very important if we want GUI elements
to be correctly notified of a change in the data they're presenting. to be correctly notified of a change in the data they're presenting.
.. attribute:: directories .. attribute:: directories
Instance of :class:`~core.directories.Directories`. It holds the current folder selection. Instance of :class:`~core.directories.Directories`. It holds the current folder selection.
.. attribute:: results .. attribute:: results
Instance of :class:`core.results.Results`. Holds the results of the latest scan. Instance of :class:`core.results.Results`. Holds the results of the latest scan.
.. attribute:: selected_dupes .. attribute:: selected_dupes
List of currently selected dupes from our :attr:`results`. Whenever the user changes its List of currently selected dupes from our :attr:`results`. Whenever the user changes its
selection at the UI level, :attr:`result_table` takes care of updating this attribute, so selection at the UI level, :attr:`result_table` takes care of updating this attribute, so
you can trust that it's always up-to-date. you can trust that it's always up-to-date.
.. attribute:: result_table .. attribute:: result_table
Instance of :mod:`meta-gui <core.gui>` table listing the results from :attr:`results` Instance of :mod:`meta-gui <core.gui>` table listing the results from :attr:`results`
""" """
#--- View interface #--- View interface
@ -154,7 +154,7 @@ class DupeGuru(Broadcaster):
PROMPT_NAME = "dupeGuru" PROMPT_NAME = "dupeGuru"
SCANNER_CLASS = scanner.Scanner SCANNER_CLASS = scanner.Scanner
def __init__(self, view): def __init__(self, view):
if view.get_default(DEBUG_MODE_PREFERENCE): if view.get_default(DEBUG_MODE_PREFERENCE):
logging.getLogger().setLevel(logging.DEBUG) logging.getLogger().setLevel(logging.DEBUG)
@ -185,14 +185,14 @@ class DupeGuru(Broadcaster):
children = [self.result_table, self.directory_tree, self.stats_label, self.details_panel] children = [self.result_table, self.directory_tree, self.stats_label, self.details_panel]
for child in children: for child in children:
child.connect() child.connect()
#--- Virtual #--- Virtual
def _prioritization_categories(self): def _prioritization_categories(self):
raise NotImplementedError() raise NotImplementedError()
def _create_result_table(self): def _create_result_table(self):
raise NotImplementedError() raise NotImplementedError()
#--- Private #--- Private
def _get_dupe_sort_key(self, dupe, get_group, key, delta): def _get_dupe_sort_key(self, dupe, get_group, key, delta):
if key == 'marked': if key == 'marked':
@ -212,7 +212,7 @@ class DupeGuru(Broadcaster):
same = cmp_value(dupe, key) == refval same = cmp_value(dupe, key) == refval
result = (same, result) result = (same, result)
return result return result
def _get_group_sort_key(self, group, key): def _get_group_sort_key(self, group, key):
if key == 'percentage': if key == 'percentage':
return group.percentage return group.percentage
@ -221,15 +221,15 @@ class DupeGuru(Broadcaster):
if key == 'marked': if key == 'marked':
return len([dupe for dupe in group.dupes if self.results.is_marked(dupe)]) return len([dupe for dupe in group.dupes if self.results.is_marked(dupe)])
return cmp_value(group.ref, key) return cmp_value(group.ref, key)
def _do_delete(self, j, link_deleted, use_hardlinks, direct_deletion): def _do_delete(self, j, link_deleted, use_hardlinks, direct_deletion):
def op(dupe): def op(dupe):
j.add_progress() j.add_progress()
return self._do_delete_dupe(dupe, link_deleted, use_hardlinks, direct_deletion) return self._do_delete_dupe(dupe, link_deleted, use_hardlinks, direct_deletion)
j.start_job(self.results.mark_count) j.start_job(self.results.mark_count)
self.results.perform_on_marked(op, True) self.results.perform_on_marked(op, True)
def _do_delete_dupe(self, dupe, link_deleted, use_hardlinks, direct_deletion): def _do_delete_dupe(self, dupe, link_deleted, use_hardlinks, direct_deletion):
if not dupe.path.exists(): if not dupe.path.exists():
return return
@ -248,11 +248,11 @@ class DupeGuru(Broadcaster):
linkfunc = os.link if use_hardlinks else os.symlink linkfunc = os.link if use_hardlinks else os.symlink
linkfunc(str(ref.path), str_path) linkfunc(str(ref.path), str_path)
self.clean_empty_dirs(dupe.path.parent()) self.clean_empty_dirs(dupe.path.parent())
def _create_file(self, path): def _create_file(self, path):
# We add fs.Folder to fileclasses in case the file we're loading contains folder paths. # We add fs.Folder to fileclasses in case the file we're loading contains folder paths.
return fs.get_file(path, self.directories.fileclasses + [fs.Folder]) return fs.get_file(path, self.directories.fileclasses + [fs.Folder])
def _get_file(self, str_path): def _get_file(self, str_path):
path = Path(str_path) path = Path(str_path)
f = self._create_file(path) f = self._create_file(path)
@ -263,7 +263,7 @@ class DupeGuru(Broadcaster):
return f return f
except EnvironmentError: except EnvironmentError:
return None return None
def _get_export_data(self): def _get_export_data(self):
columns = [col for col in self.result_table.columns.ordered_columns columns = [col for col in self.result_table.columns.ordered_columns
if col.visible and col.name != 'marked'] if col.visible and col.name != 'marked']
@ -276,20 +276,20 @@ class DupeGuru(Broadcaster):
row.insert(0, group_id) row.insert(0, group_id)
rows.append(row) rows.append(row)
return colnames, rows return colnames, rows
def _results_changed(self): def _results_changed(self):
self.selected_dupes = [d for d in self.selected_dupes self.selected_dupes = [d for d in self.selected_dupes
if self.results.get_group_of_duplicate(d) is not None] if self.results.get_group_of_duplicate(d) is not None]
self.notify('results_changed') self.notify('results_changed')
def _start_job(self, jobid, func, args=()): def _start_job(self, jobid, func, args=()):
title = JOBID2TITLE[jobid] title = JOBID2TITLE[jobid]
try: try:
self.progress_window.run(jobid, title, func, args=args) self.progress_window.run(jobid, title, func, args=args)
except job.JobInProgressError: except job.JobInProgressError:
msg = tr("A previous action is still hanging in there. You can't start a new one yet. Wait a few seconds, then try again.") msg = tr("A previous action is still hanging in there. You can't start a new one yet. Wait a few seconds, then try again.")
self.view.show_message(msg) self.view.show_message(msg)
def _job_completed(self, jobid): def _job_completed(self, jobid):
if jobid == JobType.Scan: if jobid == JobType.Scan:
self._results_changed() self._results_changed()
@ -312,7 +312,7 @@ class DupeGuru(Broadcaster):
JobType.Delete: tr("All marked files were successfully sent to Trash."), JobType.Delete: tr("All marked files were successfully sent to Trash."),
}[jobid] }[jobid]
self.view.show_message(msg) self.view.show_message(msg)
@staticmethod @staticmethod
def _remove_hardlink_dupes(files): def _remove_hardlink_dupes(files):
seen_inodes = set() seen_inodes = set()
@ -327,19 +327,19 @@ class DupeGuru(Broadcaster):
seen_inodes.add(inode) seen_inodes.add(inode)
result.append(file) result.append(file)
return result return result
def _select_dupes(self, dupes): def _select_dupes(self, dupes):
if dupes == self.selected_dupes: if dupes == self.selected_dupes:
return return
self.selected_dupes = dupes self.selected_dupes = dupes
self.notify('dupes_selected') self.notify('dupes_selected')
#--- Public #--- Public
def add_directory(self, d): def add_directory(self, d):
"""Adds folder ``d`` to :attr:`directories`. """Adds folder ``d`` to :attr:`directories`.
Shows an error message dialog if something bad happens. Shows an error message dialog if something bad happens.
:param str d: path of folder to add :param str d: path of folder to add
""" """
try: try:
@ -349,7 +349,7 @@ class DupeGuru(Broadcaster):
self.view.show_message(tr("'{}' already is in the list.").format(d)) self.view.show_message(tr("'{}' already is in the list.").format(d))
except directories.InvalidPathError: except directories.InvalidPathError:
self.view.show_message(tr("'{}' does not exist.").format(d)) self.view.show_message(tr("'{}' does not exist.").format(d))
def add_selected_to_ignore_list(self): def add_selected_to_ignore_list(self):
"""Adds :attr:`selected_dupes` to :attr:`scanner`'s ignore list. """Adds :attr:`selected_dupes` to :attr:`scanner`'s ignore list.
""" """
@ -367,10 +367,10 @@ class DupeGuru(Broadcaster):
self.scanner.ignore_list.Ignore(str(other.path), str(dupe.path)) self.scanner.ignore_list.Ignore(str(other.path), str(dupe.path))
self.remove_duplicates(dupes) self.remove_duplicates(dupes)
self.ignore_list_dialog.refresh() self.ignore_list_dialog.refresh()
def apply_filter(self, filter): def apply_filter(self, filter):
"""Apply a filter ``filter`` to the results so that it shows only dupe groups that match it. """Apply a filter ``filter`` to the results so that it shows only dupe groups that match it.
:param str filter: filter to apply :param str filter: filter to apply
""" """
self.results.apply_filter(None) self.results.apply_filter(None)
@ -379,12 +379,12 @@ class DupeGuru(Broadcaster):
filter = escape(filter, '*', '.') filter = escape(filter, '*', '.')
self.results.apply_filter(filter) self.results.apply_filter(filter)
self._results_changed() self._results_changed()
def clean_empty_dirs(self, path): def clean_empty_dirs(self, path):
if self.options['clean_empty_dirs']: if self.options['clean_empty_dirs']:
while delete_if_empty(path, ['.DS_Store']): while delete_if_empty(path, ['.DS_Store']):
path = path.parent() path = path.parent()
def copy_or_move(self, dupe, copy: bool, destination: str, dest_type: DestType): def copy_or_move(self, dupe, copy: bool, destination: str, dest_type: DestType):
source_path = dupe.path source_path = dupe.path
location_path = first(p for p in self.directories if dupe.path in p) location_path = first(p for p in self.directories if dupe.path in p)
@ -406,20 +406,20 @@ class DupeGuru(Broadcaster):
else: else:
smart_move(source_path, dest_path) smart_move(source_path, dest_path)
self.clean_empty_dirs(source_path.parent()) self.clean_empty_dirs(source_path.parent())
def copy_or_move_marked(self, copy): def copy_or_move_marked(self, copy):
"""Start an async move (or copy) job on marked duplicates. """Start an async move (or copy) job on marked duplicates.
:param bool copy: If True, duplicates will be copied instead of moved :param bool copy: If True, duplicates will be copied instead of moved
""" """
def do(j): def do(j):
def op(dupe): def op(dupe):
j.add_progress() j.add_progress()
self.copy_or_move(dupe, copy, destination, desttype) self.copy_or_move(dupe, copy, destination, desttype)
j.start_job(self.results.mark_count) j.start_job(self.results.mark_count)
self.results.perform_on_marked(op, not copy) self.results.perform_on_marked(op, not copy)
if not self.results.mark_count: if not self.results.mark_count:
self.view.show_message(MSG_NO_MARKED_DUPES) self.view.show_message(MSG_NO_MARKED_DUPES)
return return
@ -430,7 +430,7 @@ class DupeGuru(Broadcaster):
desttype = self.options['copymove_dest_type'] desttype = self.options['copymove_dest_type']
jobid = JobType.Copy if copy else JobType.Move jobid = JobType.Copy if copy else JobType.Move
self._start_job(jobid, do) self._start_job(jobid, do)
def delete_marked(self): def delete_marked(self):
"""Start an async job to send marked duplicates to the trash. """Start an async job to send marked duplicates to the trash.
""" """
@ -443,10 +443,10 @@ class DupeGuru(Broadcaster):
self.deletion_options.direct] self.deletion_options.direct]
logging.debug("Starting deletion job with args %r", args) logging.debug("Starting deletion job with args %r", args)
self._start_job(JobType.Delete, self._do_delete, args=args) self._start_job(JobType.Delete, self._do_delete, args=args)
def export_to_xhtml(self): def export_to_xhtml(self):
"""Export current results to XHTML. """Export current results to XHTML.
The configuration of the :attr:`result_table` (columns order and visibility) is used to The configuration of the :attr:`result_table` (columns order and visibility) is used to
determine how the data is presented in the export. In other words, the exported table in determine how the data is presented in the export. In other words, the exported table in
the resulting XHTML will look just like the results table. the resulting XHTML will look just like the results table.
@ -454,10 +454,10 @@ class DupeGuru(Broadcaster):
colnames, rows = self._get_export_data() colnames, rows = self._get_export_data()
export_path = export.export_to_xhtml(colnames, rows) export_path = export.export_to_xhtml(colnames, rows)
desktop.open_path(export_path) desktop.open_path(export_path)
def export_to_csv(self): def export_to_csv(self):
"""Export current results to CSV. """Export current results to CSV.
The columns and their order in the resulting CSV file is determined in the same way as in The columns and their order in the resulting CSV file is determined in the same way as in
:meth:`export_to_xhtml`. :meth:`export_to_xhtml`.
""" """
@ -465,7 +465,7 @@ class DupeGuru(Broadcaster):
if dest_file: if dest_file:
colnames, rows = self._get_export_data() colnames, rows = self._get_export_data()
export.export_to_csv(dest_file, colnames, rows) export.export_to_csv(dest_file, colnames, rows)
def get_display_info(self, dupe, group, delta=False): def get_display_info(self, dupe, group, delta=False):
def empty_data(): def empty_data():
return {c.name: '---' for c in self.result_table.COLUMNS[1:]} return {c.name: '---' for c in self.result_table.COLUMNS[1:]}
@ -476,10 +476,10 @@ class DupeGuru(Broadcaster):
except Exception as e: except Exception as e:
logging.warning("Exception on GetDisplayInfo for %s: %s", str(dupe.path), str(e)) logging.warning("Exception on GetDisplayInfo for %s: %s", str(dupe.path), str(e))
return empty_data() return empty_data()
def invoke_custom_command(self): def invoke_custom_command(self):
"""Calls command in ``CustomCommand`` pref with ``%d`` and ``%r`` placeholders replaced. """Calls command in ``CustomCommand`` pref with ``%d`` and ``%r`` placeholders replaced.
Using the current selection, ``%d`` is replaced with the currently selected dupe and ``%r`` Using the current selection, ``%d`` is replaced with the currently selected dupe and ``%r``
is replaced with that dupe's ref file. If there's no selection, the command is not invoked. is replaced with that dupe's ref file. If there's no selection, the command is not invoked.
If the dupe is a ref, ``%d`` and ``%r`` will be the same. If the dupe is a ref, ``%d`` and ``%r`` will be the same.
@ -506,10 +506,10 @@ class DupeGuru(Broadcaster):
subprocess.Popen(exename + args, shell=True, cwd=path) subprocess.Popen(exename + args, shell=True, cwd=path)
else: else:
subprocess.Popen(cmd, shell=True) subprocess.Popen(cmd, shell=True)
def load(self): def load(self):
"""Load directory selection and ignore list from files in appdata. """Load directory selection and ignore list from files in appdata.
This method is called during startup so that directory selection and ignore list, which This method is called during startup so that directory selection and ignore list, which
is persistent data, is the same as when the last session was closed (when :meth:`save` was is persistent data, is the same as when the last session was closed (when :meth:`save` was
called). called).
@ -519,19 +519,19 @@ class DupeGuru(Broadcaster):
p = op.join(self.appdata, 'ignore_list.xml') p = op.join(self.appdata, 'ignore_list.xml')
self.scanner.ignore_list.load_from_xml(p) self.scanner.ignore_list.load_from_xml(p)
self.ignore_list_dialog.refresh() self.ignore_list_dialog.refresh()
def load_from(self, filename): def load_from(self, filename):
"""Start an async job to load results from ``filename``. """Start an async job to load results from ``filename``.
:param str filename: path of the XML file (created with :meth:`save_as`) to load :param str filename: path of the XML file (created with :meth:`save_as`) to load
""" """
def do(j): def do(j):
self.results.load_from_xml(filename, self._get_file, j) self.results.load_from_xml(filename, self._get_file, j)
self._start_job(JobType.Load, do) self._start_job(JobType.Load, do)
def make_selected_reference(self): def make_selected_reference(self):
"""Promote :attr:`selected_dupes` to reference position within their respective groups. """Promote :attr:`selected_dupes` to reference position within their respective groups.
Each selected dupe will become the :attr:`~core.engine.Group.ref` of its group. If there's Each selected dupe will become the :attr:`~core.engine.Group.ref` of its group. If there's
more than one dupe selected for the same group, only the first (in the order currently shown more than one dupe selected for the same group, only the first (in the order currently shown
in :attr:`result_table`) dupe will be promoted. in :attr:`result_table`) dupe will be promoted.
@ -560,28 +560,28 @@ class DupeGuru(Broadcaster):
# do is to keep our selection index-wise (different dupe selection, but same index # do is to keep our selection index-wise (different dupe selection, but same index
# selection). # selection).
self.notify('results_changed_but_keep_selection') self.notify('results_changed_but_keep_selection')
def mark_all(self): def mark_all(self):
"""Set all dupes in the results as marked. """Set all dupes in the results as marked.
""" """
self.results.mark_all() self.results.mark_all()
self.notify('marking_changed') self.notify('marking_changed')
def mark_none(self): def mark_none(self):
"""Set all dupes in the results as unmarked. """Set all dupes in the results as unmarked.
""" """
self.results.mark_none() self.results.mark_none()
self.notify('marking_changed') self.notify('marking_changed')
def mark_invert(self): def mark_invert(self):
"""Invert the marked state of all dupes in the results. """Invert the marked state of all dupes in the results.
""" """
self.results.mark_invert() self.results.mark_invert()
self.notify('marking_changed') self.notify('marking_changed')
def mark_dupe(self, dupe, marked): def mark_dupe(self, dupe, marked):
"""Change marked status of ``dupe``. """Change marked status of ``dupe``.
:param dupe: dupe to mark/unmark :param dupe: dupe to mark/unmark
:type dupe: :class:`~core.fs.File` :type dupe: :class:`~core.fs.File`
:param bool marked: True = mark, False = unmark :param bool marked: True = mark, False = unmark
@ -591,7 +591,7 @@ class DupeGuru(Broadcaster):
else: else:
self.results.unmark(dupe) self.results.unmark(dupe)
self.notify('marking_changed') self.notify('marking_changed')
def open_selected(self): def open_selected(self):
"""Open :attr:`selected_dupes` with their associated application. """Open :attr:`selected_dupes` with their associated application.
""" """
@ -600,16 +600,16 @@ class DupeGuru(Broadcaster):
return return
for dupe in self.selected_dupes: for dupe in self.selected_dupes:
desktop.open_path(dupe.path) desktop.open_path(dupe.path)
def purge_ignore_list(self): def purge_ignore_list(self):
"""Remove files that don't exist from :attr:`ignore_list`. """Remove files that don't exist from :attr:`ignore_list`.
""" """
self.scanner.ignore_list.Filter(lambda f,s:op.exists(f) and op.exists(s)) self.scanner.ignore_list.Filter(lambda f,s:op.exists(f) and op.exists(s))
self.ignore_list_dialog.refresh() self.ignore_list_dialog.refresh()
def remove_directories(self, indexes): def remove_directories(self, indexes):
"""Remove root directories at ``indexes`` from :attr:`directories`. """Remove root directories at ``indexes`` from :attr:`directories`.
:param indexes: Indexes of the directories to remove. :param indexes: Indexes of the directories to remove.
:type indexes: list of int :type indexes: list of int
""" """
@ -620,30 +620,30 @@ class DupeGuru(Broadcaster):
self.notify('directories_changed') self.notify('directories_changed')
except IndexError: except IndexError:
pass pass
def remove_duplicates(self, duplicates): def remove_duplicates(self, duplicates):
"""Remove ``duplicates`` from :attr:`results`. """Remove ``duplicates`` from :attr:`results`.
Calls :meth:`~core.results.Results.remove_duplicates` and send appropriate notifications. Calls :meth:`~core.results.Results.remove_duplicates` and send appropriate notifications.
:param duplicates: duplicates to remove. :param duplicates: duplicates to remove.
:type duplicates: list of :class:`~core.fs.File` :type duplicates: list of :class:`~core.fs.File`
""" """
self.results.remove_duplicates(self.without_ref(duplicates)) self.results.remove_duplicates(self.without_ref(duplicates))
self.notify('results_changed_but_keep_selection') self.notify('results_changed_but_keep_selection')
def remove_marked(self): def remove_marked(self):
"""Removed marked duplicates from the results (without touching the files themselves). """Removed marked duplicates from the results (without touching the files themselves).
""" """
if not self.results.mark_count: if not self.results.mark_count:
self.view.show_message(MSG_NO_MARKED_DUPES) self.view.show_message(MSG_NO_MARKED_DUPES)
return return
msg = tr("You are about to remove %d files from results. Continue?") msg = tr("You are about to remove %d files from results. Continue?")
if not self.view.ask_yes_no(msg % self.results.mark_count): if not self.view.ask_yes_no(msg % self.results.mark_count):
return return
self.results.perform_on_marked(lambda x:None, True) self.results.perform_on_marked(lambda x:None, True)
self._results_changed() self._results_changed()
def remove_selected(self): def remove_selected(self):
"""Removed :attr:`selected_dupes` from the results (without touching the files themselves). """Removed :attr:`selected_dupes` from the results (without touching the files themselves).
""" """
@ -651,16 +651,16 @@ class DupeGuru(Broadcaster):
if not dupes: if not dupes:
self.view.show_message(MSG_NO_SELECTED_DUPES) self.view.show_message(MSG_NO_SELECTED_DUPES)
return return
msg = tr("You are about to remove %d files from results. Continue?") msg = tr("You are about to remove %d files from results. Continue?")
if not self.view.ask_yes_no(msg % len(dupes)): if not self.view.ask_yes_no(msg % len(dupes)):
return return
self.remove_duplicates(dupes) self.remove_duplicates(dupes)
def rename_selected(self, newname): def rename_selected(self, newname):
"""Renames the selected dupes's file to ``newname``. """Renames the selected dupes's file to ``newname``.
If there's more than one selected dupes, the first one is used. If there's more than one selected dupes, the first one is used.
:param str newname: The filename to rename the dupe's file to. :param str newname: The filename to rename the dupe's file to.
""" """
try: try:
@ -670,13 +670,13 @@ class DupeGuru(Broadcaster):
except (IndexError, fs.FSError) as e: except (IndexError, fs.FSError) as e:
logging.warning("dupeGuru Warning: %s" % str(e)) logging.warning("dupeGuru Warning: %s" % str(e))
return False return False
def reprioritize_groups(self, sort_key): def reprioritize_groups(self, sort_key):
"""Sort dupes in each group (in :attr:`results`) according to ``sort_key``. """Sort dupes in each group (in :attr:`results`) according to ``sort_key``.
Called by the re-prioritize dialog. Calls :meth:`~core.engine.Group.prioritize` and, once Called by the re-prioritize dialog. Calls :meth:`~core.engine.Group.prioritize` and, once
the sorting is done, show a message that confirms the action. the sorting is done, show a message that confirms the action.
:param sort_key: The key being sent to :meth:`~core.engine.Group.prioritize` :param sort_key: The key being sent to :meth:`~core.engine.Group.prioritize`
:type sort_key: f(dupe) :type sort_key: f(dupe)
""" """
@ -687,11 +687,11 @@ class DupeGuru(Broadcaster):
self._results_changed() self._results_changed()
msg = tr("{} duplicate groups were changed by the re-prioritization.").format(count) msg = tr("{} duplicate groups were changed by the re-prioritization.").format(count)
self.view.show_message(msg) self.view.show_message(msg)
def reveal_selected(self): def reveal_selected(self):
if self.selected_dupes: if self.selected_dupes:
desktop.reveal_path(self.selected_dupes[0].path) desktop.reveal_path(self.selected_dupes[0].path)
def save(self): def save(self):
if not op.exists(self.appdata): if not op.exists(self.appdata):
os.makedirs(self.appdata) os.makedirs(self.appdata)
@ -699,17 +699,17 @@ class DupeGuru(Broadcaster):
p = op.join(self.appdata, 'ignore_list.xml') p = op.join(self.appdata, 'ignore_list.xml')
self.scanner.ignore_list.save_to_xml(p) self.scanner.ignore_list.save_to_xml(p)
self.notify('save_session') self.notify('save_session')
def save_as(self, filename): def save_as(self, filename):
"""Save results in ``filename``. """Save results in ``filename``.
:param str filename: path of the file to save results (as XML) to. :param str filename: path of the file to save results (as XML) to.
""" """
self.results.save_to_xml(filename) self.results.save_to_xml(filename)
def start_scanning(self): def start_scanning(self):
"""Starts an async job to scan for duplicates. """Starts an async job to scan for duplicates.
Scans folders selected in :attr:`directories` and put the results in :attr:`results` Scans folders selected in :attr:`directories` and put the results in :attr:`results`
""" """
def do(j): def do(j):
@ -722,14 +722,14 @@ class DupeGuru(Broadcaster):
files = self._remove_hardlink_dupes(files) files = self._remove_hardlink_dupes(files)
logging.info('Scanning %d files' % len(files)) logging.info('Scanning %d files' % len(files))
self.results.groups = self.scanner.get_dupe_groups(files, j) self.results.groups = self.scanner.get_dupe_groups(files, j)
if not self.directories.has_any_file(): if not self.directories.has_any_file():
self.view.show_message(tr("The selected directories contain no scannable file.")) self.view.show_message(tr("The selected directories contain no scannable file."))
return return
self.results.groups = [] self.results.groups = []
self._results_changed() self._results_changed()
self._start_job(JobType.Scan, do) self._start_job(JobType.Scan, do)
def toggle_selected_mark_state(self): def toggle_selected_mark_state(self):
selected = self.without_ref(self.selected_dupes) selected = self.without_ref(self.selected_dupes)
if not selected: if not selected:
@ -741,12 +741,12 @@ class DupeGuru(Broadcaster):
for dupe in selected: for dupe in selected:
markfunc(dupe) markfunc(dupe)
self.notify('marking_changed') self.notify('marking_changed')
def without_ref(self, dupes): def without_ref(self, dupes):
"""Returns ``dupes`` with all reference elements removed. """Returns ``dupes`` with all reference elements removed.
""" """
return [dupe for dupe in dupes if self.results.get_group_of_duplicate(dupe).ref is not dupe] return [dupe for dupe in dupes if self.results.get_group_of_duplicate(dupe).ref is not dupe]
def get_default(self, key, fallback_value=None): def get_default(self, key, fallback_value=None):
result = nonone(self.view.get_default(key), fallback_value) result = nonone(self.view.get_default(key), fallback_value)
if fallback_value is not None and not isinstance(result, type(fallback_value)): if fallback_value is not None and not isinstance(result, type(fallback_value)):
@ -756,10 +756,10 @@ class DupeGuru(Broadcaster):
except Exception: except Exception:
result = fallback_value result = fallback_value
return result return result
def set_default(self, key, value): def set_default(self, key, value):
self.view.set_default(key, value) self.view.set_default(key, value)
#--- Properties #--- Properties
@property @property
def stat_line(self): def stat_line(self):
@ -767,4 +767,4 @@ class DupeGuru(Broadcaster):
if self.scanner.discarded_file_count: if self.scanner.discarded_file_count:
result = tr("%s (%d discarded)") % (result, self.scanner.discarded_file_count) result = tr("%s (%d discarded)") % (result, self.scanner.discarded_file_count)
return result return result

70
core/directories.py
View File

@ -1,15 +1,15 @@
# Created By: Virgil Dupras # Created By: Virgil Dupras
# Created On: 2006/02/27 # Created On: 2006/02/27
# Copyright 2014 Hardcoded Software (http://www.hardcoded.net) # Copyright 2014 Hardcoded Software (http://www.hardcoded.net)
# #
# This software is licensed under the "BSD" License as described in the "LICENSE" file, # This software is licensed under the "BSD" License as described in the "LICENSE" file,
# which should be included with this package. The terms are also available at # which should be included with this package. The terms are also available at
# http://www.hardcoded.net/licenses/bsd_license # http://www.hardcoded.net/licenses/bsd_license
from xml.etree import ElementTree as ET from xml.etree import ElementTree as ET
import logging import logging
from jobprogress import job from hscommon.jobprogress import job
from hscommon.path import Path from hscommon.path import Path
from hscommon.util import FileOrPath from hscommon.util import FileOrPath
@ -24,7 +24,7 @@ __all__ = [
class DirectoryState: class DirectoryState:
"""Enum describing how a folder should be considered. """Enum describing how a folder should be considered.
* DirectoryState.Normal: Scan all files normally * DirectoryState.Normal: Scan all files normally
* DirectoryState.Reference: Scan files, but make sure never to delete any of them * DirectoryState.Reference: Scan files, but make sure never to delete any of them
* DirectoryState.Excluded: Don't scan this folder * DirectoryState.Excluded: Don't scan this folder
@ -41,10 +41,10 @@ class InvalidPathError(Exception):
class Directories: class Directories:
"""Holds user folder selection. """Holds user folder selection.
Manages the selection that the user make through the folder selection dialog. It also manages Manages the selection that the user make through the folder selection dialog. It also manages
folder states, and how recursion applies to them. folder states, and how recursion applies to them.
Then, when the user starts the scan, :meth:`get_files` is called to retrieve all files (wrapped Then, when the user starts the scan, :meth:`get_files` is called to retrieve all files (wrapped
in :mod:`core.fs`) that have to be scanned according to the chosen folders/states. in :mod:`core.fs`) that have to be scanned according to the chosen folders/states.
""" """
@ -55,28 +55,28 @@ class Directories:
self.states = {} self.states = {}
self.fileclasses = fileclasses self.fileclasses = fileclasses
self.folderclass = fs.Folder self.folderclass = fs.Folder
def __contains__(self, path): def __contains__(self, path):
for p in self._dirs: for p in self._dirs:
if path in p: if path in p:
return True return True
return False return False
def __delitem__(self,key): def __delitem__(self,key):
self._dirs.__delitem__(key) self._dirs.__delitem__(key)
def __getitem__(self,key): def __getitem__(self,key):
return self._dirs.__getitem__(key) return self._dirs.__getitem__(key)
def __len__(self): def __len__(self):
return len(self._dirs) return len(self._dirs)
#---Private #---Private
def _default_state_for_path(self, path): def _default_state_for_path(self, path):
# Override this in subclasses to specify the state of some special folders. # Override this in subclasses to specify the state of some special folders.
if path.name.startswith('.'): # hidden if path.name.startswith('.'): # hidden
return DirectoryState.Excluded return DirectoryState.Excluded
def _get_files(self, from_path, j): def _get_files(self, from_path, j):
j.check_if_cancelled() j.check_if_cancelled()
state = self.get_state(from_path) state = self.get_state(from_path)
@ -102,7 +102,7 @@ class Directories:
yield file yield file
except (EnvironmentError, fs.InvalidPath): except (EnvironmentError, fs.InvalidPath):
pass pass
def _get_folders(self, from_folder, j): def _get_folders(self, from_folder, j):
j.check_if_cancelled() j.check_if_cancelled()
try: try:
@ -116,16 +116,16 @@ class Directories:
yield from_folder yield from_folder
except (EnvironmentError, fs.InvalidPath): except (EnvironmentError, fs.InvalidPath):
pass pass
#---Public #---Public
def add_path(self, path): def add_path(self, path):
"""Adds ``path`` to self, if not already there. """Adds ``path`` to self, if not already there.
Raises :exc:`AlreadyThereError` if ``path`` is already in self. If path is a directory Raises :exc:`AlreadyThereError` if ``path`` is already in self. If path is a directory
containing some of the directories already present in self, ``path`` will be added, but all containing some of the directories already present in self, ``path`` will be added, but all
directories under it will be removed. Can also raise :exc:`InvalidPathError` if ``path`` directories under it will be removed. Can also raise :exc:`InvalidPathError` if ``path``
does not exist. does not exist.
:param Path path: path to add :param Path path: path to add
""" """
if path in self: if path in self:
@ -134,11 +134,11 @@ class Directories:
raise InvalidPathError() raise InvalidPathError()
self._dirs = [p for p in self._dirs if p not in path] self._dirs = [p for p in self._dirs if p not in path]
self._dirs.append(path) self._dirs.append(path)
@staticmethod @staticmethod
def get_subfolders(path): def get_subfolders(path):
"""Returns a sorted list of paths corresponding to subfolders in ``path``. """Returns a sorted list of paths corresponding to subfolders in ``path``.
:param Path path: get subfolders from there :param Path path: get subfolders from there
:rtype: list of Path :rtype: list of Path
""" """
@ -148,29 +148,29 @@ class Directories:
return subpaths return subpaths
except EnvironmentError: except EnvironmentError:
return [] return []
def get_files(self, j=job.nulljob): def get_files(self, j=job.nulljob):
"""Returns a list of all files that are not excluded. """Returns a list of all files that are not excluded.
Returned files also have their ``is_ref`` attr set if applicable. Returned files also have their ``is_ref`` attr set if applicable.
""" """
for path in self._dirs: for path in self._dirs:
for file in self._get_files(path, j): for file in self._get_files(path, j):
yield file yield file
def get_folders(self, j=job.nulljob): def get_folders(self, j=job.nulljob):
"""Returns a list of all folders that are not excluded. """Returns a list of all folders that are not excluded.
Returned folders also have their ``is_ref`` attr set if applicable. Returned folders also have their ``is_ref`` attr set if applicable.
""" """
for path in self._dirs: for path in self._dirs:
from_folder = self.folderclass(path) from_folder = self.folderclass(path)
for folder in self._get_folders(from_folder, j): for folder in self._get_folders(from_folder, j):
yield folder yield folder
def get_state(self, path): def get_state(self, path):
"""Returns the state of ``path``. """Returns the state of ``path``.
:rtype: :class:`DirectoryState` :rtype: :class:`DirectoryState`
""" """
if path in self.states: if path in self.states:
@ -183,12 +183,12 @@ class Directories:
return self.get_state(parent) return self.get_state(parent)
else: else:
return DirectoryState.Normal return DirectoryState.Normal
def has_any_file(self): def has_any_file(self):
"""Returns whether selected folders contain any file. """Returns whether selected folders contain any file.
Because it stops at the first file it finds, it's much faster than get_files(). Because it stops at the first file it finds, it's much faster than get_files().
:rtype: bool :rtype: bool
""" """
try: try:
@ -196,10 +196,10 @@ class Directories:
return True return True
except StopIteration: except StopIteration:
return False return False
def load_from_file(self, infile): def load_from_file(self, infile):
"""Load folder selection from ``infile``. """Load folder selection from ``infile``.
:param file infile: path or file pointer to XML generated through :meth:`save_to_file` :param file infile: path or file pointer to XML generated through :meth:`save_to_file`
""" """
try: try:
@ -222,10 +222,10 @@ class Directories:
path = attrib['path'] path = attrib['path']
state = attrib['value'] state = attrib['value']
self.states[Path(path)] = int(state) self.states[Path(path)] = int(state)
def save_to_file(self, outfile): def save_to_file(self, outfile):
"""Save folder selection as XML to ``outfile``. """Save folder selection as XML to ``outfile``.
:param file outfile: path or file pointer to XML file to save to. :param file outfile: path or file pointer to XML file to save to.
""" """
with FileOrPath(outfile, 'wb') as fp: with FileOrPath(outfile, 'wb') as fp:
@ -239,10 +239,10 @@ class Directories:
state_node.set('value', str(state)) state_node.set('value', str(state))
tree = ET.ElementTree(root) tree = ET.ElementTree(root)
tree.write(fp, encoding='utf-8') tree.write(fp, encoding='utf-8')
def set_state(self, path, state): def set_state(self, path, state):
"""Set the state of folder at ``path``. """Set the state of folder at ``path``.
:param Path path: path of the target folder :param Path path: path of the target folder
:param state: state to set folder to :param state: state to set folder to
:type state: :class:`DirectoryState` :type state: :class:`DirectoryState`
@ -253,4 +253,4 @@ class Directories:
if path.is_parent_of(iter_path): if path.is_parent_of(iter_path):
del self.states[iter_path] del self.states[iter_path]
self.states[path] = state self.states[path] = state

98
core/engine.py
View File

@ -1,9 +1,9 @@
# Created By: Virgil Dupras # Created By: Virgil Dupras
# Created On: 2006/01/29 # Created On: 2006/01/29
# Copyright 2014 Hardcoded Software (http://www.hardcoded.net) # Copyright 2014 Hardcoded Software (http://www.hardcoded.net)
# #
# This software is licensed under the "BSD" License as described in the "LICENSE" file, # This software is licensed under the "BSD" License as described in the "LICENSE" file,
# which should be included with this package. The terms are also available at # which should be included with this package. The terms are also available at
# http://www.hardcoded.net/licenses/bsd_license # http://www.hardcoded.net/licenses/bsd_license
import difflib import difflib
@ -15,7 +15,7 @@ from unicodedata import normalize
from hscommon.util import flatten, multi_replace from hscommon.util import flatten, multi_replace
from hscommon.trans import tr from hscommon.trans import tr
from jobprogress import job from hscommon.jobprogress import job
(WEIGHT_WORDS, (WEIGHT_WORDS,
MATCH_SIMILAR_WORDS, MATCH_SIMILAR_WORDS,
@ -45,7 +45,7 @@ def unpack_fields(fields):
def compare(first, second, flags=()): def compare(first, second, flags=()):
"""Returns the % of words that match between ``first`` and ``second`` """Returns the % of words that match between ``first`` and ``second``
The result is a ``int`` in the range 0..100. The result is a ``int`` in the range 0..100.
``first`` and ``second`` can be either a string or a list (of words). ``first`` and ``second`` can be either a string or a list (of words).
""" """
@ -53,7 +53,7 @@ def compare(first, second, flags=()):
return 0 return 0
if any(isinstance(element, list) for element in first): if any(isinstance(element, list) for element in first):
return compare_fields(first, second, flags) return compare_fields(first, second, flags)
second = second[:] #We must use a copy of second because we remove items from it second = second[:] #We must use a copy of second because we remove items from it
match_similar = MATCH_SIMILAR_WORDS in flags match_similar = MATCH_SIMILAR_WORDS in flags
weight_words = WEIGHT_WORDS in flags weight_words = WEIGHT_WORDS in flags
joined = first + second joined = first + second
@ -77,9 +77,9 @@ def compare(first, second, flags=()):
def compare_fields(first, second, flags=()): def compare_fields(first, second, flags=()):
"""Returns the score for the lowest matching :ref:`fields`. """Returns the score for the lowest matching :ref:`fields`.
``first`` and ``second`` must be lists of lists of string. Each sub-list is then compared with ``first`` and ``second`` must be lists of lists of string. Each sub-list is then compared with
:func:`compare`. :func:`compare`.
""" """
if len(first) != len(second): if len(first) != len(second):
return 0 return 0
@ -104,10 +104,10 @@ def compare_fields(first, second, flags=()):
def build_word_dict(objects, j=job.nulljob): def build_word_dict(objects, j=job.nulljob):
"""Returns a dict of objects mapped by their words. """Returns a dict of objects mapped by their words.
objects must have a ``words`` attribute being a list of strings or a list of lists of strings objects must have a ``words`` attribute being a list of strings or a list of lists of strings
(:ref:`fields`). (:ref:`fields`).
The result will be a dict with words as keys, lists of objects as values. The result will be a dict with words as keys, lists of objects as values.
""" """
result = defaultdict(set) result = defaultdict(set)
@ -118,7 +118,7 @@ def build_word_dict(objects, j=job.nulljob):
def merge_similar_words(word_dict): def merge_similar_words(word_dict):
"""Take all keys in ``word_dict`` that are similar, and merge them together. """Take all keys in ``word_dict`` that are similar, and merge them together.
``word_dict`` has been built with :func:`build_word_dict`. Similarity is computed with Python's ``word_dict`` has been built with :func:`build_word_dict`. Similarity is computed with Python's
``difflib.get_close_matches()``, which computes the number of edits that are necessary to make ``difflib.get_close_matches()``, which computes the number of edits that are necessary to make
a word equal to the other. a word equal to the other.
@ -138,9 +138,9 @@ def merge_similar_words(word_dict):
def reduce_common_words(word_dict, threshold): def reduce_common_words(word_dict, threshold):
"""Remove all objects from ``word_dict`` values where the object count >= ``threshold`` """Remove all objects from ``word_dict`` values where the object count >= ``threshold``
``word_dict`` has been built with :func:`build_word_dict`. ``word_dict`` has been built with :func:`build_word_dict`.
The exception to this removal are the objects where all the words of the object are common. The exception to this removal are the objects where all the words of the object are common.
Because if we remove them, we will miss some duplicates! Because if we remove them, we will miss some duplicates!
""" """
@ -181,17 +181,17 @@ class Match(namedtuple('Match', 'first second percentage')):
exact scan methods, such as Contents scans, this will always be 100. exact scan methods, such as Contents scans, this will always be 100.
""" """
__slots__ = () __slots__ = ()
def get_match(first, second, flags=()): def get_match(first, second, flags=()):
#it is assumed here that first and second both have a "words" attribute #it is assumed here that first and second both have a "words" attribute
percentage = compare(first.words, second.words, flags) percentage = compare(first.words, second.words, flags)
return Match(first, second, percentage) return Match(first, second, percentage)
def getmatches( def getmatches(
objects, min_match_percentage=0, match_similar_words=False, weight_words=False, objects, min_match_percentage=0, match_similar_words=False, weight_words=False,
no_field_order=False, j=job.nulljob): no_field_order=False, j=job.nulljob):
"""Returns a list of :class:`Match` within ``objects`` after fuzzily matching their words. """Returns a list of :class:`Match` within ``objects`` after fuzzily matching their words.
:param objects: List of :class:`~core.fs.File` to match. :param objects: List of :class:`~core.fs.File` to match.
:param int min_match_percentage: minimum % of words that have to match. :param int min_match_percentage: minimum % of words that have to match.
:param bool match_similar_words: make similar words (see :func:`merge_similar_words`) match. :param bool match_similar_words: make similar words (see :func:`merge_similar_words`) match.
@ -246,7 +246,7 @@ def getmatches(
def getmatches_by_contents(files, sizeattr='size', partial=False, j=job.nulljob): def getmatches_by_contents(files, sizeattr='size', partial=False, j=job.nulljob):
"""Returns a list of :class:`Match` within ``files`` if their contents is the same. """Returns a list of :class:`Match` within ``files`` if their contents is the same.
:param str sizeattr: attibute name of the :class:`~core.fs.file` that returns the size of the :param str sizeattr: attibute name of the :class:`~core.fs.file` that returns the size of the
file to use for comparison. file to use for comparison.
:param bool partial: if true, will use the "md5partial" attribute instead of "md5" to compute :param bool partial: if true, will use the "md5partial" attribute instead of "md5" to compute
@ -278,44 +278,44 @@ class Group:
This manages match pairs into groups and ensures that all files in the group match to each This manages match pairs into groups and ensures that all files in the group match to each
other. other.
.. attribute:: ref .. attribute:: ref
The "reference" file, which is the file among the group that isn't going to be deleted. The "reference" file, which is the file among the group that isn't going to be deleted.
.. attribute:: ordered .. attribute:: ordered
Ordered list of duplicates in the group (including the :attr:`ref`). Ordered list of duplicates in the group (including the :attr:`ref`).
.. attribute:: unordered .. attribute:: unordered
Set duplicates in the group (including the :attr:`ref`). Set duplicates in the group (including the :attr:`ref`).
.. attribute:: dupes .. attribute:: dupes
An ordered list of the group's duplicate, without :attr:`ref`. Equivalent to An ordered list of the group's duplicate, without :attr:`ref`. Equivalent to
``ordered[1:]`` ``ordered[1:]``
.. attribute:: percentage .. attribute:: percentage
Average match percentage of match pairs containing :attr:`ref`. Average match percentage of match pairs containing :attr:`ref`.
""" """
#---Override #---Override
def __init__(self): def __init__(self):
self._clear() self._clear()
def __contains__(self, item): def __contains__(self, item):
return item in self.unordered return item in self.unordered
def __getitem__(self, key): def __getitem__(self, key):
return self.ordered.__getitem__(key) return self.ordered.__getitem__(key)
def __iter__(self): def __iter__(self):
return iter(self.ordered) return iter(self.ordered)
def __len__(self): def __len__(self):
return len(self.ordered) return len(self.ordered)
#---Private #---Private
def _clear(self): def _clear(self):
self._percentage = None self._percentage = None
@ -324,22 +324,22 @@ class Group:
self.candidates = defaultdict(set) self.candidates = defaultdict(set)
self.ordered = [] self.ordered = []
self.unordered = set() self.unordered = set()
def _get_matches_for_ref(self): def _get_matches_for_ref(self):
if self._matches_for_ref is None: if self._matches_for_ref is None:
ref = self.ref ref = self.ref
self._matches_for_ref = [match for match in self.matches if ref in match] self._matches_for_ref = [match for match in self.matches if ref in match]
return self._matches_for_ref return self._matches_for_ref
#---Public #---Public
def add_match(self, match): def add_match(self, match):
"""Adds ``match`` to internal match list and possibly add duplicates to the group. """Adds ``match`` to internal match list and possibly add duplicates to the group.
A duplicate can only be considered as such if it matches all other duplicates in the group. A duplicate can only be considered as such if it matches all other duplicates in the group.
This method registers that pair (A, B) represented in ``match`` as possible candidates and, This method registers that pair (A, B) represented in ``match`` as possible candidates and,
if A and/or B end up matching every other duplicates in the group, add these duplicates to if A and/or B end up matching every other duplicates in the group, add these duplicates to
the group. the group.
:param tuple match: pair of :class:`~core.fs.File` to add :param tuple match: pair of :class:`~core.fs.File` to add
""" """
def add_candidate(item, match): def add_candidate(item, match):
@ -348,7 +348,7 @@ class Group:
if self.unordered <= matches: if self.unordered <= matches:
self.ordered.append(item) self.ordered.append(item)
self.unordered.add(item) self.unordered.add(item)
if match in self.matches: if match in self.matches:
return return
self.matches.add(match) self.matches.add(match)
@ -359,17 +359,17 @@ class Group:
add_candidate(second, first) add_candidate(second, first)
self._percentage = None self._percentage = None
self._matches_for_ref = None self._matches_for_ref = None
def discard_matches(self): def discard_matches(self):
"""Remove all recorded matches that didn't result in a duplicate being added to the group. """Remove all recorded matches that didn't result in a duplicate being added to the group.
You can call this after the duplicate scanning process to free a bit of memory. You can call this after the duplicate scanning process to free a bit of memory.
""" """
discarded = set(m for m in self.matches if not all(obj in self.unordered for obj in [m.first, m.second])) discarded = set(m for m in self.matches if not all(obj in self.unordered for obj in [m.first, m.second]))
self.matches -= discarded self.matches -= discarded
self.candidates = defaultdict(set) self.candidates = defaultdict(set)
return discarded return discarded
def get_match_of(self, item): def get_match_of(self, item):
"""Returns the match pair between ``item`` and :attr:`ref`. """Returns the match pair between ``item`` and :attr:`ref`.
""" """
@ -378,10 +378,10 @@ class Group:
for m in self._get_matches_for_ref(): for m in self._get_matches_for_ref():
if item in m: if item in m:
return m return m
def prioritize(self, key_func, tie_breaker=None): def prioritize(self, key_func, tie_breaker=None):
"""Reorders :attr:`ordered` according to ``key_func``. """Reorders :attr:`ordered` according to ``key_func``.
:param key_func: Key (f(x)) to be used for sorting :param key_func: Key (f(x)) to be used for sorting
:param tie_breaker: function to be used to select the reference position in case the top :param tie_breaker: function to be used to select the reference position in case the top
duplicates have the same key_func() result. duplicates have the same key_func() result.
@ -405,7 +405,7 @@ class Group:
self.switch_ref(ref) self.switch_ref(ref)
return True return True
return changed return changed
def remove_dupe(self, item, discard_matches=True): def remove_dupe(self, item, discard_matches=True):
try: try:
self.ordered.remove(item) self.ordered.remove(item)
@ -419,7 +419,7 @@ class Group:
self._clear() self._clear()
except ValueError: except ValueError:
pass pass
def switch_ref(self, with_dupe): def switch_ref(self, with_dupe):
"""Make the :attr:`ref` dupe of the group switch position with ``with_dupe``. """Make the :attr:`ref` dupe of the group switch position with ``with_dupe``.
""" """
@ -433,9 +433,9 @@ class Group:
return True return True
except ValueError: except ValueError:
return False return False
dupes = property(lambda self: self[1:]) dupes = property(lambda self: self[1:])
@property @property
def percentage(self): def percentage(self):
if self._percentage is None: if self._percentage is None:
@ -445,16 +445,16 @@ class Group:
else: else:
self._percentage = 0 self._percentage = 0
return self._percentage return self._percentage
@property @property
def ref(self): def ref(self):
if self: if self:
return self[0] return self[0]
def get_groups(matches, j=job.nulljob): def get_groups(matches, j=job.nulljob):
"""Returns a list of :class:`Group` from ``matches``. """Returns a list of :class:`Group` from ``matches``.
Create groups out of match pairs in the smartest way possible. Create groups out of match pairs in the smartest way possible.
""" """
matches.sort(key=lambda match: -match.percentage) matches.sort(key=lambda match: -match.percentage)

98
core/results.py
View File

@ -1,9 +1,9 @@
# Created By: Virgil Dupras # Created By: Virgil Dupras
# Created On: 2006/02/23 # Created On: 2006/02/23
# Copyright 2014 Hardcoded Software (http://www.hardcoded.net) # Copyright 2014 Hardcoded Software (http://www.hardcoded.net)
# #
# This software is licensed under the "BSD" License as described in the "LICENSE" file, # This software is licensed under the "BSD" License as described in the "LICENSE" file,
# which should be included with this package. The terms are also available at # which should be included with this package. The terms are also available at
# http://www.hardcoded.net/licenses/bsd_license # http://www.hardcoded.net/licenses/bsd_license
import logging import logging
@ -12,7 +12,7 @@ import os
import os.path as op import os.path as op
from xml.etree import ElementTree as ET from xml.etree import ElementTree as ET
from jobprogress.job import nulljob from hscommon.jobprogress.job import nulljob
from hscommon.conflict import get_conflicted_name from hscommon.conflict import get_conflicted_name
from hscommon.util import flatten, nonone, FileOrPath, format_size from hscommon.util import flatten, nonone, FileOrPath, format_size
from hscommon.trans import tr from hscommon.trans import tr
@ -22,15 +22,15 @@ from .markable import Markable
class Results(Markable): class Results(Markable):
"""Manages a collection of duplicate :class:`~core.engine.Group`. """Manages a collection of duplicate :class:`~core.engine.Group`.
This class takes care or marking, sorting and filtering duplicate groups. This class takes care or marking, sorting and filtering duplicate groups.
.. attribute:: groups .. attribute:: groups
The list of :class:`~core.engine.Group` contained managed by this instance. The list of :class:`~core.engine.Group` contained managed by this instance.
.. attribute:: dupes .. attribute:: dupes
A list of all duplicates (:class:`~core.fs.File` instances), without ref, contained in the A list of all duplicates (:class:`~core.fs.File` instances), without ref, contained in the
currently managed :attr:`groups`. currently managed :attr:`groups`.
""" """
@ -50,16 +50,16 @@ class Results(Markable):
self.app = app self.app = app
self.problems = [] # (dupe, error_msg) self.problems = [] # (dupe, error_msg)
self.is_modified = False self.is_modified = False
def _did_mark(self, dupe): def _did_mark(self, dupe):
self.__marked_size += dupe.size self.__marked_size += dupe.size
def _did_unmark(self, dupe): def _did_unmark(self, dupe):
self.__marked_size -= dupe.size self.__marked_size -= dupe.size
def _get_markable_count(self): def _get_markable_count(self):
return self.__total_count return self.__total_count
def _is_markable(self, dupe): def _is_markable(self, dupe):
if dupe.is_ref: if dupe.is_ref:
return False return False
@ -71,25 +71,25 @@ class Results(Markable):
if self.__filtered_dupes and dupe not in self.__filtered_dupes: if self.__filtered_dupes and dupe not in self.__filtered_dupes:
return False return False
return True return True
def mark_all(self): def mark_all(self):
if self.__filters: if self.__filters:
self.mark_multiple(self.__filtered_dupes) self.mark_multiple(self.__filtered_dupes)
else: else:
Markable.mark_all(self) Markable.mark_all(self)
def mark_invert(self): def mark_invert(self):
if self.__filters: if self.__filters:
self.mark_toggle_multiple(self.__filtered_dupes) self.mark_toggle_multiple(self.__filtered_dupes)
else: else:
Markable.mark_invert(self) Markable.mark_invert(self)
def mark_none(self): def mark_none(self):
if self.__filters: if self.__filters:
self.unmark_multiple(self.__filtered_dupes) self.unmark_multiple(self.__filtered_dupes)
else: else:
Markable.mark_none(self) Markable.mark_none(self)
#---Private #---Private
def __get_dupe_list(self): def __get_dupe_list(self):
if self.__dupes is None: if self.__dupes is None:
@ -103,13 +103,13 @@ class Results(Markable):
if sd: if sd:
self.sort_dupes(sd[0], sd[1], sd[2]) self.sort_dupes(sd[0], sd[1], sd[2])
return self.__dupes return self.__dupes
def __get_groups(self): def __get_groups(self):
if self.__filtered_groups is None: if self.__filtered_groups is None:
return self.__groups return self.__groups
else: else:
return self.__filtered_groups return self.__filtered_groups
def __get_stat_line(self): def __get_stat_line(self):
if self.__filtered_dupes is None: if self.__filtered_dupes is None:
mark_count = self.mark_count mark_count = self.mark_count
@ -132,7 +132,7 @@ class Results(Markable):
if self.__filters: if self.__filters:
result += tr(" filter: %s") % ' --> '.join(self.__filters) result += tr(" filter: %s") % ' --> '.join(self.__filters)
return result return result
def __recalculate_stats(self): def __recalculate_stats(self):
self.__total_size = 0 self.__total_size = 0
self.__total_count = 0 self.__total_count = 0
@ -140,7 +140,7 @@ class Results(Markable):
markable = [dupe for dupe in group.dupes if self._is_markable(dupe)] markable = [dupe for dupe in group.dupes if self._is_markable(dupe)]
self.__total_count += len(markable) self.__total_count += len(markable)
self.__total_size += sum(dupe.size for dupe in markable) self.__total_size += sum(dupe.size for dupe in markable)
def __set_groups(self, new_groups): def __set_groups(self, new_groups):
self.mark_none() self.mark_none()
self.__groups = new_groups self.__groups = new_groups
@ -155,18 +155,18 @@ class Results(Markable):
self.apply_filter(None) self.apply_filter(None)
for filter_str in old_filters: for filter_str in old_filters:
self.apply_filter(filter_str) self.apply_filter(filter_str)
#---Public #---Public
def apply_filter(self, filter_str): def apply_filter(self, filter_str):
"""Applies a filter ``filter_str`` to :attr:`groups` """Applies a filter ``filter_str`` to :attr:`groups`
When you apply the filter, only dupes with the filename matching ``filter_str`` will be in When you apply the filter, only dupes with the filename matching ``filter_str`` will be in
in the results. To cancel the filter, just call apply_filter with ``filter_str`` to None, in the results. To cancel the filter, just call apply_filter with ``filter_str`` to None,
and the results will go back to normal. and the results will go back to normal.
If call apply_filter on a filtered results, the filter will be applied If call apply_filter on a filtered results, the filter will be applied
*on the filtered results*. *on the filtered results*.
:param str filter_str: a string containing a regexp to filter dupes with. :param str filter_str: a string containing a regexp to filter dupes with.
""" """
if not filter_str: if not filter_str:
@ -193,7 +193,7 @@ class Results(Markable):
if sd: if sd:
self.sort_groups(sd[0], sd[1]) self.sort_groups(sd[0], sd[1])
self.__dupes = None self.__dupes = None
def get_group_of_duplicate(self, dupe): def get_group_of_duplicate(self, dupe):
"""Returns :class:`~core.engine.Group` in which ``dupe`` belongs. """Returns :class:`~core.engine.Group` in which ``dupe`` belongs.
""" """
@ -201,12 +201,12 @@ class Results(Markable):
return self.__group_of_duplicate[dupe] return self.__group_of_duplicate[dupe]
except (TypeError, KeyError): except (TypeError, KeyError):
return None return None
is_markable = _is_markable is_markable = _is_markable
def load_from_xml(self, infile, get_file, j=nulljob): def load_from_xml(self, infile, get_file, j=nulljob):
"""Load results from ``infile``. """Load results from ``infile``.
:param infile: a file or path pointing to an XML file created with :meth:`save_to_xml`. :param infile: a file or path pointing to an XML file created with :meth:`save_to_xml`.
:param get_file: a function f(path) returning a :class:`~core.fs.File` wrapping the path. :param get_file: a function f(path) returning a :class:`~core.fs.File` wrapping the path.
:param j: A :ref:`job progress instance <jobs>`. :param j: A :ref:`job progress instance <jobs>`.
@ -217,7 +217,7 @@ class Results(Markable):
for other_file in other_files: for other_file in other_files:
group.add_match(engine.get_match(ref_file, other_file)) group.add_match(engine.get_match(ref_file, other_file))
do_match(other_files[0], other_files[1:], group) do_match(other_files[0], other_files[1:], group)
self.apply_filter(None) self.apply_filter(None)
try: try:
root = ET.parse(infile).getroot() root = ET.parse(infile).getroot()
@ -255,13 +255,13 @@ class Results(Markable):
do_match(dupes[0], dupes[1:], group) do_match(dupes[0], dupes[1:], group)
group.prioritize(lambda x: dupes.index(x)) group.prioritize(lambda x: dupes.index(x))
if len(group): if len(group):
groups.append(group) groups.append(group)
j.add_progress() j.add_progress()
self.groups = groups self.groups = groups
for dupe_file in marked: for dupe_file in marked:
self.mark(dupe_file) self.mark(dupe_file)
self.is_modified = False self.is_modified = False
def make_ref(self, dupe): def make_ref(self, dupe):
"""Make ``dupe`` take the :attr:`~core.engine.Group.ref` position of its group. """Make ``dupe`` take the :attr:`~core.engine.Group.ref` position of its group.
""" """
@ -279,13 +279,13 @@ class Results(Markable):
self.__dupes = None self.__dupes = None
self.is_modified = True self.is_modified = True
return True return True
def perform_on_marked(self, func, remove_from_results): def perform_on_marked(self, func, remove_from_results):
"""Performs ``func`` on all marked dupes. """Performs ``func`` on all marked dupes.
If an ``EnvironmentError`` is raised during the call, the problematic dupe is added to If an ``EnvironmentError`` is raised during the call, the problematic dupe is added to
self.problems. self.problems.
:param bool remove_from_results: If true, dupes which had ``func`` applied and didn't cause :param bool remove_from_results: If true, dupes which had ``func`` applied and didn't cause
any problem. any problem.
""" """
@ -303,10 +303,10 @@ class Results(Markable):
self.mark_none() self.mark_none()
for dupe, _ in self.problems: for dupe, _ in self.problems:
self.mark(dupe) self.mark(dupe)
def remove_duplicates(self, dupes): def remove_duplicates(self, dupes):
"""Remove ``dupes`` from their respective :class:`~core.engine.Group`. """Remove ``dupes`` from their respective :class:`~core.engine.Group`.
Also, remove the group from :attr:`groups` if it ends up empty. Also, remove the group from :attr:`groups` if it ends up empty.
""" """
affected_groups = set() affected_groups = set()
@ -331,10 +331,10 @@ class Results(Markable):
group.discard_matches() group.discard_matches()
self.__dupes = None self.__dupes = None
self.is_modified = bool(self.__groups) self.is_modified = bool(self.__groups)
def save_to_xml(self, outfile): def save_to_xml(self, outfile):
"""Save results to ``outfile`` in XML. """Save results to ``outfile`` in XML.
:param outfile: file object or path. :param outfile: file object or path.
""" """
self.apply_filter(None) self.apply_filter(None)
@ -362,11 +362,11 @@ class Results(Markable):
match_elem.set('second', str(dupe2index[match.second])) match_elem.set('second', str(dupe2index[match.second]))
match_elem.set('percentage', str(int(match.percentage))) match_elem.set('percentage', str(int(match.percentage)))
tree = ET.ElementTree(root) tree = ET.ElementTree(root)
def do_write(outfile): def do_write(outfile):
with FileOrPath(outfile, 'wb') as fp: with FileOrPath(outfile, 'wb') as fp:
tree.write(fp, encoding='utf-8') tree.write(fp, encoding='utf-8')
try: try:
do_write(outfile) do_write(outfile)
except IOError as e: except IOError as e:
@ -381,10 +381,10 @@ class Results(Markable):
else: else:
raise raise
self.is_modified = False self.is_modified = False
def sort_dupes(self, key, asc=True, delta=False): def sort_dupes(self, key, asc=True, delta=False):
"""Sort :attr:`dupes` according to ``key``. """Sort :attr:`dupes` according to ``key``.
:param str key: key attribute name to sort with. :param str key: key attribute name to sort with.
:param bool asc: If false, sorting is reversed. :param bool asc: If false, sorting is reversed.
:param bool delta: If true, sorting occurs using :ref:`delta values <deltavalues>`. :param bool delta: If true, sorting occurs using :ref:`delta values <deltavalues>`.
@ -394,19 +394,19 @@ class Results(Markable):
keyfunc = lambda d: self.app._get_dupe_sort_key(d, lambda: self.get_group_of_duplicate(d), key, delta) keyfunc = lambda d: self.app._get_dupe_sort_key(d, lambda: self.get_group_of_duplicate(d), key, delta)
self.__dupes.sort(key=keyfunc, reverse=not asc) self.__dupes.sort(key=keyfunc, reverse=not asc)
self.__dupes_sort_descriptor = (key,asc,delta) self.__dupes_sort_descriptor = (key,asc,delta)
def sort_groups(self, key, asc=True): def sort_groups(self, key, asc=True):
"""Sort :attr:`groups` according to ``key``. """Sort :attr:`groups` according to ``key``.
The :attr:`~core.engine.Group.ref` of each group is used to extract values for sorting. The :attr:`~core.engine.Group.ref` of each group is used to extract values for sorting.
:param str key: key attribute name to sort with. :param str key: key attribute name to sort with.
:param bool asc: If false, sorting is reversed. :param bool asc: If false, sorting is reversed.
""" """
keyfunc = lambda g: self.app._get_group_sort_key(g, key) keyfunc = lambda g: self.app._get_group_sort_key(g, key)
self.groups.sort(key=keyfunc, reverse=not asc) self.groups.sort(key=keyfunc, reverse=not asc)
self.__groups_sort_descriptor = (key,asc) self.__groups_sort_descriptor = (key,asc)
#---Properties #---Properties
dupes = property(__get_dupe_list) dupes = property(__get_dupe_list)
groups = property(__get_groups, __set_groups) groups = property(__get_groups, __set_groups)

22
core/scanner.py
View File

@ -1,16 +1,16 @@
# Created By: Virgil Dupras # Created By: Virgil Dupras
# Created On: 2006/03/03 # Created On: 2006/03/03
# Copyright 2014 Hardcoded Software (http://www.hardcoded.net) # Copyright 2014 Hardcoded Software (http://www.hardcoded.net)
# #
# This software is licensed under the "BSD" License as described in the "LICENSE" file, # This software is licensed under the "BSD" License as described in the "LICENSE" file,
# which should be included with this package. The terms are also available at # which should be included with this package. The terms are also available at
# http://www.hardcoded.net/licenses/bsd_license # http://www.hardcoded.net/licenses/bsd_license
import logging import logging
import re import re
import os.path as op import os.path as op
from jobprogress import job from hscommon.jobprogress import job
from hscommon.util import dedupe, rem_file_ext, get_file_ext from hscommon.util import dedupe, rem_file_ext, get_file_ext
from hscommon.trans import tr from hscommon.trans import tr
@ -29,7 +29,7 @@ class ScanType:
Folders = 4 Folders = 4
Contents = 5 Contents = 5
ContentsAudio = 6 ContentsAudio = 6
#PE #PE
FuzzyBlock = 10 FuzzyBlock = 10
ExifTimestamp = 11 ExifTimestamp = 11
@ -72,7 +72,7 @@ class Scanner:
def __init__(self): def __init__(self):
self.ignore_list = IgnoreList() self.ignore_list = IgnoreList()
self.discarded_file_count = 0 self.discarded_file_count = 0
def _getmatches(self, files, j): def _getmatches(self, files, j):
if self.size_threshold: if self.size_threshold:
j = j.start_subjob([2, 8]) j = j.start_subjob([2, 8])
@ -100,11 +100,11 @@ class Scanner:
logging.debug("Reading metadata of {}".format(str(f.path))) logging.debug("Reading metadata of {}".format(str(f.path)))
f.words = func(f) f.words = func(f)
return engine.getmatches(files, j=j, **kw) return engine.getmatches(files, j=j, **kw)
@staticmethod @staticmethod
def _key_func(dupe): def _key_func(dupe):
return -dupe.size return -dupe.size
@staticmethod @staticmethod
def _tie_breaker(ref, dupe): def _tie_breaker(ref, dupe):
refname = rem_file_ext(ref.name).lower() refname = rem_file_ext(ref.name).lower()
@ -118,7 +118,7 @@ class Scanner:
if is_same_with_digit(refname, dupename): if is_same_with_digit(refname, dupename):
return True return True
return len(dupe.path) > len(ref.path) return len(dupe.path) > len(ref.path)
def get_dupe_groups(self, files, j=job.nulljob): def get_dupe_groups(self, files, j=job.nulljob):
j = j.start_subjob([8, 2]) j = j.start_subjob([8, 2])
for f in (f for f in files if not hasattr(f, 'is_ref')): for f in (f for f in files if not hasattr(f, 'is_ref')):
@ -152,7 +152,7 @@ class Scanner:
if self.ignore_list: if self.ignore_list:
j = j.start_subjob(2) j = j.start_subjob(2)
iter_matches = j.iter_with_progress(matches, tr("Processed %d/%d matches against the ignore list")) iter_matches = j.iter_with_progress(matches, tr("Processed %d/%d matches against the ignore list"))
matches = [m for m in iter_matches matches = [m for m in iter_matches
if not self.ignore_list.AreIgnored(str(m.first.path), str(m.second.path))] if not self.ignore_list.AreIgnored(str(m.first.path), str(m.second.path))]
logging.info('Grouping matches') logging.info('Grouping matches')
groups = engine.get_groups(matches, j) groups = engine.get_groups(matches, j)
@ -177,7 +177,7 @@ class Scanner:
for g in groups: for g in groups:
g.prioritize(self._key_func, self._tie_breaker) g.prioritize(self._key_func, self._tie_breaker)
return groups return groups
match_similar_words = False match_similar_words = False
min_match_percentage = 80 min_match_percentage = 80
mix_file_kind = True mix_file_kind = True

100
core/tests/app_test.py
View File

@ -1,9 +1,9 @@
# Created By: Virgil Dupras # Created By: Virgil Dupras
# Created On: 2007-06-23 # Created On: 2007-06-23
# Copyright 2014 Hardcoded Software (http://www.hardcoded.net) # Copyright 2014 Hardcoded Software (http://www.hardcoded.net)
# #
# This software is licensed under the "BSD" License as described in the "LICENSE" file, # This software is licensed under the "BSD" License as described in the "LICENSE" file,
# which should be included with this package. The terms are also available at # which should be included with this package. The terms are also available at
# http://www.hardcoded.net/licenses/bsd_license # http://www.hardcoded.net/licenses/bsd_license
import os import os
@ -15,7 +15,7 @@ from hscommon.path import Path
import hscommon.conflict import hscommon.conflict
import hscommon.util import hscommon.util
from hscommon.testutil import CallLogger, eq_, log_calls from hscommon.testutil import CallLogger, eq_, log_calls
from jobprogress.job import Job from hscommon.jobprogress.job import Job
from .base import DupeGuru, TestApp from .base import DupeGuru, TestApp
from .results_test import GetTestGroups from .results_test import GetTestGroups
@ -36,7 +36,7 @@ class TestCaseDupeGuru:
assert call['filter_str'] is None assert call['filter_str'] is None
call = dgapp.results.apply_filter.calls[1] call = dgapp.results.apply_filter.calls[1]
eq_('foo', call['filter_str']) eq_('foo', call['filter_str'])
def test_apply_filter_escapes_regexp(self, monkeypatch): def test_apply_filter_escapes_regexp(self, monkeypatch):
dgapp = TestApp().app dgapp = TestApp().app
monkeypatch.setattr(dgapp.results, 'apply_filter', log_calls(dgapp.results.apply_filter)) monkeypatch.setattr(dgapp.results, 'apply_filter', log_calls(dgapp.results.apply_filter))
@ -50,7 +50,7 @@ class TestCaseDupeGuru:
dgapp.apply_filter('(abc)') dgapp.apply_filter('(abc)')
call = dgapp.results.apply_filter.calls[5] call = dgapp.results.apply_filter.calls[5]
eq_('(abc)', call['filter_str']) eq_('(abc)', call['filter_str'])
def test_copy_or_move(self, tmpdir, monkeypatch): def test_copy_or_move(self, tmpdir, monkeypatch):
# The goal here is just to have a test for a previous blowup I had. I know my test coverage # The goal here is just to have a test for a previous blowup I had. I know my test coverage
# for this unit is pathetic. What's done is done. My approach now is to add tests for # for this unit is pathetic. What's done is done. My approach now is to add tests for
@ -69,7 +69,7 @@ class TestCaseDupeGuru:
call = hscommon.conflict.smart_copy.calls[0] call = hscommon.conflict.smart_copy.calls[0]
eq_(call['dest_path'], op.join('some_destination', 'foo')) eq_(call['dest_path'], op.join('some_destination', 'foo'))
eq_(call['source_path'], f.path) eq_(call['source_path'], f.path)
def test_copy_or_move_clean_empty_dirs(self, tmpdir, monkeypatch): def test_copy_or_move_clean_empty_dirs(self, tmpdir, monkeypatch):
tmppath = Path(str(tmpdir)) tmppath = Path(str(tmpdir))
sourcepath = tmppath['source'] sourcepath = tmppath['source']
@ -83,13 +83,13 @@ class TestCaseDupeGuru:
calls = app.clean_empty_dirs.calls calls = app.clean_empty_dirs.calls
eq_(1, len(calls)) eq_(1, len(calls))
eq_(sourcepath, calls[0]['path']) eq_(sourcepath, calls[0]['path'])
def test_Scan_with_objects_evaluating_to_false(self): def test_Scan_with_objects_evaluating_to_false(self):
class FakeFile(fs.File): class FakeFile(fs.File):
def __bool__(self): def __bool__(self):
return False return False
# At some point, any() was used in a wrong way that made Scan() wrongly return 1 # At some point, any() was used in a wrong way that made Scan() wrongly return 1
app = TestApp().app app = TestApp().app
f1, f2 = [FakeFile('foo') for i in range(2)] f1, f2 = [FakeFile('foo') for i in range(2)]
@ -97,7 +97,7 @@ class TestCaseDupeGuru:
assert not (bool(f1) and bool(f2)) assert not (bool(f1) and bool(f2))
add_fake_files_to_directories(app.directories, [f1, f2]) add_fake_files_to_directories(app.directories, [f1, f2])
app.start_scanning() # no exception app.start_scanning() # no exception
@mark.skipif("not hasattr(os, 'link')") @mark.skipif("not hasattr(os, 'link')")
def test_ignore_hardlink_matches(self, tmpdir): def test_ignore_hardlink_matches(self, tmpdir):
# If the ignore_hardlink_matches option is set, don't match files hardlinking to the same # If the ignore_hardlink_matches option is set, don't match files hardlinking to the same
@ -111,7 +111,7 @@ class TestCaseDupeGuru:
app.options['ignore_hardlink_matches'] = True app.options['ignore_hardlink_matches'] = True
app.start_scanning() app.start_scanning()
eq_(len(app.results.groups), 0) eq_(len(app.results.groups), 0)
def test_rename_when_nothing_is_selected(self): def test_rename_when_nothing_is_selected(self):
# Issue #140 # Issue #140
# It's possible that rename operation has its selected row swept off from under it, thus # It's possible that rename operation has its selected row swept off from under it, thus
@ -127,11 +127,11 @@ class TestCaseDupeGuru_clean_empty_dirs:
# XXX This monkeypatch is temporary. will be fixed in a better monkeypatcher. # XXX This monkeypatch is temporary. will be fixed in a better monkeypatcher.
monkeypatch.setattr(app, 'delete_if_empty', hscommon.util.delete_if_empty) monkeypatch.setattr(app, 'delete_if_empty', hscommon.util.delete_if_empty)
self.app = TestApp().app self.app = TestApp().app
def test_option_off(self, do_setup): def test_option_off(self, do_setup):
self.app.clean_empty_dirs(Path('/foo/bar')) self.app.clean_empty_dirs(Path('/foo/bar'))
eq_(0, len(hscommon.util.delete_if_empty.calls)) eq_(0, len(hscommon.util.delete_if_empty.calls))
def test_option_on(self, do_setup): def test_option_on(self, do_setup):
self.app.options['clean_empty_dirs'] = True self.app.options['clean_empty_dirs'] = True
self.app.clean_empty_dirs(Path('/foo/bar')) self.app.clean_empty_dirs(Path('/foo/bar'))
@ -139,13 +139,13 @@ class TestCaseDupeGuru_clean_empty_dirs:
eq_(1, len(calls)) eq_(1, len(calls))
eq_(Path('/foo/bar'), calls[0]['path']) eq_(Path('/foo/bar'), calls[0]['path'])
eq_(['.DS_Store'], calls[0]['files_to_delete']) eq_(['.DS_Store'], calls[0]['files_to_delete'])
def test_recurse_up(self, do_setup, monkeypatch): def test_recurse_up(self, do_setup, monkeypatch):
# delete_if_empty must be recursively called up in the path until it returns False # delete_if_empty must be recursively called up in the path until it returns False
@log_calls @log_calls
def mock_delete_if_empty(path, files_to_delete=[]): def mock_delete_if_empty(path, files_to_delete=[]):
return len(path) > 1 return len(path) > 1
monkeypatch.setattr(hscommon.util, 'delete_if_empty', mock_delete_if_empty) monkeypatch.setattr(hscommon.util, 'delete_if_empty', mock_delete_if_empty)
# XXX This monkeypatch is temporary. will be fixed in a better monkeypatcher. # XXX This monkeypatch is temporary. will be fixed in a better monkeypatcher.
monkeypatch.setattr(app, 'delete_if_empty', mock_delete_if_empty) monkeypatch.setattr(app, 'delete_if_empty', mock_delete_if_empty)
@ -156,7 +156,7 @@ class TestCaseDupeGuru_clean_empty_dirs:
eq_(Path('not-empty/empty/empty'), calls[0]['path']) eq_(Path('not-empty/empty/empty'), calls[0]['path'])
eq_(Path('not-empty/empty'), calls[1]['path']) eq_(Path('not-empty/empty'), calls[1]['path'])
eq_(Path('not-empty'), calls[2]['path']) eq_(Path('not-empty'), calls[2]['path'])
class TestCaseDupeGuruWithResults: class TestCaseDupeGuruWithResults:
def pytest_funcarg__do_setup(self, request): def pytest_funcarg__do_setup(self, request):
@ -173,7 +173,7 @@ class TestCaseDupeGuruWithResults:
tmppath['foo'].mkdir() tmppath['foo'].mkdir()
tmppath['bar'].mkdir() tmppath['bar'].mkdir()
self.app.directories.add_path(tmppath) self.app.directories.add_path(tmppath)
def test_GetObjects(self, do_setup): def test_GetObjects(self, do_setup):
objects = self.objects objects = self.objects
groups = self.groups groups = self.groups
@ -186,7 +186,7 @@ class TestCaseDupeGuruWithResults:
r = self.rtable[4] r = self.rtable[4]
assert r._group is groups[1] assert r._group is groups[1]
assert r._dupe is objects[4] assert r._dupe is objects[4]
def test_GetObjects_after_sort(self, do_setup): def test_GetObjects_after_sort(self, do_setup):
objects = self.objects objects = self.objects
groups = self.groups[:] # we need an un-sorted reference groups = self.groups[:] # we need an un-sorted reference
@ -194,14 +194,14 @@ class TestCaseDupeGuruWithResults:
r = self.rtable[1] r = self.rtable[1]
assert r._group is groups[1] assert r._group is groups[1]
assert r._dupe is objects[4] assert r._dupe is objects[4]
def test_selected_result_node_paths_after_deletion(self, do_setup): def test_selected_result_node_paths_after_deletion(self, do_setup):
# cases where the selected dupes aren't there are correctly handled # cases where the selected dupes aren't there are correctly handled
self.rtable.select([1, 2, 3]) self.rtable.select([1, 2, 3])
self.app.remove_selected() self.app.remove_selected()
# The first 2 dupes have been removed. The 3rd one is a ref. it stays there, in first pos. # The first 2 dupes have been removed. The 3rd one is a ref. it stays there, in first pos.
eq_(self.rtable.selected_indexes, [1]) # no exception eq_(self.rtable.selected_indexes, [1]) # no exception
def test_selectResultNodePaths(self, do_setup): def test_selectResultNodePaths(self, do_setup):
app = self.app app = self.app
objects = self.objects objects = self.objects
@ -209,7 +209,7 @@ class TestCaseDupeGuruWithResults:
eq_(len(app.selected_dupes), 2) eq_(len(app.selected_dupes), 2)
assert app.selected_dupes[0] is objects[1] assert app.selected_dupes[0] is objects[1]
assert app.selected_dupes[1] is objects[2] assert app.selected_dupes[1] is objects[2]
def test_selectResultNodePaths_with_ref(self, do_setup): def test_selectResultNodePaths_with_ref(self, do_setup):
app = self.app app = self.app
objects = self.objects objects = self.objects
@ -218,26 +218,26 @@ class TestCaseDupeGuruWithResults:
assert app.selected_dupes[0] is objects[1] assert app.selected_dupes[0] is objects[1]
assert app.selected_dupes[1] is objects[2] assert app.selected_dupes[1] is objects[2]
assert app.selected_dupes[2] is self.groups[1].ref assert app.selected_dupes[2] is self.groups[1].ref
def test_selectResultNodePaths_after_sort(self, do_setup): def test_selectResultNodePaths_after_sort(self, do_setup):
app = self.app app = self.app
objects = self.objects objects = self.objects
groups = self.groups[:] #To keep the old order in memory groups = self.groups[:] #To keep the old order in memory
self.rtable.sort('name', False) #0 self.rtable.sort('name', False) #0
#Now, the group order is supposed to be reversed #Now, the group order is supposed to be reversed
self.rtable.select([1, 2, 3]) self.rtable.select([1, 2, 3])
eq_(len(app.selected_dupes), 3) eq_(len(app.selected_dupes), 3)
assert app.selected_dupes[0] is objects[4] assert app.selected_dupes[0] is objects[4]
assert app.selected_dupes[1] is groups[0].ref assert app.selected_dupes[1] is groups[0].ref
assert app.selected_dupes[2] is objects[1] assert app.selected_dupes[2] is objects[1]
def test_selected_powermarker_node_paths(self, do_setup): def test_selected_powermarker_node_paths(self, do_setup):
# app.selected_dupes is correctly converted into paths # app.selected_dupes is correctly converted into paths
self.rtable.power_marker = True self.rtable.power_marker = True
self.rtable.select([0, 1, 2]) self.rtable.select([0, 1, 2])
self.rtable.power_marker = False self.rtable.power_marker = False
eq_(self.rtable.selected_indexes, [1, 2, 4]) eq_(self.rtable.selected_indexes, [1, 2, 4])
def test_selected_powermarker_node_paths_after_deletion(self, do_setup): def test_selected_powermarker_node_paths_after_deletion(self, do_setup):
# cases where the selected dupes aren't there are correctly handled # cases where the selected dupes aren't there are correctly handled
app = self.app app = self.app
@ -245,7 +245,7 @@ class TestCaseDupeGuruWithResults:
self.rtable.select([0, 1, 2]) self.rtable.select([0, 1, 2])
app.remove_selected() app.remove_selected()
eq_(self.rtable.selected_indexes, []) # no exception eq_(self.rtable.selected_indexes, []) # no exception
def test_selectPowerMarkerRows_after_sort(self, do_setup): def test_selectPowerMarkerRows_after_sort(self, do_setup):
app = self.app app = self.app
objects = self.objects objects = self.objects
@ -256,7 +256,7 @@ class TestCaseDupeGuruWithResults:
assert app.selected_dupes[0] is objects[4] assert app.selected_dupes[0] is objects[4]
assert app.selected_dupes[1] is objects[2] assert app.selected_dupes[1] is objects[2]
assert app.selected_dupes[2] is objects[1] assert app.selected_dupes[2] is objects[1]
def test_toggle_selected_mark_state(self, do_setup): def test_toggle_selected_mark_state(self, do_setup):
app = self.app app = self.app
objects = self.objects objects = self.objects
@ -270,7 +270,7 @@ class TestCaseDupeGuruWithResults:
assert not app.results.is_marked(objects[2]) assert not app.results.is_marked(objects[2])
assert not app.results.is_marked(objects[3]) assert not app.results.is_marked(objects[3])
assert app.results.is_marked(objects[4]) assert app.results.is_marked(objects[4])
def test_toggle_selected_mark_state_with_different_selected_state(self, do_setup): def test_toggle_selected_mark_state_with_different_selected_state(self, do_setup):
# When marking selected dupes with a heterogenous selection, mark all selected dupes. When # When marking selected dupes with a heterogenous selection, mark all selected dupes. When
# it's homogenous, simply toggle. # it's homogenous, simply toggle.
@ -285,7 +285,7 @@ class TestCaseDupeGuruWithResults:
eq_(app.results.mark_count, 2) eq_(app.results.mark_count, 2)
app.toggle_selected_mark_state() app.toggle_selected_mark_state()
eq_(app.results.mark_count, 0) eq_(app.results.mark_count, 0)
def test_refreshDetailsWithSelected(self, do_setup): def test_refreshDetailsWithSelected(self, do_setup):
self.rtable.select([1, 4]) self.rtable.select([1, 4])
eq_(self.dpanel.row(0), ('Filename', 'bar bleh', 'foo bar')) eq_(self.dpanel.row(0), ('Filename', 'bar bleh', 'foo bar'))
@ -293,7 +293,7 @@ class TestCaseDupeGuruWithResults:
self.rtable.select([]) self.rtable.select([])
eq_(self.dpanel.row(0), ('Filename', '---', '---')) eq_(self.dpanel.row(0), ('Filename', '---', '---'))
self.dpanel.view.check_gui_calls(['refresh']) self.dpanel.view.check_gui_calls(['refresh'])
def test_makeSelectedReference(self, do_setup): def test_makeSelectedReference(self, do_setup):
app = self.app app = self.app
objects = self.objects objects = self.objects
@ -302,7 +302,7 @@ class TestCaseDupeGuruWithResults:
app.make_selected_reference() app.make_selected_reference()
assert groups[0].ref is objects[1] assert groups[0].ref is objects[1]
assert groups[1].ref is objects[4] assert groups[1].ref is objects[4]
def test_makeSelectedReference_by_selecting_two_dupes_in_the_same_group(self, do_setup): def test_makeSelectedReference_by_selecting_two_dupes_in_the_same_group(self, do_setup):
app = self.app app = self.app
objects = self.objects objects = self.objects
@ -312,7 +312,7 @@ class TestCaseDupeGuruWithResults:
app.make_selected_reference() app.make_selected_reference()
assert groups[0].ref is objects[1] assert groups[0].ref is objects[1]
assert groups[1].ref is objects[4] assert groups[1].ref is objects[4]
def test_removeSelected(self, do_setup): def test_removeSelected(self, do_setup):
app = self.app app = self.app
self.rtable.select([1, 4]) self.rtable.select([1, 4])
@ -320,7 +320,7 @@ class TestCaseDupeGuruWithResults:
eq_(len(app.results.dupes), 1) # the first path is now selected eq_(len(app.results.dupes), 1) # the first path is now selected
app.remove_selected() app.remove_selected()
eq_(len(app.results.dupes), 0) eq_(len(app.results.dupes), 0)
def test_addDirectory_simple(self, do_setup): def test_addDirectory_simple(self, do_setup):
# There's already a directory in self.app, so adding another once makes 2 of em # There's already a directory in self.app, so adding another once makes 2 of em
app = self.app app = self.app
@ -328,7 +328,7 @@ class TestCaseDupeGuruWithResults:
otherpath = Path(op.dirname(__file__)) otherpath = Path(op.dirname(__file__))
app.add_directory(otherpath) app.add_directory(otherpath)
eq_(len(app.directories), 2) eq_(len(app.directories), 2)
def test_addDirectory_already_there(self, do_setup): def test_addDirectory_already_there(self, do_setup):
app = self.app app = self.app
otherpath = Path(op.dirname(__file__)) otherpath = Path(op.dirname(__file__))
@ -336,13 +336,13 @@ class TestCaseDupeGuruWithResults:
app.add_directory(otherpath) app.add_directory(otherpath)
eq_(len(app.view.messages), 1) eq_(len(app.view.messages), 1)
assert "already" in app.view.messages[0] assert "already" in app.view.messages[0]
def test_addDirectory_does_not_exist(self, do_setup): def test_addDirectory_does_not_exist(self, do_setup):
app = self.app app = self.app
app.add_directory('/does_not_exist') app.add_directory('/does_not_exist')
eq_(len(app.view.messages), 1) eq_(len(app.view.messages), 1)
assert "exist" in app.view.messages[0] assert "exist" in app.view.messages[0]
def test_ignore(self, do_setup): def test_ignore(self, do_setup):
app = self.app app = self.app
self.rtable.select([4]) #The dupe of the second, 2 sized group self.rtable.select([4]) #The dupe of the second, 2 sized group
@ -352,7 +352,7 @@ class TestCaseDupeGuruWithResults:
app.add_selected_to_ignore_list() app.add_selected_to_ignore_list()
#BOTH the ref and the other dupe should have been added #BOTH the ref and the other dupe should have been added
eq_(len(app.scanner.ignore_list), 3) eq_(len(app.scanner.ignore_list), 3)
def test_purgeIgnoreList(self, do_setup, tmpdir): def test_purgeIgnoreList(self, do_setup, tmpdir):
app = self.app app = self.app
p1 = str(tmpdir.join('file1')) p1 = str(tmpdir.join('file1'))
@ -367,19 +367,19 @@ class TestCaseDupeGuruWithResults:
eq_(1,len(app.scanner.ignore_list)) eq_(1,len(app.scanner.ignore_list))
assert app.scanner.ignore_list.AreIgnored(p1,p2) assert app.scanner.ignore_list.AreIgnored(p1,p2)
assert not app.scanner.ignore_list.AreIgnored(dne,p1) assert not app.scanner.ignore_list.AreIgnored(dne,p1)
def test_only_unicode_is_added_to_ignore_list(self, do_setup): def test_only_unicode_is_added_to_ignore_list(self, do_setup):
def FakeIgnore(first,second): def FakeIgnore(first,second):
if not isinstance(first,str): if not isinstance(first,str):
self.fail() self.fail()
if not isinstance(second,str): if not isinstance(second,str):
self.fail() self.fail()
app = self.app app = self.app
app.scanner.ignore_list.Ignore = FakeIgnore app.scanner.ignore_list.Ignore = FakeIgnore
self.rtable.select([4]) self.rtable.select([4])
app.add_selected_to_ignore_list() app.add_selected_to_ignore_list()
def test_cancel_scan_with_previous_results(self, do_setup): def test_cancel_scan_with_previous_results(self, do_setup):
# When doing a scan with results being present prior to the scan, correctly invalidate the # When doing a scan with results being present prior to the scan, correctly invalidate the
# results table. # results table.
@ -388,7 +388,7 @@ class TestCaseDupeGuruWithResults:
add_fake_files_to_directories(app.directories, self.objects) # We want the scan to at least start add_fake_files_to_directories(app.directories, self.objects) # We want the scan to at least start
app.start_scanning() # will be cancelled immediately app.start_scanning() # will be cancelled immediately
eq_(len(self.rtable), 0) eq_(len(self.rtable), 0)
def test_selected_dupes_after_removal(self, do_setup): def test_selected_dupes_after_removal(self, do_setup):
# Purge the app's `selected_dupes` attribute when removing dupes, or else it might cause a # Purge the app's `selected_dupes` attribute when removing dupes, or else it might cause a
# crash later with None refs. # crash later with None refs.
@ -398,7 +398,7 @@ class TestCaseDupeGuruWithResults:
app.remove_marked() app.remove_marked()
eq_(len(self.rtable), 0) eq_(len(self.rtable), 0)
eq_(app.selected_dupes, []) eq_(app.selected_dupes, [])
def test_dont_crash_on_delta_powermarker_dupecount_sort(self, do_setup): def test_dont_crash_on_delta_powermarker_dupecount_sort(self, do_setup):
# Don't crash when sorting by dupe count or percentage while delta+powermarker are enabled. # Don't crash when sorting by dupe count or percentage while delta+powermarker are enabled.
# Ref #238 # Ref #238
@ -410,7 +410,7 @@ class TestCaseDupeGuruWithResults:
# don't crash # don't crash
self.rtable.sort('percentage', False) self.rtable.sort('percentage', False)
# don't crash # don't crash
class TestCaseDupeGuru_renameSelected: class TestCaseDupeGuru_renameSelected:
def pytest_funcarg__do_setup(self, request): def pytest_funcarg__do_setup(self, request):
@ -437,7 +437,7 @@ class TestCaseDupeGuru_renameSelected:
self.groups = groups self.groups = groups
self.p = p self.p = p
self.files = files self.files = files
def test_simple(self, do_setup): def test_simple(self, do_setup):
app = self.app app = self.app
g = self.groups[0] g = self.groups[0]
@ -447,7 +447,7 @@ class TestCaseDupeGuru_renameSelected:
assert 'renamed' in names assert 'renamed' in names
assert 'foo bar 2' not in names assert 'foo bar 2' not in names
eq_(g.dupes[0].name, 'renamed') eq_(g.dupes[0].name, 'renamed')
def test_none_selected(self, do_setup, monkeypatch): def test_none_selected(self, do_setup, monkeypatch):
app = self.app app = self.app
g = self.groups[0] g = self.groups[0]
@ -460,7 +460,7 @@ class TestCaseDupeGuru_renameSelected:
assert 'renamed' not in names assert 'renamed' not in names
assert 'foo bar 2' in names assert 'foo bar 2' in names
eq_(g.dupes[0].name, 'foo bar 2') eq_(g.dupes[0].name, 'foo bar 2')
def test_name_already_exists(self, do_setup, monkeypatch): def test_name_already_exists(self, do_setup, monkeypatch):
app = self.app app = self.app
g = self.groups[0] g = self.groups[0]
@ -473,7 +473,7 @@ class TestCaseDupeGuru_renameSelected:
assert 'foo bar 1' in names assert 'foo bar 1' in names
assert 'foo bar 2' in names assert 'foo bar 2' in names
eq_(g.dupes[0].name, 'foo bar 2') eq_(g.dupes[0].name, 'foo bar 2')
class TestAppWithDirectoriesInTree: class TestAppWithDirectoriesInTree:
def pytest_funcarg__do_setup(self, request): def pytest_funcarg__do_setup(self, request):
@ -487,7 +487,7 @@ class TestAppWithDirectoriesInTree:
self.dtree = app.dtree self.dtree = app.dtree
self.dtree.add_directory(p) self.dtree.add_directory(p)
self.dtree.view.clear_calls() self.dtree.view.clear_calls()
def test_set_root_as_ref_makes_subfolders_ref_as_well(self, do_setup): def test_set_root_as_ref_makes_subfolders_ref_as_well(self, do_setup):
# Setting a node state to something also affect subnodes. These subnodes must be correctly # Setting a node state to something also affect subnodes. These subnodes must be correctly
# refreshed. # refreshed.
@ -500,4 +500,4 @@ class TestAppWithDirectoriesInTree:
subnode = node[0] subnode = node[0]
eq_(subnode.state, 1) eq_(subnode.state, 1)
self.dtree.view.check_gui_calls(['refresh_states']) self.dtree.view.check_gui_calls(['refresh_states'])

50
core/tests/base.py
View File

@ -1,16 +1,16 @@
# Created By: Virgil Dupras # Created By: Virgil Dupras
# Created On: 2011/09/07 # Created On: 2011/09/07
# Copyright 2014 Hardcoded Software (http://www.hardcoded.net) # Copyright 2014 Hardcoded Software (http://www.hardcoded.net)
# #
# This software is licensed under the "BSD" License as described in the "LICENSE" file, # This software is licensed under the "BSD" License as described in the "LICENSE" file,
# which should be included with this package. The terms are also available at # which should be included with this package. The terms are also available at
# http://www.hardcoded.net/licenses/bsd_license # http://www.hardcoded.net/licenses/bsd_license
from hscommon.testutil import TestApp as TestAppBase, eq_, with_app from hscommon.testutil import TestApp as TestAppBase, eq_, with_app
from hscommon.path import Path from hscommon.path import Path
from hscommon.util import get_file_ext, format_size from hscommon.util import get_file_ext, format_size
from hscommon.gui.column import Column from hscommon.gui.column import Column
from jobprogress.job import nulljob, JobCancelled from hscommon.jobprogress.job import nulljob, JobCancelled
from .. import engine from .. import engine
from .. import prioritize from .. import prioritize
@ -23,28 +23,28 @@ from ..gui.prioritize_dialog import PrioritizeDialog
class DupeGuruView: class DupeGuruView:
JOB = nulljob JOB = nulljob
def __init__(self): def __init__(self):
self.messages = [] self.messages = []
def start_job(self, jobid, func, args=()): def start_job(self, jobid, func, args=()):
try: try:
func(self.JOB, *args) func(self.JOB, *args)
except JobCancelled: except JobCancelled:
return return
def get_default(self, key_name): def get_default(self, key_name):
return None return None
def set_default(self, key_name, value): def set_default(self, key_name, value):
pass pass
def show_message(self, msg): def show_message(self, msg):
self.messages.append(msg) self.messages.append(msg)
def ask_yes_no(self, prompt): def ask_yes_no(self, prompt):
return True # always answer yes return True # always answer yes
class ResultTable(ResultTableBase): class ResultTable(ResultTableBase):
COLUMNS = [ COLUMNS = [
@ -55,21 +55,21 @@ class ResultTable(ResultTableBase):
Column('extension', 'Kind'), Column('extension', 'Kind'),
] ]
DELTA_COLUMNS = {'size', } DELTA_COLUMNS = {'size', }
class DupeGuru(DupeGuruBase): class DupeGuru(DupeGuruBase):
NAME = 'dupeGuru' NAME = 'dupeGuru'
METADATA_TO_READ = ['size'] METADATA_TO_READ = ['size']
def __init__(self): def __init__(self):
DupeGuruBase.__init__(self, DupeGuruView()) DupeGuruBase.__init__(self, DupeGuruView())
self.appdata = '/tmp' self.appdata = '/tmp'
def _prioritization_categories(self): def _prioritization_categories(self):
return prioritize.all_categories() return prioritize.all_categories()
def _create_result_table(self): def _create_result_table(self):
return ResultTable(self) return ResultTable(self)
class NamedObject: class NamedObject:
def __init__(self, name="foobar", with_words=False, size=1, folder=None): def __init__(self, name="foobar", with_words=False, size=1, folder=None):
@ -83,10 +83,10 @@ class NamedObject:
if with_words: if with_words:
self.words = getwords(name) self.words = getwords(name)
self.is_ref = False self.is_ref = False
def __bool__(self): def __bool__(self):
return False #Make sure that operations are made correctly when the bool value of files is false. return False #Make sure that operations are made correctly when the bool value of files is false.
def get_display_info(self, group, delta): def get_display_info(self, group, delta):
size = self.size size = self.size
m = group.get_match_of(self) m = group.get_match_of(self)
@ -99,19 +99,19 @@ class NamedObject:
'size': format_size(size, 0, 1, False), 'size': format_size(size, 0, 1, False),
'extension': self.extension if hasattr(self, 'extension') else '---', 'extension': self.extension if hasattr(self, 'extension') else '---',
} }
@property @property
def path(self): def path(self):
return self._folder[self.name] return self._folder[self.name]
@property @property
def folder_path(self): def folder_path(self):
return self.path.parent() return self.path.parent()
@property @property
def extension(self): def extension(self):
return get_file_ext(self.name) return get_file_ext(self.name)
# Returns a group set that looks like that: # Returns a group set that looks like that:
# "foo bar" (1) # "foo bar" (1)
# "bar bleh" (1024) # "bar bleh" (1024)
@ -135,7 +135,7 @@ class TestApp(TestAppBase):
if hasattr(gui, 'columns'): # tables if hasattr(gui, 'columns'): # tables
gui.columns.view = self.make_logger() gui.columns.view = self.make_logger()
return gui return gui
TestAppBase.__init__(self) TestAppBase.__init__(self)
make_gui = self.make_gui make_gui = self.make_gui
self.app = DupeGuru() self.app = DupeGuru()
@ -153,14 +153,14 @@ class TestApp(TestAppBase):
link_gui(self.app.progress_window) link_gui(self.app.progress_window)
link_gui(self.app.progress_window.jobdesc_textfield) link_gui(self.app.progress_window.jobdesc_textfield)
link_gui(self.app.progress_window.progressdesc_textfield) link_gui(self.app.progress_window.progressdesc_textfield)
#--- Helpers #--- Helpers
def select_pri_criterion(self, name): def select_pri_criterion(self, name):
# Select a main prioritize criterion by name instead of by index. Makes tests more # Select a main prioritize criterion by name instead of by index. Makes tests more
# maintainable. # maintainable.
index = self.pdialog.category_list.index(name) index = self.pdialog.category_list.index(name)
self.pdialog.category_list.select(index) self.pdialog.category_list.select(index)
def add_pri_criterion(self, name, index): def add_pri_criterion(self, name, index):
self.select_pri_criterion(name) self.select_pri_criterion(name)
self.pdialog.criteria_list.select([index]) self.pdialog.criteria_list.select([index])

216
core/tests/engine_test.py
View File

@ -1,14 +1,14 @@
# Created By: Virgil Dupras # Created By: Virgil Dupras
# Created On: 2006/01/29 # Created On: 2006/01/29
# Copyright 2014 Hardcoded Software (http://www.hardcoded.net) # Copyright 2014 Hardcoded Software (http://www.hardcoded.net)
# #
# This software is licensed under the "BSD" License as described in the "LICENSE" file, # This software is licensed under the "BSD" License as described in the "LICENSE" file,
# which should be included with this package. The terms are also available at # which should be included with this package. The terms are also available at
# http://www.hardcoded.net/licenses/bsd_license # http://www.hardcoded.net/licenses/bsd_license
import sys import sys
from jobprogress import job from hscommon.jobprogress import job
from hscommon.util import first from hscommon.util import first
from hscommon.testutil import eq_, log_calls from hscommon.testutil import eq_, log_calls
@ -48,119 +48,119 @@ class TestCasegetwords:
def test_spaces(self): def test_spaces(self):
eq_(['a', 'b', 'c', 'd'], getwords("a b c d")) eq_(['a', 'b', 'c', 'd'], getwords("a b c d"))
eq_(['a', 'b', 'c', 'd'], getwords(" a b c d ")) eq_(['a', 'b', 'c', 'd'], getwords(" a b c d "))
def test_splitter_chars(self): def test_splitter_chars(self):
eq_( eq_(
[chr(i) for i in range(ord('a'),ord('z')+1)], [chr(i) for i in range(ord('a'),ord('z')+1)],
getwords("a-b_c&d+e(f)g;h\\i[j]k{l}m:n.o,p<q>r/s?t~u!v@w#x$y*z") getwords("a-b_c&d+e(f)g;h\\i[j]k{l}m:n.o,p<q>r/s?t~u!v@w#x$y*z")
) )
def test_joiner_chars(self): def test_joiner_chars(self):
eq_(["aec"], getwords("a'e\u0301c")) eq_(["aec"], getwords("a'e\u0301c"))
def test_empty(self): def test_empty(self):
eq_([], getwords('')) eq_([], getwords(''))
def test_returns_lowercase(self): def test_returns_lowercase(self):
eq_(['foo', 'bar'], getwords('FOO BAR')) eq_(['foo', 'bar'], getwords('FOO BAR'))
def test_decompose_unicode(self): def test_decompose_unicode(self):
eq_(getwords('foo\xe9bar'), ['fooebar']) eq_(getwords('foo\xe9bar'), ['fooebar'])
class TestCasegetfields: class TestCasegetfields:
def test_simple(self): def test_simple(self):
eq_([['a', 'b'], ['c', 'd', 'e']], getfields('a b - c d e')) eq_([['a', 'b'], ['c', 'd', 'e']], getfields('a b - c d e'))
def test_empty(self): def test_empty(self):
eq_([], getfields('')) eq_([], getfields(''))
def test_cleans_empty_fields(self): def test_cleans_empty_fields(self):
expected = [['a', 'bc', 'def']] expected = [['a', 'bc', 'def']]
actual = getfields(' - a bc def') actual = getfields(' - a bc def')
eq_(expected, actual) eq_(expected, actual)
expected = [['bc', 'def']] expected = [['bc', 'def']]
class TestCaseunpack_fields: class TestCaseunpack_fields:
def test_with_fields(self): def test_with_fields(self):
expected = ['a', 'b', 'c', 'd', 'e', 'f'] expected = ['a', 'b', 'c', 'd', 'e', 'f']
actual = unpack_fields([['a'], ['b', 'c'], ['d', 'e', 'f']]) actual = unpack_fields([['a'], ['b', 'c'], ['d', 'e', 'f']])
eq_(expected, actual) eq_(expected, actual)
def test_without_fields(self): def test_without_fields(self):
expected = ['a', 'b', 'c', 'd', 'e', 'f'] expected = ['a', 'b', 'c', 'd', 'e', 'f']
actual = unpack_fields(['a', 'b', 'c', 'd', 'e', 'f']) actual = unpack_fields(['a', 'b', 'c', 'd', 'e', 'f'])
eq_(expected, actual) eq_(expected, actual)
def test_empty(self): def test_empty(self):
eq_([], unpack_fields([])) eq_([], unpack_fields([]))
class TestCaseWordCompare: class TestCaseWordCompare:
def test_list(self): def test_list(self):
eq_(100, compare(['a', 'b', 'c', 'd'],['a', 'b', 'c', 'd'])) eq_(100, compare(['a', 'b', 'c', 'd'],['a', 'b', 'c', 'd']))
eq_(86, compare(['a', 'b', 'c', 'd'],['a', 'b', 'c'])) eq_(86, compare(['a', 'b', 'c', 'd'],['a', 'b', 'c']))
def test_unordered(self): def test_unordered(self):
#Sometimes, users don't want fuzzy matching too much When they set the slider #Sometimes, users don't want fuzzy matching too much When they set the slider
#to 100, they don't expect a filename with the same words, but not the same order, to match. #to 100, they don't expect a filename with the same words, but not the same order, to match.
#Thus, we want to return 99 in that case. #Thus, we want to return 99 in that case.
eq_(99, compare(['a', 'b', 'c', 'd'], ['d', 'b', 'c', 'a'])) eq_(99, compare(['a', 'b', 'c', 'd'], ['d', 'b', 'c', 'a']))
def test_word_occurs_twice(self): def test_word_occurs_twice(self):
#if a word occurs twice in first, but once in second, we want the word to be only counted once #if a word occurs twice in first, but once in second, we want the word to be only counted once
eq_(89, compare(['a', 'b', 'c', 'd', 'a'], ['d', 'b', 'c', 'a'])) eq_(89, compare(['a', 'b', 'c', 'd', 'a'], ['d', 'b', 'c', 'a']))
def test_uses_copy_of_lists(self): def test_uses_copy_of_lists(self):
first = ['foo', 'bar'] first = ['foo', 'bar']
second = ['bar', 'bleh'] second = ['bar', 'bleh']
compare(first, second) compare(first, second)
eq_(['foo', 'bar'], first) eq_(['foo', 'bar'], first)
eq_(['bar', 'bleh'], second) eq_(['bar', 'bleh'], second)
def test_word_weight(self): def test_word_weight(self):
eq_(int((6.0 / 13.0) * 100), compare(['foo', 'bar'], ['bar', 'bleh'], (WEIGHT_WORDS, ))) eq_(int((6.0 / 13.0) * 100), compare(['foo', 'bar'], ['bar', 'bleh'], (WEIGHT_WORDS, )))
def test_similar_words(self): def test_similar_words(self):
eq_(100, compare(['the', 'white', 'stripes'],['the', 'whites', 'stripe'], (MATCH_SIMILAR_WORDS, ))) eq_(100, compare(['the', 'white', 'stripes'],['the', 'whites', 'stripe'], (MATCH_SIMILAR_WORDS, )))
def test_empty(self): def test_empty(self):
eq_(0, compare([], [])) eq_(0, compare([], []))
def test_with_fields(self): def test_with_fields(self):
eq_(67, compare([['a', 'b'], ['c', 'd', 'e']], [['a', 'b'], ['c', 'd', 'f']])) eq_(67, compare([['a', 'b'], ['c', 'd', 'e']], [['a', 'b'], ['c', 'd', 'f']]))
def test_propagate_flags_with_fields(self, monkeypatch): def test_propagate_flags_with_fields(self, monkeypatch):
def mock_compare(first, second, flags): def mock_compare(first, second, flags):
eq_((0, 1, 2, 3, 5), flags) eq_((0, 1, 2, 3, 5), flags)
monkeypatch.setattr(engine, 'compare_fields', mock_compare) monkeypatch.setattr(engine, 'compare_fields', mock_compare)
compare([['a']], [['a']], (0, 1, 2, 3, 5)) compare([['a']], [['a']], (0, 1, 2, 3, 5))
class TestCaseWordCompareWithFields: class TestCaseWordCompareWithFields:
def test_simple(self): def test_simple(self):
eq_(67, compare_fields([['a', 'b'], ['c', 'd', 'e']], [['a', 'b'], ['c', 'd', 'f']])) eq_(67, compare_fields([['a', 'b'], ['c', 'd', 'e']], [['a', 'b'], ['c', 'd', 'f']]))
def test_empty(self): def test_empty(self):
eq_(0, compare_fields([], [])) eq_(0, compare_fields([], []))
def test_different_length(self): def test_different_length(self):
eq_(0, compare_fields([['a'], ['b']], [['a'], ['b'], ['c']])) eq_(0, compare_fields([['a'], ['b']], [['a'], ['b'], ['c']]))
def test_propagates_flags(self, monkeypatch): def test_propagates_flags(self, monkeypatch):
def mock_compare(first, second, flags): def mock_compare(first, second, flags):
eq_((0, 1, 2, 3, 5), flags) eq_((0, 1, 2, 3, 5), flags)
monkeypatch.setattr(engine, 'compare_fields', mock_compare) monkeypatch.setattr(engine, 'compare_fields', mock_compare)
compare_fields([['a']], [['a']],(0, 1, 2, 3, 5)) compare_fields([['a']], [['a']],(0, 1, 2, 3, 5))
def test_order(self): def test_order(self):
first = [['a', 'b'], ['c', 'd', 'e']] first = [['a', 'b'], ['c', 'd', 'e']]
second = [['c', 'd', 'f'], ['a', 'b']] second = [['c', 'd', 'f'], ['a', 'b']]
eq_(0, compare_fields(first, second)) eq_(0, compare_fields(first, second))
def test_no_order(self): def test_no_order(self):
first = [['a','b'],['c','d','e']] first = [['a','b'],['c','d','e']]
second = [['c','d','f'],['a','b']] second = [['c','d','f'],['a','b']]
@ -168,10 +168,10 @@ class TestCaseWordCompareWithFields:
first = [['a','b'],['a','b']] #a field can only be matched once. first = [['a','b'],['a','b']] #a field can only be matched once.
second = [['c','d','f'],['a','b']] second = [['c','d','f'],['a','b']]
eq_(0, compare_fields(first, second, (NO_FIELD_ORDER, ))) eq_(0, compare_fields(first, second, (NO_FIELD_ORDER, )))
first = [['a','b'],['a','b','c']] first = [['a','b'],['a','b','c']]
second = [['c','d','f'],['a','b']] second = [['c','d','f'],['a','b']]
eq_(33, compare_fields(first, second, (NO_FIELD_ORDER, ))) eq_(33, compare_fields(first, second, (NO_FIELD_ORDER, )))
def test_compare_fields_without_order_doesnt_alter_fields(self): def test_compare_fields_without_order_doesnt_alter_fields(self):
#The NO_ORDER comp type altered the fields! #The NO_ORDER comp type altered the fields!
first = [['a','b'],['c','d','e']] first = [['a','b'],['c','d','e']]
@ -179,7 +179,7 @@ class TestCaseWordCompareWithFields:
eq_(67, compare_fields(first, second, (NO_FIELD_ORDER, ))) eq_(67, compare_fields(first, second, (NO_FIELD_ORDER, )))
eq_([['a','b'],['c','d','e']],first) eq_([['a','b'],['c','d','e']],first)
eq_([['c','d','f'],['a','b']],second) eq_([['c','d','f'],['a','b']],second)
class TestCasebuild_word_dict: class TestCasebuild_word_dict:
def test_with_standard_words(self): def test_with_standard_words(self):
@ -199,30 +199,30 @@ class TestCasebuild_word_dict:
assert l[2] in d['baz'] assert l[2] in d['baz']
eq_(1,len(d['bleh'])) eq_(1,len(d['bleh']))
assert l[2] in d['bleh'] assert l[2] in d['bleh']
def test_unpack_fields(self): def test_unpack_fields(self):
o = NamedObject('') o = NamedObject('')
o.words = [['foo','bar'],['baz']] o.words = [['foo','bar'],['baz']]
d = build_word_dict([o]) d = build_word_dict([o])
eq_(3,len(d)) eq_(3,len(d))
eq_(1,len(d['foo'])) eq_(1,len(d['foo']))
def test_words_are_unaltered(self): def test_words_are_unaltered(self):
o = NamedObject('') o = NamedObject('')
o.words = [['foo','bar'],['baz']] o.words = [['foo','bar'],['baz']]
build_word_dict([o]) build_word_dict([o])
eq_([['foo','bar'],['baz']],o.words) eq_([['foo','bar'],['baz']],o.words)
def test_object_instances_can_only_be_once_in_words_object_list(self): def test_object_instances_can_only_be_once_in_words_object_list(self):
o = NamedObject('foo foo',True) o = NamedObject('foo foo',True)
d = build_word_dict([o]) d = build_word_dict([o])
eq_(1,len(d['foo'])) eq_(1,len(d['foo']))
def test_job(self): def test_job(self):
def do_progress(p,d=''): def do_progress(p,d=''):
self.log.append(p) self.log.append(p)
return True return True
j = job.Job(1,do_progress) j = job.Job(1,do_progress)
self.log = [] self.log = []
s = "foo bar" s = "foo bar"
@ -230,7 +230,7 @@ class TestCasebuild_word_dict:
# We don't have intermediate log because iter_with_progress is called with every > 1 # We don't have intermediate log because iter_with_progress is called with every > 1
eq_(0,self.log[0]) eq_(0,self.log[0])
eq_(100,self.log[1]) eq_(100,self.log[1])
class TestCasemerge_similar_words: class TestCasemerge_similar_words:
def test_some_similar_words(self): def test_some_similar_words(self):
@ -242,8 +242,8 @@ class TestCasemerge_similar_words:
merge_similar_words(d) merge_similar_words(d)
eq_(1,len(d)) eq_(1,len(d))
eq_(3,len(d['foobar'])) eq_(3,len(d['foobar']))
class TestCasereduce_common_words: class TestCasereduce_common_words:
def test_typical(self): def test_typical(self):
@ -254,7 +254,7 @@ class TestCasereduce_common_words:
reduce_common_words(d, 50) reduce_common_words(d, 50)
assert 'foo' not in d assert 'foo' not in d
eq_(49,len(d['bar'])) eq_(49,len(d['bar']))
def test_dont_remove_objects_with_only_common_words(self): def test_dont_remove_objects_with_only_common_words(self):
d = { d = {
'common': set([NamedObject("common uncommon",True) for i in range(50)] + [NamedObject("common",True)]), 'common': set([NamedObject("common uncommon",True) for i in range(50)] + [NamedObject("common",True)]),
@ -263,7 +263,7 @@ class TestCasereduce_common_words:
reduce_common_words(d, 50) reduce_common_words(d, 50)
eq_(1,len(d['common'])) eq_(1,len(d['common']))
eq_(1,len(d['uncommon'])) eq_(1,len(d['uncommon']))
def test_values_still_are_set_instances(self): def test_values_still_are_set_instances(self):
d = { d = {
'common': set([NamedObject("common uncommon",True) for i in range(50)] + [NamedObject("common",True)]), 'common': set([NamedObject("common uncommon",True) for i in range(50)] + [NamedObject("common",True)]),
@ -272,7 +272,7 @@ class TestCasereduce_common_words:
reduce_common_words(d, 50) reduce_common_words(d, 50)
assert isinstance(d['common'],set) assert isinstance(d['common'],set)
assert isinstance(d['uncommon'],set) assert isinstance(d['uncommon'],set)
def test_dont_raise_KeyError_when_a_word_has_been_removed(self): def test_dont_raise_KeyError_when_a_word_has_been_removed(self):
#If a word has been removed by the reduce, an object in a subsequent common word that #If a word has been removed by the reduce, an object in a subsequent common word that
#contains the word that has been removed would cause a KeyError. #contains the word that has been removed would cause a KeyError.
@ -285,14 +285,14 @@ class TestCasereduce_common_words:
reduce_common_words(d, 50) reduce_common_words(d, 50)
except KeyError: except KeyError:
self.fail() self.fail()
def test_unpack_fields(self): def test_unpack_fields(self):
#object.words may be fields. #object.words may be fields.
def create_it(): def create_it():
o = NamedObject('') o = NamedObject('')
o.words = [['foo','bar'],['baz']] o.words = [['foo','bar'],['baz']]
return o return o
d = { d = {
'foo': set([create_it() for i in range(50)]) 'foo': set([create_it() for i in range(50)])
} }
@ -300,7 +300,7 @@ class TestCasereduce_common_words:
reduce_common_words(d, 50) reduce_common_words(d, 50)
except TypeError: except TypeError:
self.fail("must support fields.") self.fail("must support fields.")
def test_consider_a_reduced_common_word_common_even_after_reduction(self): def test_consider_a_reduced_common_word_common_even_after_reduction(self):
#There was a bug in the code that causeda word that has already been reduced not to #There was a bug in the code that causeda word that has already been reduced not to
#be counted as a common word for subsequent words. For example, if 'foo' is processed #be counted as a common word for subsequent words. For example, if 'foo' is processed
@ -316,7 +316,7 @@ class TestCasereduce_common_words:
eq_(1,len(d['foo'])) eq_(1,len(d['foo']))
eq_(1,len(d['bar'])) eq_(1,len(d['bar']))
eq_(49,len(d['baz'])) eq_(49,len(d['baz']))
class TestCaseget_match: class TestCaseget_match:
def test_simple(self): def test_simple(self):
@ -328,7 +328,7 @@ class TestCaseget_match:
eq_(['bar','bleh'],m.second.words) eq_(['bar','bleh'],m.second.words)
assert m.first is o1 assert m.first is o1
assert m.second is o2 assert m.second is o2
def test_in(self): def test_in(self):
o1 = NamedObject("foo",True) o1 = NamedObject("foo",True)
o2 = NamedObject("bar",True) o2 = NamedObject("bar",True)
@ -336,15 +336,15 @@ class TestCaseget_match:
assert o1 in m assert o1 in m
assert o2 in m assert o2 in m
assert object() not in m assert object() not in m
def test_word_weight(self): def test_word_weight(self):
eq_(int((6.0 / 13.0) * 100),get_match(NamedObject("foo bar",True),NamedObject("bar bleh",True),(WEIGHT_WORDS,)).percentage) eq_(int((6.0 / 13.0) * 100),get_match(NamedObject("foo bar",True),NamedObject("bar bleh",True),(WEIGHT_WORDS,)).percentage)
class TestCaseGetMatches: class TestCaseGetMatches:
def test_empty(self): def test_empty(self):
eq_(getmatches([]), []) eq_(getmatches([]), [])
def test_simple(self): def test_simple(self):
l = [NamedObject("foo bar"),NamedObject("bar bleh"),NamedObject("a b c foo")] l = [NamedObject("foo bar"),NamedObject("bar bleh"),NamedObject("a b c foo")]
r = getmatches(l) r = getmatches(l)
@ -353,7 +353,7 @@ class TestCaseGetMatches:
assert_match(m, 'foo bar', 'bar bleh') assert_match(m, 'foo bar', 'bar bleh')
m = first(m for m in r if m.percentage == 33) #"foo bar" and "a b c foo" m = first(m for m in r if m.percentage == 33) #"foo bar" and "a b c foo"
assert_match(m, 'foo bar', 'a b c foo') assert_match(m, 'foo bar', 'a b c foo')
def test_null_and_unrelated_objects(self): def test_null_and_unrelated_objects(self):
l = [NamedObject("foo bar"),NamedObject("bar bleh"),NamedObject(""),NamedObject("unrelated object")] l = [NamedObject("foo bar"),NamedObject("bar bleh"),NamedObject(""),NamedObject("unrelated object")]
r = getmatches(l) r = getmatches(l)
@ -361,22 +361,22 @@ class TestCaseGetMatches:
m = r[0] m = r[0]
eq_(m.percentage, 50) eq_(m.percentage, 50)
assert_match(m, 'foo bar', 'bar bleh') assert_match(m, 'foo bar', 'bar bleh')
def test_twice_the_same_word(self): def test_twice_the_same_word(self):
l = [NamedObject("foo foo bar"),NamedObject("bar bleh")] l = [NamedObject("foo foo bar"),NamedObject("bar bleh")]
r = getmatches(l) r = getmatches(l)
eq_(1,len(r)) eq_(1,len(r))
def test_twice_the_same_word_when_preworded(self): def test_twice_the_same_word_when_preworded(self):
l = [NamedObject("foo foo bar",True),NamedObject("bar bleh",True)] l = [NamedObject("foo foo bar",True),NamedObject("bar bleh",True)]
r = getmatches(l) r = getmatches(l)
eq_(1,len(r)) eq_(1,len(r))
def test_two_words_match(self): def test_two_words_match(self):
l = [NamedObject("foo bar"),NamedObject("foo bar bleh")] l = [NamedObject("foo bar"),NamedObject("foo bar bleh")]
r = getmatches(l) r = getmatches(l)
eq_(1,len(r)) eq_(1,len(r))
def test_match_files_with_only_common_words(self): def test_match_files_with_only_common_words(self):
#If a word occurs more than 50 times, it is excluded from the matching process #If a word occurs more than 50 times, it is excluded from the matching process
#The problem with the common_word_threshold is that the files containing only common #The problem with the common_word_threshold is that the files containing only common
@ -385,18 +385,18 @@ class TestCaseGetMatches:
l = [NamedObject("foo") for i in range(50)] l = [NamedObject("foo") for i in range(50)]
r = getmatches(l) r = getmatches(l)
eq_(1225,len(r)) eq_(1225,len(r))
def test_use_words_already_there_if_there(self): def test_use_words_already_there_if_there(self):
o1 = NamedObject('foo') o1 = NamedObject('foo')
o2 = NamedObject('bar') o2 = NamedObject('bar')
o2.words = ['foo'] o2.words = ['foo']
eq_(1, len(getmatches([o1,o2]))) eq_(1, len(getmatches([o1,o2])))
def test_job(self): def test_job(self):
def do_progress(p,d=''): def do_progress(p,d=''):
self.log.append(p) self.log.append(p)
return True return True
j = job.Job(1,do_progress) j = job.Job(1,do_progress)
self.log = [] self.log = []
s = "foo bar" s = "foo bar"
@ -404,12 +404,12 @@ class TestCaseGetMatches:
assert len(self.log) > 2 assert len(self.log) > 2
eq_(0,self.log[0]) eq_(0,self.log[0])
eq_(100,self.log[-1]) eq_(100,self.log[-1])
def test_weight_words(self): def test_weight_words(self):
l = [NamedObject("foo bar"),NamedObject("bar bleh")] l = [NamedObject("foo bar"),NamedObject("bar bleh")]
m = getmatches(l, weight_words=True)[0] m = getmatches(l, weight_words=True)[0]
eq_(int((6.0 / 13.0) * 100),m.percentage) eq_(int((6.0 / 13.0) * 100),m.percentage)
def test_similar_word(self): def test_similar_word(self):
l = [NamedObject("foobar"),NamedObject("foobars")] l = [NamedObject("foobar"),NamedObject("foobars")]
eq_(len(getmatches(l, match_similar_words=True)), 1) eq_(len(getmatches(l, match_similar_words=True)), 1)
@ -420,16 +420,16 @@ class TestCaseGetMatches:
eq_(len(getmatches(l, match_similar_words=True)), 1) eq_(len(getmatches(l, match_similar_words=True)), 1)
l = [NamedObject("foobar"),NamedObject("foosbar")] l = [NamedObject("foobar"),NamedObject("foosbar")]
eq_(len(getmatches(l, match_similar_words=True)), 1) eq_(len(getmatches(l, match_similar_words=True)), 1)
def test_single_object_with_similar_words(self): def test_single_object_with_similar_words(self):
l = [NamedObject("foo foos")] l = [NamedObject("foo foos")]
eq_(len(getmatches(l, match_similar_words=True)), 0) eq_(len(getmatches(l, match_similar_words=True)), 0)
def test_double_words_get_counted_only_once(self): def test_double_words_get_counted_only_once(self):
l = [NamedObject("foo bar foo bleh"),NamedObject("foo bar bleh bar")] l = [NamedObject("foo bar foo bleh"),NamedObject("foo bar bleh bar")]
m = getmatches(l)[0] m = getmatches(l)[0]
eq_(75,m.percentage) eq_(75,m.percentage)
def test_with_fields(self): def test_with_fields(self):
o1 = NamedObject("foo bar - foo bleh") o1 = NamedObject("foo bar - foo bleh")
o2 = NamedObject("foo bar - bleh bar") o2 = NamedObject("foo bar - bleh bar")
@ -437,7 +437,7 @@ class TestCaseGetMatches:
o2.words = getfields(o2.name) o2.words = getfields(o2.name)
m = getmatches([o1, o2])[0] m = getmatches([o1, o2])[0]
eq_(50, m.percentage) eq_(50, m.percentage)
def test_with_fields_no_order(self): def test_with_fields_no_order(self):
o1 = NamedObject("foo bar - foo bleh") o1 = NamedObject("foo bar - foo bleh")
o2 = NamedObject("bleh bang - foo bar") o2 = NamedObject("bleh bang - foo bar")
@ -445,11 +445,11 @@ class TestCaseGetMatches:
o2.words = getfields(o2.name) o2.words = getfields(o2.name)
m = getmatches([o1, o2], no_field_order=True)[0] m = getmatches([o1, o2], no_field_order=True)[0]
eq_(m.percentage, 50) eq_(m.percentage, 50)
def test_only_match_similar_when_the_option_is_set(self): def test_only_match_similar_when_the_option_is_set(self):
l = [NamedObject("foobar"),NamedObject("foobars")] l = [NamedObject("foobar"),NamedObject("foobars")]
eq_(len(getmatches(l, match_similar_words=False)), 0) eq_(len(getmatches(l, match_similar_words=False)), 0)
def test_dont_recurse_do_match(self): def test_dont_recurse_do_match(self):
# with nosetests, the stack is increased. The number has to be high enough not to be failing falsely # with nosetests, the stack is increased. The number has to be high enough not to be failing falsely
sys.setrecursionlimit(100) sys.setrecursionlimit(100)
@ -460,19 +460,19 @@ class TestCaseGetMatches:
self.fail() self.fail()
finally: finally:
sys.setrecursionlimit(1000) sys.setrecursionlimit(1000)
def test_min_match_percentage(self): def test_min_match_percentage(self):
l = [NamedObject("foo bar"),NamedObject("bar bleh"),NamedObject("a b c foo")] l = [NamedObject("foo bar"),NamedObject("bar bleh"),NamedObject("a b c foo")]
r = getmatches(l, min_match_percentage=50) r = getmatches(l, min_match_percentage=50)
eq_(1,len(r)) #Only "foo bar" / "bar bleh" should match eq_(1,len(r)) #Only "foo bar" / "bar bleh" should match
def test_MemoryError(self, monkeypatch): def test_MemoryError(self, monkeypatch):
@log_calls @log_calls
def mocked_match(first, second, flags): def mocked_match(first, second, flags):
if len(mocked_match.calls) > 42: if len(mocked_match.calls) > 42:
raise MemoryError() raise MemoryError()
return Match(first, second, 0) return Match(first, second, 0)
objects = [NamedObject() for i in range(10)] # results in 45 matches objects = [NamedObject() for i in range(10)] # results in 45 matches
monkeypatch.setattr(engine, 'get_match', mocked_match) monkeypatch.setattr(engine, 'get_match', mocked_match)
try: try:
@ -480,13 +480,13 @@ class TestCaseGetMatches:
except MemoryError: except MemoryError:
self.fail('MemorryError must be handled') self.fail('MemorryError must be handled')
eq_(42, len(r)) eq_(42, len(r))
class TestCaseGetMatchesByContents: class TestCaseGetMatchesByContents:
def test_dont_compare_empty_files(self): def test_dont_compare_empty_files(self):
o1, o2 = no(size=0), no(size=0) o1, o2 = no(size=0), no(size=0)
assert not getmatches_by_contents([o1, o2]) assert not getmatches_by_contents([o1, o2])
class TestCaseGroup: class TestCaseGroup:
def test_empy(self): def test_empy(self):
@ -494,7 +494,7 @@ class TestCaseGroup:
eq_(None,g.ref) eq_(None,g.ref)
eq_([],g.dupes) eq_([],g.dupes)
eq_(0,len(g.matches)) eq_(0,len(g.matches))
def test_add_match(self): def test_add_match(self):
g = Group() g = Group()
m = get_match(NamedObject("foo",True),NamedObject("bar",True)) m = get_match(NamedObject("foo",True),NamedObject("bar",True))
@ -503,7 +503,7 @@ class TestCaseGroup:
eq_([m.second],g.dupes) eq_([m.second],g.dupes)
eq_(1,len(g.matches)) eq_(1,len(g.matches))
assert m in g.matches assert m in g.matches
def test_multiple_add_match(self): def test_multiple_add_match(self):
g = Group() g = Group()
o1 = NamedObject("a",True) o1 = NamedObject("a",True)
@ -529,13 +529,13 @@ class TestCaseGroup:
g.add_match(get_match(o3,o4)) g.add_match(get_match(o3,o4))
eq_([o2,o3,o4],g.dupes) eq_([o2,o3,o4],g.dupes)
eq_(6,len(g.matches)) eq_(6,len(g.matches))
def test_len(self): def test_len(self):
g = Group() g = Group()
eq_(0,len(g)) eq_(0,len(g))
g.add_match(get_match(NamedObject("foo",True),NamedObject("bar",True))) g.add_match(get_match(NamedObject("foo",True),NamedObject("bar",True)))
eq_(2,len(g)) eq_(2,len(g))
def test_add_same_match_twice(self): def test_add_same_match_twice(self):
g = Group() g = Group()
m = get_match(NamedObject("foo",True),NamedObject("foo",True)) m = get_match(NamedObject("foo",True),NamedObject("foo",True))
@ -545,7 +545,7 @@ class TestCaseGroup:
g.add_match(m) g.add_match(m)
eq_(2,len(g)) eq_(2,len(g))
eq_(1,len(g.matches)) eq_(1,len(g.matches))
def test_in(self): def test_in(self):
g = Group() g = Group()
o1 = NamedObject("foo",True) o1 = NamedObject("foo",True)
@ -554,7 +554,7 @@ class TestCaseGroup:
g.add_match(get_match(o1,o2)) g.add_match(get_match(o1,o2))
assert o1 in g assert o1 in g
assert o2 in g assert o2 in g
def test_remove(self): def test_remove(self):
g = Group() g = Group()
o1 = NamedObject("foo",True) o1 = NamedObject("foo",True)
@ -571,7 +571,7 @@ class TestCaseGroup:
g.remove_dupe(o1) g.remove_dupe(o1)
eq_(0,len(g.matches)) eq_(0,len(g.matches))
eq_(0,len(g)) eq_(0,len(g))
def test_remove_with_ref_dupes(self): def test_remove_with_ref_dupes(self):
g = Group() g = Group()
o1 = NamedObject("foo",True) o1 = NamedObject("foo",True)
@ -584,7 +584,7 @@ class TestCaseGroup:
o2.is_ref = True o2.is_ref = True
g.remove_dupe(o3) g.remove_dupe(o3)
eq_(0,len(g)) eq_(0,len(g))
def test_switch_ref(self): def test_switch_ref(self):
o1 = NamedObject(with_words=True) o1 = NamedObject(with_words=True)
o2 = NamedObject(with_words=True) o2 = NamedObject(with_words=True)
@ -598,7 +598,7 @@ class TestCaseGroup:
assert o2 is g.ref assert o2 is g.ref
g.switch_ref(NamedObject('',True)) g.switch_ref(NamedObject('',True))
assert o2 is g.ref assert o2 is g.ref
def test_switch_ref_from_ref_dir(self): def test_switch_ref_from_ref_dir(self):
# When the ref dupe is from a ref dir, switch_ref() does nothing # When the ref dupe is from a ref dir, switch_ref() does nothing
o1 = no(with_words=True) o1 = no(with_words=True)
@ -608,7 +608,7 @@ class TestCaseGroup:
g.add_match(get_match(o1, o2)) g.add_match(get_match(o1, o2))
g.switch_ref(o2) g.switch_ref(o2)
assert o1 is g.ref assert o1 is g.ref
def test_get_match_of(self): def test_get_match_of(self):
g = Group() g = Group()
for m in get_match_triangle(): for m in get_match_triangle():
@ -619,7 +619,7 @@ class TestCaseGroup:
assert o in m assert o in m
assert g.get_match_of(NamedObject('',True)) is None assert g.get_match_of(NamedObject('',True)) is None
assert g.get_match_of(g.ref) is None assert g.get_match_of(g.ref) is None
def test_percentage(self): def test_percentage(self):
#percentage should return the avg percentage in relation to the ref #percentage should return the avg percentage in relation to the ref
m1,m2,m3 = get_match_triangle() m1,m2,m3 = get_match_triangle()
@ -638,11 +638,11 @@ class TestCaseGroup:
g.add_match(m1) g.add_match(m1)
g.add_match(m2) g.add_match(m2)
eq_(66,g.percentage) eq_(66,g.percentage)
def test_percentage_on_empty_group(self): def test_percentage_on_empty_group(self):
g = Group() g = Group()
eq_(0,g.percentage) eq_(0,g.percentage)
def test_prioritize(self): def test_prioritize(self):
m1,m2,m3 = get_match_triangle() m1,m2,m3 = get_match_triangle()
o1 = m1.first o1 = m1.first
@ -658,7 +658,7 @@ class TestCaseGroup:
assert o1 is g.ref assert o1 is g.ref
assert g.prioritize(lambda x:x.name) assert g.prioritize(lambda x:x.name)
assert o3 is g.ref assert o3 is g.ref
def test_prioritize_with_tie_breaker(self): def test_prioritize_with_tie_breaker(self):
# if the ref has the same key as one or more of the dupe, run the tie_breaker func among them # if the ref has the same key as one or more of the dupe, run the tie_breaker func among them
g = get_test_group() g = get_test_group()
@ -666,9 +666,9 @@ class TestCaseGroup:
tie_breaker = lambda ref, dupe: dupe is o3 tie_breaker = lambda ref, dupe: dupe is o3
g.prioritize(lambda x:0, tie_breaker) g.prioritize(lambda x:0, tie_breaker)
assert g.ref is o3 assert g.ref is o3
def test_prioritize_with_tie_breaker_runs_on_all_dupes(self): def test_prioritize_with_tie_breaker_runs_on_all_dupes(self):
# Even if a dupe is chosen to switch with ref with a tie breaker, we still run the tie breaker # Even if a dupe is chosen to switch with ref with a tie breaker, we still run the tie breaker
# with other dupes and the newly chosen ref # with other dupes and the newly chosen ref
g = get_test_group() g = get_test_group()
o1, o2, o3 = g.ordered o1, o2, o3 = g.ordered
@ -678,7 +678,7 @@ class TestCaseGroup:
tie_breaker = lambda ref, dupe: dupe.foo > ref.foo tie_breaker = lambda ref, dupe: dupe.foo > ref.foo
g.prioritize(lambda x:0, tie_breaker) g.prioritize(lambda x:0, tie_breaker)
assert g.ref is o3 assert g.ref is o3
def test_prioritize_with_tie_breaker_runs_only_on_tie_dupes(self): def test_prioritize_with_tie_breaker_runs_only_on_tie_dupes(self):
# The tie breaker only runs on dupes that had the same value for the key_func # The tie breaker only runs on dupes that had the same value for the key_func
g = get_test_group() g = get_test_group()
@ -693,7 +693,7 @@ class TestCaseGroup:
tie_breaker = lambda ref, dupe: dupe.bar > ref.bar tie_breaker = lambda ref, dupe: dupe.bar > ref.bar
g.prioritize(key_func, tie_breaker) g.prioritize(key_func, tie_breaker)
assert g.ref is o2 assert g.ref is o2
def test_prioritize_with_ref_dupe(self): def test_prioritize_with_ref_dupe(self):
# when the ref dupe of a group is from a ref dir, make it stay on top. # when the ref dupe of a group is from a ref dir, make it stay on top.
g = get_test_group() g = get_test_group()
@ -702,7 +702,7 @@ class TestCaseGroup:
o2.size = 2 o2.size = 2
g.prioritize(lambda x: -x.size) g.prioritize(lambda x: -x.size)
assert g.ref is o1 assert g.ref is o1
def test_prioritize_nothing_changes(self): def test_prioritize_nothing_changes(self):
# prioritize() returns False when nothing changes in the group. # prioritize() returns False when nothing changes in the group.
g = get_test_group() g = get_test_group()
@ -710,14 +710,14 @@ class TestCaseGroup:
g[1].name = 'b' g[1].name = 'b'
g[2].name = 'c' g[2].name = 'c'
assert not g.prioritize(lambda x:x.name) assert not g.prioritize(lambda x:x.name)
def test_list_like(self): def test_list_like(self):
g = Group() g = Group()
o1,o2 = (NamedObject("foo",True),NamedObject("bar",True)) o1,o2 = (NamedObject("foo",True),NamedObject("bar",True))
g.add_match(get_match(o1,o2)) g.add_match(get_match(o1,o2))
assert g[0] is o1 assert g[0] is o1
assert g[1] is o2 assert g[1] is o2
def test_discard_matches(self): def test_discard_matches(self):
g = Group() g = Group()
o1,o2,o3 = (NamedObject("foo",True),NamedObject("bar",True),NamedObject("baz",True)) o1,o2,o3 = (NamedObject("foo",True),NamedObject("bar",True),NamedObject("baz",True))
@ -726,13 +726,13 @@ class TestCaseGroup:
g.discard_matches() g.discard_matches()
eq_(1,len(g.matches)) eq_(1,len(g.matches))
eq_(0,len(g.candidates)) eq_(0,len(g.candidates))
class TestCaseget_groups: class TestCaseget_groups:
def test_empty(self): def test_empty(self):
r = get_groups([]) r = get_groups([])
eq_([],r) eq_([],r)
def test_simple(self): def test_simple(self):
l = [NamedObject("foo bar"),NamedObject("bar bleh")] l = [NamedObject("foo bar"),NamedObject("bar bleh")]
matches = getmatches(l) matches = getmatches(l)
@ -742,7 +742,7 @@ class TestCaseget_groups:
g = r[0] g = r[0]
assert g.ref is m.first assert g.ref is m.first
eq_([m.second],g.dupes) eq_([m.second],g.dupes)
def test_group_with_multiple_matches(self): def test_group_with_multiple_matches(self):
#This results in 3 matches #This results in 3 matches
l = [NamedObject("foo"),NamedObject("foo"),NamedObject("foo")] l = [NamedObject("foo"),NamedObject("foo"),NamedObject("foo")]
@ -751,7 +751,7 @@ class TestCaseget_groups:
eq_(1,len(r)) eq_(1,len(r))
g = r[0] g = r[0]
eq_(3,len(g)) eq_(3,len(g))
def test_must_choose_a_group(self): def test_must_choose_a_group(self):
l = [NamedObject("a b"),NamedObject("a b"),NamedObject("b c"),NamedObject("c d"),NamedObject("c d")] l = [NamedObject("a b"),NamedObject("a b"),NamedObject("b c"),NamedObject("c d"),NamedObject("c d")]
#There will be 2 groups here: group "a b" and group "c d" #There will be 2 groups here: group "a b" and group "c d"
@ -760,7 +760,7 @@ class TestCaseget_groups:
r = get_groups(matches) r = get_groups(matches)
eq_(2,len(r)) eq_(2,len(r))
eq_(5,len(r[0])+len(r[1])) eq_(5,len(r[0])+len(r[1]))
def test_should_all_go_in_the_same_group(self): def test_should_all_go_in_the_same_group(self):
l = [NamedObject("a b"),NamedObject("a b"),NamedObject("a b"),NamedObject("a b")] l = [NamedObject("a b"),NamedObject("a b"),NamedObject("a b"),NamedObject("a b")]
#There will be 2 groups here: group "a b" and group "c d" #There will be 2 groups here: group "a b" and group "c d"
@ -768,7 +768,7 @@ class TestCaseget_groups:
matches = getmatches(l) matches = getmatches(l)
r = get_groups(matches) r = get_groups(matches)
eq_(1,len(r)) eq_(1,len(r))
def test_give_priority_to_matches_with_higher_percentage(self): def test_give_priority_to_matches_with_higher_percentage(self):
o1 = NamedObject(with_words=True) o1 = NamedObject(with_words=True)
o2 = NamedObject(with_words=True) o2 = NamedObject(with_words=True)
@ -782,14 +782,14 @@ class TestCaseget_groups:
assert o1 not in g assert o1 not in g
assert o2 in g assert o2 in g
assert o3 in g assert o3 in g
def test_four_sized_group(self): def test_four_sized_group(self):
l = [NamedObject("foobar") for i in range(4)] l = [NamedObject("foobar") for i in range(4)]
m = getmatches(l) m = getmatches(l)
r = get_groups(m) r = get_groups(m)
eq_(1,len(r)) eq_(1,len(r))
eq_(4,len(r[0])) eq_(4,len(r[0]))
def test_referenced_by_ref2(self): def test_referenced_by_ref2(self):
o1 = NamedObject(with_words=True) o1 = NamedObject(with_words=True)
o2 = NamedObject(with_words=True) o2 = NamedObject(with_words=True)
@ -799,12 +799,12 @@ class TestCaseget_groups:
m3 = get_match(o3,o2) m3 = get_match(o3,o2)
r = get_groups([m1,m2,m3]) r = get_groups([m1,m2,m3])
eq_(3,len(r[0])) eq_(3,len(r[0]))
def test_job(self): def test_job(self):
def do_progress(p,d=''): def do_progress(p,d=''):
self.log.append(p) self.log.append(p)
return True return True
self.log = [] self.log = []
j = job.Job(1,do_progress) j = job.Job(1,do_progress)
m1,m2,m3 = get_match_triangle() m1,m2,m3 = get_match_triangle()
@ -813,7 +813,7 @@ class TestCaseget_groups:
get_groups([m1,m2,m3,m4],j) get_groups([m1,m2,m3,m4],j)
eq_(0,self.log[0]) eq_(0,self.log[0])
eq_(100,self.log[-1]) eq_(100,self.log[-1])
def test_group_admissible_discarded_dupes(self): def test_group_admissible_discarded_dupes(self):
# If, with a (A, B, C, D) set, all match with A, but C and D don't match with B and that the # If, with a (A, B, C, D) set, all match with A, but C and D don't match with B and that the
# (A, B) match is the highest (thus resulting in an (A, B) group), still match C and D # (A, B) match is the highest (thus resulting in an (A, B) group), still match C and D
@ -830,4 +830,4 @@ class TestCaseget_groups:
assert B in g1 assert B in g1
assert C in g2 assert C in g2
assert D in g2 assert D in g2

18
core/tests/scanner_test.py
View File

@ -1,12 +1,12 @@
# Created By: Virgil Dupras # Created By: Virgil Dupras
# Created On: 2006/03/03 # Created On: 2006/03/03
# Copyright 2014 Hardcoded Software (http://www.hardcoded.net) # Copyright 2014 Hardcoded Software (http://www.hardcoded.net)
# #
# This software is licensed under the "BSD" License as described in the "LICENSE" file, # This software is licensed under the "BSD" License as described in the "LICENSE" file,
# which should be included with this package. The terms are also available at # which should be included with this package. The terms are also available at
# http://www.hardcoded.net/licenses/bsd_license # http://www.hardcoded.net/licenses/bsd_license
from jobprogress import job from hscommon.jobprogress import job
from hscommon.path import Path from hscommon.path import Path
from hscommon.testutil import eq_ from hscommon.testutil import eq_
@ -25,10 +25,10 @@ class NamedObject:
self.size = size self.size = size
self.path = path self.path = path
self.words = getwords(name) self.words = getwords(name)
def __repr__(self): def __repr__(self):
return '<NamedObject %r %r>' % (self.name, self.path) return '<NamedObject %r %r>' % (self.name, self.path)
no = NamedObject no = NamedObject
@ -384,7 +384,7 @@ def test_file_evaluates_to_false(fake_fileexists):
class FalseNamedObject(NamedObject): class FalseNamedObject(NamedObject):
def __bool__(self): def __bool__(self):
return False return False
s = Scanner() s = Scanner()
f1 = FalseNamedObject('foobar', path='p1') f1 = FalseNamedObject('foobar', path='p1')
@ -445,7 +445,7 @@ def test_tie_breaker_same_name_plus_digit(fake_fileexists):
assert group.ref is o5 assert group.ref is o5
def test_partial_group_match(fake_fileexists): def test_partial_group_match(fake_fileexists):
# Count the number of discarded matches (when a file doesn't match all other dupes of the # Count the number of discarded matches (when a file doesn't match all other dupes of the
# group) in Scanner.discarded_file_count # group) in Scanner.discarded_file_count
s = Scanner() s = Scanner()
o1, o2, o3 = no('a b'), no('a'), no('b') o1, o2, o3 = no('a b'), no('a'), no('b')
@ -476,7 +476,7 @@ def test_dont_group_files_that_dont_exist(tmpdir):
file2.path.remove() file2.path.remove()
return [Match(file1, file2, 100)] return [Match(file1, file2, 100)]
s._getmatches = getmatches s._getmatches = getmatches
assert not s.get_dupe_groups([file1, file2]) assert not s.get_dupe_groups([file1, file2])
def test_folder_scan_exclude_subfolder_matches(fake_fileexists): def test_folder_scan_exclude_subfolder_matches(fake_fileexists):

26
core_pe/matchblock.py
View File

@ -1,9 +1,9 @@
# Created By: Virgil Dupras # Created By: Virgil Dupras
# Created On: 2007/02/25 # Created On: 2007/02/25
# Copyright 2014 Hardcoded Software (http://www.hardcoded.net) # Copyright 2014 Hardcoded Software (http://www.hardcoded.net)
# #
# This software is licensed under the "BSD" License as described in the "LICENSE" file, # This software is licensed under the "BSD" License as described in the "LICENSE" file,
# which should be included with this package. The terms are also available at # which should be included with this package. The terms are also available at
# http://www.hardcoded.net/licenses/bsd_license # http://www.hardcoded.net/licenses/bsd_license
import logging import logging
@ -12,7 +12,7 @@ from itertools import combinations
from hscommon.util import extract from hscommon.util import extract
from hscommon.trans import tr from hscommon.trans import tr
from jobprogress import job from hscommon.jobprogress import job
from core.engine import Match from core.engine import Match
from .block import avgdiff, DifferentBlockCountError, NoBlocksError from .block import avgdiff, DifferentBlockCountError, NoBlocksError
@ -132,14 +132,14 @@ def async_compare(ref_ids, other_ids, dbname, threshold, picinfo):
results.append((ref_id, other_id, percentage)) results.append((ref_id, other_id, percentage))
cache.close() cache.close()
return results return results
def getmatches(pictures, cache_path, threshold=75, match_scaled=False, j=job.nulljob): def getmatches(pictures, cache_path, threshold=75, match_scaled=False, j=job.nulljob):
def get_picinfo(p): def get_picinfo(p):
if match_scaled: if match_scaled:
return (None, p.is_ref) return (None, p.is_ref)
else: else:
return (p.dimensions, p.is_ref) return (p.dimensions, p.is_ref)
def collect_results(collect_all=False): def collect_results(collect_all=False):
# collect results and wait until the queue is small enough to accomodate a new results. # collect results and wait until the queue is small enough to accomodate a new results.
nonlocal async_results, matches, comparison_count nonlocal async_results, matches, comparison_count
@ -152,7 +152,7 @@ def getmatches(pictures, cache_path, threshold=75, match_scaled=False, j=job.nul
comparison_count += 1 comparison_count += 1
progress_msg = tr("Performed %d/%d chunk matches") % (comparison_count, len(comparisons_to_do)) progress_msg = tr("Performed %d/%d chunk matches") % (comparison_count, len(comparisons_to_do))
j.set_progress(comparison_count, progress_msg) j.set_progress(comparison_count, progress_msg)
j = j.start_subjob([3, 7]) j = j.start_subjob([3, 7])
pictures = prepare_pictures(pictures, cache_path, with_dimensions=not match_scaled, j=j) pictures = prepare_pictures(pictures, cache_path, with_dimensions=not match_scaled, j=j)
j = j.start_subjob([9, 1], tr("Preparing for matching")) j = j.start_subjob([9, 1], tr("Preparing for matching"))
@ -188,9 +188,14 @@ def getmatches(pictures, cache_path, threshold=75, match_scaled=False, j=job.nul
collect_results() collect_results()
collect_results(collect_all=True) collect_results(collect_all=True)
pool.close() pool.close()
result = [] result = []
for ref_id, other_id, percentage in j.iter_with_progress(matches, tr("Verified %d/%d matches"), every=10): myiter = j.iter_with_progress(
matches,
tr("Verified %d/%d matches"),
every=10
)
for ref_id, other_id, percentage in myiter:
ref = id2picture[ref_id] ref = id2picture[ref_id]
other = id2picture[other_id] other = id2picture[other_id]
if percentage == 100 and ref.md5 != other.md5: if percentage == 100 and ref.md5 != other.md5:
@ -201,4 +206,5 @@ def getmatches(pictures, cache_path, threshold=75, match_scaled=False, j=job.nul
result.append(get_match(ref, other, percentage)) result.append(get_match(ref, other, percentage))
return result return result
multiprocessing.freeze_support() multiprocessing.freeze_support()

0
hscommon/__init__.py Executable file → Normal file
View File

57
hscommon/gui/progress_window.py
View File

@ -1,61 +1,58 @@
# Created On: 2013/07/01 # Created On: 2013/07/01
# Copyright 2014 Hardcoded Software (http://www.hardcoded.net) # Copyright 2014 Hardcoded Software (http://www.hardcoded.net)
# #
# This software is licensed under the "BSD" License as described in the "LICENSE" file, # This software is licensed under the "BSD" License as described in the "LICENSE" file,
# which should be included with this package. The terms are also available at # which should be included with this package. The terms are also available at
# http://www.hardcoded.net/licenses/bsd_license # http://www.hardcoded.net/licenses/bsd_license
from jobprogress.performer import ThreadedJobPerformer from ..jobprogress.performer import ThreadedJobPerformer
from .base import GUIObject from .base import GUIObject
from .text_field import TextField from .text_field import TextField
class ProgressWindowView: class ProgressWindowView:
"""Expected interface for :class:`ProgressWindow`'s view. """Expected interface for :class:`ProgressWindow`'s view.
*Not actually used in the code. For documentation purposes only.* *Not actually used in the code. For documentation purposes only.*
Our view, some kind window with a progress bar, two labels and a cancel button, is expected Our view, some kind window with a progress bar, two labels and a cancel button, is expected
to properly respond to its callbacks. to properly respond to its callbacks.
It's also expected to call :meth:`ProgressWindow.cancel` when the cancel button is clicked. It's also expected to call :meth:`ProgressWindow.cancel` when the cancel button is clicked.
""" """
def show(self): def show(self):
"""Show the dialog. """Show the dialog.
""" """
def close(self): def close(self):
"""Close the dialog. """Close the dialog.
""" """
def set_progress(self, progress): def set_progress(self, progress):
"""Set the progress of the progress bar to ``progress``. """Set the progress of the progress bar to ``progress``.
Not all jobs are equally responsive on their job progress report and it is recommended that Not all jobs are equally responsive on their job progress report and it is recommended that
you put your progressbar in "indeterminate" mode as long as you haven't received the first you put your progressbar in "indeterminate" mode as long as you haven't received the first
``set_progress()`` call to avoid letting the user think that the app is frozen. ``set_progress()`` call to avoid letting the user think that the app is frozen.
:param int progress: a value between ``0`` and ``100``. :param int progress: a value between ``0`` and ``100``.
""" """
class ProgressWindow(GUIObject, ThreadedJobPerformer): class ProgressWindow(GUIObject, ThreadedJobPerformer):
"""Cross-toolkit GUI-enabled progress window. """Cross-toolkit GUI-enabled progress window.
This class allows you to run a long running, `job enabled`_ function in a separate thread and This class allows you to run a long running, job enabled function in a separate thread and
allow the user to follow its progress with a progress dialog. allow the user to follow its progress with a progress dialog.
To use it, you start your long-running job with :meth:`run` and then have your UI layer To use it, you start your long-running job with :meth:`run` and then have your UI layer
regularly call :meth:`pulse` to refresh the job status in the UI. It is advised that you call regularly call :meth:`pulse` to refresh the job status in the UI. It is advised that you call
:meth:`pulse` in the main thread because GUI toolkit usually only support calling UI-related :meth:`pulse` in the main thread because GUI toolkit usually only support calling UI-related
functions from the main thread. functions from the main thread.
We subclass :class:`.GUIObject` and ``ThreadedJobPerformer`` (from the ``jobprogress`` library). We subclass :class:`.GUIObject` and :class:`ThreadedJobPerformer`.
Expected view: :class:`ProgressWindowView`. Expected view: :class:`ProgressWindowView`.
:param finishfunc: A function ``f(jobid)`` that is called when a job is completed. ``jobid`` is :param finishfunc: A function ``f(jobid)`` that is called when a job is completed. ``jobid`` is
an arbitrary id passed to :meth:`run`. an arbitrary id passed to :meth:`run`.
.. _job enabled: https://pypi.python.org/pypi/jobprogress
""" """
def __init__(self, finish_func): def __init__(self, finish_func):
# finish_func(jobid) is the function that is called when a job is completed. # finish_func(jobid) is the function that is called when a job is completed.
@ -68,7 +65,7 @@ class ProgressWindow(GUIObject, ThreadedJobPerformer):
#: during its course. #: during its course.
self.progressdesc_textfield = TextField() self.progressdesc_textfield = TextField()
self.jobid = None self.jobid = None
def cancel(self): def cancel(self):
"""Call for a user-initiated job cancellation. """Call for a user-initiated job cancellation.
""" """
@ -77,13 +74,13 @@ class ProgressWindow(GUIObject, ThreadedJobPerformer):
# we verify that the job is still running. # we verify that the job is still running.
if self._job_running: if self._job_running:
self.job_cancelled = True self.job_cancelled = True
def pulse(self): def pulse(self):
"""Update progress reports in the GUI. """Update progress reports in the GUI.
Call this regularly from the GUI main run loop. The values might change before Call this regularly from the GUI main run loop. The values might change before
:meth:`ProgressWindowView.set_progress` happens. :meth:`ProgressWindowView.set_progress` happens.
If the job is finished, ``pulse()`` will take care of closing the window and re-raising any If the job is finished, ``pulse()`` will take care of closing the window and re-raising any
exception that might have been raised during the job (in the main thread this time). If exception that might have been raised during the job (in the main thread this time). If
there was no exception, ``finish_func(jobid)`` is called to let you take appropriate action. there was no exception, ``finish_func(jobid)`` is called to let you take appropriate action.
@ -101,13 +98,13 @@ class ProgressWindow(GUIObject, ThreadedJobPerformer):
if last_desc: if last_desc:
self.progressdesc_textfield.text = last_desc self.progressdesc_textfield.text = last_desc
self.view.set_progress(last_progress) self.view.set_progress(last_progress)
def run(self, jobid, title, target, args=()): def run(self, jobid, title, target, args=()):
"""Starts a threaded job. """Starts a threaded job.
The ``target`` function will be sent, as its first argument, a ``Job`` instance (from the The ``target`` function will be sent, as its first argument, a :class:`Job` instance which
``jobprogress`` library) which it can use to report on its progress. it can use to report on its progress.
:param jobid: Arbitrary identifier which will be passed to ``finish_func()`` at the end. :param jobid: Arbitrary identifier which will be passed to ``finish_func()`` at the end.
:param title: A title for the task you're starting. :param title: A title for the task you're starting.
:param target: The function that does your famous long running job. :param target: The function that does your famous long running job.
@ -122,4 +119,4 @@ class ProgressWindow(GUIObject, ThreadedJobPerformer):
self.run_threaded(target, args) self.run_threaded(target, args)
self.jobdesc_textfield.text = title self.jobdesc_textfield.text = title
self.view.show() self.view.show()

0
hscommon/jobprogress/__init__.py Normal file
View File

160
hscommon/jobprogress/job.py Normal file
View File

@ -0,0 +1,160 @@
# Created By: Virgil Dupras
# Created On: 2004/12/20
# Copyright 2011 Hardcoded Software (http://www.hardcoded.net)
# This software is licensed under the "BSD" License as described in the "LICENSE" file,
# which should be included with this package. The terms are also available at
# http://www.hardcoded.net/licenses/bsd_license
class JobCancelled(Exception):
"The user has cancelled the job"
class JobInProgressError(Exception):
"A job is already being performed, you can't perform more than one at the same time."
class JobCountError(Exception):
"The number of jobs started have exceeded the number of jobs allowed"
class Job:
"""Manages a job's progression and return it's progression through a callback.
Note that this class is not foolproof. For example, you could call
start_subjob, and then call add_progress from the parent job, and nothing
would stop you from doing it. However, it would mess your progression
because it is the sub job that is supposed to drive the progression.
Another example would be to start a subjob, then start another, and call
add_progress from the old subjob. Once again, it would mess your progression.
There are no stops because it would remove the lightweight aspect of the
class (A Job would need to have a Parent instead of just a callback,
and the parent could be None. A lot of checks for nothing.).
Another one is that nothing stops you from calling add_progress right after
SkipJob.
"""
#---Magic functions
def __init__(self, job_proportions, callback):
"""Initialize the Job with 'jobcount' jobs. Start every job with
start_job(). Every time the job progress is updated, 'callback' is called
'callback' takes a 'progress' int param, and a optional 'desc'
parameter. Callback must return false if the job must be cancelled.
"""
if not hasattr(callback, '__call__'):
raise TypeError("'callback' MUST be set when creating a Job")
if isinstance(job_proportions, int):
job_proportions = [1] * job_proportions
self._job_proportions = list(job_proportions)
self._jobcount = sum(job_proportions)
self._callback = callback
self._current_job = 0
self._passed_jobs = 0
self._progress = 0
self._currmax = 1
#---Private
def _subjob_callback(self, progress, desc=''):
"""This is the callback passed to children jobs.
"""
self.set_progress(progress, desc)
return True #if JobCancelled has to be raised, it will be at the highest level
def _do_update(self, desc):
"""Calls the callback function with a % progress as a parameter.
The parameter is a int in the 0-100 range.
"""
if self._current_job:
passed_progress = self._passed_jobs * self._currmax
current_progress = self._current_job * self._progress
total_progress = self._jobcount * self._currmax
progress = ((passed_progress + current_progress) * 100) // total_progress
else:
progress = -1 # indeterminate
# It's possible that callback doesn't support a desc arg
result = self._callback(progress, desc) if desc else self._callback(progress)
if not result:
raise JobCancelled()
#---Public
def add_progress(self, progress=1, desc=''):
self.set_progress(self._progress + progress, desc)
def check_if_cancelled(self):
self._do_update('')
def iter_with_progress(self, sequence, desc_format=None, every=1):
''' Iterate through sequence while automatically adding progress.
'''
desc = ''
if desc_format:
desc = desc_format % (0, len(sequence))
self.start_job(len(sequence), desc)
for i, element in enumerate(sequence, start=1):
yield element
if i % every == 0:
if desc_format:
desc = desc_format % (i, len(sequence))
self.add_progress(progress=every, desc=desc)
if desc_format:
desc = desc_format % (len(sequence), len(sequence))
self.set_progress(100, desc)
def start_job(self, max_progress=100, desc=''):
"""Begin work on the next job. You must not call start_job more than
'jobcount' (in __init__) times.
'max' is the job units you are to perform.
'desc' is the description of the job.
"""
self._passed_jobs += self._current_job
try:
self._current_job = self._job_proportions.pop(0)
except IndexError:
raise JobCountError()
self._progress = 0
self._currmax = max(1, max_progress)
self._do_update(desc)
def start_subjob(self, job_proportions, desc=''):
"""Starts a sub job. Use this when you want to split a job into
multiple smaller jobs. Pretty handy when starting a process where you
know how many subjobs you will have, but don't know the work unit count
for every of them.
returns the Job object
"""
self.start_job(100, desc)
return Job(job_proportions, self._subjob_callback)
def set_progress(self, progress, desc=''):
"""Sets the progress of the current job to 'progress', and call the
callback
"""
self._progress = progress
if self._progress > self._currmax:
self._progress = self._currmax
if self._progress < 0:
self._progress = 0
self._do_update(desc)
class NullJob:
def __init__(self, *args, **kwargs):
pass
def add_progress(self, *args, **kwargs):
pass
def check_if_cancelled(self):
pass
def iter_with_progress(self, sequence, *args, **kwargs):
return iter(sequence)
def start_job(self, *args, **kwargs):
pass
def start_subjob(self, *args, **kwargs):
return NullJob()
def set_progress(self, *args, **kwargs):
pass
nulljob = NullJob()

72
hscommon/jobprogress/performer.py Normal file
View File

@ -0,0 +1,72 @@
# Created By: Virgil Dupras
# Created On: 2010-11-19
# Copyright 2011 Hardcoded Software (http://www.hardcoded.net)
#
# This software is licensed under the "BSD" License as described in the "LICENSE" file,
# which should be included with this package. The terms are also available at
# http://www.hardcoded.net/licenses/bsd_license
from threading import Thread
import sys
from .job import Job, JobInProgressError, JobCancelled
class ThreadedJobPerformer:
"""Run threaded jobs and track progress.
To run a threaded job, first create a job with _create_job(), then call _run_threaded(), with
your work function as a parameter.
Example:
j = self._create_job()
self._run_threaded(self.some_work_func, (arg1, arg2, j))
"""
_job_running = False
last_error = None
#--- Protected
def create_job(self):
if self._job_running:
raise JobInProgressError()
self.last_progress = -1
self.last_desc = ''
self.job_cancelled = False
return Job(1, self._update_progress)
def _async_run(self, *args):
target = args[0]
args = tuple(args[1:])
self._job_running = True
self.last_error = None
try:
target(*args)
except JobCancelled:
pass
except Exception as e:
self.last_error = e
self.last_traceback = sys.exc_info()[2]
finally:
self._job_running = False
self.last_progress = None
def reraise_if_error(self):
"""Reraises the error that happened in the thread if any.
Call this after the caller of run_threaded detected that self._job_running returned to False
"""
if self.last_error is not None:
raise self.last_error.with_traceback(self.last_traceback)
def _update_progress(self, newprogress, newdesc=''):
self.last_progress = newprogress
if newdesc:
self.last_desc = newdesc
return not self.job_cancelled
def run_threaded(self, target, args=()):
if self._job_running:
raise JobInProgressError()
args = (target, ) + args
Thread(target=self._async_run, args=args).start()

52
hscommon/jobprogress/qt.py Normal file
View File

@ -0,0 +1,52 @@
# Created By: Virgil Dupras
# Created On: 2009-09-14
# Copyright 2011 Hardcoded Software (http://www.hardcoded.net)
#
# This software is licensed under the "BSD" License as described in the "LICENSE" file,
# which should be included with this package. The terms are also available at
# http://www.hardcoded.net/licenses/bsd_license
from PyQt4.QtCore import pyqtSignal, Qt, QTimer
from PyQt4.QtGui import QProgressDialog
from . import job, performer
class Progress(QProgressDialog, performer.ThreadedJobPerformer):
finished = pyqtSignal(['QString'])
def __init__(self, parent):
flags = Qt.CustomizeWindowHint | Qt.WindowTitleHint | Qt.WindowSystemMenuHint
QProgressDialog.__init__(self, '', "Cancel", 0, 100, parent, flags)
self.setModal(True)
self.setAutoReset(False)
self.setAutoClose(False)
self._timer = QTimer()
self._jobid = ''
self._timer.timeout.connect(self.updateProgress)
def updateProgress(self):
# the values might change before setValue happens
last_progress = self.last_progress
last_desc = self.last_desc
if not self._job_running or last_progress is None:
self._timer.stop()
self.close()
if not self.job_cancelled:
self.finished.emit(self._jobid)
return
if self.wasCanceled():
self.job_cancelled = True
return
if last_desc:
self.setLabelText(last_desc)
self.setValue(last_progress)
def run(self, jobid, title, target, args=()):
self._jobid = jobid
self.reset()
self.setLabelText('')
self.run_threaded(target, args)
self.setWindowTitle(title)
self.show()
self._timer.start(500)

0
hscommon/path.py Executable file → Normal file
View File

14
package.py
View File

@ -1,9 +1,9 @@
# Created By: Virgil Dupras # Created By: Virgil Dupras
# Created On: 2009-12-30 # Created On: 2009-12-30
# Copyright 2014 Hardcoded Software (http://www.hardcoded.net) # Copyright 2014 Hardcoded Software (http://www.hardcoded.net)
# #
# This software is licensed under the "BSD" License as described in the "LICENSE" file, # This software is licensed under the "BSD" License as described in the "LICENSE" file,
# which should be included with this package. The terms are also available at # which should be included with this package. The terms are also available at
# http://www.hardcoded.net/licenses/bsd_license # http://www.hardcoded.net/licenses/bsd_license
import os import os
@ -42,10 +42,10 @@ def package_windows(edition, dev):
add_to_pythonpath('.') add_to_pythonpath('.')
app_version = get_module_version('core_{}'.format(edition)) app_version = get_module_version('core_{}'.format(edition))
distdir = 'dist' distdir = 'dist'
if op.exists(distdir): if op.exists(distdir):
shutil.rmtree(distdir) shutil.rmtree(distdir)
if not dev: if not dev:
# Copy qt plugins # Copy qt plugins
plugin_dest = distdir plugin_dest = distdir
@ -129,7 +129,7 @@ def package_debian_distribution(edition, distribution):
ed = lambda s: s.format(edition) ed = lambda s: s.format(edition)
destpath = op.join('build', 'dupeguru-{0}-{1}'.format(edition, version)) destpath = op.join('build', 'dupeguru-{0}-{1}'.format(edition, version))
srcpath = op.join(destpath, 'src') srcpath = op.join(destpath, 'src')
packages = ['hscommon', 'core', ed('core_{0}'), 'qtlib', 'qt', 'send2trash', 'jobprogress'] packages = ['hscommon', 'core', ed('core_{0}'), 'qtlib', 'qt', 'send2trash']
if edition == 'me': if edition == 'me':
packages.append('hsaudiotag') packages.append('hsaudiotag')
copy_files_to_package(srcpath, packages, with_so=False) copy_files_to_package(srcpath, packages, with_so=False)
@ -171,7 +171,7 @@ def package_arch(edition):
print("Packaging for Arch") print("Packaging for Arch")
ed = lambda s: s.format(edition) ed = lambda s: s.format(edition)
srcpath = op.join('build', ed('dupeguru-{}-arch')) srcpath = op.join('build', ed('dupeguru-{}-arch'))
packages = ['hscommon', 'core', ed('core_{0}'), 'qtlib', 'qt', 'send2trash', 'jobprogress'] packages = ['hscommon', 'core', ed('core_{0}'), 'qtlib', 'qt', 'send2trash']
if edition == 'me': if edition == 'me':
packages.append('hsaudiotag') packages.append('hsaudiotag')
copy_files_to_package(srcpath, packages, with_so=True) copy_files_to_package(srcpath, packages, with_so=True)

1
requirements.txt
View File

@ -1,4 +1,3 @@
jobprogress>=1.0.4
Send2Trash>=1.3.0 Send2Trash>=1.3.0
sphinx>=1.2.2 sphinx>=1.2.2
polib>=1.0.4 polib>=1.0.4