arsenetar/dupeguru
1
0
Fork 0
mirror of https://github.com/arsenetar/dupeguru.git synced 2026-01-22 06:37:17 +00:00
Code Releases Wiki Activity

Integrated the jobprogress library into hscommon

Browse Source 
I have a fix to make in it and it's really silly to pretend that this
lib is of any use to anybody outside HS apps. Bringing it back here will
make things more simple.
This commit is contained in:
Virgil Dupras
2014-10-05 16:31:16 -04:00
parent 87c2fa2573
commit ac32305532
21 changed files with 775 additions and 487 deletions
Download Patch File Download Diff File Expand all files Collapse all files

100
core/tests/app_test.py
View File

@@ -1,9 +1,9 @@
# Created By: Virgil Dupras
# Created On: 2007-06-23
# Copyright 2014 Hardcoded Software (http://www.hardcoded.net)
#
# This software is licensed under the "BSD" License as described in the "LICENSE" file,
# which should be included with this package. The terms are also available at
#
# This software is licensed under the "BSD" License as described in the "LICENSE" file,
# which should be included with this package. The terms are also available at
# http://www.hardcoded.net/licenses/bsd_license
import os
@@ -15,7 +15,7 @@ from hscommon.path import Path
import hscommon.conflict
import hscommon.util
from hscommon.testutil import CallLogger, eq_, log_calls
from jobprogress.job import Job
from hscommon.jobprogress.job import Job
from .base import DupeGuru, TestApp
from .results_test import GetTestGroups
@@ -36,7 +36,7 @@ class TestCaseDupeGuru:
assert call['filter_str'] is None
call = dgapp.results.apply_filter.calls[1]
eq_('foo', call['filter_str'])
def test_apply_filter_escapes_regexp(self, monkeypatch):
dgapp = TestApp().app
monkeypatch.setattr(dgapp.results, 'apply_filter', log_calls(dgapp.results.apply_filter))
@@ -50,7 +50,7 @@ class TestCaseDupeGuru:
dgapp.apply_filter('(abc)')
call = dgapp.results.apply_filter.calls[5]
eq_('(abc)', call['filter_str'])
def test_copy_or_move(self, tmpdir, monkeypatch):
# The goal here is just to have a test for a previous blowup I had. I know my test coverage
# for this unit is pathetic. What's done is done. My approach now is to add tests for
@@ -69,7 +69,7 @@ class TestCaseDupeGuru:
call = hscommon.conflict.smart_copy.calls[0]
eq_(call['dest_path'], op.join('some_destination', 'foo'))
eq_(call['source_path'], f.path)
def test_copy_or_move_clean_empty_dirs(self, tmpdir, monkeypatch):
tmppath = Path(str(tmpdir))
sourcepath = tmppath['source']
@@ -83,13 +83,13 @@ class TestCaseDupeGuru:
calls = app.clean_empty_dirs.calls
eq_(1, len(calls))
eq_(sourcepath, calls[0]['path'])
def test_Scan_with_objects_evaluating_to_false(self):
class FakeFile(fs.File):
def __bool__(self):
return False
# At some point, any() was used in a wrong way that made Scan() wrongly return 1
app = TestApp().app
f1, f2 = [FakeFile('foo') for i in range(2)]
@@ -97,7 +97,7 @@ class TestCaseDupeGuru:
assert not (bool(f1) and bool(f2))
add_fake_files_to_directories(app.directories, [f1, f2])
app.start_scanning() # no exception
@mark.skipif("not hasattr(os, 'link')")
def test_ignore_hardlink_matches(self, tmpdir):
# If the ignore_hardlink_matches option is set, don't match files hardlinking to the same
@@ -111,7 +111,7 @@ class TestCaseDupeGuru:
app.options['ignore_hardlink_matches'] = True
app.start_scanning()
eq_(len(app.results.groups), 0)
def test_rename_when_nothing_is_selected(self):
# Issue #140
# It's possible that rename operation has its selected row swept off from under it, thus
@@ -127,11 +127,11 @@ class TestCaseDupeGuru_clean_empty_dirs:
# XXX This monkeypatch is temporary. will be fixed in a better monkeypatcher.
monkeypatch.setattr(app, 'delete_if_empty', hscommon.util.delete_if_empty)
self.app = TestApp().app
def test_option_off(self, do_setup):
self.app.clean_empty_dirs(Path('/foo/bar'))
eq_(0, len(hscommon.util.delete_if_empty.calls))
def test_option_on(self, do_setup):
self.app.options['clean_empty_dirs'] = True
self.app.clean_empty_dirs(Path('/foo/bar'))
@@ -139,13 +139,13 @@ class TestCaseDupeGuru_clean_empty_dirs:
eq_(1, len(calls))
eq_(Path('/foo/bar'), calls[0]['path'])
eq_(['.DS_Store'], calls[0]['files_to_delete'])
def test_recurse_up(self, do_setup, monkeypatch):
# delete_if_empty must be recursively called up in the path until it returns False
@log_calls
def mock_delete_if_empty(path, files_to_delete=[]):
return len(path) > 1
monkeypatch.setattr(hscommon.util, 'delete_if_empty', mock_delete_if_empty)
# XXX This monkeypatch is temporary. will be fixed in a better monkeypatcher.
monkeypatch.setattr(app, 'delete_if_empty', mock_delete_if_empty)
@@ -156,7 +156,7 @@ class TestCaseDupeGuru_clean_empty_dirs:
eq_(Path('not-empty/empty/empty'), calls[0]['path'])
eq_(Path('not-empty/empty'), calls[1]['path'])
eq_(Path('not-empty'), calls[2]['path'])
class TestCaseDupeGuruWithResults:
def pytest_funcarg__do_setup(self, request):
@@ -173,7 +173,7 @@ class TestCaseDupeGuruWithResults:
tmppath['foo'].mkdir()
tmppath['bar'].mkdir()
self.app.directories.add_path(tmppath)
def test_GetObjects(self, do_setup):
objects = self.objects
groups = self.groups
@@ -186,7 +186,7 @@ class TestCaseDupeGuruWithResults:
r = self.rtable[4]
assert r._group is groups[1]
assert r._dupe is objects[4]
def test_GetObjects_after_sort(self, do_setup):
objects = self.objects
groups = self.groups[:] # we need an un-sorted reference
@@ -194,14 +194,14 @@ class TestCaseDupeGuruWithResults:
r = self.rtable[1]
assert r._group is groups[1]
assert r._dupe is objects[4]
def test_selected_result_node_paths_after_deletion(self, do_setup):
# cases where the selected dupes aren't there are correctly handled
self.rtable.select([1, 2, 3])
self.app.remove_selected()
# The first 2 dupes have been removed. The 3rd one is a ref. it stays there, in first pos.
eq_(self.rtable.selected_indexes, [1]) # no exception
def test_selectResultNodePaths(self, do_setup):
app = self.app
objects = self.objects
@@ -209,7 +209,7 @@ class TestCaseDupeGuruWithResults:
eq_(len(app.selected_dupes), 2)
assert app.selected_dupes[0] is objects[1]
assert app.selected_dupes[1] is objects[2]
def test_selectResultNodePaths_with_ref(self, do_setup):
app = self.app
objects = self.objects
@@ -218,26 +218,26 @@ class TestCaseDupeGuruWithResults:
assert app.selected_dupes[0] is objects[1]
assert app.selected_dupes[1] is objects[2]
assert app.selected_dupes[2] is self.groups[1].ref
def test_selectResultNodePaths_after_sort(self, do_setup):
app = self.app
objects = self.objects
groups = self.groups[:] #To keep the old order in memory
self.rtable.sort('name', False) #0
self.rtable.sort('name', False) #0
#Now, the group order is supposed to be reversed
self.rtable.select([1, 2, 3])
eq_(len(app.selected_dupes), 3)
assert app.selected_dupes[0] is objects[4]
assert app.selected_dupes[1] is groups[0].ref
assert app.selected_dupes[2] is objects[1]
def test_selected_powermarker_node_paths(self, do_setup):
# app.selected_dupes is correctly converted into paths
self.rtable.power_marker = True
self.rtable.select([0, 1, 2])
self.rtable.power_marker = False
eq_(self.rtable.selected_indexes, [1, 2, 4])
def test_selected_powermarker_node_paths_after_deletion(self, do_setup):
# cases where the selected dupes aren't there are correctly handled
app = self.app
@@ -245,7 +245,7 @@ class TestCaseDupeGuruWithResults:
self.rtable.select([0, 1, 2])
app.remove_selected()
eq_(self.rtable.selected_indexes, []) # no exception
def test_selectPowerMarkerRows_after_sort(self, do_setup):
app = self.app
objects = self.objects
@@ -256,7 +256,7 @@ class TestCaseDupeGuruWithResults:
assert app.selected_dupes[0] is objects[4]
assert app.selected_dupes[1] is objects[2]
assert app.selected_dupes[2] is objects[1]
def test_toggle_selected_mark_state(self, do_setup):
app = self.app
objects = self.objects
@@ -270,7 +270,7 @@ class TestCaseDupeGuruWithResults:
assert not app.results.is_marked(objects[2])
assert not app.results.is_marked(objects[3])
assert app.results.is_marked(objects[4])
def test_toggle_selected_mark_state_with_different_selected_state(self, do_setup):
# When marking selected dupes with a heterogenous selection, mark all selected dupes. When
# it's homogenous, simply toggle.
@@ -285,7 +285,7 @@ class TestCaseDupeGuruWithResults:
eq_(app.results.mark_count, 2)
app.toggle_selected_mark_state()
eq_(app.results.mark_count, 0)
def test_refreshDetailsWithSelected(self, do_setup):
self.rtable.select([1, 4])
eq_(self.dpanel.row(0), ('Filename', 'bar bleh', 'foo bar'))
@@ -293,7 +293,7 @@ class TestCaseDupeGuruWithResults:
self.rtable.select([])
eq_(self.dpanel.row(0), ('Filename', '---', '---'))
self.dpanel.view.check_gui_calls(['refresh'])
def test_makeSelectedReference(self, do_setup):
app = self.app
objects = self.objects
@@ -302,7 +302,7 @@ class TestCaseDupeGuruWithResults:
app.make_selected_reference()
assert groups[0].ref is objects[1]
assert groups[1].ref is objects[4]
def test_makeSelectedReference_by_selecting_two_dupes_in_the_same_group(self, do_setup):
app = self.app
objects = self.objects
@@ -312,7 +312,7 @@ class TestCaseDupeGuruWithResults:
app.make_selected_reference()
assert groups[0].ref is objects[1]
assert groups[1].ref is objects[4]
def test_removeSelected(self, do_setup):
app = self.app
self.rtable.select([1, 4])
@@ -320,7 +320,7 @@ class TestCaseDupeGuruWithResults:
eq_(len(app.results.dupes), 1) # the first path is now selected
app.remove_selected()
eq_(len(app.results.dupes), 0)
def test_addDirectory_simple(self, do_setup):
# There's already a directory in self.app, so adding another once makes 2 of em
app = self.app
@@ -328,7 +328,7 @@ class TestCaseDupeGuruWithResults:
otherpath = Path(op.dirname(__file__))
app.add_directory(otherpath)
eq_(len(app.directories), 2)
def test_addDirectory_already_there(self, do_setup):
app = self.app
otherpath = Path(op.dirname(__file__))
@@ -336,13 +336,13 @@ class TestCaseDupeGuruWithResults:
app.add_directory(otherpath)
eq_(len(app.view.messages), 1)
assert "already" in app.view.messages[0]
def test_addDirectory_does_not_exist(self, do_setup):
app = self.app
app.add_directory('/does_not_exist')
eq_(len(app.view.messages), 1)
assert "exist" in app.view.messages[0]
def test_ignore(self, do_setup):
app = self.app
self.rtable.select([4]) #The dupe of the second, 2 sized group
@@ -352,7 +352,7 @@ class TestCaseDupeGuruWithResults:
app.add_selected_to_ignore_list()
#BOTH the ref and the other dupe should have been added
eq_(len(app.scanner.ignore_list), 3)
def test_purgeIgnoreList(self, do_setup, tmpdir):
app = self.app
p1 = str(tmpdir.join('file1'))
@@ -367,19 +367,19 @@ class TestCaseDupeGuruWithResults:
eq_(1,len(app.scanner.ignore_list))
assert app.scanner.ignore_list.AreIgnored(p1,p2)
assert not app.scanner.ignore_list.AreIgnored(dne,p1)
def test_only_unicode_is_added_to_ignore_list(self, do_setup):
def FakeIgnore(first,second):
if not isinstance(first,str):
self.fail()
if not isinstance(second,str):
self.fail()
app = self.app
app.scanner.ignore_list.Ignore = FakeIgnore
self.rtable.select([4])
app.add_selected_to_ignore_list()
def test_cancel_scan_with_previous_results(self, do_setup):
# When doing a scan with results being present prior to the scan, correctly invalidate the
# results table.
@@ -388,7 +388,7 @@ class TestCaseDupeGuruWithResults:
add_fake_files_to_directories(app.directories, self.objects) # We want the scan to at least start
app.start_scanning() # will be cancelled immediately
eq_(len(self.rtable), 0)
def test_selected_dupes_after_removal(self, do_setup):
# Purge the app's `selected_dupes` attribute when removing dupes, or else it might cause a
# crash later with None refs.
@@ -398,7 +398,7 @@ class TestCaseDupeGuruWithResults:
app.remove_marked()
eq_(len(self.rtable), 0)
eq_(app.selected_dupes, [])
def test_dont_crash_on_delta_powermarker_dupecount_sort(self, do_setup):
# Don't crash when sorting by dupe count or percentage while delta+powermarker are enabled.
# Ref #238
@@ -410,7 +410,7 @@ class TestCaseDupeGuruWithResults:
# don't crash
self.rtable.sort('percentage', False)
# don't crash
class TestCaseDupeGuru_renameSelected:
def pytest_funcarg__do_setup(self, request):
@@ -437,7 +437,7 @@ class TestCaseDupeGuru_renameSelected:
self.groups = groups
self.p = p
self.files = files
def test_simple(self, do_setup):
app = self.app
g = self.groups[0]
@@ -447,7 +447,7 @@ class TestCaseDupeGuru_renameSelected:
assert 'renamed' in names
assert 'foo bar 2' not in names
eq_(g.dupes[0].name, 'renamed')
def test_none_selected(self, do_setup, monkeypatch):
app = self.app
g = self.groups[0]
@@ -460,7 +460,7 @@ class TestCaseDupeGuru_renameSelected:
assert 'renamed' not in names
assert 'foo bar 2' in names
eq_(g.dupes[0].name, 'foo bar 2')
def test_name_already_exists(self, do_setup, monkeypatch):
app = self.app
g = self.groups[0]
@@ -473,7 +473,7 @@ class TestCaseDupeGuru_renameSelected:
assert 'foo bar 1' in names
assert 'foo bar 2' in names
eq_(g.dupes[0].name, 'foo bar 2')
class TestAppWithDirectoriesInTree:
def pytest_funcarg__do_setup(self, request):
@@ -487,7 +487,7 @@ class TestAppWithDirectoriesInTree:
self.dtree = app.dtree
self.dtree.add_directory(p)
self.dtree.view.clear_calls()
def test_set_root_as_ref_makes_subfolders_ref_as_well(self, do_setup):
# Setting a node state to something also affect subnodes. These subnodes must be correctly
# refreshed.
@@ -500,4 +500,4 @@ class TestAppWithDirectoriesInTree:
subnode = node[0]
eq_(subnode.state, 1)
self.dtree.view.check_gui_calls(['refresh_states'])

50
core/tests/base.py
View File

@@ -1,16 +1,16 @@
# Created By: Virgil Dupras
# Created On: 2011/09/07
# Copyright 2014 Hardcoded Software (http://www.hardcoded.net)
#
# This software is licensed under the "BSD" License as described in the "LICENSE" file,
# which should be included with this package. The terms are also available at
#
# This software is licensed under the "BSD" License as described in the "LICENSE" file,
# which should be included with this package. The terms are also available at
# http://www.hardcoded.net/licenses/bsd_license
from hscommon.testutil import TestApp as TestAppBase, eq_, with_app
from hscommon.path import Path
from hscommon.util import get_file_ext, format_size
from hscommon.gui.column import Column
from jobprogress.job import nulljob, JobCancelled
from hscommon.jobprogress.job import nulljob, JobCancelled
from .. import engine
from .. import prioritize
@@ -23,28 +23,28 @@ from ..gui.prioritize_dialog import PrioritizeDialog
class DupeGuruView:
JOB = nulljob
def __init__(self):
self.messages = []
def start_job(self, jobid, func, args=()):
try:
func(self.JOB, *args)
except JobCancelled:
return
def get_default(self, key_name):
return None
def set_default(self, key_name, value):
pass
def show_message(self, msg):
self.messages.append(msg)
def ask_yes_no(self, prompt):
return True # always answer yes
class ResultTable(ResultTableBase):
COLUMNS = [
@@ -55,21 +55,21 @@ class ResultTable(ResultTableBase):
Column('extension', 'Kind'),
]
DELTA_COLUMNS = {'size', }
class DupeGuru(DupeGuruBase):
NAME = 'dupeGuru'
METADATA_TO_READ = ['size']
def __init__(self):
DupeGuruBase.__init__(self, DupeGuruView())
self.appdata = '/tmp'
def _prioritization_categories(self):
return prioritize.all_categories()
def _create_result_table(self):
return ResultTable(self)
class NamedObject:
def __init__(self, name="foobar", with_words=False, size=1, folder=None):
@@ -83,10 +83,10 @@ class NamedObject:
if with_words:
self.words = getwords(name)
self.is_ref = False
def __bool__(self):
return False #Make sure that operations are made correctly when the bool value of files is false.
def get_display_info(self, group, delta):
size = self.size
m = group.get_match_of(self)
@@ -99,19 +99,19 @@ class NamedObject:
'size': format_size(size, 0, 1, False),
'extension': self.extension if hasattr(self, 'extension') else '---',
}
@property
def path(self):
return self._folder[self.name]
@property
def folder_path(self):
return self.path.parent()
@property
def extension(self):
return get_file_ext(self.name)
# Returns a group set that looks like that:
# "foo bar" (1)
# "bar bleh" (1024)
@@ -135,7 +135,7 @@ class TestApp(TestAppBase):
if hasattr(gui, 'columns'): # tables
gui.columns.view = self.make_logger()
return gui
TestAppBase.__init__(self)
make_gui = self.make_gui
self.app = DupeGuru()
@@ -153,14 +153,14 @@ class TestApp(TestAppBase):
link_gui(self.app.progress_window)
link_gui(self.app.progress_window.jobdesc_textfield)
link_gui(self.app.progress_window.progressdesc_textfield)
#--- Helpers
def select_pri_criterion(self, name):
# Select a main prioritize criterion by name instead of by index. Makes tests more
# maintainable.
index = self.pdialog.category_list.index(name)
self.pdialog.category_list.select(index)
def add_pri_criterion(self, name, index):
self.select_pri_criterion(name)
self.pdialog.criteria_list.select([index])

216
core/tests/engine_test.py
View File

@@ -1,14 +1,14 @@
# Created By: Virgil Dupras
# Created On: 2006/01/29
# Copyright 2014 Hardcoded Software (http://www.hardcoded.net)
#
# This software is licensed under the "BSD" License as described in the "LICENSE" file,
# which should be included with this package. The terms are also available at
#
# This software is licensed under the "BSD" License as described in the "LICENSE" file,
# which should be included with this package. The terms are also available at
# http://www.hardcoded.net/licenses/bsd_license
import sys
from jobprogress import job
from hscommon.jobprogress import job
from hscommon.util import first
from hscommon.testutil import eq_, log_calls
@@ -48,119 +48,119 @@ class TestCasegetwords:
def test_spaces(self):
eq_(['a', 'b', 'c', 'd'], getwords("a b c d"))
eq_(['a', 'b', 'c', 'd'], getwords(" a b c d "))
def test_splitter_chars(self):
eq_(
[chr(i) for i in range(ord('a'),ord('z')+1)],
getwords("a-b_c&d+e(f)g;h\\i[j]k{l}m:n.o,p<q>r/s?t~u!v@w#x$y*z")
)
def test_joiner_chars(self):
eq_(["aec"], getwords("a'e\u0301c"))
def test_empty(self):
eq_([], getwords(''))
def test_returns_lowercase(self):
eq_(['foo', 'bar'], getwords('FOO BAR'))
def test_decompose_unicode(self):
eq_(getwords('foo\xe9bar'), ['fooebar'])
class TestCasegetfields:
def test_simple(self):
eq_([['a', 'b'], ['c', 'd', 'e']], getfields('a b - c d e'))
def test_empty(self):
eq_([], getfields(''))
def test_cleans_empty_fields(self):
expected = [['a', 'bc', 'def']]
actual = getfields(' - a bc def')
eq_(expected, actual)
expected = [['bc', 'def']]
class TestCaseunpack_fields:
def test_with_fields(self):
expected = ['a', 'b', 'c', 'd', 'e', 'f']
actual = unpack_fields([['a'], ['b', 'c'], ['d', 'e', 'f']])
eq_(expected, actual)
def test_without_fields(self):
expected = ['a', 'b', 'c', 'd', 'e', 'f']
actual = unpack_fields(['a', 'b', 'c', 'd', 'e', 'f'])
eq_(expected, actual)
def test_empty(self):
eq_([], unpack_fields([]))
class TestCaseWordCompare:
def test_list(self):
eq_(100, compare(['a', 'b', 'c', 'd'],['a', 'b', 'c', 'd']))
eq_(86, compare(['a', 'b', 'c', 'd'],['a', 'b', 'c']))
def test_unordered(self):
#Sometimes, users don't want fuzzy matching too much When they set the slider
#to 100, they don't expect a filename with the same words, but not the same order, to match.
#Thus, we want to return 99 in that case.
eq_(99, compare(['a', 'b', 'c', 'd'], ['d', 'b', 'c', 'a']))
def test_word_occurs_twice(self):
#if a word occurs twice in first, but once in second, we want the word to be only counted once
eq_(89, compare(['a', 'b', 'c', 'd', 'a'], ['d', 'b', 'c', 'a']))
def test_uses_copy_of_lists(self):
first = ['foo', 'bar']
second = ['bar', 'bleh']
compare(first, second)
eq_(['foo', 'bar'], first)
eq_(['bar', 'bleh'], second)
def test_word_weight(self):
eq_(int((6.0 / 13.0) * 100), compare(['foo', 'bar'], ['bar', 'bleh'], (WEIGHT_WORDS, )))
def test_similar_words(self):
eq_(100, compare(['the', 'white', 'stripes'],['the', 'whites', 'stripe'], (MATCH_SIMILAR_WORDS, )))
def test_empty(self):
eq_(0, compare([], []))
def test_with_fields(self):
eq_(67, compare([['a', 'b'], ['c', 'd', 'e']], [['a', 'b'], ['c', 'd', 'f']]))
def test_propagate_flags_with_fields(self, monkeypatch):
def mock_compare(first, second, flags):
eq_((0, 1, 2, 3, 5), flags)
monkeypatch.setattr(engine, 'compare_fields', mock_compare)
compare([['a']], [['a']], (0, 1, 2, 3, 5))
class TestCaseWordCompareWithFields:
def test_simple(self):
eq_(67, compare_fields([['a', 'b'], ['c', 'd', 'e']], [['a', 'b'], ['c', 'd', 'f']]))
def test_empty(self):
eq_(0, compare_fields([], []))
def test_different_length(self):
eq_(0, compare_fields([['a'], ['b']], [['a'], ['b'], ['c']]))
def test_propagates_flags(self, monkeypatch):
def mock_compare(first, second, flags):
eq_((0, 1, 2, 3, 5), flags)
monkeypatch.setattr(engine, 'compare_fields', mock_compare)
compare_fields([['a']], [['a']],(0, 1, 2, 3, 5))
def test_order(self):
first = [['a', 'b'], ['c', 'd', 'e']]
second = [['c', 'd', 'f'], ['a', 'b']]
eq_(0, compare_fields(first, second))
def test_no_order(self):
first = [['a','b'],['c','d','e']]
second = [['c','d','f'],['a','b']]
@@ -168,10 +168,10 @@ class TestCaseWordCompareWithFields:
first = [['a','b'],['a','b']] #a field can only be matched once.
second = [['c','d','f'],['a','b']]
eq_(0, compare_fields(first, second, (NO_FIELD_ORDER, )))
first = [['a','b'],['a','b','c']]
first = [['a','b'],['a','b','c']]
second = [['c','d','f'],['a','b']]
eq_(33, compare_fields(first, second, (NO_FIELD_ORDER, )))
def test_compare_fields_without_order_doesnt_alter_fields(self):
#The NO_ORDER comp type altered the fields!
first = [['a','b'],['c','d','e']]
@@ -179,7 +179,7 @@ class TestCaseWordCompareWithFields:
eq_(67, compare_fields(first, second, (NO_FIELD_ORDER, )))
eq_([['a','b'],['c','d','e']],first)
eq_([['c','d','f'],['a','b']],second)
class TestCasebuild_word_dict:
def test_with_standard_words(self):
@@ -199,30 +199,30 @@ class TestCasebuild_word_dict:
assert l[2] in d['baz']
eq_(1,len(d['bleh']))
assert l[2] in d['bleh']
def test_unpack_fields(self):
o = NamedObject('')
o.words = [['foo','bar'],['baz']]
d = build_word_dict([o])
eq_(3,len(d))
eq_(1,len(d['foo']))
def test_words_are_unaltered(self):
o = NamedObject('')
o.words = [['foo','bar'],['baz']]
build_word_dict([o])
eq_([['foo','bar'],['baz']],o.words)
def test_object_instances_can_only_be_once_in_words_object_list(self):
o = NamedObject('foo foo',True)
d = build_word_dict([o])
eq_(1,len(d['foo']))
def test_job(self):
def do_progress(p,d=''):
self.log.append(p)
return True
j = job.Job(1,do_progress)
self.log = []
s = "foo bar"
@@ -230,7 +230,7 @@ class TestCasebuild_word_dict:
# We don't have intermediate log because iter_with_progress is called with every > 1
eq_(0,self.log[0])
eq_(100,self.log[1])
class TestCasemerge_similar_words:
def test_some_similar_words(self):
@@ -242,8 +242,8 @@ class TestCasemerge_similar_words:
merge_similar_words(d)
eq_(1,len(d))
eq_(3,len(d['foobar']))
class TestCasereduce_common_words:
def test_typical(self):
@@ -254,7 +254,7 @@ class TestCasereduce_common_words:
reduce_common_words(d, 50)
assert 'foo' not in d
eq_(49,len(d['bar']))
def test_dont_remove_objects_with_only_common_words(self):
d = {
'common': set([NamedObject("common uncommon",True) for i in range(50)] + [NamedObject("common",True)]),
@@ -263,7 +263,7 @@ class TestCasereduce_common_words:
reduce_common_words(d, 50)
eq_(1,len(d['common']))
eq_(1,len(d['uncommon']))
def test_values_still_are_set_instances(self):
d = {
'common': set([NamedObject("common uncommon",True) for i in range(50)] + [NamedObject("common",True)]),
@@ -272,7 +272,7 @@ class TestCasereduce_common_words:
reduce_common_words(d, 50)
assert isinstance(d['common'],set)
assert isinstance(d['uncommon'],set)
def test_dont_raise_KeyError_when_a_word_has_been_removed(self):
#If a word has been removed by the reduce, an object in a subsequent common word that
#contains the word that has been removed would cause a KeyError.
@@ -285,14 +285,14 @@ class TestCasereduce_common_words:
reduce_common_words(d, 50)
except KeyError:
self.fail()
def test_unpack_fields(self):
#object.words may be fields.
def create_it():
o = NamedObject('')
o.words = [['foo','bar'],['baz']]
return o
d = {
'foo': set([create_it() for i in range(50)])
}
@@ -300,7 +300,7 @@ class TestCasereduce_common_words:
reduce_common_words(d, 50)
except TypeError:
self.fail("must support fields.")
def test_consider_a_reduced_common_word_common_even_after_reduction(self):
#There was a bug in the code that causeda word that has already been reduced not to
#be counted as a common word for subsequent words. For example, if 'foo' is processed
@@ -316,7 +316,7 @@ class TestCasereduce_common_words:
eq_(1,len(d['foo']))
eq_(1,len(d['bar']))
eq_(49,len(d['baz']))
class TestCaseget_match:
def test_simple(self):
@@ -328,7 +328,7 @@ class TestCaseget_match:
eq_(['bar','bleh'],m.second.words)
assert m.first is o1
assert m.second is o2
def test_in(self):
o1 = NamedObject("foo",True)
o2 = NamedObject("bar",True)
@@ -336,15 +336,15 @@ class TestCaseget_match:
assert o1 in m
assert o2 in m
assert object() not in m
def test_word_weight(self):
eq_(int((6.0 / 13.0) * 100),get_match(NamedObject("foo bar",True),NamedObject("bar bleh",True),(WEIGHT_WORDS,)).percentage)
class TestCaseGetMatches:
def test_empty(self):
eq_(getmatches([]), [])
def test_simple(self):
l = [NamedObject("foo bar"),NamedObject("bar bleh"),NamedObject("a b c foo")]
r = getmatches(l)
@@ -353,7 +353,7 @@ class TestCaseGetMatches:
assert_match(m, 'foo bar', 'bar bleh')
m = first(m for m in r if m.percentage == 33) #"foo bar" and "a b c foo"
assert_match(m, 'foo bar', 'a b c foo')
def test_null_and_unrelated_objects(self):
l = [NamedObject("foo bar"),NamedObject("bar bleh"),NamedObject(""),NamedObject("unrelated object")]
r = getmatches(l)
@@ -361,22 +361,22 @@ class TestCaseGetMatches:
m = r[0]
eq_(m.percentage, 50)
assert_match(m, 'foo bar', 'bar bleh')
def test_twice_the_same_word(self):
l = [NamedObject("foo foo bar"),NamedObject("bar bleh")]
r = getmatches(l)
eq_(1,len(r))
def test_twice_the_same_word_when_preworded(self):
l = [NamedObject("foo foo bar",True),NamedObject("bar bleh",True)]
r = getmatches(l)
eq_(1,len(r))
def test_two_words_match(self):
l = [NamedObject("foo bar"),NamedObject("foo bar bleh")]
r = getmatches(l)
eq_(1,len(r))
def test_match_files_with_only_common_words(self):
#If a word occurs more than 50 times, it is excluded from the matching process
#The problem with the common_word_threshold is that the files containing only common
@@ -385,18 +385,18 @@ class TestCaseGetMatches:
l = [NamedObject("foo") for i in range(50)]
r = getmatches(l)
eq_(1225,len(r))
def test_use_words_already_there_if_there(self):
o1 = NamedObject('foo')
o2 = NamedObject('bar')
o2.words = ['foo']
eq_(1, len(getmatches([o1,o2])))
def test_job(self):
def do_progress(p,d=''):
self.log.append(p)
return True
j = job.Job(1,do_progress)
self.log = []
s = "foo bar"
@@ -404,12 +404,12 @@ class TestCaseGetMatches:
assert len(self.log) > 2
eq_(0,self.log[0])
eq_(100,self.log[-1])
def test_weight_words(self):
l = [NamedObject("foo bar"),NamedObject("bar bleh")]
m = getmatches(l, weight_words=True)[0]
eq_(int((6.0 / 13.0) * 100),m.percentage)
def test_similar_word(self):
l = [NamedObject("foobar"),NamedObject("foobars")]
eq_(len(getmatches(l, match_similar_words=True)), 1)
@@ -420,16 +420,16 @@ class TestCaseGetMatches:
eq_(len(getmatches(l, match_similar_words=True)), 1)
l = [NamedObject("foobar"),NamedObject("foosbar")]
eq_(len(getmatches(l, match_similar_words=True)), 1)
def test_single_object_with_similar_words(self):
l = [NamedObject("foo foos")]
eq_(len(getmatches(l, match_similar_words=True)), 0)
def test_double_words_get_counted_only_once(self):
l = [NamedObject("foo bar foo bleh"),NamedObject("foo bar bleh bar")]
m = getmatches(l)[0]
eq_(75,m.percentage)
def test_with_fields(self):
o1 = NamedObject("foo bar - foo bleh")
o2 = NamedObject("foo bar - bleh bar")
@@ -437,7 +437,7 @@ class TestCaseGetMatches:
o2.words = getfields(o2.name)
m = getmatches([o1, o2])[0]
eq_(50, m.percentage)
def test_with_fields_no_order(self):
o1 = NamedObject("foo bar - foo bleh")
o2 = NamedObject("bleh bang - foo bar")
@@ -445,11 +445,11 @@ class TestCaseGetMatches:
o2.words = getfields(o2.name)
m = getmatches([o1, o2], no_field_order=True)[0]
eq_(m.percentage, 50)
def test_only_match_similar_when_the_option_is_set(self):
l = [NamedObject("foobar"),NamedObject("foobars")]
eq_(len(getmatches(l, match_similar_words=False)), 0)
def test_dont_recurse_do_match(self):
# with nosetests, the stack is increased. The number has to be high enough not to be failing falsely
sys.setrecursionlimit(100)
@@ -460,19 +460,19 @@ class TestCaseGetMatches:
self.fail()
finally:
sys.setrecursionlimit(1000)
def test_min_match_percentage(self):
l = [NamedObject("foo bar"),NamedObject("bar bleh"),NamedObject("a b c foo")]
r = getmatches(l, min_match_percentage=50)
eq_(1,len(r)) #Only "foo bar" / "bar bleh" should match
def test_MemoryError(self, monkeypatch):
@log_calls
def mocked_match(first, second, flags):
if len(mocked_match.calls) > 42:
raise MemoryError()
return Match(first, second, 0)
objects = [NamedObject() for i in range(10)] # results in 45 matches
monkeypatch.setattr(engine, 'get_match', mocked_match)
try:
@@ -480,13 +480,13 @@ class TestCaseGetMatches:
except MemoryError:
self.fail('MemorryError must be handled')
eq_(42, len(r))
class TestCaseGetMatchesByContents:
def test_dont_compare_empty_files(self):
o1, o2 = no(size=0), no(size=0)
assert not getmatches_by_contents([o1, o2])
class TestCaseGroup:
def test_empy(self):
@@ -494,7 +494,7 @@ class TestCaseGroup:
eq_(None,g.ref)
eq_([],g.dupes)
eq_(0,len(g.matches))
def test_add_match(self):
g = Group()
m = get_match(NamedObject("foo",True),NamedObject("bar",True))
@@ -503,7 +503,7 @@ class TestCaseGroup:
eq_([m.second],g.dupes)
eq_(1,len(g.matches))
assert m in g.matches
def test_multiple_add_match(self):
g = Group()
o1 = NamedObject("a",True)
@@ -529,13 +529,13 @@ class TestCaseGroup:
g.add_match(get_match(o3,o4))
eq_([o2,o3,o4],g.dupes)
eq_(6,len(g.matches))
def test_len(self):
g = Group()
eq_(0,len(g))
g.add_match(get_match(NamedObject("foo",True),NamedObject("bar",True)))
eq_(2,len(g))
def test_add_same_match_twice(self):
g = Group()
m = get_match(NamedObject("foo",True),NamedObject("foo",True))
@@ -545,7 +545,7 @@ class TestCaseGroup:
g.add_match(m)
eq_(2,len(g))
eq_(1,len(g.matches))
def test_in(self):
g = Group()
o1 = NamedObject("foo",True)
@@ -554,7 +554,7 @@ class TestCaseGroup:
g.add_match(get_match(o1,o2))
assert o1 in g
assert o2 in g
def test_remove(self):
g = Group()
o1 = NamedObject("foo",True)
@@ -571,7 +571,7 @@ class TestCaseGroup:
g.remove_dupe(o1)
eq_(0,len(g.matches))
eq_(0,len(g))
def test_remove_with_ref_dupes(self):
g = Group()
o1 = NamedObject("foo",True)
@@ -584,7 +584,7 @@ class TestCaseGroup:
o2.is_ref = True
g.remove_dupe(o3)
eq_(0,len(g))
def test_switch_ref(self):
o1 = NamedObject(with_words=True)
o2 = NamedObject(with_words=True)
@@ -598,7 +598,7 @@ class TestCaseGroup:
assert o2 is g.ref
g.switch_ref(NamedObject('',True))
assert o2 is g.ref
def test_switch_ref_from_ref_dir(self):
# When the ref dupe is from a ref dir, switch_ref() does nothing
o1 = no(with_words=True)
@@ -608,7 +608,7 @@ class TestCaseGroup:
g.add_match(get_match(o1, o2))
g.switch_ref(o2)
assert o1 is g.ref
def test_get_match_of(self):
g = Group()
for m in get_match_triangle():
@@ -619,7 +619,7 @@ class TestCaseGroup:
assert o in m
assert g.get_match_of(NamedObject('',True)) is None
assert g.get_match_of(g.ref) is None
def test_percentage(self):
#percentage should return the avg percentage in relation to the ref
m1,m2,m3 = get_match_triangle()
@@ -638,11 +638,11 @@ class TestCaseGroup:
g.add_match(m1)
g.add_match(m2)
eq_(66,g.percentage)
def test_percentage_on_empty_group(self):
g = Group()
eq_(0,g.percentage)
def test_prioritize(self):
m1,m2,m3 = get_match_triangle()
o1 = m1.first
@@ -658,7 +658,7 @@ class TestCaseGroup:
assert o1 is g.ref
assert g.prioritize(lambda x:x.name)
assert o3 is g.ref
def test_prioritize_with_tie_breaker(self):
# if the ref has the same key as one or more of the dupe, run the tie_breaker func among them
g = get_test_group()
@@ -666,9 +666,9 @@ class TestCaseGroup:
tie_breaker = lambda ref, dupe: dupe is o3
g.prioritize(lambda x:0, tie_breaker)
assert g.ref is o3
def test_prioritize_with_tie_breaker_runs_on_all_dupes(self):
# Even if a dupe is chosen to switch with ref with a tie breaker, we still run the tie breaker
# Even if a dupe is chosen to switch with ref with a tie breaker, we still run the tie breaker
# with other dupes and the newly chosen ref
g = get_test_group()
o1, o2, o3 = g.ordered
@@ -678,7 +678,7 @@ class TestCaseGroup:
tie_breaker = lambda ref, dupe: dupe.foo > ref.foo
g.prioritize(lambda x:0, tie_breaker)
assert g.ref is o3
def test_prioritize_with_tie_breaker_runs_only_on_tie_dupes(self):
# The tie breaker only runs on dupes that had the same value for the key_func
g = get_test_group()
@@ -693,7 +693,7 @@ class TestCaseGroup:
tie_breaker = lambda ref, dupe: dupe.bar > ref.bar
g.prioritize(key_func, tie_breaker)
assert g.ref is o2
def test_prioritize_with_ref_dupe(self):
# when the ref dupe of a group is from a ref dir, make it stay on top.
g = get_test_group()
@@ -702,7 +702,7 @@ class TestCaseGroup:
o2.size = 2
g.prioritize(lambda x: -x.size)
assert g.ref is o1
def test_prioritize_nothing_changes(self):
# prioritize() returns False when nothing changes in the group.
g = get_test_group()
@@ -710,14 +710,14 @@ class TestCaseGroup:
g[1].name = 'b'
g[2].name = 'c'
assert not g.prioritize(lambda x:x.name)
def test_list_like(self):
g = Group()
o1,o2 = (NamedObject("foo",True),NamedObject("bar",True))
g.add_match(get_match(o1,o2))
assert g[0] is o1
assert g[1] is o2
def test_discard_matches(self):
g = Group()
o1,o2,o3 = (NamedObject("foo",True),NamedObject("bar",True),NamedObject("baz",True))
@@ -726,13 +726,13 @@ class TestCaseGroup:
g.discard_matches()
eq_(1,len(g.matches))
eq_(0,len(g.candidates))
class TestCaseget_groups:
def test_empty(self):
r = get_groups([])
eq_([],r)
def test_simple(self):
l = [NamedObject("foo bar"),NamedObject("bar bleh")]
matches = getmatches(l)
@@ -742,7 +742,7 @@ class TestCaseget_groups:
g = r[0]
assert g.ref is m.first
eq_([m.second],g.dupes)
def test_group_with_multiple_matches(self):
#This results in 3 matches
l = [NamedObject("foo"),NamedObject("foo"),NamedObject("foo")]
@@ -751,7 +751,7 @@ class TestCaseget_groups:
eq_(1,len(r))
g = r[0]
eq_(3,len(g))
def test_must_choose_a_group(self):
l = [NamedObject("a b"),NamedObject("a b"),NamedObject("b c"),NamedObject("c d"),NamedObject("c d")]
#There will be 2 groups here: group "a b" and group "c d"
@@ -760,7 +760,7 @@ class TestCaseget_groups:
r = get_groups(matches)
eq_(2,len(r))
eq_(5,len(r[0])+len(r[1]))
def test_should_all_go_in_the_same_group(self):
l = [NamedObject("a b"),NamedObject("a b"),NamedObject("a b"),NamedObject("a b")]
#There will be 2 groups here: group "a b" and group "c d"
@@ -768,7 +768,7 @@ class TestCaseget_groups:
matches = getmatches(l)
r = get_groups(matches)
eq_(1,len(r))
def test_give_priority_to_matches_with_higher_percentage(self):
o1 = NamedObject(with_words=True)
o2 = NamedObject(with_words=True)
@@ -782,14 +782,14 @@ class TestCaseget_groups:
assert o1 not in g
assert o2 in g
assert o3 in g
def test_four_sized_group(self):
l = [NamedObject("foobar") for i in range(4)]
m = getmatches(l)
r = get_groups(m)
eq_(1,len(r))
eq_(4,len(r[0]))
def test_referenced_by_ref2(self):
o1 = NamedObject(with_words=True)
o2 = NamedObject(with_words=True)
@@ -799,12 +799,12 @@ class TestCaseget_groups:
m3 = get_match(o3,o2)
r = get_groups([m1,m2,m3])
eq_(3,len(r[0]))
def test_job(self):
def do_progress(p,d=''):
self.log.append(p)
return True
self.log = []
j = job.Job(1,do_progress)
m1,m2,m3 = get_match_triangle()
@@ -813,7 +813,7 @@ class TestCaseget_groups:
get_groups([m1,m2,m3,m4],j)
eq_(0,self.log[0])
eq_(100,self.log[-1])
def test_group_admissible_discarded_dupes(self):
# If, with a (A, B, C, D) set, all match with A, but C and D don't match with B and that the
# (A, B) match is the highest (thus resulting in an (A, B) group), still match C and D
@@ -830,4 +830,4 @@ class TestCaseget_groups:
assert B in g1
assert C in g2
assert D in g2

18
core/tests/scanner_test.py
View File

@@ -1,12 +1,12 @@
# Created By: Virgil Dupras
# Created On: 2006/03/03
# Copyright 2014 Hardcoded Software (http://www.hardcoded.net)
#
# This software is licensed under the "BSD" License as described in the "LICENSE" file,
# which should be included with this package. The terms are also available at
#
# This software is licensed under the "BSD" License as described in the "LICENSE" file,
# which should be included with this package. The terms are also available at
# http://www.hardcoded.net/licenses/bsd_license
from jobprogress import job
from hscommon.jobprogress import job
from hscommon.path import Path
from hscommon.testutil import eq_
@@ -25,10 +25,10 @@ class NamedObject:
self.size = size
self.path = path
self.words = getwords(name)
def __repr__(self):
return '<NamedObject %r %r>' % (self.name, self.path)
no = NamedObject
@@ -384,7 +384,7 @@ def test_file_evaluates_to_false(fake_fileexists):
class FalseNamedObject(NamedObject):
def __bool__(self):
return False
s = Scanner()
f1 = FalseNamedObject('foobar', path='p1')
@@ -445,7 +445,7 @@ def test_tie_breaker_same_name_plus_digit(fake_fileexists):
assert group.ref is o5
def test_partial_group_match(fake_fileexists):
# Count the number of discarded matches (when a file doesn't match all other dupes of the
# Count the number of discarded matches (when a file doesn't match all other dupes of the
# group) in Scanner.discarded_file_count
s = Scanner()
o1, o2, o3 = no('a b'), no('a'), no('b')
@@ -476,7 +476,7 @@ def test_dont_group_files_that_dont_exist(tmpdir):
file2.path.remove()
return [Match(file1, file2, 100)]
s._getmatches = getmatches
assert not s.get_dupe_groups([file1, file2])
def test_folder_scan_exclude_subfolder_matches(fake_fileexists):