arsenetar/dupeguru
1
0
Fork 0
mirror of https://github.com/arsenetar/dupeguru.git synced 2026-01-22 06:37:17 +00:00
Code Releases Wiki Activity

Refactoring: Path API compatibility with pathlib

Browse Source 
Refactored dupeGuru to make hscommon.path's API a bit close to pathlib's
API. It's not 100% compatible yet, but it's much better than before.

This is more of a hscommon refactoring than a dupeguru one, but since
duepGuru is the main user of Path, it was the driver behind the
refactoring.

This refactoring also see the introduction of @pathify, which ensure
Path arguments. Previously, we were often unsure of whether the caller
of a function was passing a Path or a str. This problem is now solved
and this allows us to remove hscommon.io, an ill-conceived attempt to
solve that same ambiguity problem.

Fixes #235.
This commit is contained in:
Virgil Dupras
2013-11-16 12:06:16 -05:00
parent e8c42740cf
commit 10dbfa9b38
24 changed files with 353 additions and 313 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
core/tests/app_test.py
View File

@@ -11,7 +11,6 @@ import os.path as op
import logging
from pytest import mark
from hscommon import io
from hscommon.path import Path
import hscommon.conflict
import hscommon.util
@@ -57,7 +56,7 @@ class TestCaseDupeGuru:
# for this unit is pathetic. What's done is done. My approach now is to add tests for
# every change I want to make. The blowup was caused by a missing import.
p = Path(str(tmpdir))
io.open(p + 'foo', 'w').close()
p['foo'].open('w').close()
monkeypatch.setattr(hscommon.conflict, 'smart_copy', log_calls(lambda source_path, dest_path: None))
# XXX This monkeypatch is temporary. will be fixed in a better monkeypatcher.
monkeypatch.setattr(app, 'smart_copy', hscommon.conflict.smart_copy)
@@ -73,14 +72,14 @@ class TestCaseDupeGuru:
def test_copy_or_move_clean_empty_dirs(self, tmpdir, monkeypatch):
tmppath = Path(str(tmpdir))
sourcepath = tmppath + 'source'
io.mkdir(sourcepath)
io.open(sourcepath + 'myfile', 'w')
sourcepath = tmppath['source']
sourcepath.mkdir()
sourcepath['myfile'].open('w')
app = TestApp().app
app.directories.add_path(tmppath)
[myfile] = app.directories.get_files()
monkeypatch.setattr(app, 'clean_empty_dirs', log_calls(lambda path: None))
app.copy_or_move(myfile, False, tmppath + 'dest', 0)
app.copy_or_move(myfile, False, tmppath['dest'], 0)
calls = app.clean_empty_dirs.calls
eq_(1, len(calls))
eq_(sourcepath, calls[0]['path'])
@@ -104,8 +103,8 @@ class TestCaseDupeGuru:
# If the ignore_hardlink_matches option is set, don't match files hardlinking to the same
# inode.
tmppath = Path(str(tmpdir))
io.open(tmppath + 'myfile', 'w').write('foo')
os.link(str(tmppath + 'myfile'), str(tmppath + 'hardlink'))
tmppath['myfile'].open('w').write('foo')
os.link(str(tmppath['myfile']), str(tmppath['hardlink']))
app = TestApp().app
app.directories.add_path(tmppath)
app.scanner.scan_type = ScanType.Contents
@@ -171,8 +170,8 @@ class TestCaseDupeGuruWithResults:
self.rtable.refresh()
tmpdir = request.getfuncargvalue('tmpdir')
tmppath = Path(str(tmpdir))
io.mkdir(tmppath + 'foo')
io.mkdir(tmppath + 'bar')
tmppath['foo'].mkdir()
tmppath['bar'].mkdir()
self.app.directories.add_path(tmppath)
def test_GetObjects(self, do_setup):
@@ -417,11 +416,11 @@ class TestCaseDupeGuru_renameSelected:
def pytest_funcarg__do_setup(self, request):
tmpdir = request.getfuncargvalue('tmpdir')
p = Path(str(tmpdir))
fp = open(str(p + 'foo bar 1'),mode='w')
fp = open(str(p['foo bar 1']),mode='w')
fp.close()
fp = open(str(p + 'foo bar 2'),mode='w')
fp = open(str(p['foo bar 2']),mode='w')
fp.close()
fp = open(str(p + 'foo bar 3'),mode='w')
fp = open(str(p['foo bar 3']),mode='w')
fp.close()
files = fs.get_files(p)
for f in files:
@@ -444,7 +443,7 @@ class TestCaseDupeGuru_renameSelected:
g = self.groups[0]
self.rtable.select([1])
assert app.rename_selected('renamed')
names = io.listdir(self.p)
names = [p.name for p in self.p.listdir()]
assert 'renamed' in names
assert 'foo bar 2' not in names
eq_(g.dupes[0].name, 'renamed')
@@ -457,7 +456,7 @@ class TestCaseDupeGuru_renameSelected:
assert not app.rename_selected('renamed')
msg = logging.warning.calls[0]['msg']
eq_('dupeGuru Warning: list index out of range', msg)
names = io.listdir(self.p)
names = [p.name for p in self.p.listdir()]
assert 'renamed' not in names
assert 'foo bar 2' in names
eq_(g.dupes[0].name, 'foo bar 2')
@@ -470,7 +469,7 @@ class TestCaseDupeGuru_renameSelected:
assert not app.rename_selected('foo bar 1')
msg = logging.warning.calls[0]['msg']
assert msg.startswith('dupeGuru Warning: \'foo bar 1\' already exists in')
names = io.listdir(self.p)
names = [p.name for p in self.p.listdir()]
assert 'foo bar 1' in names
assert 'foo bar 2' in names
eq_(g.dupes[0].name, 'foo bar 2')
@@ -480,9 +479,9 @@ class TestAppWithDirectoriesInTree:
def pytest_funcarg__do_setup(self, request):
tmpdir = request.getfuncargvalue('tmpdir')
p = Path(str(tmpdir))
io.mkdir(p + 'sub1')
io.mkdir(p + 'sub2')
io.mkdir(p + 'sub3')
p['sub1'].mkdir()
p['sub2'].mkdir()
p['sub3'].mkdir()
app = TestApp()
self.app = app.app
self.dtree = app.dtree

4
core/tests/base.py
View File

@@ -102,11 +102,11 @@ class NamedObject:
@property
def path(self):
return self._folder + self.name
return self._folder[self.name]
@property
def folder_path(self):
return self.path[:-1]
return self.path.parent()
@property
def extension(self):

139
core/tests/directories_test.py
View File

@@ -12,7 +12,6 @@ import tempfile
import shutil
from pytest import raises
from hscommon import io
from hscommon.path import Path
from hscommon.testutil import eq_
@@ -20,27 +19,27 @@ from ..directories import *
def create_fake_fs(rootpath):
# We have it as a separate function because other units are using it.
rootpath = rootpath + 'fs'
io.mkdir(rootpath)
io.mkdir(rootpath + 'dir1')
io.mkdir(rootpath + 'dir2')
io.mkdir(rootpath + 'dir3')
fp = io.open(rootpath + 'file1.test', 'w')
rootpath = rootpath['fs']
rootpath.mkdir()
rootpath['dir1'].mkdir()
rootpath['dir2'].mkdir()
rootpath['dir3'].mkdir()
fp = rootpath['file1.test'].open('w')
fp.write('1')
fp.close()
fp = io.open(rootpath + 'file2.test', 'w')
fp = rootpath['file2.test'].open('w')
fp.write('12')
fp.close()
fp = io.open(rootpath + 'file3.test', 'w')
fp = rootpath['file3.test'].open('w')
fp.write('123')
fp.close()
fp = io.open(rootpath + ('dir1', 'file1.test'), 'w')
fp = rootpath['dir1']['file1.test'].open('w')
fp.write('1')
fp.close()
fp = io.open(rootpath + ('dir2', 'file2.test'), 'w')
fp = rootpath['dir2']['file2.test'].open('w')
fp.write('12')
fp.close()
fp = io.open(rootpath + ('dir3', 'file3.test'), 'w')
fp = rootpath['dir3']['file3.test'].open('w')
fp.write('123')
fp.close()
return rootpath
@@ -50,9 +49,9 @@ def setup_module(module):
# and another with a more complex structure.
testpath = Path(tempfile.mkdtemp())
module.testpath = testpath
rootpath = testpath + 'onefile'
io.mkdir(rootpath)
fp = io.open(rootpath + 'test.txt', 'w')
rootpath = testpath['onefile']
rootpath.mkdir()
fp = rootpath['test.txt'].open('w')
fp.write('test_data')
fp.close()
create_fake_fs(testpath)
@@ -67,30 +66,30 @@ def test_empty():
def test_add_path():
d = Directories()
p = testpath + 'onefile'
p = testpath['onefile']
d.add_path(p)
eq_(1,len(d))
assert p in d
assert (p + 'foobar') in d
assert p[:-1] not in d
p = testpath + 'fs'
assert (p['foobar']) in d
assert p.parent() not in d
p = testpath['fs']
d.add_path(p)
eq_(2,len(d))
assert p in d
def test_AddPath_when_path_is_already_there():
d = Directories()
p = testpath + 'onefile'
p = testpath['onefile']
d.add_path(p)
with raises(AlreadyThereError):
d.add_path(p)
with raises(AlreadyThereError):
d.add_path(p + 'foobar')
d.add_path(p['foobar'])
eq_(1, len(d))
def test_add_path_containing_paths_already_there():
d = Directories()
d.add_path(testpath + 'onefile')
d.add_path(testpath['onefile'])
eq_(1, len(d))
d.add_path(testpath)
eq_(len(d), 1)
@@ -98,7 +97,7 @@ def test_add_path_containing_paths_already_there():
def test_AddPath_non_latin(tmpdir):
p = Path(str(tmpdir))
to_add = p + 'unicode\u201a'
to_add = p['unicode\u201a']
os.mkdir(str(to_add))
d = Directories()
try:
@@ -108,24 +107,24 @@ def test_AddPath_non_latin(tmpdir):
def test_del():
d = Directories()
d.add_path(testpath + 'onefile')
d.add_path(testpath['onefile'])
try:
del d[1]
assert False
except IndexError:
pass
d.add_path(testpath + 'fs')
d.add_path(testpath['fs'])
del d[1]
eq_(1, len(d))
def test_states():
d = Directories()
p = testpath + 'onefile'
p = testpath['onefile']
d.add_path(p)
eq_(DirectoryState.Normal ,d.get_state(p))
d.set_state(p, DirectoryState.Reference)
eq_(DirectoryState.Reference ,d.get_state(p))
eq_(DirectoryState.Reference ,d.get_state(p + 'dir1'))
eq_(DirectoryState.Reference ,d.get_state(p['dir1']))
eq_(1,len(d.states))
eq_(p,list(d.states.keys())[0])
eq_(DirectoryState.Reference ,d.states[p])
@@ -133,67 +132,67 @@ def test_states():
def test_get_state_with_path_not_there():
# When the path's not there, just return DirectoryState.Normal
d = Directories()
d.add_path(testpath + 'onefile')
d.add_path(testpath['onefile'])
eq_(d.get_state(testpath), DirectoryState.Normal)
def test_states_remain_when_larger_directory_eat_smaller_ones():
d = Directories()
p = testpath + 'onefile'
p = testpath['onefile']
d.add_path(p)
d.set_state(p, DirectoryState.Excluded)
d.add_path(testpath)
d.set_state(testpath, DirectoryState.Reference)
eq_(DirectoryState.Excluded ,d.get_state(p))
eq_(DirectoryState.Excluded ,d.get_state(p + 'dir1'))
eq_(DirectoryState.Excluded ,d.get_state(p['dir1']))
eq_(DirectoryState.Reference ,d.get_state(testpath))
def test_set_state_keep_state_dict_size_to_minimum():
d = Directories()
p = testpath + 'fs'
p = testpath['fs']
d.add_path(p)
d.set_state(p, DirectoryState.Reference)
d.set_state(p + 'dir1', DirectoryState.Reference)
d.set_state(p['dir1'], DirectoryState.Reference)
eq_(1,len(d.states))
eq_(DirectoryState.Reference ,d.get_state(p + 'dir1'))
d.set_state(p + 'dir1', DirectoryState.Normal)
eq_(DirectoryState.Reference ,d.get_state(p['dir1']))
d.set_state(p['dir1'], DirectoryState.Normal)
eq_(2,len(d.states))
eq_(DirectoryState.Normal ,d.get_state(p + 'dir1'))
d.set_state(p + 'dir1', DirectoryState.Reference)
eq_(DirectoryState.Normal ,d.get_state(p['dir1']))
d.set_state(p['dir1'], DirectoryState.Reference)
eq_(1,len(d.states))
eq_(DirectoryState.Reference ,d.get_state(p + 'dir1'))
eq_(DirectoryState.Reference ,d.get_state(p['dir1']))
def test_get_files():
d = Directories()
p = testpath + 'fs'
p = testpath['fs']
d.add_path(p)
d.set_state(p + 'dir1', DirectoryState.Reference)
d.set_state(p + 'dir2', DirectoryState.Excluded)
d.set_state(p['dir1'], DirectoryState.Reference)
d.set_state(p['dir2'], DirectoryState.Excluded)
files = list(d.get_files())
eq_(5, len(files))
for f in files:
if f.path[:-1] == p + 'dir1':
if f.path.parent() == p['dir1']:
assert f.is_ref
else:
assert not f.is_ref
def test_get_folders():
d = Directories()
p = testpath + 'fs'
p = testpath['fs']
d.add_path(p)
d.set_state(p + 'dir1', DirectoryState.Reference)
d.set_state(p + 'dir2', DirectoryState.Excluded)
d.set_state(p['dir1'], DirectoryState.Reference)
d.set_state(p['dir2'], DirectoryState.Excluded)
folders = list(d.get_folders())
eq_(len(folders), 3)
ref = [f for f in folders if f.is_ref]
not_ref = [f for f in folders if not f.is_ref]
eq_(len(ref), 1)
eq_(ref[0].path, p + 'dir1')
eq_(ref[0].path, p['dir1'])
eq_(len(not_ref), 2)
eq_(ref[0].size, 1)
def test_get_files_with_inherited_exclusion():
d = Directories()
p = testpath + 'onefile'
p = testpath['onefile']
d.add_path(p)
d.set_state(p, DirectoryState.Excluded)
eq_([], list(d.get_files()))
@@ -202,19 +201,19 @@ def test_save_and_load(tmpdir):
d1 = Directories()
d2 = Directories()
p1 = Path(str(tmpdir.join('p1')))
io.mkdir(p1)
p1.mkdir()
p2 = Path(str(tmpdir.join('p2')))
io.mkdir(p2)
p2.mkdir()
d1.add_path(p1)
d1.add_path(p2)
d1.set_state(p1, DirectoryState.Reference)
d1.set_state(p1 + 'dir1', DirectoryState.Excluded)
d1.set_state(p1['dir1'], DirectoryState.Excluded)
tmpxml = str(tmpdir.join('directories_testunit.xml'))
d1.save_to_file(tmpxml)
d2.load_from_file(tmpxml)
eq_(2, len(d2))
eq_(DirectoryState.Reference ,d2.get_state(p1))
eq_(DirectoryState.Excluded ,d2.get_state(p1 + 'dir1'))
eq_(DirectoryState.Excluded ,d2.get_state(p1['dir1']))
def test_invalid_path():
d = Directories()
@@ -234,12 +233,12 @@ def test_load_from_file_with_invalid_path(tmpdir):
#This test simulates a load from file resulting in a
#InvalidPath raise. Other directories must be loaded.
d1 = Directories()
d1.add_path(testpath + 'onefile')
d1.add_path(testpath['onefile'])
#Will raise InvalidPath upon loading
p = Path(str(tmpdir.join('toremove')))
io.mkdir(p)
p.mkdir()
d1.add_path(p)
io.rmdir(p)
p.rmdir()
tmpxml = str(tmpdir.join('directories_testunit.xml'))
d1.save_to_file(tmpxml)
d2 = Directories()
@@ -248,11 +247,11 @@ def test_load_from_file_with_invalid_path(tmpdir):
def test_unicode_save(tmpdir):
d = Directories()
p1 = Path(str(tmpdir)) + 'hello\xe9'
io.mkdir(p1)
io.mkdir(p1 + 'foo\xe9')
p1 = Path(str(tmpdir))['hello\xe9']
p1.mkdir()
p1['foo\xe9'].mkdir()
d.add_path(p1)
d.set_state(p1 + 'foo\xe9', DirectoryState.Excluded)
d.set_state(p1['foo\xe9'], DirectoryState.Excluded)
tmpxml = str(tmpdir.join('directories_testunit.xml'))
try:
d.save_to_file(tmpxml)
@@ -261,12 +260,12 @@ def test_unicode_save(tmpdir):
def test_get_files_refreshes_its_directories():
d = Directories()
p = testpath + 'fs'
p = testpath['fs']
d.add_path(p)
files = d.get_files()
eq_(6, len(list(files)))
time.sleep(1)
os.remove(str(p + ('dir1','file1.test')))
os.remove(str(p['dir1']['file1.test']))
files = d.get_files()
eq_(5, len(list(files)))
@@ -274,14 +273,14 @@ def test_get_files_does_not_choke_on_non_existing_directories(tmpdir):
d = Directories()
p = Path(str(tmpdir))
d.add_path(p)
io.rmtree(p)
p.rmtree()
eq_([], list(d.get_files()))
def test_get_state_returns_excluded_by_default_for_hidden_directories(tmpdir):
d = Directories()
p = Path(str(tmpdir))
hidden_dir_path = p + '.foo'
io.mkdir(p + '.foo')
hidden_dir_path = p['.foo']
p['.foo'].mkdir()
d.add_path(p)
eq_(d.get_state(hidden_dir_path), DirectoryState.Excluded)
# But it can be overriden
@@ -297,16 +296,16 @@ def test_default_path_state_override(tmpdir):
d = MyDirectories()
p1 = Path(str(tmpdir))
io.mkdir(p1 + 'foobar')
io.open(p1 + 'foobar/somefile', 'w').close()
io.mkdir(p1 + 'foobaz')
io.open(p1 + 'foobaz/somefile', 'w').close()
p1['foobar'].mkdir()
p1['foobar/somefile'].open('w').close()
p1['foobaz'].mkdir()
p1['foobaz/somefile'].open('w').close()
d.add_path(p1)
eq_(d.get_state(p1 + 'foobaz'), DirectoryState.Normal)
eq_(d.get_state(p1 + 'foobar'), DirectoryState.Excluded)
eq_(d.get_state(p1['foobaz']), DirectoryState.Normal)
eq_(d.get_state(p1['foobar']), DirectoryState.Excluded)
eq_(len(list(d.get_files())), 1) # only the 'foobaz' file is there
# However, the default state can be changed
d.set_state(p1 + 'foobar', DirectoryState.Normal)
eq_(d.get_state(p1 + 'foobar'), DirectoryState.Normal)
d.set_state(p1['foobar'], DirectoryState.Normal)
eq_(d.get_state(p1['foobar']), DirectoryState.Normal)
eq_(len(list(d.get_files())), 2)

12
core/tests/fs_test.py
View File

@@ -25,12 +25,12 @@ def test_md5_aggregate_subfiles_sorted(tmpdir):
#same order everytime.
p = create_fake_fs(Path(str(tmpdir)))
b = fs.Folder(p)
md51 = fs.File(p + ('dir1', 'file1.test')).md5
md52 = fs.File(p + ('dir2', 'file2.test')).md5
md53 = fs.File(p + ('dir3', 'file3.test')).md5
md54 = fs.File(p + 'file1.test').md5
md55 = fs.File(p + 'file2.test').md5
md56 = fs.File(p + 'file3.test').md5
md51 = fs.File(p['dir1']['file1.test']).md5
md52 = fs.File(p['dir2']['file2.test']).md5
md53 = fs.File(p['dir3']['file3.test']).md5
md54 = fs.File(p['file1.test']).md5
md55 = fs.File(p['file2.test']).md5
md56 = fs.File(p['file3.test']).md5
# The expected md5 is the md5 of md5s for folders and the direct md5 for files
folder_md51 = hashlib.md5(md51).digest()
folder_md52 = hashlib.md5(md52).digest()

10
core/tests/scanner_test.py
View File

@@ -7,7 +7,6 @@
# http://www.hardcoded.net/licenses/bsd_license
from jobprogress import job
from hscommon import io
from hscommon.path import Path
from hscommon.testutil import eq_
@@ -21,7 +20,7 @@ class NamedObject:
if path is None:
path = Path(name)
else:
path = Path(path) + name
path = Path(path)[name]
self.name = name
self.size = size
self.path = path
@@ -37,7 +36,6 @@ def pytest_funcarg__fake_fileexists(request):
# This is a hack to avoid invalidating all previous tests since the scanner started to test
# for file existence before doing the match grouping.
monkeypatch = request.getfuncargvalue('monkeypatch')
monkeypatch.setattr(io, 'exists', lambda _: True)
monkeypatch.setattr(Path, 'exists', lambda _: True)
def test_empty(fake_fileexists):
@@ -471,11 +469,11 @@ def test_dont_group_files_that_dont_exist(tmpdir):
s = Scanner()
s.scan_type = ScanType.Contents
p = Path(str(tmpdir))
io.open(p + 'file1', 'w').write('foo')
io.open(p + 'file2', 'w').write('foo')
p['file1'].open('w').write('foo')
p['file2'].open('w').write('foo')
file1, file2 = fs.get_files(p)
def getmatches(*args, **kw):
io.remove(file2.path)
file2.path.remove()
return [Match(file1, file2, 100)]
s._getmatches = getmatches