arsenetar
/
dupeguru
mirror of https://github.com/arsenetar/dupeguru.git
1
0
Fork 0
Code Releases Wiki Activity

Added tox configuration 

... and fixed pep8 warnings. There's a lot of them that are still
ignored, but that's because it's too much of a step to take at once.
This commit is contained in:
Virgil Dupras 2014-10-13 15:08:59 -04:00
parent 24643a9b5d
commit 2166a0996c
46 changed files with 794 additions and 612 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -6,6 +6,7 @@
*.waf*
.lock-waf*
.idea
.tox
build
dist

121
build.py
View File

@ -18,10 +18,12 @@ import compileall
from setuptools import setup, Extension
from hscommon import sphinxgen
from hscommon.build import (add_to_pythonpath, print_and_do, copy_packages, filereplace,
from hscommon.build import (
add_to_pythonpath, print_and_do, copy_packages, filereplace,
get_module_version, move_all, copy_all, OSXAppStructure,
build_cocoalib_xibless, fix_qt_resource_file, build_cocoa_ext, copy_embeddable_python_dylib,
collect_stdlib_dependencies, copy)
collect_stdlib_dependencies, copy
)
from hscommon import loc
from hscommon.plat import ISOSX, ISLINUX
from hscommon.util import ensure_folder, delete_files_with_pattern
@ -29,24 +31,42 @@ from hscommon.util import ensure_folder, delete_files_with_pattern
def parse_args():
usage = "usage: %prog [options]"
parser = OptionParser(usage=usage)
parser.add_option('--clean', action='store_true', dest='clean',
help="Clean build folder before building")
parser.add_option('--doc', action='store_true', dest='doc',
help="Build only the help file")
parser.add_option('--loc', action='store_true', dest='loc',
help="Build only localization")
parser.add_option('--cocoa-ext', action='store_true', dest='cocoa_ext',
help="Build only Cocoa extensions")
parser.add_option('--cocoa-compile', action='store_true', dest='cocoa_compile',
help="Build only Cocoa executable")
parser.add_option('--xibless', action='store_true', dest='xibless',
help="Build only xibless UIs")
parser.add_option('--updatepot', action='store_true', dest='updatepot',
help="Generate .pot files from source code.")
parser.add_option('--mergepot', action='store_true', dest='mergepot',
help="Update all .po files based on .pot files.")
parser.add_option('--normpo', action='store_true', dest='normpo',
help="Normalize all PO files (do this before commit).")
parser.add_option(
'--clean', action='store_true', dest='clean',
help="Clean build folder before building"
)
parser.add_option(
'--doc', action='store_true', dest='doc',
help="Build only the help file"
)
parser.add_option(
'--loc', action='store_true', dest='loc',
help="Build only localization"
)
parser.add_option(
'--cocoa-ext', action='store_true', dest='cocoa_ext',
help="Build only Cocoa extensions"
)
parser.add_option(
'--cocoa-compile', action='store_true', dest='cocoa_compile',
help="Build only Cocoa executable"
)
parser.add_option(
'--xibless', action='store_true', dest='xibless',
help="Build only xibless UIs"
)
parser.add_option(
'--updatepot', action='store_true', dest='updatepot',
help="Generate .pot files from source code."
)
parser.add_option(
'--mergepot', action='store_true', dest='mergepot',
help="Update all .po files based on .pot files."
)
parser.add_option(
'--normpo', action='store_true', dest='normpo',
help="Normalize all PO files (do this before commit)."
)
(options, args) = parser.parse_args()
return options
@ -75,12 +95,20 @@ def build_xibless(edition, dest='cocoa/autogen'):
('preferences_panel.py', 'PreferencesPanel_UI'),
]
for srcname, dstname in FNPAIRS:
xibless.generate(op.join('cocoa', 'base', 'ui', srcname), op.join(dest, dstname),
localizationTable='Localizable', args={'edition': edition})
xibless.generate(
op.join('cocoa', 'base', 'ui', srcname), op.join(dest, dstname),
localizationTable='Localizable', args={'edition': edition}
)
if edition == 'pe':
xibless.generate('cocoa/pe/ui/details_panel.py', op.join(dest, 'DetailsPanel_UI'), localizationTable='Localizable')
xibless.generate(
'cocoa/pe/ui/details_panel.py', op.join(dest, 'DetailsPanel_UI'),
localizationTable='Localizable'
)
else:
xibless.generate('cocoa/base/ui/details_panel.py', op.join(dest, 'DetailsPanel_UI'), localizationTable='Localizable')
xibless.generate(
'cocoa/base/ui/details_panel.py', op.join(dest, 'DetailsPanel_UI'),
localizationTable='Localizable'
)
def build_cocoa(edition, dev):
print("Creating OS X app structure")
@ -119,7 +147,7 @@ def build_cocoa(edition, dev):
if edition == 'pe':
# ModuleFinder can't seem to correctly detect the multiprocessing dependency, so we have
# to manually specify it.
extra_deps=['multiprocessing']
extra_deps = ['multiprocessing']
collect_stdlib_dependencies('build/dg_cocoa.py', pydep_folder, extra_deps=extra_deps)
del sys.path[0]
# Views are not referenced by python code, so they're not found by the collector.
@ -225,8 +253,10 @@ def build_updatepot():
os.remove(cocoalib_pot)
loc.strings2pot(op.join('cocoalib', 'en.lproj', 'cocoalib.strings'), cocoalib_pot)
print("Enhancing ui.pot with Cocoa's strings files")
loc.strings2pot(op.join('cocoa', 'base', 'en.lproj', 'Localizable.strings'),
op.join('locale', 'ui.pot'))
loc.strings2pot(
op.join('cocoa', 'base', 'en.lproj', 'Localizable.strings'),
op.join('locale', 'ui.pot')
)
def build_mergepot():
print("Updating .po files using .pot files")
@ -243,11 +273,15 @@ def build_cocoa_proxy_module():
print("Building Cocoa Proxy")
import objp.p2o
objp.p2o.generate_python_proxy_code('cocoalib/cocoa/CocoaProxy.h', 'build/CocoaProxy.m')
build_cocoa_ext("CocoaProxy", 'cocoalib/cocoa',
['cocoalib/cocoa/CocoaProxy.m', 'build/CocoaProxy.m', 'build/ObjP.m',
'cocoalib/HSErrorReportWindow.m', 'cocoa/autogen/HSErrorReportWindow_UI.m'],
build_cocoa_ext(
"CocoaProxy", 'cocoalib/cocoa',
[
'cocoalib/cocoa/CocoaProxy.m', 'build/CocoaProxy.m', 'build/ObjP.m',
'cocoalib/HSErrorReportWindow.m', 'cocoa/autogen/HSErrorReportWindow_UI.m'
],
['AppKit', 'CoreServices'],
['cocoalib', 'cocoa/autogen'])
['cocoalib', 'cocoa/autogen']
)
def build_cocoa_bridging_interfaces(edition):
print("Building Cocoa Bridging Interfaces")
@ -255,9 +289,11 @@ def build_cocoa_bridging_interfaces(edition):
import objp.p2o
add_to_pythonpath('cocoa')
add_to_pythonpath('cocoalib')
from cocoa.inter import (PyGUIObject, GUIObjectView, PyColumns, ColumnsView, PyOutline,
from cocoa.inter import (
PyGUIObject, GUIObjectView, PyColumns, ColumnsView, PyOutline,
OutlineView, PySelectableList, SelectableListView, PyTable, TableView, PyBaseApp,
PyTextField, ProgressWindowView, PyProgressWindow)
PyTextField, ProgressWindowView, PyProgressWindow
)
from inter.deletion_options import PyDeletionOptions, DeletionOptionsView
from inter.details_panel import PyDetailsPanel, DetailsPanelView
from inter.directory_outline import PyDirectoryOutline, DirectoryOutlineView
@ -269,16 +305,20 @@ def build_cocoa_bridging_interfaces(edition):
from inter.stats_label import PyStatsLabel, StatsLabelView
from inter.app import PyDupeGuruBase, DupeGuruView
appmod = importlib.import_module('inter.app_{}'.format(edition))
allclasses = [PyGUIObject, PyColumns, PyOutline, PySelectableList, PyTable, PyBaseApp,
allclasses = [
PyGUIObject, PyColumns, PyOutline, PySelectableList, PyTable, PyBaseApp,
PyDetailsPanel, PyDirectoryOutline, PyPrioritizeDialog, PyPrioritizeList, PyProblemDialog,
PyIgnoreListDialog, PyDeletionOptions, PyResultTable, PyStatsLabel, PyDupeGuruBase,
PyTextField, PyProgressWindow, appmod.PyDupeGuru]
PyTextField, PyProgressWindow, appmod.PyDupeGuru
]
for class_ in allclasses:
objp.o2p.generate_objc_code(class_, 'cocoa/autogen', inherit=True)
allclasses = [GUIObjectView, ColumnsView, OutlineView, SelectableListView, TableView,
allclasses = [
GUIObjectView, ColumnsView, OutlineView, SelectableListView, TableView,
DetailsPanelView, DirectoryOutlineView, PrioritizeDialogView, PrioritizeListView,
IgnoreListDialogView, DeletionOptionsView, ResultTableView, StatsLabelView,
ProgressWindowView, DupeGuruView]
ProgressWindowView, DupeGuruView
]
clsspecs = [objp.o2p.spec_from_python_class(class_) for class_ in allclasses]
objp.p2o.generate_python_proxy_code_from_clsspec(clsspecs, 'build/CocoaViews.m')
build_cocoa_ext('CocoaViews', 'cocoa/inter', ['build/CocoaViews.m', 'build/ObjP.m'])
@ -297,11 +337,12 @@ def build_pe_modules(ui):
extra_link_args=[
"-framework", "CoreFoundation",
"-framework", "Foundation",
"-framework", "ApplicationServices",]
"-framework", "ApplicationServices",
]
))
setup(
script_args = ['build_ext', '--inplace'],
ext_modules = exts,
script_args=['build_ext', '--inplace'],
ext_modules=exts,
)
move_all('_block_qt*', op.join('qt', 'pe'))
move_all('_block*', 'core_pe')

26
configure.py
View File

@ -1,12 +1,11 @@
# Created By: Virgil Dupras
# Created On: 2009-12-30
# Copyright 2014 Hardcoded Software (http://www.hardcoded.net)
#
# This software is licensed under the "BSD" License as described in the "LICENSE" file,
# which should be included with this package. The terms are also available at
#
# This software is licensed under the "BSD" License as described in the "LICENSE" file,
# which should be included with this package. The terms are also available at
# http://www.hardcoded.net/licenses/bsd_license
import sys
from optparse import OptionParser
import json
@ -29,11 +28,18 @@ def main(options):
if __name__ == '__main__':
usage = "usage: %prog [options]"
parser = OptionParser(usage=usage)
parser.add_option('--edition', dest='edition',
help="dupeGuru edition to build (se, me or pe). Default is se.")
parser.add_option('--ui', dest='ui',
help="Type of UI to build. 'qt' or 'cocoa'. Default is determined by your system.")
parser.add_option('--dev', action='store_true', dest='dev', default=False,
help="If this flag is set, will configure for dev builds.")
parser.add_option(
'--edition', dest='edition',
help="dupeGuru edition to build (se, me or pe). Default is se."
)
parser.add_option(
'--ui', dest='ui',
help="Type of UI to build. 'qt' or 'cocoa'. Default is determined by your system."
)
parser.add_option(
'--dev', action='store_true', dest='dev', default=False,
help="If this flag is set, will configure for dev builds."
)
(options, args) = parser.parse_args()
main(options)

39
core/app.py
View File

@ -38,8 +38,10 @@ DEBUG_MODE_PREFERENCE = 'DebugMode'
MSG_NO_MARKED_DUPES = tr("There are no marked duplicates. Nothing has been done.")
MSG_NO_SELECTED_DUPES = tr("There are no selected duplicates. Nothing has been done.")
MSG_MANY_FILES_TO_OPEN = tr("You're about to open many files at once. Depending on what those "
"files are opened with, doing so can create quite a mess. Continue?")
MSG_MANY_FILES_TO_OPEN = tr(
"You're about to open many files at once. Depending on what those "
"files are opened with, doing so can create quite a mess. Continue?"
)
class DestType:
Direct = 0
@ -265,8 +267,10 @@ class DupeGuru(Broadcaster):
return None
def _get_export_data(self):
columns = [col for col in self.result_table.columns.ordered_columns
if col.visible and col.name != 'marked']
columns = [
col for col in self.result_table.columns.ordered_columns
if col.visible and col.name != 'marked'
]
colnames = [col.display for col in columns]
rows = []
for group_id, group in enumerate(self.results.groups):
@ -278,8 +282,10 @@ class DupeGuru(Broadcaster):
return colnames, rows
def _results_changed(self):
self.selected_dupes = [d for d in self.selected_dupes
if self.results.get_group_of_duplicate(d) is not None]
self.selected_dupes = [
d for d in self.selected_dupes
if self.results.get_group_of_duplicate(d) is not None
]
self.notify('results_changed')
def _start_job(self, jobid, func, args=()):
@ -287,7 +293,10 @@ class DupeGuru(Broadcaster):
try:
self.progress_window.run(jobid, title, func, args=args)
except job.JobInProgressError:
msg = tr("A previous action is still hanging in there. You can't start a new one yet. Wait a few seconds, then try again.")
msg = tr(
"A previous action is still hanging in there. You can't start a new one yet. Wait "
"a few seconds, then try again."
)
self.view.show_message(msg)
def _job_completed(self, jobid):
@ -439,8 +448,10 @@ class DupeGuru(Broadcaster):
return
if not self.deletion_options.show(self.results.mark_count):
return
args = [self.deletion_options.link_deleted, self.deletion_options.use_hardlinks,
self.deletion_options.direct]
args = [
self.deletion_options.link_deleted, self.deletion_options.use_hardlinks,
self.deletion_options.direct
]
logging.debug("Starting deletion job with args %r", args)
self._start_job(JobType.Delete, self._do_delete, args=args)
@ -550,8 +561,10 @@ class DupeGuru(Broadcaster):
# If no group was changed, however, we don't touch the selection.
if not self.result_table.power_marker:
if changed_groups:
self.selected_dupes = [d for d in self.selected_dupes
if self.results.get_group_of_duplicate(d).ref is d]
self.selected_dupes = [
d for d in self.selected_dupes
if self.results.get_group_of_duplicate(d).ref is d
]
self.notify('results_changed')
else:
# If we're in "Dupes Only" mode (previously called Power Marker), things are a bit
@ -604,7 +617,7 @@ class DupeGuru(Broadcaster):
def purge_ignore_list(self):
"""Remove files that don't exist from :attr:`ignore_list`.
"""
self.scanner.ignore_list.Filter(lambda f,s:op.exists(f) and op.exists(s))
self.scanner.ignore_list.Filter(lambda f, s: op.exists(f) and op.exists(s))
self.ignore_list_dialog.refresh()
def remove_directories(self, indexes):
@ -641,7 +654,7 @@ class DupeGuru(Broadcaster):
msg = tr("You are about to remove %d files from results. Continue?")
if not self.view.ask_yes_no(msg % self.results.mark_count):
return
self.results.perform_on_marked(lambda x:None, True)
self.results.perform_on_marked(lambda x: None, True)
self._results_changed()
def remove_selected(self):

9
core/directories.py
View File

@ -62,10 +62,10 @@ class Directories:
return True
return False
def __delitem__(self,key):
def __delitem__(self, key):
self._dirs.__delitem__(key)
def __getitem__(self,key):
def __getitem__(self, key):
return self._dirs.__getitem__(key)
def __len__(self):
@ -95,7 +95,8 @@ class Directories:
file.is_ref = state == DirectoryState.Reference
filepaths.add(file.path)
yield file
# it's possible that a folder (bundle) gets into the file list. in that case, we don't want to recurse into it
# it's possible that a folder (bundle) gets into the file list. in that case, we don't
# want to recurse into it
subfolders = [p for p in from_path.listdir() if not p.islink() and p.isdir() and p not in filepaths]
for subfolder in subfolders:
for file in self._get_files(subfolder, j):
@ -144,7 +145,7 @@ class Directories:
"""
try:
subpaths = [p for p in path.listdir() if p.isdir()]
subpaths.sort(key=lambda x:x.name.lower())
subpaths.sort(key=lambda x: x.name.lower())
return subpaths
except EnvironmentError:
return []

14
core/engine.py
View File

@ -17,9 +17,11 @@ from hscommon.util import flatten, multi_replace
from hscommon.trans import tr
from hscommon.jobprogress import job
(WEIGHT_WORDS,
MATCH_SIMILAR_WORDS,
NO_FIELD_ORDER) = range(3)
(
WEIGHT_WORDS,
MATCH_SIMILAR_WORDS,
NO_FIELD_ORDER,
) = range(3)
JOB_REFRESH_RATE = 100
@ -259,6 +261,7 @@ def getmatches_by_contents(files, sizeattr='size', partial=False, j=job.nulljob)
filesize = getattr(file, sizeattr)
if filesize:
size2files[filesize].add(file)
del files
possible_matches = [files for files in size2files.values() if len(files) > 1]
del size2files
result = []
@ -495,7 +498,10 @@ def get_groups(matches, j=job.nulljob):
matched_files = set(flatten(groups))
orphan_matches = []
for group in groups:
orphan_matches += set(m for m in group.discard_matches() if not any(obj in matched_files for obj in [m.first, m.second]))
orphan_matches += {
m for m in group.discard_matches()
if not any(obj in matched_files for obj in [m.first, m.second])
}
if groups and orphan_matches:
groups += get_groups(orphan_matches) # no job, as it isn't supposed to take a long time
return groups

76
core/export.py
View File

@ -1,9 +1,9 @@
# Created By: Virgil Dupras
# Created On: 2006/09/16
# Copyright 2014 Hardcoded Software (http://www.hardcoded.net)
#
# This software is licensed under the "BSD" License as described in the "LICENSE" file,
# which should be included with this package. The terms are also available at
#
# This software is licensed under the "BSD" License as described in the "LICENSE" file,
# which should be included with this package. The terms are also available at
# http://www.hardcoded.net/licenses/bsd_license
import os.path as op
@ -19,56 +19,56 @@ MAIN_TEMPLATE = """
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta content="text/html; charset=utf-8" http-equiv="Content-Type"/>
<title>dupeGuru Results</title>
<style type="text/css">
<title>dupeGuru Results</title>
<style type="text/css">
BODY
{
background-color:white;
background-color:white;
}
BODY,A,P,UL,TABLE,TR,TD
{
font-family:Tahoma,Arial,sans-serif;
font-size:10pt;
color: #4477AA;
font-family:Tahoma,Arial,sans-serif;
font-size:10pt;
color: #4477AA;
}
TABLE
{
background-color: #225588;
margin-left: auto;
margin-right: auto;
width: 90%;
background-color: #225588;
margin-left: auto;
margin-right: auto;
width: 90%;
}
TR
TR
{
background-color: white;
}
TH
{
font-weight: bold;
color: black;
background-color: #C8D6E5;
TH
{
font-weight: bold;
color: black;
background-color: #C8D6E5;
}
TH TD
TH TD
{
color:black;
}
TD
TD
{
padding-left: 2pt;
}
TD.rightelem
{
text-align:right;
/*padding-left:0pt;*/
padding-right: 2pt;
width: 17%;
text-align:right;
/*padding-left:0pt;*/
padding-right: 2pt;
width: 17%;
}
TD.indented
@ -78,19 +78,19 @@ TD.indented
H1
{
font-family:&quot;Courier New&quot;,monospace;
color:#6699CC;
font-size:18pt;
color:#6da500;
border-color: #70A0CF;
border-width: 1pt;
border-style: solid;
margin-top: 16pt;
margin-left: 5%;
margin-right: 5%;
padding-top: 2pt;
padding-bottom:2pt;
text-align: center;
font-family:&quot;Courier New&quot;,monospace;
color:#6699CC;
font-size:18pt;
color:#6da500;
border-color: #70A0CF;
border-width: 1pt;
border-style: solid;
margin-top: 16pt;
margin-left: 5%;
margin-right: 5%;
padding-top: 2pt;
padding-bottom:2pt;
text-align: center;
}
</style>
</head>

63
core/fs.py
View File

@ -1,9 +1,9 @@
# Created By: Virgil Dupras
# Created On: 2009-10-22
# Copyright 2014 Hardcoded Software (http://www.hardcoded.net)
#
# This software is licensed under the "BSD" License as described in the "LICENSE" file,
# which should be included with this package. The terms are also available at
#
# This software is licensed under the "BSD" License as described in the "LICENSE" file,
# which should be included with this package. The terms are also available at
# http://www.hardcoded.net/licenses/bsd_license
# This is a fork from hsfs. The reason for this fork is that hsfs has been designed for musicGuru
@ -32,6 +32,7 @@ NOT_SET = object()
class FSError(Exception):
cls_message = "An error has occured on '{name}' in '{parent}'"
def __init__(self, fsobject, parent=None):
message = self.cls_message
if isinstance(fsobject, str):
@ -42,7 +43,7 @@ class FSError(Exception):
name = ''
parentname = str(parent) if parent is not None else ''
Exception.__init__(self, message.format(name=name, parent=parentname))
class AlreadyExistsError(FSError):
"The directory or file name we're trying to add already exists"
@ -57,7 +58,7 @@ class InvalidDestinationError(FSError):
cls_message = "'{name}' is an invalid destination for this operation."
class OperationError(FSError):
"""A copy/move/delete operation has been called, but the checkup after the
"""A copy/move/delete operation has been called, but the checkup after the
operation shows that it didn't work."""
cls_message = "Operation on '{name}' failed."
@ -74,15 +75,15 @@ class File:
# files, I saved 35% memory usage with "unread" files (no _read_info() call) and gains become
# even greater when we take into account read attributes (70%!). Yeah, it's worth it.
__slots__ = ('path', 'is_ref', 'words') + tuple(INITIAL_INFO.keys())
def __init__(self, path):
self.path = path
for attrname in self.INITIAL_INFO:
setattr(self, attrname, NOT_SET)
def __repr__(self):
return "<{} {}>".format(self.__class__.__name__, str(self.path))
def __getattribute__(self, attrname):
result = object.__getattribute__(self, attrname)
if result is NOT_SET:
@ -94,12 +95,12 @@ class File:
if result is NOT_SET:
result = self.INITIAL_INFO[attrname]
return result
#This offset is where we should start reading the file to get a partial md5
#For audio file, it should be where audio data starts
def _get_md5partial_offset_and_size(self):
return (0x4000, 0x4000) #16Kb
def _read_info(self, field):
if field in ('size', 'mtime'):
stats = self.path.stat()
@ -129,24 +130,24 @@ class File:
fp.close()
except Exception:
pass
def _read_all_info(self, attrnames=None):
"""Cache all possible info.
If `attrnames` is not None, caches only attrnames.
"""
if attrnames is None:
attrnames = self.INITIAL_INFO.keys()
for attrname in attrnames:
getattr(self, attrname)
#--- Public
@classmethod
def can_handle(cls, path):
"""Returns whether this file wrapper class can handle ``path``.
"""
return not path.islink() and path.isfile()
def rename(self, newname):
if newname == self.name:
return
@ -160,42 +161,42 @@ class File:
if not destpath.exists():
raise OperationError(self)
self.path = destpath
def get_display_info(self, group, delta):
"""Returns a display-ready dict of dupe's data.
"""
raise NotImplementedError()
#--- Properties
@property
def extension(self):
return get_file_ext(self.name)
@property
def name(self):
return self.path.name
@property
def folder_path(self):
return self.path.parent()
class Folder(File):
"""A wrapper around a folder path.
It has the size/md5 info of a File, but it's value are the sum of its subitems.
"""
__slots__ = File.__slots__ + ('_subfolders', )
def __init__(self, path):
File.__init__(self, path)
self._subfolders = None
def _all_items(self):
folders = self.subfolders
files = get_files(self.path)
return folders + files
def _read_info(self, field):
if field in {'size', 'mtime'}:
size = sum((f.size for f in self._all_items()), 0)
@ -208,31 +209,31 @@ class Folder(File):
# different md5 if a file gets moved in a different subdirectory.
def get_dir_md5_concat():
items = self._all_items()
items.sort(key=lambda f:f.path)
items.sort(key=lambda f: f.path)
md5s = [getattr(f, field) for f in items]
return b''.join(md5s)
md5 = hashlib.md5(get_dir_md5_concat())
digest = md5.digest()
setattr(self, field, digest)
@property
def subfolders(self):
if self._subfolders is None:
subfolders = [p for p in self.path.listdir() if not p.islink() and p.isdir()]
self._subfolders = [self.__class__(p) for p in subfolders]
return self._subfolders
@classmethod
def can_handle(cls, path):
return not path.islink() and path.isdir()
def get_file(path, fileclasses=[File]):
"""Wraps ``path`` around its appropriate :class:`File` class.
Whether a class is "appropriate" is decided by :meth:`File.can_handle`
:param Path path: path to wrap
:param fileclasses: List of candidate :class:`File` classes
"""
@ -242,7 +243,7 @@ def get_file(path, fileclasses=[File]):
def get_files(path, fileclasses=[File]):
"""Returns a list of :class:`File` for each file contained in ``path``.
:param Path path: path to scan
:param fileclasses: List of candidate :class:`File` classes
"""

3
core/gui/__init__.py
View File

@ -12,4 +12,5 @@ either Cocoa's ``NSTableView`` or Qt's ``QTableView``. It tells them which cell
blue, which is supposed to be orange, does the sorting logic, holds selection, etc..
.. _cross-toolkit: http://www.hardcoded.net/articles/cross-toolkit-software
"""
"""

19
core/gui/base.py
View File

@ -1,31 +1,30 @@
# Created By: Virgil Dupras
# Created On: 2010-02-06
# Copyright 2014 Hardcoded Software (http://www.hardcoded.net)
#
# This software is licensed under the "BSD" License as described in the "LICENSE" file,
# which should be included with this package. The terms are also available at
#
# This software is licensed under the "BSD" License as described in the "LICENSE" file,
# which should be included with this package. The terms are also available at
# http://www.hardcoded.net/licenses/bsd_license
from hscommon.notify import Listener
from hscommon.gui.base import NoopGUI
class DupeGuruGUIObject(Listener):
def __init__(self, app):
Listener.__init__(self, app)
self.app = app
def directories_changed(self):
pass
def dupes_selected(self):
pass
def marking_changed(self):
pass
def results_changed(self):
pass
def results_changed_but_keep_selection(self):
pass

25
core/gui/prioritize_dialog.py
View File

@ -1,9 +1,9 @@
# Created By: Virgil Dupras
# Created On: 2011-09-06
# Copyright 2014 Hardcoded Software (http://www.hardcoded.net)
#
# This software is licensed under the "BSD" License as described in the "LICENSE" file,
# which should be included with this package. The terms are also available at
#
# This software is licensed under the "BSD" License as described in the "LICENSE" file,
# which should be included with this package. The terms are also available at
# http://www.hardcoded.net/licenses/bsd_license
from hscommon.gui.base import GUIObject
@ -13,7 +13,7 @@ class CriterionCategoryList(GUISelectableList):
def __init__(self, dialog):
self.dialog = dialog
GUISelectableList.__init__(self, [c.NAME for c in dialog.categories])
def _update_selection(self):
self.dialog.select_category(self.dialog.categories[self.selected_index])
GUISelectableList._update_selection(self)
@ -22,10 +22,10 @@ class PrioritizationList(GUISelectableList):
def __init__(self, dialog):
self.dialog = dialog
GUISelectableList.__init__(self)
def _refresh_contents(self):
self[:] = [crit.display for crit in self.dialog.prioritizations]
def move_indexes(self, indexes, dest_index):
indexes.sort()
prilist = self.dialog.prioritizations
@ -34,7 +34,7 @@ class PrioritizationList(GUISelectableList):
del prilist[i]
prilist[dest_index:dest_index] = selected
self._refresh_contents()
def remove_selected(self):
prilist = self.dialog.prioritizations
for i in sorted(self.selected_indexes, reverse=True):
@ -51,15 +51,15 @@ class PrioritizeDialog(GUIObject):
self.criteria_list = GUISelectableList()
self.prioritizations = []
self.prioritization_list = PrioritizationList(self)
#--- Override
def _view_updated(self):
self.category_list.select(0)
#--- Private
def _sort_key(self, dupe):
return tuple(crit.sort_key(dupe) for crit in self.prioritizations)
#--- Public
def select_category(self, category):
self.criteria = category.criteria_list()
@ -71,10 +71,11 @@ class PrioritizeDialog(GUIObject):
return
crit = self.criteria[self.criteria_list.selected_index]
self.prioritizations.append(crit)
del crit
self.prioritization_list[:] = [crit.display for crit in self.prioritizations]
def remove_selected(self):
self.prioritization_list.remove_selected()
def perform_reprioritization(self):
self.app.reprioritize_groups(self._sort_key)

58
core/ignore.py
View File

@ -1,9 +1,9 @@
# Created By: Virgil Dupras
# Created On: 2006/05/02
# Copyright 2014 Hardcoded Software (http://www.hardcoded.net)
#
# This software is licensed under the "BSD" License as described in the "LICENSE" file,
# which should be included with this package. The terms are also available at
#
# This software is licensed under the "BSD" License as described in the "LICENSE" file,
# which should be included with this package. The terms are also available at
# http://www.hardcoded.net/licenses/bsd_license
from xml.etree import ElementTree as ET
@ -12,7 +12,7 @@ from hscommon.util import FileOrPath
class IgnoreList:
"""An ignore list implementation that is iterable, filterable and exportable to XML.
Call Ignore to add an ignore list entry, and AreIgnore to check if 2 items are in the list.
When iterated, 2 sized tuples will be returned, the tuples containing 2 items ignored together.
"""
@ -20,43 +20,43 @@ class IgnoreList:
def __init__(self):
self._ignored = {}
self._count = 0
def __iter__(self):
for first,seconds in self._ignored.items():
for first, seconds in self._ignored.items():
for second in seconds:
yield (first,second)
yield (first, second)
def __len__(self):
return self._count
#---Public
def AreIgnored(self,first,second):
def do_check(first,second):
def AreIgnored(self, first, second):
def do_check(first, second):
try:
matches = self._ignored[first]
return second in matches
except KeyError:
return False
return do_check(first,second) or do_check(second,first)
return do_check(first, second) or do_check(second, first)
def Clear(self):
self._ignored = {}
self._count = 0
def Filter(self,func):
def Filter(self, func):
"""Applies a filter on all ignored items, and remove all matches where func(first,second)
doesn't return True.
"""
filtered = IgnoreList()
for first,second in self:
if func(first,second):
filtered.Ignore(first,second)
for first, second in self:
if func(first, second):
filtered.Ignore(first, second)
self._ignored = filtered._ignored
self._count = filtered._count
def Ignore(self,first,second):
if self.AreIgnored(first,second):
def Ignore(self, first, second):
if self.AreIgnored(first, second):
return
try:
matches = self._ignored[first]
@ -70,7 +70,7 @@ class IgnoreList:
matches.add(second)
self._ignored[first] = matches
self._count += 1
def remove(self, first, second):
def inner(first, second):
try:
@ -85,14 +85,14 @@ class IgnoreList:
return False
except KeyError:
return False
if not inner(first, second):
if not inner(second, first):
raise ValueError()
def load_from_xml(self, infile):
"""Loads the ignore list from a XML created with save_to_xml.
infile can be a file object or a filename.
"""
try:
@ -109,10 +109,10 @@ class IgnoreList:
subfile_path = sfn.get('path')
if subfile_path:
self.Ignore(file_path, subfile_path)
def save_to_xml(self, outfile):
"""Create a XML file that can be used by load_from_xml.
outfile can be a file object or a filename.
"""
root = ET.Element('ignore_list')
@ -125,5 +125,5 @@ class IgnoreList:
tree = ET.ElementTree(root)
with FileOrPath(outfile, 'wb') as fp:
tree.write(fp, encoding='utf-8')

17
core/results.py
View File

@ -96,7 +96,10 @@ class Results(Markable):
self.__dupes = flatten(group.dupes for group in self.groups)
if None in self.__dupes:
# This is debug logging to try to figure out #44
logging.warning("There is a None value in the Results' dupe list. dupes: %r groups: %r", self.__dupes, self.groups)
logging.warning(
"There is a None value in the Results' dupe list. dupes: %r groups: %r",
self.__dupes, self.groups
)
if self.__filtered_dupes:
self.__dupes = [dupe for dupe in self.__dupes if dupe in self.__filtered_dupes]
sd = self.__dupes_sort_descriptor
@ -249,7 +252,8 @@ class Results(Markable):
second_file = dupes[int(attrs['second'])]
percentage = int(attrs['percentage'])
group.add_match(engine.Match(first_file, second_file, percentage))
except (IndexError, KeyError, ValueError): # Covers missing attr, non-int values and indexes out of bounds
except (IndexError, KeyError, ValueError):
# Covers missing attr, non-int values and indexes out of bounds
pass
if (not group.matches) and (len(dupes) >= 2):
do_match(dupes[0], dupes[1:], group)
@ -393,7 +397,7 @@ class Results(Markable):
self.__get_dupe_list()
keyfunc = lambda d: self.app._get_dupe_sort_key(d, lambda: self.get_group_of_duplicate(d), key, delta)
self.__dupes.sort(key=keyfunc, reverse=not asc)
self.__dupes_sort_descriptor = (key,asc,delta)
self.__dupes_sort_descriptor = (key, asc, delta)
def sort_groups(self, key, asc=True):
"""Sort :attr:`groups` according to ``key``.
@ -405,9 +409,10 @@ class Results(Markable):
"""
keyfunc = lambda g: self.app._get_group_sort_key(g, key)
self.groups.sort(key=keyfunc, reverse=not asc)
self.__groups_sort_descriptor = (key,asc)
self.__groups_sort_descriptor = (key, asc)
#---Properties
dupes = property(__get_dupe_list)
groups = property(__get_groups, __set_groups)
dupes = property(__get_dupe_list)
groups = property(__get_groups, __set_groups)
stat_line = property(__get_stat_line)

29
core/scanner.py
View File

@ -81,7 +81,9 @@ class Scanner:
files = [f for f in files if f.size >= self.size_threshold]
if self.scan_type in {ScanType.Contents, ScanType.ContentsAudio, ScanType.Folders}:
sizeattr = 'audiosize' if self.scan_type == ScanType.ContentsAudio else 'size'
return engine.getmatches_by_contents(files, sizeattr, partial=self.scan_type==ScanType.ContentsAudio, j=j)
return engine.getmatches_by_contents(
files, sizeattr, partial=self.scan_type == ScanType.ContentsAudio, j=j
)
else:
j = j.start_subjob([2, 8])
kw = {}
@ -94,7 +96,11 @@ class Scanner:
func = {
ScanType.Filename: lambda f: engine.getwords(rem_file_ext(f.name)),
ScanType.Fields: lambda f: engine.getfields(rem_file_ext(f.name)),
ScanType.Tag: lambda f: [engine.getwords(str(getattr(f, attrname))) for attrname in SCANNABLE_TAGS if attrname in self.scanned_tags],
ScanType.Tag: lambda f: [
engine.getwords(str(getattr(f, attrname)))
for attrname in SCANNABLE_TAGS
if attrname in self.scanned_tags
],
}[self.scan_type]
for f in j.iter_with_progress(files, tr("Read metadata of %d/%d files")):
logging.debug("Reading metadata of {}".format(str(f.path)))
@ -152,8 +158,10 @@ class Scanner:
if self.ignore_list:
j = j.start_subjob(2)
iter_matches = j.iter_with_progress(matches, tr("Processed %d/%d matches against the ignore list"))
matches = [m for m in iter_matches
if not self.ignore_list.AreIgnored(str(m.first.path), str(m.second.path))]
matches = [
m for m in iter_matches
if not self.ignore_list.AreIgnored(str(m.first.path), str(m.second.path))
]
logging.info('Grouping matches')
groups = engine.get_groups(matches, j)
matched_files = dedupe([m.first for m in matches] + [m.second for m in matches])
@ -178,10 +186,11 @@ class Scanner:
g.prioritize(self._key_func, self._tie_breaker)
return groups
match_similar_words = False
match_similar_words = False
min_match_percentage = 80
mix_file_kind = True
scan_type = ScanType.Filename
scanned_tags = {'artist', 'title'}
size_threshold = 0
word_weighting = False
mix_file_kind = True
scan_type = ScanType.Filename
scanned_tags = {'artist', 'title'}
size_threshold = 0
word_weighting = False

3
core_me/__init__.py
View File

@ -1,2 +1,3 @@
__version__ = '6.8.0'
__appname__ = 'dupeGuru Music Edition'
__appname__ = 'dupeGuru Music Edition'

20
core_me/app.py
View File

@ -1,8 +1,8 @@
# Created On: 2011/09/20
# Copyright 2014 Hardcoded Software (http://www.hardcoded.net)
#
# This software is licensed under the "BSD" License as described in the "LICENSE" file,
# which should be included with this package. The terms are also available at
#
# This software is licensed under the "BSD" License as described in the "LICENSE" file,
# which should be included with this package. The terms are also available at
# http://www.hardcoded.net/licenses/bsd_license
from core.app import DupeGuru as DupeGuruBase
@ -13,28 +13,30 @@ from .result_table import ResultTable
class DupeGuru(DupeGuruBase):
NAME = __appname__
METADATA_TO_READ = ['size', 'mtime', 'duration', 'bitrate', 'samplerate', 'title', 'artist',
'album', 'genre', 'year', 'track', 'comment']
METADATA_TO_READ = [
'size', 'mtime', 'duration', 'bitrate', 'samplerate', 'title', 'artist',
'album', 'genre', 'year', 'track', 'comment'
]
def __init__(self, view):
DupeGuruBase.__init__(self, view)
self.scanner = scanner.ScannerME()
self.directories.fileclasses = [fs.MusicFile]
def _get_dupe_sort_key(self, dupe, get_group, key, delta):
if key == 'folder_path':
dupe_folder_path = getattr(dupe, 'display_folder_path', dupe.folder_path)
return str(dupe_folder_path).lower()
return DupeGuruBase._get_dupe_sort_key(self, dupe, get_group, key, delta)
def _get_group_sort_key(self, group, key):
if key == 'folder_path':
dupe_folder_path = getattr(group.ref, 'display_folder_path', group.ref.folder_path)
return str(dupe_folder_path).lower()
return DupeGuruBase._get_group_sort_key(self, group, key)
def _prioritization_categories(self):
return prioritize.all_categories()
def _create_result_table(self):
return ResultTable(self)

44
core_me/fs.py
View File

@ -1,9 +1,9 @@
# Created By: Virgil Dupras
# Created On: 2009-10-23
# Copyright 2014 Hardcoded Software (http://www.hardcoded.net)
#
# This software is licensed under the "BSD" License as described in the "LICENSE" file,
# which should be included with this package. The terms are also available at
#
# This software is licensed under the "BSD" License as described in the "LICENSE" file,
# which should be included with this package. The terms are also available at
# http://www.hardcoded.net/licenses/bsd_license
from hsaudiotag import auto
@ -12,32 +12,34 @@ from hscommon.util import get_file_ext, format_size, format_time
from core.app import format_timestamp, format_perc, format_words, format_dupe_count
from core import fs
TAG_FIELDS = {'audiosize', 'duration', 'bitrate', 'samplerate', 'title', 'artist',
'album', 'genre', 'year', 'track', 'comment'}
TAG_FIELDS = {
'audiosize', 'duration', 'bitrate', 'samplerate', 'title', 'artist',
'album', 'genre', 'year', 'track', 'comment'
}
class MusicFile(fs.File):
INITIAL_INFO = fs.File.INITIAL_INFO.copy()
INITIAL_INFO.update({
'audiosize': 0,
'bitrate' : 0,
'duration' : 0,
'samplerate':0,
'artist' : '',
'album' : '',
'title' : '',
'genre' : '',
'comment' : '',
'year' : '',
'track' : 0,
'bitrate': 0,
'duration': 0,
'samplerate': 0,
'artist': '',
'album': '',
'title': '',
'genre': '',
'comment': '',
'year': '',
'track': 0,
})
__slots__ = fs.File.__slots__ + tuple(INITIAL_INFO.keys())
@classmethod
def can_handle(cls, path):
if not fs.File.can_handle(path):
return False
return get_file_ext(path.name) in auto.EXT2CLASS
def get_display_info(self, group, delta):
size = self.size
duration = self.duration
@ -67,7 +69,7 @@ class MusicFile(fs.File):
'bitrate': str(bitrate),
'samplerate': str(samplerate),
'extension': self.extension,
'mtime': format_timestamp(mtime,delta and m),
'mtime': format_timestamp(mtime, delta and m),
'title': self.title,
'artist': self.artist,
'album': self.album,
@ -79,11 +81,11 @@ class MusicFile(fs.File):
'words': format_words(self.words) if hasattr(self, 'words') else '',
'dupe_count': format_dupe_count(dupe_count),
}
def _get_md5partial_offset_and_size(self):
f = auto.File(str(self.path))
return (f.audio_offset, f.audio_size)
def _read_info(self, field):
fs.File._read_info(self, field)
if field in TAG_FIELDS:
@ -99,4 +101,4 @@ class MusicFile(fs.File):
self.comment = f.comment
self.year = f.year
self.track = f.track

25
core_me/prioritize.py
View File

@ -1,35 +1,40 @@