Converted to py3k. There's probably some bugs still. So far, I managed to run dupeGuru SE under pyobjc and qt.

This commit is contained in:
Virgil Dupras 2010-08-11 16:39:06 +02:00
parent fb79daad6a
commit 854d194f88
57 changed files with 394 additions and 338 deletions

View File

@ -20,10 +20,10 @@ from hscommon.build import add_to_pythonpath, print_and_do, build_all_qt_ui, cop
def build_cocoa(edition, dev, help_destpath): def build_cocoa(edition, dev, help_destpath):
if not dev: if not dev:
print "Building help index" print("Building help index")
os.system('open -a /Developer/Applications/Utilities/Help\\ Indexer.app {0}'.format(help_destpath)) os.system('open -a /Developer/Applications/Utilities/Help\\ Indexer.app {0}'.format(help_destpath))
print "Building dg_cocoa.plugin" print("Building dg_cocoa.plugin")
if op.exists('build'): if op.exists('build'):
shutil.rmtree('build') shutil.rmtree('build')
os.mkdir('build') os.mkdir('build')
@ -54,7 +54,7 @@ def build_cocoa(edition, dev, help_destpath):
pthpath = op.join(pluginpath, 'Contents/Resources/dev.pth') pthpath = op.join(pluginpath, 'Contents/Resources/dev.pth')
open(pthpath, 'w').write(op.abspath('.')) open(pthpath, 'w').write(op.abspath('.'))
os.chdir(cocoa_project_path) os.chdir(cocoa_project_path)
print "Building the XCode project" print("Building the XCode project")
args = [] args = []
if dev: if dev:
args.append('-configuration dev') args.append('-configuration dev')
@ -68,10 +68,10 @@ def build_qt(edition, dev):
build_all_qt_ui(op.join('qtlib', 'ui')) build_all_qt_ui(op.join('qtlib', 'ui'))
build_all_qt_ui(op.join('qt', 'base')) build_all_qt_ui(op.join('qt', 'base'))
build_all_qt_ui(op.join('qt', edition)) build_all_qt_ui(op.join('qt', edition))
print_and_do("pyrcc4 {0} > {1}".format(op.join('qt', 'base', 'dg.qrc'), op.join('qt', 'base', 'dg_rc.py'))) print_and_do("pyrcc4 -py3 {0} > {1}".format(op.join('qt', 'base', 'dg.qrc'), op.join('qt', 'base', 'dg_rc.py')))
if edition == 'pe': if edition == 'pe':
os.chdir(op.join('qt', edition)) os.chdir(op.join('qt', edition))
os.system('python gen.py') os.system('python3 gen.py')
os.chdir(op.join('..', '..')) os.chdir(op.join('..', '..'))
def main(): def main():
@ -79,11 +79,11 @@ def main():
edition = conf['edition'] edition = conf['edition']
ui = conf['ui'] ui = conf['ui']
dev = conf['dev'] dev = conf['dev']
print "Building dupeGuru {0} with UI {1}".format(edition.upper(), ui) print("Building dupeGuru {0} with UI {1}".format(edition.upper(), ui))
if dev: if dev:
print "Building in Dev mode" print("Building in Dev mode")
add_to_pythonpath('.') add_to_pythonpath('.')
print "Generating Help" print("Generating Help")
windows = sys.platform == 'win32' windows = sys.platform == 'win32'
profile = 'win_en' if windows else 'osx_en' profile = 'win_en' if windows else 'osx_en'
help_dir = 'help_{0}'.format(edition) help_dir = 'help_{0}'.format(edition)
@ -91,10 +91,10 @@ def main():
help_basepath = op.abspath(help_dir) help_basepath = op.abspath(help_dir)
help_destpath = op.abspath(op.join(help_dir, dest_dir)) help_destpath = op.abspath(op.join(help_dir, dest_dir))
helpgen.gen(help_basepath, help_destpath, profile=profile) helpgen.gen(help_basepath, help_destpath, profile=profile)
print "Building dupeGuru" print("Building dupeGuru")
if edition == 'pe': if edition == 'pe':
os.chdir('core_pe') os.chdir('core_pe')
os.system('python gen.py') os.system('python3 gen.py')
os.chdir('..') os.chdir('..')
if ui == 'cocoa': if ui == 'cocoa':
build_cocoa(edition, dev, help_destpath) build_cocoa(edition, dev, help_destpath)

View File

@ -23,10 +23,10 @@ class PyDupeGuru(PyDupeGuruBase):
#---Information #---Information
def getSelectedDupePath(self): def getSelectedDupePath(self):
return unicode(self.py.selected_dupe_path()) return str(self.py.selected_dupe_path())
def getSelectedDupeRefPath(self): def getSelectedDupeRefPath(self):
return unicode(self.py.selected_dupe_ref_path()) return str(self.py.selected_dupe_ref_path())
#---Properties #---Properties
def setMatchScaled_(self,match_scaled): def setMatchScaled_(self,match_scaled):

View File

@ -11,8 +11,10 @@ from core.app_cocoa_inter import PyDupeGuruBase, PyDetailsPanel
from core_se.app_cocoa import DupeGuru from core_se.app_cocoa import DupeGuru
# Fix py2app imports with chokes on relative imports and other stuff # Fix py2app imports with chokes on relative imports and other stuff
from core_se import fs, data import hsutil.conflict
from lxml import etree, _elementpath import core.engine, core.fs, core.app
import core_se.fs, core_se.data
import lxml.etree, lxml._elementpath
import gzip import gzip
class PyDupeGuru(PyDupeGuruBase): class PyDupeGuru(PyDupeGuruBase):

View File

@ -18,7 +18,7 @@ def main(edition, ui, dev):
if ui not in ('cocoa', 'qt'): if ui not in ('cocoa', 'qt'):
ui = 'cocoa' if sys.platform == 'darwin' else 'qt' ui = 'cocoa' if sys.platform == 'darwin' else 'qt'
build_type = 'Dev' if dev else 'Release' build_type = 'Dev' if dev else 'Release'
print "Configuring dupeGuru {0} for UI {1} ({2})".format(edition.upper(), ui, build_type) print("Configuring dupeGuru {0} for UI {1} ({2})".format(edition.upper(), ui, build_type))
conf = { conf = {
'edition': edition, 'edition': edition,
'ui': ui, 'ui': ui,

View File

@ -6,7 +6,7 @@
# which should be included with this package. The terms are also available at # which should be included with this package. The terms are also available at
# http://www.hardcoded.net/licenses/hs_license # http://www.hardcoded.net/licenses/hs_license
from __future__ import unicode_literals
import os import os
import os.path as op import os.path as op
@ -76,7 +76,7 @@ class DupeGuru(RegistrableApplication, Broadcaster):
def _do_delete_dupe(self, dupe): def _do_delete_dupe(self, dupe):
if not io.exists(dupe.path): if not io.exists(dupe.path):
return return
send2trash(unicode(dupe.path)) # Raises OSError when there's a problem send2trash(str(dupe.path)) # Raises OSError when there's a problem
self.clean_empty_dirs(dupe.path[:-1]) self.clean_empty_dirs(dupe.path[:-1])
def _do_load(self, j): def _do_load(self, j):
@ -100,7 +100,7 @@ class DupeGuru(RegistrableApplication, Broadcaster):
try: try:
return self.data.GetDisplayInfo(dupe, group, delta) return self.data.GetDisplayInfo(dupe, group, delta)
except Exception as e: except Exception as e:
logging.warning("Exception on GetDisplayInfo for %s: %s", unicode(dupe.path), unicode(e)) logging.warning("Exception on GetDisplayInfo for %s: %s", str(dupe.path), str(e))
return ['---'] * len(self.data.COLUMNS) return ['---'] * len(self.data.COLUMNS)
def _get_file(self, str_path): def _get_file(self, str_path):
@ -149,7 +149,7 @@ class DupeGuru(RegistrableApplication, Broadcaster):
g = self.results.get_group_of_duplicate(dupe) g = self.results.get_group_of_duplicate(dupe)
for other in g: for other in g:
if other is not dupe: if other is not dupe:
self.scanner.ignore_list.Ignore(unicode(other.path), unicode(dupe.path)) self.scanner.ignore_list.Ignore(str(other.path), str(dupe.path))
self.remove_duplicates(dupes) self.remove_duplicates(dupes)
def apply_filter(self, filter): def apply_filter(self, filter):
@ -208,7 +208,7 @@ class DupeGuru(RegistrableApplication, Broadcaster):
def export_to_xhtml(self, column_ids): def export_to_xhtml(self, column_ids):
column_ids = [colid for colid in column_ids if colid.isdigit()] column_ids = [colid for colid in column_ids if colid.isdigit()]
column_ids = map(int, column_ids) column_ids = list(map(int, column_ids))
column_ids.sort() column_ids.sort()
colnames = [col['display'] for i, col in enumerate(self.data.COLUMNS) if i in column_ids] colnames = [col['display'] for i, col in enumerate(self.data.COLUMNS) if i in column_ids]
rows = [] rows = []
@ -232,8 +232,8 @@ class DupeGuru(RegistrableApplication, Broadcaster):
dupe = self.selected_dupes[0] dupe = self.selected_dupes[0]
group = self.results.get_group_of_duplicate(dupe) group = self.results.get_group_of_duplicate(dupe)
ref = group.ref ref = group.ref
cmd = cmd.replace('%d', unicode(dupe.path)) cmd = cmd.replace('%d', str(dupe.path))
cmd = cmd.replace('%r', unicode(ref.path)) cmd = cmd.replace('%r', str(ref.path))
match = re.match(r'"([^"]+)"(.*)', cmd) match = re.match(r'"([^"]+)"(.*)', cmd)
if match is not None: if match is not None:
# This code here is because subprocess. Popen doesn't seem to accept, under Windows, # This code here is because subprocess. Popen doesn't seem to accept, under Windows,
@ -313,7 +313,7 @@ class DupeGuru(RegistrableApplication, Broadcaster):
d.rename(newname) d.rename(newname)
return True return True
except (IndexError, fs.FSError) as e: except (IndexError, fs.FSError) as e:
logging.warning("dupeGuru Warning: %s" % unicode(e)) logging.warning("dupeGuru Warning: %s" % str(e))
return False return False
def reveal_selected(self): def reveal_selected(self):

View File

@ -49,11 +49,11 @@ class DupeGuru(app.DupeGuru):
#--- Override #--- Override
@staticmethod @staticmethod
def _open_path(path): def _open_path(path):
NSWorkspace.sharedWorkspace().openFile_(unicode(path)) NSWorkspace.sharedWorkspace().openFile_(str(path))
@staticmethod @staticmethod
def _reveal_path(path): def _reveal_path(path):
NSWorkspace.sharedWorkspace().selectFile_inFileViewerRootedAtPath_(unicode(path), '') NSWorkspace.sharedWorkspace().selectFile_inFileViewerRootedAtPath_(str(path), '')
def _start_job(self, jobid, func): def _start_job(self, jobid, func):
try: try:

View File

@ -11,7 +11,7 @@ from hsutil.str import format_time, FT_DECIMAL, format_size
import time import time
def format_path(p): def format_path(p):
return unicode(p[:-1]) return str(p[:-1])
def format_timestamp(t, delta): def format_timestamp(t, delta):
if delta: if delta:
@ -38,4 +38,4 @@ def format_dupe_count(c):
return str(c) if c else '---' return str(c) if c else '---'
def cmp_value(value): def cmp_value(value):
return value.lower() if isinstance(value, basestring) else value return value.lower() if isinstance(value, str) else value

View File

@ -151,11 +151,11 @@ class Directories(object):
root = etree.Element('directories') root = etree.Element('directories')
for root_path in self: for root_path in self:
root_path_node = etree.SubElement(root, 'root_directory') root_path_node = etree.SubElement(root, 'root_directory')
root_path_node.set('path', unicode(root_path)) root_path_node.set('path', str(root_path))
for path, state in self.states.iteritems(): for path, state in self.states.items():
state_node = etree.SubElement(root, 'state') state_node = etree.SubElement(root, 'state')
state_node.set('path', unicode(path)) state_node.set('path', str(path))
state_node.set('value', unicode(state)) state_node.set('value', str(state))
tree = etree.ElementTree(root) tree = etree.ElementTree(root)
tree.write(fp, encoding='utf-8') tree.write(fp, encoding='utf-8')

View File

@ -6,7 +6,7 @@
# which should be included with this package. The terms are also available at # which should be included with this package. The terms are also available at
# http://www.hardcoded.net/licenses/hs_license # http://www.hardcoded.net/licenses/hs_license
from __future__ import division
import difflib import difflib
import itertools import itertools
import logging import logging
@ -25,15 +25,15 @@ NO_FIELD_ORDER) = range(3)
JOB_REFRESH_RATE = 100 JOB_REFRESH_RATE = 100
def getwords(s): def getwords(s):
if isinstance(s, unicode): if isinstance(s, str):
s = normalize('NFD', s) s = normalize('NFD', s)
s = multi_replace(s, "-_&+():;\\[]{}.,<>/?~!@#$*", ' ').lower() s = multi_replace(s, "-_&+():;\\[]{}.,<>/?~!@#$*", ' ').lower()
s = ''.join(c for c in s if c in string.ascii_letters + string.digits + string.whitespace) s = ''.join(c for c in s if c in string.ascii_letters + string.digits + string.whitespace)
return filter(None, s.split(' ')) # filter() is to remove empty elements return [_f for _f in s.split(' ') if _f] # remove empty elements
def getfields(s): def getfields(s):
fields = [getwords(field) for field in s.split(' - ')] fields = [getwords(field) for field in s.split(' - ')]
return filter(None, fields) return [_f for _f in fields if _f]
def unpack_fields(fields): def unpack_fields(fields):
result = [] result = []
@ -118,7 +118,7 @@ def build_word_dict(objects, j=job.nulljob):
def merge_similar_words(word_dict): def merge_similar_words(word_dict):
"""Take all keys in word_dict that are similar, and merge them together. """Take all keys in word_dict that are similar, and merge them together.
""" """
keys = word_dict.keys() keys = list(word_dict.keys())
keys.sort(key=len)# we want the shortest word to stay keys.sort(key=len)# we want the shortest word to stay
while keys: while keys:
key = keys.pop(0) key = keys.pop(0)
@ -138,7 +138,7 @@ def reduce_common_words(word_dict, threshold):
Because if we remove them, we will miss some duplicates! Because if we remove them, we will miss some duplicates!
""" """
uncommon_words = set(word for word, objects in word_dict.items() if len(objects) < threshold) uncommon_words = set(word for word, objects in word_dict.items() if len(objects) < threshold)
for word, objects in word_dict.items(): for word, objects in list(word_dict.items()):
if len(objects) < threshold: if len(objects) < threshold:
continue continue
reduced = set() reduced = set()

View File

@ -13,7 +13,7 @@ from tempfile import mkdtemp
# Yes, this is a very low-tech solution, but at least it doesn't have all these annoying dependency # Yes, this is a very low-tech solution, but at least it doesn't have all these annoying dependency
# and resource problems. # and resource problems.
MAIN_TEMPLATE = u""" MAIN_TEMPLATE = """
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<!DOCTYPE html PUBLIC '-//W3C//DTD XHTML 1.0 Strict//EN' 'http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd'> <!DOCTYPE html PUBLIC '-//W3C//DTD XHTML 1.0 Strict//EN' 'http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd'>
<html xmlns="http://www.w3.org/1999/xhtml"> <html xmlns="http://www.w3.org/1999/xhtml">
@ -104,33 +104,33 @@ $rows
</html> </html>
""" """
COLHEADERS_TEMPLATE = u"<th>{name}</th>" COLHEADERS_TEMPLATE = "<th>{name}</th>"
ROW_TEMPLATE = u""" ROW_TEMPLATE = """
<tr> <tr>
<td class="{indented}">{filename}</td>{cells} <td class="{indented}">{filename}</td>{cells}
</tr> </tr>
""" """
CELL_TEMPLATE = u"""<td>{value}</td>""" CELL_TEMPLATE = """<td>{value}</td>"""
def export_to_xhtml(colnames, rows): def export_to_xhtml(colnames, rows):
# a row is a list of values with the first value being a flag indicating if the row should be indented # a row is a list of values with the first value being a flag indicating if the row should be indented
if rows: if rows:
assert len(rows[0]) == len(colnames) + 1 # + 1 is for the "indented" flag assert len(rows[0]) == len(colnames) + 1 # + 1 is for the "indented" flag
colheaders = u''.join(COLHEADERS_TEMPLATE.format(name=name) for name in colnames) colheaders = ''.join(COLHEADERS_TEMPLATE.format(name=name) for name in colnames)
rendered_rows = [] rendered_rows = []
for row in rows: for row in rows:
# [2:] is to remove the indented flag + filename # [2:] is to remove the indented flag + filename
indented = u'indented' if row[0] else u'' indented = 'indented' if row[0] else ''
filename = row[1] filename = row[1]
cells = u''.join(CELL_TEMPLATE.format(value=value) for value in row[2:]) cells = ''.join(CELL_TEMPLATE.format(value=value) for value in row[2:])
rendered_rows.append(ROW_TEMPLATE.format(indented=indented, filename=filename, cells=cells)) rendered_rows.append(ROW_TEMPLATE.format(indented=indented, filename=filename, cells=cells))
rendered_rows = u''.join(rendered_rows) rendered_rows = ''.join(rendered_rows)
# The main template can't use format because the css code uses {} # The main template can't use format because the css code uses {}
content = MAIN_TEMPLATE.replace('$colheaders', colheaders).replace('$rows', rendered_rows) content = MAIN_TEMPLATE.replace('$colheaders', colheaders).replace('$rows', rendered_rows)
folder = mkdtemp() folder = mkdtemp()
destpath = op.join(folder, u'export.htm') destpath = op.join(folder, 'export.htm')
fp = open(destpath, 'w') fp = open(destpath, 'w')
fp.write(content.encode('utf-8')) fp.write(content.encode('utf-8'))
fp.close() fp.close()

View File

@ -12,7 +12,7 @@
# resulting needless complexity and memory usage. It's been a while since I wanted to do that fork, # resulting needless complexity and memory usage. It's been a while since I wanted to do that fork,
# and I'm doing it now. # and I'm doing it now.
from __future__ import unicode_literals
import hashlib import hashlib
import logging import logging
@ -25,13 +25,13 @@ class FSError(Exception):
cls_message = "An error has occured on '{name}' in '{parent}'" cls_message = "An error has occured on '{name}' in '{parent}'"
def __init__(self, fsobject, parent=None): def __init__(self, fsobject, parent=None):
message = self.cls_message message = self.cls_message
if isinstance(fsobject, basestring): if isinstance(fsobject, str):
name = fsobject name = fsobject
elif isinstance(fsobject, File): elif isinstance(fsobject, File):
name = fsobject.name name = fsobject.name
else: else:
name = '' name = ''
parentname = unicode(parent) if parent is not None else '' parentname = str(parent) if parent is not None else ''
Exception.__init__(self, message.format(name=name, parent=parentname)) Exception.__init__(self, message.format(name=name, parent=parentname))
@ -119,7 +119,7 @@ class File(object):
If `attrnames` is not None, caches only attrnames. If `attrnames` is not None, caches only attrnames.
""" """
if attrnames is None: if attrnames is None:
attrnames = self.INITIAL_INFO.keys() attrnames = list(self.INITIAL_INFO.keys())
for attrname in attrnames: for attrname in attrnames:
if attrname not in self.__dict__: if attrname not in self.__dict__:
self._read_info(attrname) self._read_info(attrname)

View File

@ -32,7 +32,7 @@ class DetailsPanel(GUIObject):
ref = group.ref if group is not None and group.ref is not dupe else None ref = group.ref if group is not None and group.ref is not dupe else None
l2 = self.app._get_display_info(ref, group, False) l2 = self.app._get_display_info(ref, group, False)
names = [c['display'] for c in self.app.data.COLUMNS] names = [c['display'] for c in self.app.data.COLUMNS]
self._table = zip(names, l1, l2) self._table = list(zip(names, l1, l2))
#--- Public #--- Public
def row_count(self): def row_count(self):

View File

@ -62,7 +62,7 @@ class DirectoryTree(GUIObject, Tree):
def _refresh(self): def _refresh(self):
self.clear() self.clear()
for path in self.app.directories: for path in self.app.directories:
self.append(DirectoryNode(self.app, path, unicode(path))) self.append(DirectoryNode(self.app, path, str(path)))
def add_directory(self, path): def add_directory(self, path):
self.app.add_directory(path) self.app.add_directory(path)

View File

@ -39,5 +39,5 @@ class ProblemRow(Row):
Row.__init__(self, table) Row.__init__(self, table)
self.dupe = dupe self.dupe = dupe
self.msg = msg self.msg = msg
self.path = unicode(dupe.path) self.path = str(dupe.path)

View File

@ -63,7 +63,7 @@ class ResultTree(GUIObject, Tree):
def _select_nodes(self, nodes): def _select_nodes(self, nodes):
Tree._select_nodes(self, nodes) Tree._select_nodes(self, nodes)
self.app._select_dupes(map(attrgetter('_dupe'), nodes)) self.app._select_dupes(list(map(attrgetter('_dupe'), nodes)))
#--- Private #--- Private
def _refresh(self): def _refresh(self):

View File

@ -22,7 +22,7 @@ class IgnoreList(object):
self._count = 0 self._count = 0
def __iter__(self): def __iter__(self):
for first,seconds in self._ignored.iteritems(): for first,seconds in self._ignored.items():
for second in seconds: for second in seconds:
yield (first,second) yield (first,second)

View File

@ -147,7 +147,7 @@ class Results(Markable):
self.__filters.append(filter_str) self.__filters.append(filter_str)
if self.__filtered_dupes is None: if self.__filtered_dupes is None:
self.__filtered_dupes = flatten(g[:] for g in self.groups) self.__filtered_dupes = flatten(g[:] for g in self.groups)
self.__filtered_dupes = set(dupe for dupe in self.__filtered_dupes if filter_re.search(unicode(dupe.path))) self.__filtered_dupes = set(dupe for dupe in self.__filtered_dupes if filter_re.search(str(dupe.path)))
filtered_groups = set() filtered_groups = set()
for dupe in self.__filtered_dupes: for dupe in self.__filtered_dupes:
filtered_groups.add(self.get_group_of_duplicate(dupe)) filtered_groups.add(self.get_group_of_duplicate(dupe))
@ -241,7 +241,7 @@ class Results(Markable):
func(dupe) func(dupe)
to_remove.append(dupe) to_remove.append(dupe)
except EnvironmentError as e: except EnvironmentError as e:
self.problems.append((dupe, unicode(e))) self.problems.append((dupe, str(e)))
if remove_from_results: if remove_from_results:
self.remove_duplicates(to_remove) self.remove_duplicates(to_remove)
self.mark_none() self.mark_none()
@ -285,7 +285,7 @@ class Results(Markable):
words = () words = ()
file_elem = etree.SubElement(group_elem, 'file') file_elem = etree.SubElement(group_elem, 'file')
try: try:
file_elem.set('path', unicode(d.path)) file_elem.set('path', str(d.path))
file_elem.set('words', ','.join(words)) file_elem.set('words', ','.join(words))
except ValueError: # If there's an invalid character, just skip the file except ValueError: # If there's an invalid character, just skip the file
file_elem.set('path', '') file_elem.set('path', '')
@ -293,9 +293,9 @@ class Results(Markable):
file_elem.set('marked', ('y' if self.is_marked(d) else 'n')) file_elem.set('marked', ('y' if self.is_marked(d) else 'n'))
for match in g.matches: for match in g.matches:
match_elem = etree.SubElement(group_elem, 'match') match_elem = etree.SubElement(group_elem, 'match')
match_elem.set('first', unicode(dupe2index[match.first])) match_elem.set('first', str(dupe2index[match.first]))
match_elem.set('second', unicode(dupe2index[match.second])) match_elem.set('second', str(dupe2index[match.second]))
match_elem.set('percentage', unicode(int(match.percentage))) match_elem.set('percentage', str(int(match.percentage)))
tree = etree.ElementTree(root) tree = etree.ElementTree(root)
with FileOrPath(outfile, 'wb') as fp: with FileOrPath(outfile, 'wb') as fp:
tree.write(fp, encoding='utf-8') tree.write(fp, encoding='utf-8')

View File

@ -1,109 +1,113 @@
# Created By: Virgil Dupras # Created By: Virgil Dupras
# Created On: 2006/03/03 # Created On: 2006/03/03
# Copyright 2010 Hardcoded Software (http://www.hardcoded.net) # Copyright 2010 Hardcoded Software (http://www.hardcoded.net)
# #
# This software is licensed under the "HS" License as described in the "LICENSE" file, # This software is licensed under the "HS" License as described in the "LICENSE" file,
# which should be included with this package. The terms are also available at # which should be included with this package. The terms are also available at
# http://www.hardcoded.net/licenses/hs_license # http://www.hardcoded.net/licenses/hs_license
import logging import logging
from hscommon import job from hscommon import job
from hsutil import io from hsutil import io
from hsutil.misc import dedupe from hsutil.misc import dedupe
from hsutil.str import get_file_ext, rem_file_ext from hsutil.str import get_file_ext, rem_file_ext
from . import engine from . import engine
from .ignore import IgnoreList from .ignore import IgnoreList
(SCAN_TYPE_FILENAME, (SCAN_TYPE_FILENAME,
SCAN_TYPE_FIELDS, SCAN_TYPE_FIELDS,
SCAN_TYPE_FIELDS_NO_ORDER, SCAN_TYPE_FIELDS_NO_ORDER,
SCAN_TYPE_TAG, SCAN_TYPE_TAG,
UNUSED, # Must not be removed. Constants here are what scan_type in the prefs are. UNUSED, # Must not be removed. Constants here are what scan_type in the prefs are.
SCAN_TYPE_CONTENT, SCAN_TYPE_CONTENT,
SCAN_TYPE_CONTENT_AUDIO) = range(7) SCAN_TYPE_CONTENT_AUDIO) = range(7)
SCANNABLE_TAGS = ['track', 'artist', 'album', 'title', 'genre', 'year'] SCANNABLE_TAGS = ['track', 'artist', 'album', 'title', 'genre', 'year']
class Scanner(object): class Scanner(object):
def __init__(self): def __init__(self):
self.ignore_list = IgnoreList() self.ignore_list = IgnoreList()
self.discarded_file_count = 0 self.discarded_file_count = 0
def _getmatches(self, files, j): def _getmatches(self, files, j):
if self.size_threshold: if self.size_threshold:
j = j.start_subjob([2, 8]) j = j.start_subjob([2, 8])
for f in j.iter_with_progress(files, 'Read size of %d/%d files'): for f in j.iter_with_progress(files, 'Read size of %d/%d files'):
f.size # pre-read, makes a smoother progress if read here (especially for bundles) f.size # pre-read, makes a smoother progress if read here (especially for bundles)
files = [f for f in files if f.size >= self.size_threshold] files = [f for f in files if f.size >= self.size_threshold]
if self.scan_type in (SCAN_TYPE_CONTENT, SCAN_TYPE_CONTENT_AUDIO): if self.scan_type in (SCAN_TYPE_CONTENT, SCAN_TYPE_CONTENT_AUDIO):
sizeattr = 'size' if self.scan_type == SCAN_TYPE_CONTENT else 'audiosize' sizeattr = 'size' if self.scan_type == SCAN_TYPE_CONTENT else 'audiosize'
return engine.getmatches_by_contents(files, sizeattr, partial=self.scan_type==SCAN_TYPE_CONTENT_AUDIO, j=j) return engine.getmatches_by_contents(files, sizeattr, partial=self.scan_type==SCAN_TYPE_CONTENT_AUDIO, j=j)
else: else:
j = j.start_subjob([2, 8]) j = j.start_subjob([2, 8])
kw = {} kw = {}
kw['match_similar_words'] = self.match_similar_words kw['match_similar_words'] = self.match_similar_words
kw['weight_words'] = self.word_weighting kw['weight_words'] = self.word_weighting
kw['min_match_percentage'] = self.min_match_percentage kw['min_match_percentage'] = self.min_match_percentage
if self.scan_type == SCAN_TYPE_FIELDS_NO_ORDER: if self.scan_type == SCAN_TYPE_FIELDS_NO_ORDER:
self.scan_type = SCAN_TYPE_FIELDS self.scan_type = SCAN_TYPE_FIELDS
kw['no_field_order'] = True kw['no_field_order'] = True
func = { func = {
SCAN_TYPE_FILENAME: lambda f: engine.getwords(rem_file_ext(f.name)), SCAN_TYPE_FILENAME: lambda f: engine.getwords(rem_file_ext(f.name)),
SCAN_TYPE_FIELDS: lambda f: engine.getfields(rem_file_ext(f.name)), SCAN_TYPE_FIELDS: lambda f: engine.getfields(rem_file_ext(f.name)),
SCAN_TYPE_TAG: lambda f: [engine.getwords(unicode(getattr(f, attrname))) for attrname in SCANNABLE_TAGS if attrname in self.scanned_tags], SCAN_TYPE_TAG: lambda f: [engine.getwords(str(getattr(f, attrname))) for attrname in SCANNABLE_TAGS if attrname in self.scanned_tags],
}[self.scan_type] }[self.scan_type]
for f in j.iter_with_progress(files, 'Read metadata of %d/%d files'): for f in j.iter_with_progress(files, 'Read metadata of %d/%d files'):
f.words = func(f) f.words = func(f)
return engine.getmatches(files, j=j, **kw) return engine.getmatches(files, j=j, **kw)
@staticmethod @staticmethod
def _key_func(dupe): def _key_func(dupe):
return (not dupe.is_ref, -dupe.size) return (not dupe.is_ref, -dupe.size)
@staticmethod @staticmethod
def _tie_breaker(ref, dupe): def _tie_breaker(ref, dupe):
refname = rem_file_ext(ref.name).lower() refname = rem_file_ext(ref.name).lower()
dupename = rem_file_ext(dupe.name).lower() dupename = rem_file_ext(dupe.name).lower()
if 'copy' in refname and 'copy' not in dupename: if 'copy' in dupename:
return True return False
if refname.startswith(dupename) and (refname[len(dupename):].strip().isdigit()): if 'copy' in refname:
return True return True
return len(dupe.path) > len(ref.path) if dupename.startswith(refname) and (dupename[len(refname):].strip().isdigit()):
return False
def GetDupeGroups(self, files, j=job.nulljob): if refname.startswith(dupename) and (refname[len(dupename):].strip().isdigit()):
j = j.start_subjob([8, 2]) return True
for f in [f for f in files if not hasattr(f, 'is_ref')]: return len(dupe.path) > len(ref.path)
f.is_ref = False
logging.info('Getting matches') def GetDupeGroups(self, files, j=job.nulljob):
matches = self._getmatches(files, j) j = j.start_subjob([8, 2])
logging.info('Found %d matches' % len(matches)) for f in [f for f in files if not hasattr(f, 'is_ref')]:
j.set_progress(100, 'Removing false matches') f.is_ref = False
if not self.mix_file_kind: logging.info('Getting matches')
matches = [m for m in matches if get_file_ext(m.first.name) == get_file_ext(m.second.name)] matches = self._getmatches(files, j)
matches = [m for m in matches if io.exists(m.first.path) and io.exists(m.second.path)] logging.info('Found %d matches' % len(matches))
if self.ignore_list: j.set_progress(100, 'Removing false matches')
j = j.start_subjob(2) if not self.mix_file_kind:
iter_matches = j.iter_with_progress(matches, 'Processed %d/%d matches against the ignore list') matches = [m for m in matches if get_file_ext(m.first.name) == get_file_ext(m.second.name)]
matches = [m for m in iter_matches matches = [m for m in matches if io.exists(m.first.path) and io.exists(m.second.path)]
if not self.ignore_list.AreIgnored(unicode(m.first.path), unicode(m.second.path))] if self.ignore_list:
logging.info('Grouping matches') j = j.start_subjob(2)
groups = engine.get_groups(matches, j) iter_matches = j.iter_with_progress(matches, 'Processed %d/%d matches against the ignore list')
matched_files = dedupe([m.first for m in matches] + [m.second for m in matches]) matches = [m for m in iter_matches
self.discarded_file_count = len(matched_files) - sum(len(g) for g in groups) if not self.ignore_list.AreIgnored(str(m.first.path), str(m.second.path))]
groups = [g for g in groups if any(not f.is_ref for f in g)] logging.info('Grouping matches')
logging.info('Created %d groups' % len(groups)) groups = engine.get_groups(matches, j)
j.set_progress(100, 'Doing group prioritization') matched_files = dedupe([m.first for m in matches] + [m.second for m in matches])
for g in groups: self.discarded_file_count = len(matched_files) - sum(len(g) for g in groups)
g.prioritize(self._key_func, self._tie_breaker) groups = [g for g in groups if any(not f.is_ref for f in g)]
return groups logging.info('Created %d groups' % len(groups))
j.set_progress(100, 'Doing group prioritization')
match_similar_words = False for g in groups:
min_match_percentage = 80 g.prioritize(self._key_func, self._tie_breaker)
mix_file_kind = True return groups
scan_type = SCAN_TYPE_FILENAME
scanned_tags = set(['artist', 'title']) match_similar_words = False
size_threshold = 0 min_match_percentage = 80
word_weighting = False mix_file_kind = True
scan_type = SCAN_TYPE_FILENAME
scanned_tags = set(['artist', 'title'])
size_threshold = 0
word_weighting = False

View File

@ -109,7 +109,7 @@ class TCDupeGuru(TestCase):
def test_Scan_with_objects_evaluating_to_false(self): def test_Scan_with_objects_evaluating_to_false(self):
class FakeFile(fs.File): class FakeFile(fs.File):
def __nonzero__(self): def __bool__(self):
return False return False
@ -200,11 +200,11 @@ class TCDupeGuruWithResults(TestCase):
if expected is not None: if expected is not None:
expected = set(expected) expected = set(expected)
not_called = expected - calls not_called = expected - calls
assert not not_called, u"These calls haven't been made: {0}".format(not_called) assert not not_called, "These calls haven't been made: {0}".format(not_called)
if not_expected is not None: if not_expected is not None:
not_expected = set(not_expected) not_expected = set(not_expected)
called = not_expected & calls called = not_expected & calls
assert not called, u"These calls shouldn't have been made: {0}".format(called) assert not called, "These calls shouldn't have been made: {0}".format(called)
gui.clear_calls() gui.clear_calls()
def clear_gui_calls(self): def clear_gui_calls(self):
@ -409,9 +409,9 @@ class TCDupeGuruWithResults(TestCase):
def test_only_unicode_is_added_to_ignore_list(self): def test_only_unicode_is_added_to_ignore_list(self):
def FakeIgnore(first,second): def FakeIgnore(first,second):
if not isinstance(first,unicode): if not isinstance(first,str):
self.fail() self.fail()
if not isinstance(second,unicode): if not isinstance(second,str):
self.fail() self.fail()
app = self.app app = self.app
@ -423,11 +423,11 @@ class TCDupeGuruWithResults(TestCase):
class TCDupeGuru_renameSelected(TestCase): class TCDupeGuru_renameSelected(TestCase):
def setUp(self): def setUp(self):
p = self.tmppath() p = self.tmppath()
fp = open(unicode(p + 'foo bar 1'),mode='w') fp = open(str(p + 'foo bar 1'),mode='w')
fp.close() fp.close()
fp = open(unicode(p + 'foo bar 2'),mode='w') fp = open(str(p + 'foo bar 2'),mode='w')
fp.close() fp.close()
fp = open(unicode(p + 'foo bar 3'),mode='w') fp = open(str(p + 'foo bar 3'),mode='w')
fp.close() fp.close()
files = fs.get_files(p) files = fs.get_files(p)
matches = engine.getmatches(files) matches = engine.getmatches(files)

View File

@ -82,8 +82,8 @@ class TCDirectories(TestCase):
def test_AddPath_non_latin(self): def test_AddPath_non_latin(self):
p = Path(self.tmpdir()) p = Path(self.tmpdir())
to_add = p + u'unicode\u201a' to_add = p + 'unicode\u201a'
os.mkdir(unicode(to_add)) os.mkdir(str(to_add))
d = Directories() d = Directories()
try: try:
d.add_path(to_add) d.add_path(to_add)
@ -111,7 +111,7 @@ class TCDirectories(TestCase):
self.assertEqual(STATE_REFERENCE,d.get_state(p)) self.assertEqual(STATE_REFERENCE,d.get_state(p))
self.assertEqual(STATE_REFERENCE,d.get_state(p + 'dir1')) self.assertEqual(STATE_REFERENCE,d.get_state(p + 'dir1'))
self.assertEqual(1,len(d.states)) self.assertEqual(1,len(d.states))
self.assertEqual(p,d.states.keys()[0]) self.assertEqual(p,list(d.states.keys())[0])
self.assertEqual(STATE_REFERENCE,d.states[p]) self.assertEqual(STATE_REFERENCE,d.states[p])
def test_get_state_with_path_not_there(self): def test_get_state_with_path_not_there(self):
@ -213,11 +213,11 @@ class TCDirectories(TestCase):
def test_unicode_save(self): def test_unicode_save(self):
d = Directories() d = Directories()
p1 = self.tmppath() + u'hello\xe9' p1 = self.tmppath() + 'hello\xe9'
io.mkdir(p1) io.mkdir(p1)
io.mkdir(p1 + u'foo\xe9') io.mkdir(p1 + 'foo\xe9')
d.add_path(p1) d.add_path(p1)
d.set_state(p1 + u'foo\xe9', STATE_EXCLUDED) d.set_state(p1 + 'foo\xe9', STATE_EXCLUDED)
tmpxml = op.join(self.tmpdir(), 'directories_testunit.xml') tmpxml = op.join(self.tmpdir(), 'directories_testunit.xml')
try: try:
d.save_to_file(tmpxml) d.save_to_file(tmpxml)

View File

@ -62,12 +62,12 @@ class TCgetwords(TestCase):
def test_splitter_chars(self): def test_splitter_chars(self):
self.assertEqual( self.assertEqual(
[chr(i) for i in xrange(ord('a'),ord('z')+1)], [chr(i) for i in range(ord('a'),ord('z')+1)],
getwords("a-b_c&d+e(f)g;h\\i[j]k{l}m:n.o,p<q>r/s?t~u!v@w#x$y*z") getwords("a-b_c&d+e(f)g;h\\i[j]k{l}m:n.o,p<q>r/s?t~u!v@w#x$y*z")
) )
def test_joiner_chars(self): def test_joiner_chars(self):
self.assertEqual(["aec"], getwords(u"a'e\u0301c")) self.assertEqual(["aec"], getwords("a'e\u0301c"))
def test_empty(self): def test_empty(self):
self.assertEqual([], getwords('')) self.assertEqual([], getwords(''))
@ -76,7 +76,7 @@ class TCgetwords(TestCase):
self.assertEqual(['foo', 'bar'], getwords('FOO BAR')) self.assertEqual(['foo', 'bar'], getwords('FOO BAR'))
def test_decompose_unicode(self): def test_decompose_unicode(self):
self.assertEqual(getwords(u'foo\xe9bar'), ['fooebar']) self.assertEqual(getwords('foo\xe9bar'), ['fooebar'])
class TCgetfields(TestCase): class TCgetfields(TestCase):
@ -768,7 +768,7 @@ class TCget_groups(TestCase):
self.assert_(o3 in g) self.assert_(o3 in g)
def test_four_sized_group(self): def test_four_sized_group(self):
l = [NamedObject("foobar") for i in xrange(4)] l = [NamedObject("foobar") for i in range(4)]
m = getmatches(l) m = getmatches(l)
r = get_groups(m) r = get_groups(m)
self.assertEqual(1,len(r)) self.assertEqual(1,len(r))

View File

@ -6,7 +6,7 @@
# which should be included with this package. The terms are also available at # which should be included with this package. The terms are also available at
# http://www.hardcoded.net/licenses/hs_license # http://www.hardcoded.net/licenses/hs_license
import cStringIO import io
from lxml import etree from lxml import etree
from hsutil.testutil import eq_ from hsutil.testutil import eq_
@ -59,7 +59,7 @@ def test_save_to_xml():
il.Ignore('foo','bar') il.Ignore('foo','bar')
il.Ignore('foo','bleh') il.Ignore('foo','bleh')
il.Ignore('bleh','bar') il.Ignore('bleh','bar')
f = cStringIO.StringIO() f = io.BytesIO()
il.save_to_xml(f) il.save_to_xml(f)
f.seek(0) f.seek(0)
doc = etree.parse(f) doc = etree.parse(f)
@ -76,19 +76,19 @@ def test_SaveThenLoad():
il.Ignore('foo', 'bar') il.Ignore('foo', 'bar')
il.Ignore('foo', 'bleh') il.Ignore('foo', 'bleh')
il.Ignore('bleh', 'bar') il.Ignore('bleh', 'bar')
il.Ignore(u'\u00e9', 'bar') il.Ignore('\u00e9', 'bar')
f = cStringIO.StringIO() f = io.BytesIO()
il.save_to_xml(f) il.save_to_xml(f)
f.seek(0) f.seek(0)
f.seek(0) f.seek(0)
il = IgnoreList() il = IgnoreList()
il.load_from_xml(f) il.load_from_xml(f)
eq_(4,len(il)) eq_(4,len(il))
assert il.AreIgnored(u'\u00e9','bar') assert il.AreIgnored('\u00e9','bar')
def test_LoadXML_with_empty_file_tags(): def test_LoadXML_with_empty_file_tags():
f = cStringIO.StringIO() f = io.BytesIO()
f.write('<?xml version="1.0" encoding="utf-8"?><ignore_list><file><file/></file></ignore_list>') f.write(b'<?xml version="1.0" encoding="utf-8"?><ignore_list><file><file/></file></ignore_list>')
f.seek(0) f.seek(0)
il = IgnoreList() il = IgnoreList()
il.load_from_xml(f) il.load_from_xml(f)
@ -130,12 +130,12 @@ def test_filter():
def test_save_with_non_ascii_items(): def test_save_with_non_ascii_items():
il = IgnoreList() il = IgnoreList()
il.Ignore(u'\xac', u'\xbf') il.Ignore('\xac', '\xbf')
f = cStringIO.StringIO() f = io.BytesIO()
try: try:
il.save_to_xml(f) il.save_to_xml(f)
except Exception as e: except Exception as e:
raise AssertionError(unicode(e)) raise AssertionError(str(e))
def test_len(): def test_len():
il = IgnoreList() il = IgnoreList()

View File

@ -7,7 +7,7 @@
# which should be included with this package. The terms are also available at # which should be included with this package. The terms are also available at
# http://www.hardcoded.net/licenses/hs_license # http://www.hardcoded.net/licenses/hs_license
import StringIO import io
import os.path as op import os.path as op
from lxml import etree from lxml import etree
@ -25,7 +25,7 @@ class NamedObject(engine_test.NamedObject):
path = property(lambda x:Path('basepath') + x.name) path = property(lambda x:Path('basepath') + x.name)
is_ref = False is_ref = False
def __nonzero__(self): def __bool__(self):
return False #Make sure that operations are made correctly when the bool value of files is false. return False #Make sure that operations are made correctly when the bool value of files is false.
# Returns a group set that looks like that: # Returns a group set that looks like that:
@ -63,7 +63,7 @@ class TCResultsEmpty(TestCase):
self.assert_(self.results.get_group_of_duplicate('foo') is None) self.assert_(self.results.get_group_of_duplicate('foo') is None)
def test_save_to_xml(self): def test_save_to_xml(self):
f = StringIO.StringIO() f = io.BytesIO()
self.results.save_to_xml(f) self.results.save_to_xml(f)
f.seek(0) f.seek(0)
doc = etree.parse(f) doc = etree.parse(f)
@ -324,7 +324,7 @@ class TCResultsMarkings(TestCase):
def test_SaveXML(self): def test_SaveXML(self):
self.results.mark(self.objects[1]) self.results.mark(self.objects[1])
self.results.mark_invert() self.results.mark_invert()
f = StringIO.StringIO() f = io.BytesIO()
self.results.save_to_xml(f) self.results.save_to_xml(f)
f.seek(0) f.seek(0)
doc = etree.parse(f) doc = etree.parse(f)
@ -345,7 +345,7 @@ class TCResultsMarkings(TestCase):
self.objects[4].name = 'ibabtu 2' #we can't have 2 files with the same path self.objects[4].name = 'ibabtu 2' #we can't have 2 files with the same path
self.results.mark(self.objects[1]) self.results.mark(self.objects[1])
self.results.mark_invert() self.results.mark_invert()
f = StringIO.StringIO() f = io.BytesIO()
self.results.save_to_xml(f) self.results.save_to_xml(f)
f.seek(0) f.seek(0)
r = Results(data) r = Results(data)
@ -369,7 +369,7 @@ class TCResultsXML(TestCase):
def test_save_to_xml(self): def test_save_to_xml(self):
self.objects[0].is_ref = True self.objects[0].is_ref = True
self.objects[0].words = [['foo','bar']] self.objects[0].words = [['foo','bar']]
f = StringIO.StringIO() f = io.BytesIO()
self.results.save_to_xml(f) self.results.save_to_xml(f)
f.seek(0) f.seek(0)
doc = etree.parse(f) doc = etree.parse(f)
@ -408,7 +408,7 @@ class TCResultsXML(TestCase):
self.objects[0].is_ref = True self.objects[0].is_ref = True
self.objects[4].name = 'ibabtu 2' #we can't have 2 files with the same path self.objects[4].name = 'ibabtu 2' #we can't have 2 files with the same path
f = StringIO.StringIO() f = io.BytesIO()
self.results.save_to_xml(f) self.results.save_to_xml(f)
f.seek(0) f.seek(0)
r = Results(data) r = Results(data)
@ -451,7 +451,7 @@ class TCResultsXML(TestCase):
return [f for f in self.objects if str(f.path) == path][0] return [f for f in self.objects if str(f.path) == path][0]
self.objects[4].name = 'ibabtu 2' #we can't have 2 files with the same path self.objects[4].name = 'ibabtu 2' #we can't have 2 files with the same path
f = StringIO.StringIO() f = io.BytesIO()
self.results.save_to_xml(f) self.results.save_to_xml(f)
f.seek(0) f.seek(0)
r = Results(data) r = Results(data)
@ -490,7 +490,7 @@ class TCResultsXML(TestCase):
match_node.set('percentage', 'baz') match_node.set('percentage', 'baz')
group_node = etree.SubElement(root, 'foobar') #invalid group group_node = etree.SubElement(root, 'foobar') #invalid group
group_node = etree.SubElement(root, 'group') #empty group group_node = etree.SubElement(root, 'group') #empty group
f = StringIO.StringIO() f = io.BytesIO()
tree = etree.ElementTree(root) tree = etree.ElementTree(root)
tree.write(f, encoding='utf-8') tree.write(f, encoding='utf-8')
f.seek(0) f.seek(0)
@ -501,30 +501,30 @@ class TCResultsXML(TestCase):
def test_xml_non_ascii(self): def test_xml_non_ascii(self):
def get_file(path): def get_file(path):
if path == op.join('basepath',u'\xe9foo bar'): if path == op.join('basepath','\xe9foo bar'):
return objects[0] return objects[0]
if path == op.join('basepath',u'bar bleh'): if path == op.join('basepath','bar bleh'):
return objects[1] return objects[1]
objects = [NamedObject(u"\xe9foo bar",True),NamedObject("bar bleh",True)] objects = [NamedObject("\xe9foo bar",True),NamedObject("bar bleh",True)]
matches = engine.getmatches(objects) #we should have 5 matches matches = engine.getmatches(objects) #we should have 5 matches
groups = engine.get_groups(matches) #We should have 2 groups groups = engine.get_groups(matches) #We should have 2 groups
for g in groups: for g in groups:
g.prioritize(lambda x:objects.index(x)) #We want the dupes to be in the same order as the list is g.prioritize(lambda x:objects.index(x)) #We want the dupes to be in the same order as the list is
results = Results(data) results = Results(data)
results.groups = groups results.groups = groups
f = StringIO.StringIO() f = io.BytesIO()
results.save_to_xml(f) results.save_to_xml(f)
f.seek(0) f.seek(0)
r = Results(data) r = Results(data)
r.load_from_xml(f,get_file) r.load_from_xml(f,get_file)
g = r.groups[0] g = r.groups[0]
self.assertEqual(u"\xe9foo bar",g[0].name) self.assertEqual("\xe9foo bar",g[0].name)
self.assertEqual(['efoo','bar'],g[0].words) self.assertEqual(['efoo','bar'],g[0].words)
def test_load_invalid_xml(self): def test_load_invalid_xml(self):
f = StringIO.StringIO() f = io.BytesIO()
f.write('<this is invalid') f.write(b'<this is invalid')
f.seek(0) f.seek(0)
r = Results(data) r = Results(data)
r.load_from_xml(f,None) r.load_from_xml(f,None)
@ -546,7 +546,7 @@ class TCResultsXML(TestCase):
fake_matches.add(engine.Match(d1, d3, 43)) fake_matches.add(engine.Match(d1, d3, 43))
fake_matches.add(engine.Match(d2, d3, 46)) fake_matches.add(engine.Match(d2, d3, 46))
group.matches = fake_matches group.matches = fake_matches
f = StringIO.StringIO() f = io.BytesIO()
results = self.results results = self.results
results.save_to_xml(f) results.save_to_xml(f)
f.seek(0) f.seek(0)
@ -564,7 +564,7 @@ class TCResultsXML(TestCase):
def test_save_and_load(self): def test_save_and_load(self):
# previously, when reloading matches, they wouldn't be reloaded as namedtuples # previously, when reloading matches, they wouldn't be reloaded as namedtuples
f = StringIO.StringIO() f = io.BytesIO()
self.results.save_to_xml(f) self.results.save_to_xml(f)
f.seek(0) f.seek(0)
self.results.load_from_xml(f, self.get_file) self.results.load_from_xml(f, self.get_file)
@ -572,13 +572,13 @@ class TCResultsXML(TestCase):
def test_apply_filter_works_on_paths(self): def test_apply_filter_works_on_paths(self):
# apply_filter() searches on the whole path, not just on the filename. # apply_filter() searches on the whole path, not just on the filename.
self.results.apply_filter(u'basepath') self.results.apply_filter('basepath')
eq_(len(self.results.groups), 2) eq_(len(self.results.groups), 2)
def test_save_xml_with_invalid_characters(self): def test_save_xml_with_invalid_characters(self):
# Don't crash when saving files that have invalid xml characters in their path # Don't crash when saving files that have invalid xml characters in their path
self.objects[0].name = u'foo\x19' self.objects[0].name = 'foo\x19'
self.results.save_to_xml(StringIO.StringIO()) # don't crash self.results.save_to_xml(io.BytesIO()) # don't crash
class TCResultsFilter(TestCase): class TCResultsFilter(TestCase):

View File

@ -25,6 +25,9 @@ class NamedObject(object):
self.path = Path('') self.path = Path('')
self.words = getwords(name) self.words = getwords(name)
def __repr__(self):
return '<NamedObject %r>' % self.name
no = NamedObject no = NamedObject
@ -297,8 +300,8 @@ class ScannerTestFakeFiles(TestCase):
s.scanned_tags = set(['title']) s.scanned_tags = set(['title'])
o1 = no('foo') o1 = no('foo')
o2 = no('bar') o2 = no('bar')
o1.title = u'foobar\u00e9' o1.title = 'foobar\u00e9'
o2.title = u'foobar\u00e9' o2.title = 'foobar\u00e9'
try: try:
r = s.GetDupeGroups([o1, o2]) r = s.GetDupeGroups([o1, o2])
except UnicodeEncodeError: except UnicodeEncodeError:
@ -362,11 +365,11 @@ class ScannerTestFakeFiles(TestCase):
f1 = no('foobar') f1 = no('foobar')
f2 = no('foobar') f2 = no('foobar')
f3 = no('foobar') f3 = no('foobar')
f1.path = Path(u'foo1\u00e9') f1.path = Path('foo1\u00e9')
f2.path = Path(u'foo2\u00e9') f2.path = Path('foo2\u00e9')
f3.path = Path(u'foo3\u00e9') f3.path = Path('foo3\u00e9')
s.ignore_list.Ignore(unicode(f1.path),unicode(f2.path)) s.ignore_list.Ignore(str(f1.path),str(f2.path))
s.ignore_list.Ignore(unicode(f1.path),unicode(f3.path)) s.ignore_list.Ignore(str(f1.path),str(f3.path))
r = s.GetDupeGroups([f1,f2,f3]) r = s.GetDupeGroups([f1,f2,f3])
eq_(len(r), 1) eq_(len(r), 1)
g = r[0] g = r[0]
@ -379,7 +382,7 @@ class ScannerTestFakeFiles(TestCase):
# A very wrong way to use any() was added at some point, causing resulting group list # A very wrong way to use any() was added at some point, causing resulting group list
# to be empty. # to be empty.
class FalseNamedObject(NamedObject): class FalseNamedObject(NamedObject):
def __nonzero__(self): def __bool__(self):
return False return False

View File

@ -41,7 +41,7 @@ class DupeGuruME(DupeGuruBase):
try: try:
track.delete(timeout=0) track.delete(timeout=0)
except CommandError as e: except CommandError as e:
logging.warning('Error while trying to remove a track from iTunes: %s' % unicode(e)) logging.warning('Error while trying to remove a track from iTunes: %s' % str(e))
self._start_job(JOB_REMOVE_DEAD_TRACKS, do) self._start_job(JOB_REMOVE_DEAD_TRACKS, do)

View File

@ -42,12 +42,12 @@ class Mp3File(MusicFile):
HANDLED_EXTS = set(['mp3']) HANDLED_EXTS = set(['mp3'])
def _read_info(self, field): def _read_info(self, field):
if field == 'md5partial': if field == 'md5partial':
fileinfo = mpeg.Mpeg(unicode(self.path)) fileinfo = mpeg.Mpeg(str(self.path))
self._md5partial_offset = fileinfo.audio_offset self._md5partial_offset = fileinfo.audio_offset
self._md5partial_size = fileinfo.audio_size self._md5partial_size = fileinfo.audio_size
MusicFile._read_info(self, field) MusicFile._read_info(self, field)
if field in TAG_FIELDS: if field in TAG_FIELDS:
fileinfo = mpeg.Mpeg(unicode(self.path)) fileinfo = mpeg.Mpeg(str(self.path))
self.audiosize = fileinfo.audio_size self.audiosize = fileinfo.audio_size
self.bitrate = fileinfo.bitrate self.bitrate = fileinfo.bitrate
self.duration = fileinfo.duration self.duration = fileinfo.duration
@ -70,12 +70,12 @@ class WmaFile(MusicFile):
HANDLED_EXTS = set(['wma']) HANDLED_EXTS = set(['wma'])
def _read_info(self, field): def _read_info(self, field):
if field == 'md5partial': if field == 'md5partial':
dec = wma.WMADecoder(unicode(self.path)) dec = wma.WMADecoder(str(self.path))
self._md5partial_offset = dec.audio_offset self._md5partial_offset = dec.audio_offset
self._md5partial_size = dec.audio_size self._md5partial_size = dec.audio_size
MusicFile._read_info(self, field) MusicFile._read_info(self, field)
if field in TAG_FIELDS: if field in TAG_FIELDS:
dec = wma.WMADecoder(unicode(self.path)) dec = wma.WMADecoder(str(self.path))
self.audiosize = dec.audio_size self.audiosize = dec.audio_size
self.bitrate = dec.bitrate self.bitrate = dec.bitrate
self.duration = dec.duration self.duration = dec.duration
@ -92,13 +92,13 @@ class Mp4File(MusicFile):
HANDLED_EXTS = set(['m4a', 'm4p']) HANDLED_EXTS = set(['m4a', 'm4p'])
def _read_info(self, field): def _read_info(self, field):
if field == 'md5partial': if field == 'md5partial':
dec = mp4.File(unicode(self.path)) dec = mp4.File(str(self.path))
self._md5partial_offset = dec.audio_offset self._md5partial_offset = dec.audio_offset
self._md5partial_size = dec.audio_size self._md5partial_size = dec.audio_size
dec.close() dec.close()
MusicFile._read_info(self, field) MusicFile._read_info(self, field)
if field in TAG_FIELDS: if field in TAG_FIELDS:
dec = mp4.File(unicode(self.path)) dec = mp4.File(str(self.path))
self.audiosize = dec.audio_size self.audiosize = dec.audio_size
self.bitrate = dec.bitrate self.bitrate = dec.bitrate
self.duration = dec.duration self.duration = dec.duration
@ -116,12 +116,12 @@ class OggFile(MusicFile):
HANDLED_EXTS = set(['ogg']) HANDLED_EXTS = set(['ogg'])
def _read_info(self, field): def _read_info(self, field):
if field == 'md5partial': if field == 'md5partial':
dec = ogg.Vorbis(unicode(self.path)) dec = ogg.Vorbis(str(self.path))
self._md5partial_offset = dec.audio_offset self._md5partial_offset = dec.audio_offset
self._md5partial_size = dec.audio_size self._md5partial_size = dec.audio_size
MusicFile._read_info(self, field) MusicFile._read_info(self, field)
if field in TAG_FIELDS: if field in TAG_FIELDS:
dec = ogg.Vorbis(unicode(self.path)) dec = ogg.Vorbis(str(self.path))
self.audiosize = dec.audio_size self.audiosize = dec.audio_size
self.bitrate = dec.bitrate self.bitrate = dec.bitrate
self.duration = dec.duration self.duration = dec.duration
@ -138,12 +138,12 @@ class FlacFile(MusicFile):
HANDLED_EXTS = set(['flac']) HANDLED_EXTS = set(['flac'])
def _read_info(self, field): def _read_info(self, field):
if field == 'md5partial': if field == 'md5partial':
dec = flac.FLAC(unicode(self.path)) dec = flac.FLAC(str(self.path))
self._md5partial_offset = dec.audio_offset self._md5partial_offset = dec.audio_offset
self._md5partial_size = dec.audio_size self._md5partial_size = dec.audio_size
MusicFile._read_info(self, field) MusicFile._read_info(self, field)
if field in TAG_FIELDS: if field in TAG_FIELDS:
dec = flac.FLAC(unicode(self.path)) dec = flac.FLAC(str(self.path))
self.audiosize = dec.audio_size self.audiosize = dec.audio_size
self.bitrate = dec.bitrate self.bitrate = dec.bitrate
self.duration = dec.duration self.duration = dec.duration
@ -160,12 +160,12 @@ class AiffFile(MusicFile):
HANDLED_EXTS = set(['aif', 'aiff', 'aifc']) HANDLED_EXTS = set(['aif', 'aiff', 'aifc'])
def _read_info(self, field): def _read_info(self, field):
if field == 'md5partial': if field == 'md5partial':
dec = aiff.File(unicode(self.path)) dec = aiff.File(str(self.path))
self._md5partial_offset = dec.audio_offset self._md5partial_offset = dec.audio_offset
self._md5partial_size = dec.audio_size self._md5partial_size = dec.audio_size
MusicFile._read_info(self, field) MusicFile._read_info(self, field)
if field in TAG_FIELDS: if field in TAG_FIELDS:
dec = aiff.File(unicode(self.path)) dec = aiff.File(str(self.path))
self.audiosize = dec.audio_size self.audiosize = dec.audio_size
self.bitrate = dec.bitrate self.bitrate = dec.bitrate
self.duration = dec.duration self.duration = dec.duration

View File

@ -37,15 +37,15 @@ class Photo(fs.File):
def _read_info(self, field): def _read_info(self, field):
fs.File._read_info(self, field) fs.File._read_info(self, field)
if field == 'dimensions': if field == 'dimensions':
self.dimensions = _block_osx.get_image_size(unicode(self.path)) self.dimensions = _block_osx.get_image_size(str(self.path))
def get_blocks(self, block_count_per_side): def get_blocks(self, block_count_per_side):
try: try:
blocks = _block_osx.getblocks(unicode(self.path), block_count_per_side) blocks = _block_osx.getblocks(str(self.path), block_count_per_side)
except Exception as e: except Exception as e:
raise IOError('The reading of "%s" failed with "%s"' % (unicode(self.path), unicode(e))) raise IOError('The reading of "%s" failed with "%s"' % (str(self.path), str(e)))
if not blocks: if not blocks:
raise IOError('The picture %s could not be read' % unicode(self.path)) raise IOError('The picture %s could not be read' % str(self.path))
return blocks return blocks
@ -140,7 +140,7 @@ class DupeGuruPE(app_cocoa.DupeGuru):
photos = as_fetch(a.photo_library_album().photos, k.item) photos = as_fetch(a.photo_library_album().photos, k.item)
for photo in j.iter_with_progress(photos): for photo in j.iter_with_progress(photos):
try: try:
self.path2iphoto[unicode(photo.image_path(timeout=0))] = photo self.path2iphoto[str(photo.image_path(timeout=0))] = photo
except CommandError: except CommandError:
pass pass
except (CommandError, RuntimeError): except (CommandError, RuntimeError):
@ -151,15 +151,15 @@ class DupeGuruPE(app_cocoa.DupeGuru):
def _do_delete_dupe(self, dupe): def _do_delete_dupe(self, dupe):
if isinstance(dupe, IPhoto): if isinstance(dupe, IPhoto):
if unicode(dupe.path) in self.path2iphoto: if str(dupe.path) in self.path2iphoto:
photo = self.path2iphoto[unicode(dupe.path)] photo = self.path2iphoto[str(dupe.path)]
try: try:
a = app('iPhoto') a = app('iPhoto')
a.remove(photo, timeout=0) a.remove(photo, timeout=0)
except (CommandError, RuntimeError) as e: except (CommandError, RuntimeError) as e:
raise EnvironmentError(unicode(e)) raise EnvironmentError(str(e))
else: else:
msg = u"Could not find photo %s in iPhoto Library" % unicode(dupe.path) msg = "Could not find photo %s in iPhoto Library" % str(dupe.path)
raise EnvironmentError(msg) raise EnvironmentError(msg)
else: else:
app_cocoa.DupeGuru._do_delete_dupe(self, dupe) app_cocoa.DupeGuru._do_delete_dupe(self, dupe)

View File

@ -6,7 +6,7 @@
# which should be included with this package. The terms are also available at # which should be included with this package. The terms are also available at
# http://www.hardcoded.net/licenses/hs_license # http://www.hardcoded.net/licenses/hs_license
from _block import NoBlocksError, DifferentBlockCountError, avgdiff, getblocks2 from ._block import NoBlocksError, DifferentBlockCountError, avgdiff, getblocks2
# Converted to C # Converted to C
# def getblock(image): # def getblock(image):

View File

@ -82,7 +82,7 @@ class Cache(object):
self.con.execute(sql, [value, key]) self.con.execute(sql, [value, key])
except sqlite.OperationalError: except sqlite.OperationalError:
logging.warning('Picture cache could not set %r for key %r', value, key) logging.warning('Picture cache could not set %r for key %r', value, key)
except sqlite.DatabaseError, e: except sqlite.DatabaseError as e:
logging.warning('DatabaseError while setting %r for key %r: %s', value, key, str(e)) logging.warning('DatabaseError while setting %r for key %r: %s', value, key, str(e))
def _create_con(self, second_try=False): def _create_con(self, second_try=False):
@ -97,7 +97,7 @@ class Cache(object):
self.con.execute("select * from pictures where 1=2") self.con.execute("select * from pictures where 1=2")
except sqlite.OperationalError: # new db except sqlite.OperationalError: # new db
create_tables() create_tables()
except sqlite.DatabaseError, e: # corrupted db except sqlite.DatabaseError as e: # corrupted db
if second_try: if second_try:
raise # Something really strange is happening raise # Something really strange is happening
logging.warning('Could not create picture cache because of an error: %s', str(e)) logging.warning('Could not create picture cache because of an error: %s', str(e))

View File

@ -15,11 +15,11 @@ def move(src, dst):
return return
if op.exists(dst): if op.exists(dst):
os.remove(dst) os.remove(dst)
print 'Moving %s --> %s' % (src, dst) print('Moving %s --> %s' % (src, dst))
os.rename(src, dst) os.rename(src, dst)
os.chdir('modules') os.chdir('modules')
os.system('python setup.py build_ext --inplace') os.system('python3 setup.py build_ext --inplace')
os.chdir('..') os.chdir('..')
move(op.join('modules', '_block.so'), '_block.so') move(op.join('modules', '_block.so'), '_block.so')
move(op.join('modules', '_block.pyd'), '_block.pyd') move(op.join('modules', '_block.pyd'), '_block.pyd')

View File

@ -34,16 +34,16 @@ def prepare_pictures(pictures, cache_path, j=job.nulljob):
try: try:
for picture in j.iter_with_progress(pictures, 'Analyzed %d/%d pictures'): for picture in j.iter_with_progress(pictures, 'Analyzed %d/%d pictures'):
picture.dimensions picture.dimensions
picture.unicode_path = unicode(picture.path) picture.unicode_path = str(picture.path)
try: try:
if picture.unicode_path not in cache: if picture.unicode_path not in cache:
blocks = picture.get_blocks(BLOCK_COUNT_PER_SIDE) blocks = picture.get_blocks(BLOCK_COUNT_PER_SIDE)
cache[picture.unicode_path] = blocks cache[picture.unicode_path] = blocks
prepared.append(picture) prepared.append(picture)
except (IOError, ValueError) as e: except (IOError, ValueError) as e:
logging.warning(unicode(e)) logging.warning(str(e))
except MemoryError: except MemoryError:
logging.warning(u'Ran out of memory while reading %s of size %d' % (picture.unicode_path, picture.size)) logging.warning('Ran out of memory while reading %s of size %d' % (picture.unicode_path, picture.size))
if picture.size < 10 * 1024 * 1024: # We're really running out of memory if picture.size < 10 * 1024 * 1024: # We're really running out of memory
raise raise
except MemoryError: except MemoryError:

View File

@ -39,9 +39,9 @@ static PyObject* getblock(PyObject *image)
pg = PySequence_ITEM(ppixel, 1); pg = PySequence_ITEM(ppixel, 1);
pb = PySequence_ITEM(ppixel, 2); pb = PySequence_ITEM(ppixel, 2);
Py_DECREF(ppixel); Py_DECREF(ppixel);
r = PyInt_AsSsize_t(pr); r = PyLong_AsLong(pr);
g = PyInt_AsSsize_t(pg); g = PyLong_AsLong(pg);
b = PyInt_AsSsize_t(pb); b = PyLong_AsLong(pb);
Py_DECREF(pr); Py_DECREF(pr);
Py_DECREF(pg); Py_DECREF(pg);
Py_DECREF(pb); Py_DECREF(pb);
@ -67,14 +67,14 @@ static PyObject* getblock(PyObject *image)
*/ */
static int diff(PyObject *first, PyObject *second) static int diff(PyObject *first, PyObject *second)
{ {
Py_ssize_t r1, g1, b1, r2, b2, g2; int r1, g1, b1, r2, b2, g2;
PyObject *pr, *pg, *pb; PyObject *pr, *pg, *pb;
pr = PySequence_ITEM(first, 0); pr = PySequence_ITEM(first, 0);
pg = PySequence_ITEM(first, 1); pg = PySequence_ITEM(first, 1);
pb = PySequence_ITEM(first, 2); pb = PySequence_ITEM(first, 2);
r1 = PyInt_AsSsize_t(pr); r1 = PyLong_AsLong(pr);
g1 = PyInt_AsSsize_t(pg); g1 = PyLong_AsLong(pg);
b1 = PyInt_AsSsize_t(pb); b1 = PyLong_AsLong(pb);
Py_DECREF(pr); Py_DECREF(pr);
Py_DECREF(pg); Py_DECREF(pg);
Py_DECREF(pb); Py_DECREF(pb);
@ -82,9 +82,9 @@ static int diff(PyObject *first, PyObject *second)
pr = PySequence_ITEM(second, 0); pr = PySequence_ITEM(second, 0);
pg = PySequence_ITEM(second, 1); pg = PySequence_ITEM(second, 1);
pb = PySequence_ITEM(second, 2); pb = PySequence_ITEM(second, 2);
r2 = PyInt_AsSsize_t(pr); r2 = PyLong_AsLong(pr);
g2 = PyInt_AsSsize_t(pg); g2 = PyLong_AsLong(pg);
b2 = PyInt_AsSsize_t(pb); b2 = PyLong_AsLong(pb);
Py_DECREF(pr); Py_DECREF(pr);
Py_DECREF(pg); Py_DECREF(pg);
Py_DECREF(pb); Py_DECREF(pb);
@ -115,8 +115,8 @@ static PyObject* block_getblocks2(PyObject *self, PyObject *args)
pimage_size = PyObject_GetAttrString(image, "size"); pimage_size = PyObject_GetAttrString(image, "size");
pwidth = PySequence_ITEM(pimage_size, 0); pwidth = PySequence_ITEM(pimage_size, 0);
pheight = PySequence_ITEM(pimage_size, 1); pheight = PySequence_ITEM(pimage_size, 1);
width = PyInt_AsSsize_t(pwidth); width = PyLong_AsLong(pwidth);
height = PyInt_AsSsize_t(pheight); height = PyLong_AsLong(pheight);
Py_DECREF(pimage_size); Py_DECREF(pimage_size);
Py_DECREF(pwidth); Py_DECREF(pwidth);
Py_DECREF(pheight); Py_DECREF(pheight);
@ -147,8 +147,8 @@ static PyObject* block_getblocks2(PyObject *self, PyObject *args)
left = min(iw*block_width, width-block_width); left = min(iw*block_width, width-block_width);
right = left + block_width; right = left + block_width;
pbox = inttuple(4, left, top, right, bottom); pbox = inttuple(4, left, top, right, bottom);
pmethodname = PyString_FromString("crop"); pmethodname = PyUnicode_FromString("crop");
pcrop = PyObject_CallMethodObjArgs(image, pmethodname, pbox); pcrop = PyObject_CallMethodObjArgs(image, pmethodname, pbox, NULL);
Py_DECREF(pmethodname); Py_DECREF(pmethodname);
Py_DECREF(pbox); Py_DECREF(pbox);
if (pcrop == NULL) { if (pcrop == NULL) {
@ -207,7 +207,7 @@ static PyObject* block_avgdiff(PyObject *self, PyObject *args)
Py_DECREF(item1); Py_DECREF(item1);
Py_DECREF(item2); Py_DECREF(item2);
if ((sum > limit*iteration_count) && (iteration_count >= min_iterations)) { if ((sum > limit*iteration_count) && (iteration_count >= min_iterations)) {
return PyInt_FromSsize_t(limit + 1); return PyLong_FromLong(limit + 1);
} }
} }
@ -215,7 +215,7 @@ static PyObject* block_avgdiff(PyObject *self, PyObject *args)
if (!result && sum) { if (!result && sum) {
result = 1; result = 1;
} }
return PyInt_FromSsize_t(result); return PyLong_FromLong(result);
} }
static PyMethodDef BlockMethods[] = { static PyMethodDef BlockMethods[] = {
@ -224,16 +224,30 @@ static PyMethodDef BlockMethods[] = {
{NULL, NULL, 0, NULL} /* Sentinel */ {NULL, NULL, 0, NULL} /* Sentinel */
}; };
PyMODINIT_FUNC static struct PyModuleDef BlockDef = {
init_block(void) PyModuleDef_HEAD_INIT,
"_block",
NULL,
-1,
BlockMethods,
NULL,
NULL,
NULL,
NULL
};
PyObject *
PyInit__block(void)
{ {
PyObject *m = Py_InitModule("_block", BlockMethods); PyObject *m = PyModule_Create(&BlockDef);
if (m == NULL) { if (m == NULL) {
return; return NULL;
} }
NoBlocksError = PyErr_NewException("_block.NoBlocksError", NULL, NULL); NoBlocksError = PyErr_NewException("_block.NoBlocksError", NULL, NULL);
PyModule_AddObject(m, "NoBlocksError", NoBlocksError); PyModule_AddObject(m, "NoBlocksError", NoBlocksError);
DifferentBlockCountError = PyErr_NewException("_block.DifferentBlockCountError", NULL, NULL); DifferentBlockCountError = PyErr_NewException("_block.DifferentBlockCountError", NULL, NULL);
PyModule_AddObject(m, "DifferentBlockCountError", DifferentBlockCountError); PyModule_AddObject(m, "DifferentBlockCountError", DifferentBlockCountError);
return m;
} }

View File

@ -29,8 +29,8 @@ pystring2cfstring(PyObject *pystring)
Py_INCREF(encoded); Py_INCREF(encoded);
} }
s = (UInt8*)PyString_AS_STRING(encoded); s = (UInt8*)PyBytes_AS_STRING(encoded);
size = PyString_GET_SIZE(encoded); size = PyUnicode_GET_SIZE(encoded);
result = CFStringCreateWithBytes(NULL, s, size, kCFStringEncodingUTF8, FALSE); result = CFStringCreateWithBytes(NULL, s, size, kCFStringEncodingUTF8, FALSE);
Py_DECREF(encoded); Py_DECREF(encoded);
return result; return result;
@ -43,7 +43,7 @@ static PyObject* block_osx_get_image_size(PyObject *self, PyObject *args)
CFURLRef image_url; CFURLRef image_url;
CGImageSourceRef source; CGImageSourceRef source;
CGImageRef image; CGImageRef image;
size_t width, height; long width, height;
PyObject *pwidth, *pheight; PyObject *pwidth, *pheight;
PyObject *result; PyObject *result;
@ -72,11 +72,11 @@ static PyObject* block_osx_get_image_size(PyObject *self, PyObject *args)
CFRelease(source); CFRelease(source);
} }
pwidth = PyInt_FromSsize_t(width); pwidth = PyLong_FromLong(width);
if (pwidth == NULL) { if (pwidth == NULL) {
return NULL; return NULL;
} }
pheight = PyInt_FromSsize_t(height); pheight = PyLong_FromLong(height);
if (pheight == NULL) { if (pheight == NULL) {
return NULL; return NULL;
} }
@ -228,8 +228,24 @@ static PyMethodDef BlockOsxMethods[] = {
{NULL, NULL, 0, NULL} /* Sentinel */ {NULL, NULL, 0, NULL} /* Sentinel */
}; };
PyMODINIT_FUNC static struct PyModuleDef BlockOsxDef = {
init_block_osx(void) PyModuleDef_HEAD_INIT,
"_block_osx",
NULL,
-1,
BlockOsxMethods,
NULL,
NULL,
NULL,
NULL
};
PyObject *
PyInit__block_osx(void)
{ {
Py_InitModule("_block_osx", BlockOsxMethods); PyObject *m = PyModule_Create(&BlockOsxDef);
if (m == NULL) {
return NULL;
}
return m;
} }

View File

@ -72,8 +72,24 @@ static PyMethodDef CacheMethods[] = {
{NULL, NULL, 0, NULL} /* Sentinel */ {NULL, NULL, 0, NULL} /* Sentinel */
}; };
PyMODINIT_FUNC static struct PyModuleDef CacheDef = {
init_cache(void) PyModuleDef_HEAD_INIT,
"_cache",
NULL,
-1,
CacheMethods,
NULL,
NULL,
NULL,
NULL
};
PyObject *
PyInit__cache(void)
{ {
(void)Py_InitModule("_cache", CacheMethods); PyObject *m = PyModule_Create(&CacheDef);
} if (m == NULL) {
return NULL;
}
return m;
}

View File

@ -32,7 +32,7 @@ PyObject* inttuple(int n, ...)
result = PyTuple_New(n); result = PyTuple_New(n);
for (i=0; i<n; i++) { for (i=0; i<n; i++) {
pnumber = PyInt_FromLong(va_arg(numbers, int)); pnumber = PyLong_FromLong(va_arg(numbers, long));
if (pnumber == NULL) { if (pnumber == NULL) {
Py_DECREF(result); Py_DECREF(result);
return NULL; return NULL;

View File

@ -258,8 +258,8 @@ class TCavgdiff(unittest.TestCase):
def test_return_at_least_1_at_the_slightest_difference(self): def test_return_at_least_1_at_the_slightest_difference(self):
ref = (0,0,0) ref = (0,0,0)
b1 = (1,0,0) b1 = (1,0,0)
blocks1 = [ref for i in xrange(250)] blocks1 = [ref for i in range(250)]
blocks2 = [ref for i in xrange(250)] blocks2 = [ref for i in range(250)]
blocks2[0] = b1 blocks2[0] = b1
self.assertEqual(1,my_avgdiff(blocks1,blocks2)) self.assertEqual(1,my_avgdiff(blocks1,blocks2))

View File

@ -6,10 +6,8 @@
# which should be included with this package. The terms are also available at # which should be included with this package. The terms are also available at
# http://www.hardcoded.net/licenses/hs_license # http://www.hardcoded.net/licenses/hs_license
from StringIO import StringIO
import os.path as op import os.path as op
import os import os
import threading
from hsutil.testcase import TestCase from hsutil.testcase import TestCase

View File

@ -6,7 +6,7 @@
# which should be included with this package. The terms are also available at # which should be included with this package. The terms are also available at
# http://www.hardcoded.net/licenses/hs_license # http://www.hardcoded.net/licenses/hs_license
from __future__ import unicode_literals
import logging import logging
@ -24,17 +24,17 @@ def is_bundle(str_path):
sw = NSWorkspace.sharedWorkspace() sw = NSWorkspace.sharedWorkspace()
uti, error = sw.typeOfFile_error_(str_path, None) uti, error = sw.typeOfFile_error_(str_path, None)
if error is not None: if error is not None:
logging.warning(u'There was an error trying to detect the UTI of %s', str_path) logging.warning('There was an error trying to detect the UTI of %s', str_path)
return sw.type_conformsToType_(uti, 'com.apple.bundle') or sw.type_conformsToType_(uti, 'com.apple.package') return sw.type_conformsToType_(uti, 'com.apple.bundle') or sw.type_conformsToType_(uti, 'com.apple.package')
class Bundle(BundleBase): class Bundle(BundleBase):
@classmethod @classmethod
def can_handle(cls, path): def can_handle(cls, path):
return not io.islink(path) and io.isdir(path) and is_bundle(unicode(path)) return not io.islink(path) and io.isdir(path) and is_bundle(str(path))
class Directories(DirectoriesBase): class Directories(DirectoriesBase):
ROOT_PATH_TO_EXCLUDE = map(Path, ['/Library', '/Volumes', '/System', '/bin', '/sbin', '/opt', '/private', '/dev']) ROOT_PATH_TO_EXCLUDE = list(map(Path, ['/Library', '/Volumes', '/System', '/bin', '/sbin', '/opt', '/private', '/dev']))
HOME_PATH_TO_EXCLUDE = [Path('Library')] HOME_PATH_TO_EXCLUDE = [Path('Library')]
def __init__(self): def __init__(self):
DirectoriesBase.__init__(self, fileclasses=[Bundle, fs.File]) DirectoriesBase.__init__(self, fileclasses=[Bundle, fs.File])

View File

@ -35,7 +35,7 @@ class Bundle(fs.File):
files = fs.get_all_files(self.path) files = fs.get_all_files(self.path)
files.sort(key=lambda f:f.path) files.sort(key=lambda f:f.path)
md5s = [getattr(f, field) for f in files] md5s = [getattr(f, field) for f in files]
return ''.join(md5s) return b''.join(md5s)
md5 = hashlib.md5(get_dir_md5_concat()) md5 = hashlib.md5(get_dir_md5_concat())
digest = md5.digest() digest = md5.digest()

View File

@ -30,7 +30,7 @@ def package_windows(edition, dev):
# On Windows, PyInstaller is used to build an exe (py2exe creates a very bad looking icon) # On Windows, PyInstaller is used to build an exe (py2exe creates a very bad looking icon)
# The release version is outdated. Use at least r672 on http://svn.pyinstaller.org/trunk # The release version is outdated. Use at least r672 on http://svn.pyinstaller.org/trunk
if sys.platform != "win32": if sys.platform != "win32":
print "Qt packaging only works under Windows." print("Qt packaging only works under Windows.")
return return
add_to_pythonpath('.') add_to_pythonpath('.')
add_to_pythonpath('qt') add_to_pythonpath('qt')
@ -60,7 +60,7 @@ def package_windows(edition, dev):
help_basedir = '..\\..\\help_{0}'.format(edition) help_basedir = '..\\..\\help_{0}'.format(edition)
help_dir = 'dupeguru_{0}_help'.format(edition) if edition != 'se' else 'dupeguru_help' help_dir = 'dupeguru_{0}_help'.format(edition) if edition != 'se' else 'dupeguru_help'
help_path = op.join(help_basedir, help_dir) help_path = op.join(help_basedir, help_dir)
print "Copying {0} to dist\\help".format(help_path) print("Copying {0} to dist\\help".format(help_path))
shutil.copytree(help_path, 'dist\\help') shutil.copytree(help_path, 'dist\\help')
# AdvancedInstaller.com has to be in your PATH # AdvancedInstaller.com has to be in your PATH
@ -106,7 +106,7 @@ def main():
edition = conf['edition'] edition = conf['edition']
ui = conf['ui'] ui = conf['ui']
dev = conf['dev'] dev = conf['dev']
print "Packaging dupeGuru {0} with UI {1}".format(edition.upper(), ui) print("Packaging dupeGuru {0} with UI {1}".format(edition.upper(), ui))
if ui == 'cocoa': if ui == 'cocoa':
package_cocoa(edition) package_cocoa(edition)
elif ui == 'qt': elif ui == 'qt':
@ -115,7 +115,7 @@ def main():
elif sys.platform == "linux2": elif sys.platform == "linux2":
package_debian(edition) package_debian(edition)
else: else:
print "Qt packaging only works under Windows or Linux." print("Qt packaging only works under Windows or Linux.")
if __name__ == '__main__': if __name__ == '__main__':
main() main()

View File

@ -6,7 +6,7 @@
# which should be included with this package. The terms are also available at # which should be included with this package. The terms are also available at
# http://www.hardcoded.net/licenses/hs_license # http://www.hardcoded.net/licenses/hs_license
from __future__ import unicode_literals
import logging import logging
import os import os
@ -54,7 +54,7 @@ class DupeGuru(DupeGuruBase, QObject):
DELTA_COLUMNS = frozenset() DELTA_COLUMNS = frozenset()
def __init__(self, data_module, appid): def __init__(self, data_module, appid):
appdata = unicode(QDesktopServices.storageLocation(QDesktopServices.DataLocation)) appdata = str(QDesktopServices.storageLocation(QDesktopServices.DataLocation))
if not op.exists(appdata): if not op.exists(appdata):
os.makedirs(appdata) os.makedirs(appdata)
# For basicConfig() to work, we have to be sure that no logging has taken place before this call. # For basicConfig() to work, we have to be sure that no logging has taken place before this call.
@ -120,7 +120,7 @@ class DupeGuru(DupeGuruBase, QObject):
#--- Override #--- Override
@staticmethod @staticmethod
def _open_path(path): def _open_path(path):
url = QUrl.fromLocalFile(unicode(path)) url = QUrl.fromLocalFile(str(path))
QDesktopServices.openUrl(url) QDesktopServices.openUrl(url)
@staticmethod @staticmethod
@ -150,7 +150,7 @@ class DupeGuru(DupeGuruBase, QObject):
opname = 'copy' if copy else 'move' opname = 'copy' if copy else 'move'
title = "Select a directory to {0} marked files to".format(opname) title = "Select a directory to {0} marked files to".format(opname)
flags = QFileDialog.ShowDirsOnly flags = QFileDialog.ShowDirsOnly
destination = unicode(QFileDialog.getExistingDirectory(self.main_window, title, '', flags)) destination = str(QFileDialog.getExistingDirectory(self.main_window, title, '', flags))
if not destination: if not destination:
return return
recreate_path = self.prefs.destination_type recreate_path = self.prefs.destination_type

View File

@ -52,9 +52,9 @@ class DirectoriesDialog(QDialog, Ui_DirectoriesDialog):
# label = 'Remove' if node.parent is None else 'Exclude' # label = 'Remove' if node.parent is None else 'Exclude'
def addButtonClicked(self): def addButtonClicked(self):
title = u"Select a directory to add to the scanning list" title = "Select a directory to add to the scanning list"
flags = QFileDialog.ShowDirsOnly flags = QFileDialog.ShowDirsOnly
dirpath = unicode(QFileDialog.getExistingDirectory(self, title, self.lastAddedFolder, flags)) dirpath = str(QFileDialog.getExistingDirectory(self, title, self.lastAddedFolder, flags))
if not dirpath: if not dirpath:
return return
self.lastAddedFolder = dirpath self.lastAddedFolder = dirpath

View File

@ -6,7 +6,7 @@
# which should be included with this package. The terms are also available at # which should be included with this package. The terms are also available at
# http://www.hardcoded.net/licenses/hs_license # http://www.hardcoded.net/licenses/hs_license
import urllib import urllib.parse
from PyQt4.QtCore import QModelIndex, Qt, QRect, QEvent, QPoint, QUrl from PyQt4.QtCore import QModelIndex, Qt, QRect, QEvent, QPoint, QUrl
from PyQt4.QtGui import (QComboBox, QStyledItemDelegate, QMouseEvent, QApplication, QBrush, QStyle, from PyQt4.QtGui import (QComboBox, QStyledItemDelegate, QMouseEvent, QApplication, QBrush, QStyle,
@ -101,9 +101,9 @@ class DirectoriesModel(TreeModel):
if not mimeData.hasFormat('text/uri-list'): if not mimeData.hasFormat('text/uri-list'):
return False return False
data = str(mimeData.data('text/uri-list')) data = str(mimeData.data('text/uri-list'))
unquoted = urllib.unquote(data) unquoted = urllib.parse.unquote(data)
urls = unicode(unquoted, 'utf-8').split('\r\n') urls = str(unquoted, 'utf-8').split('\r\n')
paths = [unicode(QUrl(url).toLocalFile()) for url in urls if url] paths = [str(QUrl(url).toLocalFile()) for url in urls if url]
for path in paths: for path in paths:
self.model.add_directory(path) self.model.add_directory(path)
self.reset() self.reset()

View File

@ -16,10 +16,10 @@ from hsutil.misc import nonone
from core.app import NoScannableFileError, AllFilesAreRefError from core.app import NoScannableFileError, AllFilesAreRefError
import dg_rc from . import dg_rc
from main_window_ui import Ui_MainWindow from .main_window_ui import Ui_MainWindow
from results_model import ResultsModel from .results_model import ResultsModel
from stats_label import StatsLabel from .stats_label import StatsLabel
class MainWindow(QMainWindow, Ui_MainWindow): class MainWindow(QMainWindow, Ui_MainWindow):
def __init__(self, app): def __init__(self, app):
@ -104,7 +104,7 @@ class MainWindow(QMainWindow, Ui_MainWindow):
h = self.resultsView.header() h = self.resultsView.header()
h.setResizeMode(QHeaderView.Interactive) h.setResizeMode(QHeaderView.Interactive)
prefs = self.app.prefs prefs = self.app.prefs
attrs = zip(prefs.columns_width, prefs.columns_visible) attrs = list(zip(prefs.columns_width, prefs.columns_visible))
for index, (width, visible) in enumerate(attrs): for index, (width, visible) in enumerate(attrs):
h.resizeSection(index, width) h.resizeSection(index, width)
h.setSectionHidden(index, not visible) h.setSectionHidden(index, not visible)
@ -145,7 +145,7 @@ class MainWindow(QMainWindow, Ui_MainWindow):
answer, ok = QInputDialog.getText(self, title, msg, QLineEdit.Normal, text) answer, ok = QInputDialog.getText(self, title, msg, QLineEdit.Normal, text)
if not ok: if not ok:
return return
answer = unicode(answer) answer = str(answer)
self.app.apply_filter(answer) self.app.apply_filter(answer)
self._last_filter = answer self._last_filter = answer

View File

@ -11,10 +11,10 @@ import logging
import sys import sys
if sys.platform == 'win32': if sys.platform == 'win32':
from platform_win import * from .platform_win import *
elif sys.platform == 'darwin': elif sys.platform == 'darwin':
from platform_osx import * from .platform_osx import *
elif sys.platform == 'linux2': elif sys.platform == 'linux2':
from platform_lnx import * from .platform_lnx import *
else: else:
pass # unsupported platform pass # unsupported platform

View File

@ -7,5 +7,5 @@
# which should be included with this package. The terms are also available at # which should be included with this package. The terms are also available at
# http://www.hardcoded.net/licenses/hs_license # http://www.hardcoded.net/licenses/hs_license
INITIAL_FOLDER_IN_DIALOGS = u'/' INITIAL_FOLDER_IN_DIALOGS = '/'
HELP_PATH = '/usr/local/share/dupeguru_{0}/help' HELP_PATH = '/usr/local/share/dupeguru_{0}/help'

View File

@ -9,5 +9,5 @@
# dummy unit to allow the app to run under OSX during development # dummy unit to allow the app to run under OSX during development
INITIAL_FOLDER_IN_DIALOGS = u'/' INITIAL_FOLDER_IN_DIALOGS = '/'
HELP_PATH = '' HELP_PATH = ''

View File

@ -7,5 +7,5 @@
# which should be included with this package. The terms are also available at # which should be included with this package. The terms are also available at
# http://www.hardcoded.net/licenses/hs_license # http://www.hardcoded.net/licenses/hs_license
INITIAL_FOLDER_IN_DIALOGS = u'C:\\' INITIAL_FOLDER_IN_DIALOGS = 'C:\\'
HELP_PATH = 'help' HELP_PATH = 'help'

View File

@ -112,7 +112,7 @@ class ResultsModel(TreeModel):
return True return True
if role == Qt.EditRole: if role == Qt.EditRole:
if index.column() == 0: if index.column() == 0:
value = unicode(value.toString()) value = str(value.toString())
return self.model.rename_selected(value) return self.model.rename_selected(value)
return False return False

View File

@ -76,7 +76,7 @@ class PreferencesDialog(QDialog, Ui_PreferencesDialog):
prefs.use_regexp = ischecked(self.useRegexpBox) prefs.use_regexp = ischecked(self.useRegexpBox)
prefs.remove_empty_folders = ischecked(self.removeEmptyFoldersBox) prefs.remove_empty_folders = ischecked(self.removeEmptyFoldersBox)
prefs.destination_type = self.copyMoveDestinationComboBox.currentIndex() prefs.destination_type = self.copyMoveDestinationComboBox.currentIndex()
prefs.custom_command = unicode(self.customCommandEdit.text()) prefs.custom_command = str(self.customCommandEdit.text())
def resetToDefaults(self): def resetToDefaults(self):
self.load(preferences.Preferences()) self.load(preferences.Preferences())

View File

@ -41,14 +41,14 @@ class File(fs.File):
fs.File._read_info(self, field) fs.File._read_info(self, field)
if field == 'dimensions': if field == 'dimensions':
try: try:
im = PIL.Image.open(unicode(self.path)) im = PIL.Image.open(str(self.path))
self.dimensions = im.size self.dimensions = im.size
except IOError: except IOError:
self.dimensions = (0, 0) self.dimensions = (0, 0)
logging.warning(u"Could not read image '%s'", unicode(self.path)) logging.warning("Could not read image '%s'", str(self.path))
def get_blocks(self, block_count_per_side): def get_blocks(self, block_count_per_side):
image = QImage(unicode(self.path)) image = QImage(str(self.path))
image = image.convertToFormat(QImage.Format_RGB888) image = image.convertToFormat(QImage.Format_RGB888)
return getblocks(image, block_count_per_side) return getblocks(image, block_count_per_side)

View File

@ -28,11 +28,11 @@ class DetailsDialog(DetailsDialogBase, Ui_DetailsDialog):
group = self.app.results.get_group_of_duplicate(dupe) group = self.app.results.get_group_of_duplicate(dupe)
ref = group.ref ref = group.ref
self.selectedPixmap = QPixmap(unicode(dupe.path)) self.selectedPixmap = QPixmap(str(dupe.path))
if ref is dupe: if ref is dupe:
self.referencePixmap = None self.referencePixmap = None
else: else:
self.referencePixmap = QPixmap(unicode(ref.path)) self.referencePixmap = QPixmap(str(ref.path))
self._updateImages() self._updateImages()
def _updateImages(self): def _updateImages(self):

View File

@ -15,7 +15,7 @@ def move(src, dst):
return return
if op.exists(dst): if op.exists(dst):
os.remove(dst) os.remove(dst)
print 'Moving %s --> %s' % (src, dst) print('Moving %s --> %s' % (src, dst))
os.rename(src, dst) os.rename(src, dst)
os.chdir('modules') os.chdir('modules')

View File

@ -46,7 +46,7 @@ class PreferencesDialog(QDialog, Ui_PreferencesDialog):
prefs.use_regexp = ischecked(self.useRegexpBox) prefs.use_regexp = ischecked(self.useRegexpBox)
prefs.remove_empty_folders = ischecked(self.removeEmptyFoldersBox) prefs.remove_empty_folders = ischecked(self.removeEmptyFoldersBox)
prefs.destination_type = self.copyMoveDestinationComboBox.currentIndex() prefs.destination_type = self.copyMoveDestinationComboBox.currentIndex()
prefs.custom_command = unicode(self.customCommandEdit.text()) prefs.custom_command = str(self.customCommandEdit.text())
def resetToDefaults(self): def resetToDefaults(self):
self.load(preferences.Preferences()) self.load(preferences.Preferences())

View File

@ -48,7 +48,7 @@ class PreferencesDialog(QDialog, Ui_PreferencesDialog):
setchecked(self.useRegexpBox, prefs.use_regexp) setchecked(self.useRegexpBox, prefs.use_regexp)
setchecked(self.removeEmptyFoldersBox, prefs.remove_empty_folders) setchecked(self.removeEmptyFoldersBox, prefs.remove_empty_folders)
setchecked(self.ignoreSmallFilesBox, prefs.ignore_small_files) setchecked(self.ignoreSmallFilesBox, prefs.ignore_small_files)
self.sizeThresholdEdit.setText(unicode(prefs.small_file_threshold)) self.sizeThresholdEdit.setText(str(prefs.small_file_threshold))
self.copyMoveDestinationComboBox.setCurrentIndex(prefs.destination_type) self.copyMoveDestinationComboBox.setCurrentIndex(prefs.destination_type)
self.customCommandEdit.setText(prefs.custom_command) self.customCommandEdit.setText(prefs.custom_command)
@ -65,7 +65,7 @@ class PreferencesDialog(QDialog, Ui_PreferencesDialog):
prefs.ignore_small_files = ischecked(self.ignoreSmallFilesBox) prefs.ignore_small_files = ischecked(self.ignoreSmallFilesBox)
prefs.small_file_threshold = tryint(self.sizeThresholdEdit.text()) prefs.small_file_threshold = tryint(self.sizeThresholdEdit.text())
prefs.destination_type = self.copyMoveDestinationComboBox.currentIndex() prefs.destination_type = self.copyMoveDestinationComboBox.currentIndex()
prefs.custom_command = unicode(self.customCommandEdit.text()) prefs.custom_command = str(self.customCommandEdit.text())
def resetToDefaults(self): def resetToDefaults(self):
self.load(preferences.Preferences()) self.load(preferences.Preferences())

View File

@ -6,6 +6,8 @@
# http://www.hardcoded.net/licenses/hs_license # http://www.hardcoded.net/licenses/hs_license
import sys import sys
import sip
sip.setapi('QVariant', 1)
from PyQt4.QtCore import QCoreApplication from PyQt4.QtCore import QCoreApplication
from PyQt4.QtGui import QApplication, QIcon, QPixmap from PyQt4.QtGui import QApplication, QIcon, QPixmap

5
run.py
View File

@ -20,7 +20,7 @@ def main():
edition = conf['edition'] edition = conf['edition']
ui = conf['ui'] ui = conf['ui']
dev = conf['dev'] dev = conf['dev']
print "Running dupeGuru {0} with UI {1}".format(edition.upper(), ui) print("Running dupeGuru {0} with UI {1}".format(edition.upper(), ui))
if ui == 'cocoa': if ui == 'cocoa':
subfolder = 'dev' if dev else 'release' subfolder = 'dev' if dev else 'release'
app_path = { app_path = {
@ -32,8 +32,9 @@ def main():
elif ui == 'qt': elif ui == 'qt':
add_to_pythonpath('.') add_to_pythonpath('.')
add_to_pythonpath('qt') add_to_pythonpath('qt')
add_to_pythonpath(op.join('qt', 'base'))
os.chdir(op.join('qt', edition)) os.chdir(op.join('qt', edition))
os.system('python start.py') os.system('python3 start.py')
os.chdir('..') os.chdir('..')
if __name__ == '__main__': if __name__ == '__main__':