From 70689ce0570cce588b72bc5deca1f20e14e5970f Mon Sep 17 00:00:00 2001 From: Virgil Dupras Date: Wed, 21 Sep 2011 10:26:58 -0400 Subject: [PATCH] Removed data modules and moved their functionalities to core_*.app. --- core/app.py | 76 ++++++++++++++++++------ core/app_cocoa.py | 2 +- core/data.py | 41 ------------- core/gui/details_panel.py | 7 +-- core/gui/prioritize_dialog.py | 2 +- core/gui/result_table.py | 4 +- core/results.py | 8 +-- core/tests/base.py | 42 ++++++++++++-- core/tests/data.py | 48 ---------------- core/tests/results_test.py | 60 ++++++++++++------- core_me/app.py | 102 +++++++++++++++++++++++++++++++-- core_me/data.py | 105 ---------------------------------- core_pe/app.py | 95 ++++++++++++++++++++++++++++-- core_pe/data.py | 100 -------------------------------- core_se/app.py | 75 ++++++++++++++++++++++-- core_se/data.py | 78 ------------------------- qt/base/result_window.py | 6 +- qt/base/results_model.py | 13 ++--- 18 files changed, 417 insertions(+), 447 deletions(-) delete mode 100644 core/data.py delete mode 100644 core/tests/data.py delete mode 100644 core_me/data.py delete mode 100644 core_pe/data.py delete mode 100644 core_se/data.py diff --git a/core/app.py b/core/app.py index 1f4f9c14..9e2c37bf 100644 --- a/core/app.py +++ b/core/app.py @@ -11,6 +11,8 @@ import os.path as op import logging import subprocess import re +import time +from collections import namedtuple from send2trash import send2trash from hscommon import io @@ -18,7 +20,7 @@ from hscommon.reg import RegistrableApplication from hscommon.notify import Broadcaster from hscommon.path import Path from hscommon.conflict import smart_move, smart_copy -from hscommon.util import delete_if_empty, first, escape, nonone +from hscommon.util import delete_if_empty, first, escape, nonone, format_time_decimal from hscommon.trans import tr from . import directories, results, scanner, export, fs @@ -40,6 +42,36 @@ class DestType: Relative = 1 Absolute = 2 + +Column = namedtuple('Column', 'attr display') + +def format_timestamp(t, delta): + if delta: + return format_time_decimal(t) + else: + if t > 0: + return time.strftime('%Y/%m/%d %H:%M:%S', time.localtime(t)) + else: + return '---' + +def format_words(w): + def do_format(w): + if isinstance(w, list): + return '(%s)' % ', '.join(do_format(item) for item in w) + else: + return w.replace('\n', ' ') + + return ', '.join(do_format(item) for item in w) + +def format_perc(p): + return "%0.0f" % p + +def format_dupe_count(c): + return str(c) if c else '---' + +def cmp_value(value): + return value.lower() if isinstance(value, str) else value + class DupeGuru(RegistrableApplication, Broadcaster): #--- View interface # open_path(path) @@ -49,7 +81,7 @@ class DupeGuru(RegistrableApplication, Broadcaster): # set_default(key_name, value) # show_extra_fairware_reminder() - def __init__(self, view, data_module, appdata): + def __init__(self, view, appdata): self.view = view if self.get_default(DEBUG_MODE_PREFERENCE, False): logging.getLogger().setLevel(logging.DEBUG) @@ -62,9 +94,8 @@ class DupeGuru(RegistrableApplication, Broadcaster): self.appdata = appdata if not op.exists(self.appdata): os.makedirs(self.appdata) - self.data = data_module self.directories = directories.Directories() - self.results = results.Results(data_module) + self.results = results.Results(self) self.scanner = scanner.Scanner() self.options = { 'escape_filter_regexp': True, @@ -73,6 +104,19 @@ class DupeGuru(RegistrableApplication, Broadcaster): } self.selected_dupes = [] + #--- Virtual + def _get_display_info(self, dupe, group, delta): + raise NotImplementedError() + + def _get_dupe_sort_key(self, dupe, get_group, key, delta): + raise NotImplementedError() + + def _get_group_sort_key(self, group, key): + raise NotImplementedError() + + def _prioritization_categories(self): + raise NotImplementedError() + #--- Private def _do_delete(self, j, replace_with_hardlinks): def op(dupe): @@ -92,15 +136,6 @@ class DupeGuru(RegistrableApplication, Broadcaster): os.link(str(ref.path), str(dupe.path)) self.clean_empty_dirs(dupe.path[:-1]) - def _get_display_info(self, dupe, group, delta=False): - if (dupe is None) or (group is None): - return ['---'] * len(self.data.COLUMNS) - try: - return self.data.GetDisplayInfo(dupe, group, delta) - except Exception as e: - logging.warning("Exception on GetDisplayInfo for %s: %s", str(dupe.path), str(e)) - return ['---'] * len(self.data.COLUMNS) - def _create_file(self, path): # We add fs.Folder to fileclasses in case the file we're loading contains folder paths. return fs.get_file(path, self.directories.fileclasses + [fs.Folder]) @@ -111,7 +146,7 @@ class DupeGuru(RegistrableApplication, Broadcaster): if f is None: return None try: - f._read_all_info(attrnames=self.data.METADATA_TO_READ) + f._read_all_info(attrnames=self.METADATA_TO_READ) return f except EnvironmentError: return None @@ -232,16 +267,25 @@ class DupeGuru(RegistrableApplication, Broadcaster): column_ids = [colid for colid in column_ids if colid.isdigit()] column_ids = list(map(int, column_ids)) column_ids.sort() - colnames = [col.display for i, col in enumerate(self.data.COLUMNS) if i in column_ids] + colnames = [col.display for i, col in enumerate(self.COLUMNS) if i in column_ids] rows = [] for group in self.results.groups: for dupe in group: - data = self._get_display_info(dupe, group) + data = self.get_display_info(dupe, group) row = [data[colid] for colid in column_ids] row.insert(0, dupe is not group.ref) rows.append(row) return export.export_to_xhtml(colnames, rows) + def get_display_info(self, dupe, group, delta=False): + if (dupe is None) or (group is None): + return ['---'] * len(self.COLUMNS) + try: + return self._get_display_info(dupe, group, delta) + except Exception as e: + logging.warning("Exception on GetDisplayInfo for %s: %s", str(dupe.path), str(e)) + return ['---'] * len(self.COLUMNS) + def invoke_command(self, cmd): """Calls command `cmd` with %d and %r placeholders replaced. diff --git a/core/app_cocoa.py b/core/app_cocoa.py index 6a9a3093..08582394 100644 --- a/core/app_cocoa.py +++ b/core/app_cocoa.py @@ -138,7 +138,7 @@ class PyDupeGuruBase(PyFairware): return self.py.results.is_modified def deltaColumns(self): - return list(self.py.data.DELTA_COLUMNS) + return list(self.py.DELTA_COLUMNS) #---Properties @signature('v@:c') diff --git a/core/data.py b/core/data.py deleted file mode 100644 index a497ebf6..00000000 --- a/core/data.py +++ /dev/null @@ -1,41 +0,0 @@ -# Created By: Virgil Dupras -# Created On: 2006/03/15 -# Copyright 2011 Hardcoded Software (http://www.hardcoded.net) -# -# This software is licensed under the "BSD" License as described in the "LICENSE" file, -# which should be included with this package. The terms are also available at -# http://www.hardcoded.net/licenses/bsd_license - -from collections import namedtuple -import time - -from hscommon.util import format_time_decimal, format_size - -Column = namedtuple('Column', 'attr display') - -def format_timestamp(t, delta): - if delta: - return format_time_decimal(t) - else: - if t > 0: - return time.strftime('%Y/%m/%d %H:%M:%S', time.localtime(t)) - else: - return '---' - -def format_words(w): - def do_format(w): - if isinstance(w, list): - return '(%s)' % ', '.join(do_format(item) for item in w) - else: - return w.replace('\n', ' ') - - return ', '.join(do_format(item) for item in w) - -def format_perc(p): - return "%0.0f" % p - -def format_dupe_count(c): - return str(c) if c else '---' - -def cmp_value(value): - return value.lower() if isinstance(value, str) else value \ No newline at end of file diff --git a/core/gui/details_panel.py b/core/gui/details_panel.py index 5505fbfe..8fcea194 100644 --- a/core/gui/details_panel.py +++ b/core/gui/details_panel.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Created By: Virgil Dupras # Created On: 2010-02-05 # Copyright 2011 Hardcoded Software (http://www.hardcoded.net) @@ -27,11 +26,11 @@ class DetailsPanel(GUIObject): else: dupe = None group = None - l1 = self.app._get_display_info(dupe, group, False) + l1 = self.app.get_display_info(dupe, group, False) # we don't want the two sides of the table to display the stats for the same file ref = group.ref if group is not None and group.ref is not dupe else None - l2 = self.app._get_display_info(ref, group, False) - names = [c.display for c in self.app.data.COLUMNS] + l2 = self.app.get_display_info(ref, group, False) + names = [c.display for c in self.app.COLUMNS] self._table = list(zip(names, l1, l2)) #--- Public diff --git a/core/gui/prioritize_dialog.py b/core/gui/prioritize_dialog.py index 8c9f3b47..d68c6b7d 100644 --- a/core/gui/prioritize_dialog.py +++ b/core/gui/prioritize_dialog.py @@ -42,7 +42,7 @@ class PrioritizationList(GUISelectableList): class PrioritizeDialog: def __init__(self, view, app): self.app = app - self.categories = [cat(app.results) for cat in app.data.prioritization_categories()] + self.categories = [cat(app.results) for cat in app._prioritization_categories()] self.category_list = CriterionCategoryList(self) self.criteria = [] self.criteria_list = GUISelectableList() diff --git a/core/gui/result_table.py b/core/gui/result_table.py index 7391e01f..2e0bb667 100644 --- a/core/gui/result_table.py +++ b/core/gui/result_table.py @@ -25,13 +25,13 @@ class DupeRow(Row): @property def data(self): if self._data is None: - self._data = self._app._get_display_info(self._dupe, self._group, False) + self._data = self._app.get_display_info(self._dupe, self._group, False) return self._data @property def data_delta(self): if self._data_delta is None: - self._data_delta = self._app._get_display_info(self._dupe, self._group, True) + self._data_delta = self._app.get_display_info(self._dupe, self._group, True) return self._data_delta @property diff --git a/core/results.py b/core/results.py index 2dd116e4..6b0df2e7 100644 --- a/core/results.py +++ b/core/results.py @@ -21,7 +21,7 @@ from hscommon.trans import tr class Results(Markable): #---Override - def __init__(self, data_module): + def __init__(self, app): super(Results, self).__init__() self.__groups = [] self.__group_of_duplicate = {} @@ -33,7 +33,7 @@ class Results(Markable): self.__filtered_groups = None self.__recalculate_stats() self.__marked_size = 0 - self.data = data_module + self.app = app self.problems = [] # (dupe, error_msg) self.is_modified = False @@ -328,12 +328,12 @@ class Results(Markable): def sort_dupes(self, key, asc=True, delta=False): if not self.__dupes: self.__get_dupe_list() - keyfunc = lambda d: self.data.GetDupeSortKey(d, lambda: self.get_group_of_duplicate(d), key, delta) + keyfunc = lambda d: self.app._get_dupe_sort_key(d, lambda: self.get_group_of_duplicate(d), key, delta) self.__dupes.sort(key=keyfunc, reverse=not asc) self.__dupes_sort_descriptor = (key,asc,delta) def sort_groups(self,key,asc=True): - keyfunc = lambda g: self.data.GetGroupSortKey(g, key) + keyfunc = lambda g: self.app._get_group_sort_key(g, key) self.groups.sort(key=keyfunc, reverse=not asc) self.__groups_sort_descriptor = (key,asc) diff --git a/core/tests/base.py b/core/tests/base.py index 4c4ae1c1..c1599d7e 100644 --- a/core/tests/base.py +++ b/core/tests/base.py @@ -8,17 +8,17 @@ from hscommon.testutil import TestApp as TestAppBase, eq_, with_app from hscommon.path import Path -from hscommon.util import get_file_ext +from hscommon.util import get_file_ext, format_size from jobprogress.job import nulljob, JobCancelled from .. import engine +from .. import prioritize from ..engine import getwords -from ..app import DupeGuru as DupeGuruBase +from ..app import DupeGuru as DupeGuruBase, Column, cmp_value from ..gui.details_panel import DetailsPanel from ..gui.directory_tree import DirectoryTree from ..gui.result_table import ResultTable from ..gui.prioritize_dialog import PrioritizeDialog -from . import data class DupeGuruView: JOB = nulljob @@ -37,8 +37,42 @@ class DupeGuruView: class DupeGuru(DupeGuruBase): + COLUMNS = [ + Column('name', 'Filename'), + Column('folder_path', 'Directory'), + Column('size', 'Size (KB)'), + Column('extension', 'Kind'), + ] + DELTA_COLUMNS = {2,} + METADATA_TO_READ = ['size'] + def __init__(self): - DupeGuruBase.__init__(self, DupeGuruView(), data, '/tmp') + DupeGuruBase.__init__(self, DupeGuruView(), '/tmp') + + def _get_display_info(self, dupe, group, delta): + size = dupe.size + m = group.get_match_of(dupe) + if m and delta: + r = group.ref + size -= r.size + return [ + dupe.name, + str(dupe.folder_path), + format_size(size, 0, 1, False), + dupe.extension if hasattr(dupe, 'extension') else '---', + ] + + def _get_dupe_sort_key(self, dupe, get_group, key, delta): + r = cmp_value(getattr(dupe, self.COLUMNS[key].attr)) + if delta and (key in self.DELTA_COLUMNS): + r -= cmp_value(getattr(get_group().ref, self.COLUMNS[key].attr)) + return r + + def _get_group_sort_key(self, group, key): + return cmp_value(getattr(group.ref, self.COLUMNS[key].attr)) + + def _prioritization_categories(self): + return prioritize.all_categories() class NamedObject: def __init__(self, name="foobar", with_words=False, size=1, folder=None): diff --git a/core/tests/data.py b/core/tests/data.py deleted file mode 100644 index d312b04d..00000000 --- a/core/tests/data.py +++ /dev/null @@ -1,48 +0,0 @@ -# Created By: Virgil Dupras -# Created On: 2009-10-23 -# Copyright 2011 Hardcoded Software (http://www.hardcoded.net) -# -# This software is licensed under the "BSD" License as described in the "LICENSE" file, -# which should be included with this package. The terms are also available at -# http://www.hardcoded.net/licenses/bsd_license - -# data module for tests - -from hscommon.util import format_size -from ..data import cmp_value, Column -from .. import prioritize - -COLUMNS = [ - Column('name', 'Filename'), - Column('folder_path', 'Directory'), - Column('size', 'Size (KB)'), - Column('extension', 'Kind'), -] - -METADATA_TO_READ = ['size'] -DELTA_COLUMNS = {2,} - -def GetDisplayInfo(dupe, group, delta): - size = dupe.size - m = group.get_match_of(dupe) - if m and delta: - r = group.ref - size -= r.size - return [ - dupe.name, - str(dupe.folder_path), - format_size(size, 0, 1, False), - dupe.extension if hasattr(dupe, 'extension') else '---', - ] - -def GetDupeSortKey(dupe, get_group, key, delta): - r = cmp_value(getattr(dupe, COLUMNS[key].attr)) - if delta and (key in DELTA_COLUMNS): - r -= cmp_value(getattr(get_group().ref, COLUMNS[key].attr)) - return r - -def GetGroupSortKey(group, key): - return cmp_value(getattr(group.ref, COLUMNS[key].attr)) - -def prioritization_categories(): - return prioritize.all_categories() diff --git a/core/tests/results_test.py b/core/tests/results_test.py index f611463b..4ab6dee5 100644 --- a/core/tests/results_test.py +++ b/core/tests/results_test.py @@ -14,14 +14,14 @@ from xml.etree import ElementTree as ET from hscommon.testutil import eq_ from hscommon.util import first -from . import data from .. import engine -from .base import NamedObject, GetTestGroups +from .base import NamedObject, GetTestGroups, DupeGuru from ..results import Results class TestCaseResultsEmpty: def setup_method(self, method): - self.results = Results(data) + self.app = DupeGuru() + self.results = self.app.results def test_apply_invalid_filter(self): # If the applied filter is an invalid regexp, just ignore the filter. @@ -68,7 +68,8 @@ class TestCaseResultsEmpty: class TestCaseResultsWithSomeGroups: def setup_method(self, method): - self.results = Results(data) + self.app = DupeGuru() + self.results = self.app.results self.objects,self.matches,self.groups = GetTestGroups() self.results.groups = self.groups @@ -186,7 +187,8 @@ class TestCaseResultsWithSomeGroups: def test_sort_empty_list(self): #There was an infinite loop when sorting an empty list. - r = Results(data) + app = DupeGuru() + r = app.results r.sort_dupes(0) eq_([],r.dupes) @@ -231,7 +233,8 @@ class TestCaseResultsWithSomeGroups: class TestCaseResultsWithSavedResults: def setup_method(self, method): - self.results = Results(data) + self.app = DupeGuru() + self.results = self.app.results self.objects,self.matches,self.groups = GetTestGroups() self.results.groups = self.groups self.f = io.BytesIO() @@ -264,7 +267,8 @@ class TestCaseResultsWithSavedResults: class TestCaseResultsMarkings: def setup_method(self, method): - self.results = Results(data) + self.app = DupeGuru() + self.results = self.app.results self.objects,self.matches,self.groups = GetTestGroups() self.results.groups = self.groups @@ -407,7 +411,8 @@ class TestCaseResultsMarkings: f = io.BytesIO() self.results.save_to_xml(f) f.seek(0) - r = Results(data) + app = DupeGuru() + r = Results(app) r.load_from_xml(f,get_file) assert not r.is_marked(self.objects[0]) assert not r.is_marked(self.objects[1]) @@ -418,7 +423,8 @@ class TestCaseResultsMarkings: class TestCaseResultsXML: def setup_method(self, method): - self.results = Results(data) + self.app = DupeGuru() + self.results = self.app.results self.objects, self.matches, self.groups = GetTestGroups() self.results.groups = self.groups @@ -470,7 +476,8 @@ class TestCaseResultsXML: f = io.BytesIO() self.results.save_to_xml(f) f.seek(0) - r = Results(data) + app = DupeGuru() + r = Results(app) r.load_from_xml(f,get_file) eq_(2,len(r.groups)) g1,g2 = r.groups @@ -499,7 +506,8 @@ class TestCaseResultsXML: filename = str(tmpdir.join('dupeguru_results.xml')) self.objects[4].name = 'ibabtu 2' #we can't have 2 files with the same path self.results.save_to_xml(filename) - r = Results(data) + app = DupeGuru() + r = Results(app) r.load_from_xml(filename,get_file) eq_(2,len(r.groups)) @@ -513,7 +521,8 @@ class TestCaseResultsXML: f = io.BytesIO() self.results.save_to_xml(f) f.seek(0) - r = Results(data) + app = DupeGuru() + r = Results(app) r.load_from_xml(f,get_file) eq_(1,len(r.groups)) eq_(3,len(r.groups[0])) @@ -553,7 +562,8 @@ class TestCaseResultsXML: tree = ET.ElementTree(root) tree.write(f, encoding='utf-8') f.seek(0) - r = Results(data) + app = DupeGuru() + r = Results(app) r.load_from_xml(f, get_file) eq_(1,len(r.groups)) eq_(3,len(r.groups[0])) @@ -570,12 +580,14 @@ class TestCaseResultsXML: groups = engine.get_groups(matches) #We should have 2 groups for g in groups: g.prioritize(lambda x:objects.index(x)) #We want the dupes to be in the same order as the list is - results = Results(data) + app = DupeGuru() + results = Results(app) results.groups = groups f = io.BytesIO() results.save_to_xml(f) f.seek(0) - r = Results(data) + app = DupeGuru() + r = Results(app) r.load_from_xml(f,get_file) g = r.groups[0] eq_("\xe9foo bar",g[0].name) @@ -585,12 +597,14 @@ class TestCaseResultsXML: f = io.BytesIO() f.write(b'