arsenetar
/
dupeguru
mirror of https://github.com/arsenetar/dupeguru.git
1
0
Fork 0
Code Releases Wiki Activity

Began serious code documentation effort 

Enabled the autodoc Sphinx extension and started adding docstrings to
classes, methods, etc.. It's quickly becoming quite interesting...
This commit is contained in:
Virgil Dupras 2013-08-18 18:36:09 -04:00
parent 8a8ac027f5
commit 7e8f9036d8
15 changed files with 284 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
core/app.py
View File

@ -96,6 +96,32 @@ def cmp_value(dupe, attrname):
return value.lower() if isinstance(value, str) else value
class DupeGuru(RegistrableApplication, Broadcaster):
"""Holds everything together.
Instantiated once per running application, it holds a reference to every high-level object
whose reference needs to be held: :class:`Results`, :class:`Scanner`,
:class:`~core.directories.Directories`, :mod:`core.gui` instances, etc..
It also hosts high level methods and acts as a coordinator for all those elements.
.. attribute:: directories
Instance of :class:`~core.directories.Directories`. It holds the current folder selection.
.. attribute:: results
Instance of :class:`core.results.Results`. Holds the results of the latest scan.
.. attribute:: selected_dupes
List of currently selected dupes from our :attr:`results`. Whenever the user changes its
selection at the UI level, :attr:`result_table` takes care of updating this attribute, so
you can trust that it's always up-to-date.
.. attribute:: result_table
Instance of :mod:`meta-gui <core.gui>` table listing the results from :attr:`results`
"""
#--- View interface
# open_path(path)
# reveal_path(path)
@ -299,6 +325,12 @@ class DupeGuru(RegistrableApplication, Broadcaster):
#--- Public
def add_directory(self, d):
"""Adds folder ``d`` to :attr:`directories`.
Shows an error message dialog if something bad happens.
:param str d: path of folder to add
"""
try:
self.directories.add_path(Path(d))
self.notify('directories_changed')
@ -308,6 +340,8 @@ class DupeGuru(RegistrableApplication, Broadcaster):
self.view.show_message(tr("'{}' does not exist.").format(d))
def add_selected_to_ignore_list(self):
"""Adds :attr:`selected_dupes` to :attr:`scanner`'s ignore list.
"""
dupes = self.without_ref(self.selected_dupes)
if not dupes:
self.view.show_message(MSG_NO_SELECTED_DUPES)
@ -324,6 +358,10 @@ class DupeGuru(RegistrableApplication, Broadcaster):
self.ignore_list_dialog.refresh()
def apply_filter(self, filter):
"""Apply a filter ``filter`` to the results so that it shows only dupe groups that match it.
:param str filter: filter to apply
"""
self.results.apply_filter(None)
if self.options['escape_filter_regexp']:
filter = escape(filter, set('()[]\\.|+?^'))
@ -359,6 +397,10 @@ class DupeGuru(RegistrableApplication, Broadcaster):
self.clean_empty_dirs(source_path[:-1])
def copy_or_move_marked(self, copy):
"""Start an async move (or copy) job on marked duplicates.
:param bool copy: If True, duplicates will be copied instead of moved
"""
def do(j):
def op(dupe):
j.add_progress()
@ -381,6 +423,8 @@ class DupeGuru(RegistrableApplication, Broadcaster):
self._start_job(jobid, do)
def delete_marked(self):
"""Start an async job to send marked duplicates to the trash.
"""
if not self._check_demo():
return
if not self.results.mark_count:
@ -416,11 +460,11 @@ class DupeGuru(RegistrableApplication, Broadcaster):
return empty_data()
def invoke_custom_command(self):
"""Calls command in 'CustomCommand' pref with %d and %r placeholders replaced.
"""Calls command in ``CustomCommand`` pref with ``%d`` and ``%r`` placeholders replaced.
Using the current selection, %d is replaced with the currently selected dupe and %r is
replaced with that dupe's ref file. If there's no selection, the command is not invoked.
If the dupe is a ref, %d and %r will be the same.
Using the current selection, ``%d`` is replaced with the currently selected dupe and ``%r``
is replaced with that dupe's ref file. If there's no selection, the command is not invoked.
If the dupe is a ref, ``%d`` and ``%r`` will be the same.
"""
cmd = self.view.get_default('CustomCommand')
if not cmd:
@ -453,6 +497,10 @@ class DupeGuru(RegistrableApplication, Broadcaster):
self.ignore_list_dialog.refresh()
def load_from(self, filename):
"""Start an async job to load results from ``filename``.
:param str filename: path of the XML file (created with :meth:`save_as`) to load
"""
def do(j):
self.results.load_from_xml(filename, self._get_file, j)
self._start_job(JobType.Load, do)
@ -503,6 +551,8 @@ class DupeGuru(RegistrableApplication, Broadcaster):
self.notify('marking_changed')
def open_selected(self):
"""Open :attr:`selected_dupes` with their associated application.
"""
if len(self.selected_dupes) > 10:
if not self.view.ask_yes_no(MSG_MANY_FILES_TO_OPEN):
return
@ -527,6 +577,8 @@ class DupeGuru(RegistrableApplication, Broadcaster):
self.notify('results_changed_but_keep_selection')
def remove_marked(self):
"""Removed marked duplicates from the results (without touching the files themselves).
"""
if not self.results.mark_count:
self.view.show_message(MSG_NO_MARKED_DUPES)
return
@ -537,6 +589,8 @@ class DupeGuru(RegistrableApplication, Broadcaster):
self._results_changed()
def remove_selected(self):
"""Removed :attr:`selected_dupes` from the results (without touching the files themselves).
"""
dupes = self.without_ref(self.selected_dupes)
if not dupes:
self.view.show_message(MSG_NO_SELECTED_DUPES)
@ -577,9 +631,17 @@ class DupeGuru(RegistrableApplication, Broadcaster):
self.notify('save_session')
def save_as(self, filename):
"""Save results in ``filename``.
:param str filename: path of the file to save results (as XML) to.
"""
self.results.save_to_xml(filename)
def start_scanning(self):
"""Starts an async job to scan for duplicates.
Scans folders selected in :attr:`directories` and put the results in :attr:`results`
"""
def do(j):
j.set_progress(0, tr("Collecting files to scan"))
if self.scanner.scan_type == scanner.ScanType.Folders:
@ -611,6 +673,8 @@ class DupeGuru(RegistrableApplication, Broadcaster):
self.notify('marking_changed')
def without_ref(self, dupes):
"""Returns ``dupes`` with all reference elements removed.
"""
return [dupe for dupe in dupes if self.results.get_group_of_duplicate(dupe).ref is not dupe]
def get_default(self, key, fallback_value=None):

66
core/directories.py
View File

@ -15,7 +15,20 @@ from hscommon.util import FileOrPath
from . import fs
__all__ = [
'Directories',
'DirectoryState',
'AlreadyThereError',
'InvalidPathError',
]
class DirectoryState:
"""Enum describing how a folder should be considered.
* DirectoryState.Normal: Scan all files normally
* DirectoryState.Reference: Scan files, but make sure never to delete any of them
* DirectoryState.Excluded: Don't scan this folder
"""
Normal = 0
Reference = 1
Excluded = 2
@ -27,6 +40,14 @@ class InvalidPathError(Exception):
"""The path being added is invalid"""
class Directories:
"""Holds user folder selection.
Manages the selection that the user make through the folder selection dialog. It also manages
folder states, and how recursion applies to them.
Then, when the user starts the scan, :meth:`get_files` is called to retrieve all files (wrapped
in :mod:`core.fs`) that have to be scanned according to the chosen folders/states.
"""
#---Override
def __init__(self, fileclasses=[fs.File]):
self._dirs = []
@ -97,11 +118,14 @@ class Directories:
#---Public
def add_path(self, path):
"""Adds 'path' to self, if not already there.
"""Adds ``path`` to self, if not already there.
Raises AlreadyThereError if 'path' is already in self. If path is a directory containing
some of the directories already present in self, 'path' will be added, but all directories
under it will be removed. Can also raise InvalidPathError if 'path' does not exist.
Raises :exc:`AlreadyThereError` if ``path`` is already in self. If path is a directory
containing some of the directories already present in self, ``path`` will be added, but all
directories under it will be removed. Can also raise :exc:`InvalidPathError` if ``path``
does not exist.
:param Path path: path to add
"""
if path in self:
raise AlreadyThereError()
@ -112,7 +136,11 @@ class Directories:
@staticmethod
def get_subfolders(path):
"""returns a sorted list of paths corresponding to subfolders in `path`"""
"""Returns a sorted list of paths corresponding to subfolders in ``path``.
:param Path path: get subfolders from there
:rtype: list of Path
"""
try:
names = [name for name in path.listdir() if (path + name).isdir()]
names.sort(key=lambda x:x.lower())
@ -123,7 +151,7 @@ class Directories:
def get_files(self, j=job.nulljob):
"""Returns a list of all files that are not excluded.
Returned files also have their 'is_ref' attr set.
Returned files also have their ``is_ref`` attr set if applicable.
"""
for path in self._dirs:
for file in self._get_files(path, j):
@ -132,7 +160,7 @@ class Directories:
def get_folders(self, j=job.nulljob):
"""Returns a list of all folders that are not excluded.
Returned folders also have their 'is_ref' attr set.
Returned folders also have their ``is_ref`` attr set if applicable.
"""
for path in self._dirs:
from_folder = fs.Folder(path)
@ -140,7 +168,9 @@ class Directories:
yield folder
def get_state(self, path):
"""Returns the state of 'path' (One of the STATE_* const.)
"""Returns the state of ``path``.
:rtype: :class:`DirectoryState`
"""
if path in self.states:
return self.states[path]
@ -154,6 +184,12 @@ class Directories:
return DirectoryState.Normal
def has_any_file(self):
"""Returns whether selected folders contain any file.
Because it stops at the first file it finds, it's much faster than get_files().
:rtype: bool
"""
try:
next(self.get_files())
return True
@ -161,6 +197,10 @@ class Directories:
return False
def load_from_file(self, infile):
"""Load folder selection from ``infile``.
:param file infile: path or file pointer to XML generated through :meth:`save_to_file`
"""
try:
root = ET.parse(infile).getroot()
except Exception:
@ -183,6 +223,10 @@ class Directories:
self.set_state(Path(path), int(state))
def save_to_file(self, outfile):
"""Save folder selection as XML to ``outfile``.
:param file outfile: path or file pointer to XML file to save to.
"""
with FileOrPath(outfile, 'wb') as fp:
root = ET.Element('directories')
for root_path in self:
@ -196,6 +240,12 @@ class Directories:
tree.write(fp, encoding='utf-8')
def set_state(self, path, state):
"""Set the state of folder at ``path``.
:param Path path: path of the target folder
:param state: state to set folder to
:type state: :class:`DirectoryState`
"""
if self.get_state(path) == state:
return
# we don't want to needlessly fill self.states. if get_state returns the same thing

38
core/engine.py
View File

@ -224,6 +224,23 @@ def getmatches_by_contents(files, sizeattr='size', partial=False, j=job.nulljob)
return result
class Group:
"""A group of :class:`~core.fs.File` that match together.
This manages match pairs into groups and ensures that all files in the group match to each
other.
.. attribute:: ref
The "reference" file, which is the file among the group that isn't going to be deleted.
.. attribute:: ordered
Ordered list of duplicates in the group (including the :attr:`ref`).
.. attribute:: unordered
Set duplicates in the group (including the :attr:`ref`).
"""
#---Override
def __init__(self):
self._clear()
@ -257,6 +274,15 @@ class Group:
#---Public
def add_match(self, match):
"""Adds ``match`` to internal match list and possibly add duplicates to the group.
A duplicate can only be considered as such if it matches all other duplicates in the group.
This method registers that pair (A, B) represented in ``match`` as possible candidates and,
if A and/or B end up matching every other duplicates in the group, add these duplicates to
the group.
:param tuple match: pair of :class:`~core.fs.File` to add
"""
def add_candidate(item, match):
matches = self.candidates[item]
matches.add(match)
@ -276,12 +302,18 @@ class Group:
self._matches_for_ref = None
def discard_matches(self):
"""Remove all recorded matches that didn't result in a duplicate being added to the group.
You can call this after the duplicate scanning process to free a bit of memory.
"""
discarded = set(m for m in self.matches if not all(obj in self.unordered for obj in [m.first, m.second]))
self.matches -= discarded
self.candidates = defaultdict(set)
return discarded
def get_match_of(self, item):
"""Returns the match pair between ``item`` and :attr:`ref`.
"""
if item is self.ref:
return
for m in self._get_matches_for_ref():
@ -289,6 +321,12 @@ class Group:
return m
def prioritize(self, key_func, tie_breaker=None):
"""Reorders :attr:`ordered` according to ``key_func``.
:param key_func: Key (f(x)) to be used for sorting
:param tie_breaker: function to be used to select the reference position in case the top
duplicates have the same key_func() result.
"""
# tie_breaker(ref, dupe) --> True if dupe should be ref
# Returns True if anything changed during prioritization.
master_key_func = lambda x: (-x.is_ref, key_func(x))

30
core/fs.py
View File

@ -16,6 +16,18 @@ import logging
from hscommon.util import nonone, get_file_ext
__all__ = [
'File',
'Folder',
'get_file',
'get_files',
'FSError',
'AlreadyExistsError',
'InvalidPath',
'InvalidDestinationError',
'OperationError',
]
NOT_SET = object()
class FSError(Exception):
@ -50,6 +62,8 @@ class OperationError(FSError):
cls_message = "Operation on '{name}' failed."
class File:
"""Represents a file and holds metadata to be used for scanning.
"""
INITIAL_INFO = {
'size': 0,
'mtime': 0,
@ -129,6 +143,8 @@ class File:
#--- Public
@classmethod
def can_handle(cls, path):
"""Returns whether this file wrapper class can handle ``path``.
"""
return not path.islink() and path.isfile()
def rename(self, newname):
@ -214,11 +230,25 @@ class Folder(File):
def get_file(path, fileclasses=[File]):
"""Wraps ``path`` around its appropriate :class:`File` class.
Whether a class is "appropriate" is decided by :meth:`File.can_handle`
:param Path path: path to wrap
:param fileclasses: List of candidate :class:`File` classes
"""
for fileclass in fileclasses:
if fileclass.can_handle(path):
return fileclass(path)
def get_files(path, fileclasses=[File]):
"""Returns a list of :class:`File` for each file contained in ``path``.
Subfolders are recursively scanned.
:param Path path: path to scan
:param fileclasses: List of candidate :class:`File` classes
"""
assert all(issubclass(fileclass, File) for fileclass in fileclasses)
def combine_paths(p1, p2):
try:

15
core/gui/__init__.py
View File

@ -0,0 +1,15 @@
"""
Meta GUI elements in dupeGuru
-----------------------------
dupeGuru is designed with a `cross-toolkit`_ approach in mind. It means that its core code
(which doesn't depend on any GUI toolkit) has elements which preformat core information in a way
that makes it easy for a UI layer to consume.
For example, we have :class:`~core.gui.ResultTable` which takes information from
:class:`~core.results.Results` and mashes it in rows and columns which are ready to be fetched by
either Cocoa's ``NSTableView`` or Qt's ``QTableView``. It tells them which cell is supposed to be
blue, which is supposed to be orange, does the sorting logic, holds selection, etc..
.. _cross-toolkit: http://www.hardcoded.net/articles/cross-toolkit-software
"""

35
core/results.py
View File

@ -21,6 +21,19 @@ from . import engine
from .markable import Markable
class Results(Markable):
"""Manages a collection of duplicate :class:`~core.engine.Group`.
This class takes care or marking, sorting and filtering duplicate groups.
.. attribute:: groups
The list of :class:`~core.engine.Group` contained managed by this instance.
.. attribute:: dupes
A list of all duplicates (:class:`~core.fs.File` instances), without ref, contained in the
currently managed :attr:`groups`.
"""
#---Override
def __init__(self, app):
Markable.__init__(self)
@ -145,17 +158,17 @@ class Results(Markable):
#---Public
def apply_filter(self, filter_str):
''' Applies a filter 'filter_str' to self.groups
"""Applies a filter ``filter_str`` to :attr:`groups`
When you apply the filter, only dupes with the filename matching 'filter_str' will be in
in the results. To cancel the filter, just call apply_filter with 'filter_str' to None,
and the results will go back to normal.
When you apply the filter, only dupes with the filename matching ``filter_str`` will be in
in the results. To cancel the filter, just call apply_filter with ``filter_str`` to None,
and the results will go back to normal.
If call apply_filter on a filtered results, the filter will be applied
*on the filtered results*.
If call apply_filter on a filtered results, the filter will be applied
*on the filtered results*.
'filter_str' is a string containing a regexp to filter dupes with.
'''
:param str filter_str: a string containing a regexp to filter dupes with.
"""
if not filter_str:
self.__filtered_dupes = None
self.__filtered_groups = None
@ -276,8 +289,10 @@ class Results(Markable):
self.mark(dupe)
def remove_duplicates(self, dupes):
'''Remove 'dupes' from their respective group, and remove the group is it ends up empty.
'''
"""Remove ``dupes`` from their respective :class:`~core.engine.Group`.
Also, remove the group from :attr:`groups` if it ends up empty.
"""
affected_groups = set()
for dupe in dupes:
group = self.get_group_of_duplicate(dupe)

6
help/conf.tmpl
View File

@ -16,7 +16,9 @@ import sys, os
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#sys.path.insert(0, os.path.abspath('.'))
# for autodocs
sys.path.insert(0, os.path.abspath(os.path.join('..', '..')))
# -- General configuration -----------------------------------------------------
@ -25,7 +27,7 @@ import sys, os
# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = ['sphinx.ext.todo']
extensions = ['sphinx.ext.todo', 'sphinx.ext.autodoc']
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

5
help/en/developer/core/app.rst Normal file
View File

@ -0,0 +1,5 @@
core.app
========
.. automodule:: core.app
:members:

5
help/en/developer/core/directories.rst Normal file
View File

@ -0,0 +1,5 @@
core.directories
================
.. automodule:: core.directories
:members:

7
help/en/developer/core/engine.rst Normal file
View File

@ -0,0 +1,7 @@
core.engine
===========
.. automodule:: core.engine
.. autoclass:: core.engine.Group
:members:

5
help/en/developer/core/fs.rst Normal file
View File

@ -0,0 +1,5 @@
core.fs
=======
.. automodule:: core.fs
:members:

5
help/en/developer/core/gui.rst Normal file
View File

@ -0,0 +1,5 @@
core.gui
========
.. automodule:: core.gui
:members:

5
help/en/developer/core/results.rst Normal file
View File

@ -0,0 +1,5 @@
core.results
============
.. automodule:: core.results
:members:

13
help/en/developer.rst → help/en/developer/index.rst
View File

@ -44,3 +44,16 @@ a list of matches and returns a list of ``Group`` instances (a ``Group`` is basi
When a scan is over, the final result (the list of groups from ``get_groups()``) is placed into
``app.DupeGuru.results``, which is a ``results.Results`` instance. The ``Results`` instance is where
all the dupe marking, sorting, removing, power marking, etc. takes place.
API
---
.. toctree::
:maxdepth: 2
core/app
core/fs
core/engine
core/directories
core/results
core/gui

2
help/en/index.rst
View File

@ -54,6 +54,6 @@ Contents:
results
reprioritize
faq
developer
developer/index
changelog
credits