feat: Create longest and shortest path criteria (#1242)

* Create longest and shortest path criteria

.ctags

@@ -0,0 +1,5 @@
+-R
+--exclude=build
+--exclude=env
+--exclude=.tox
+--python-kinds=-i

.github/FUNDING.yml

@@ -0,0 +1,13 @@
+# These are supported funding model platforms
+
+github: arsenetar
+patreon: # Replace with a single Patreon username
+open_collective: # Replace with a single Open Collective username
+ko_fi: # Replace with a single Ko-fi username
+tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
+community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
+liberapay: # Replace with a single Liberapay username
+issuehunt: # Replace with a single IssueHunt username
+otechie: # Replace with a single Otechie username
+lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry
+custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,31 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: bug
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Desktop (please complete the following information):**
+ - OS: [e.g. Windows 10 / OSX 10.15 / Ubuntu 20.04 / Arch Linux]
+ - Version [e.g. 4.1.0]
+
+**Additional context**
+Add any other context about the problem here. You may include the debug log although it is normally best to attach it as a file.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: feature
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,50 @@
+name: "CodeQL"
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [master]
+  schedule:
+    - cron: "24 20 * * 2"
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: ["cpp", "python"]
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      # Initializes the CodeQL tools for scanning.
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v1
+        with:
+          languages: ${{ matrix.language }}
+          # If you wish to specify custom queries, you can do so here or in a config file.
+          # By default, queries listed here will override any specified in a config file.
+          # Prefix the list here with "+" to use these queries and those in the config file.
+          # queries: ./path/to/local/query, your-org/your-repo/queries@main
+      - if: matrix.language == 'cpp'
+        name: Build Cpp
+        run: |
+          sudo apt-get update
+          sudo apt-get install python3-pyqt5
+          make modules
+      - if: matrix.language == 'python'
+        name: Autobuild
+        uses: github/codeql-action/autobuild@v1
+      # Analysis
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v1

.github/workflows/default.yml

@@ -0,0 +1,65 @@
+# Workflow lints, and checks format in parallel then runs tests on all platforms
+
+name: Default CI/CD
+
+on:
+  push:
+  pull_request:
+    branches: [master]
+
+jobs:
+  pre-commit:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - uses: pre-commit/action@v3.0.1
+  test:
+    needs: [pre-commit]
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        python-version: [3.7, 3.8, 3.9, "3.10", "3.11", "3.12"]
+        include:
+          - os: windows-latest
+            python-version: "3.12"
+          - os: macos-latest
+            python-version: "3.12"
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install setuptools
+          pip install -r requirements.txt -r requirements-extra.txt
+      - name: Build python modules
+        run: |
+          python build.py --modules
+      - name: Run tests
+        run: |
+          pytest core hscommon
+      - name: Upload Artifacts
+        if: matrix.os == 'ubuntu-latest'
+        uses: actions/upload-artifact@v4
+        with:
+          name: modules ${{ matrix.python-version }}
+          path: build/**/*.so
+  merge-artifacts:
+    needs: [test]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Merge Artifacts
+        uses: actions/upload-artifact/merge@v4
+        with:
+          name: modules
+          pattern: modules*
+          delete-merged: true

.github/workflows/tx-push.yml

@@ -0,0 +1,26 @@
+# Push translation source to Transifex
+name: Transifex Sync
+
+on:
+  push:
+    branches:
+      - master
+    paths:
+      - locale/*.pot
+
+env:
+  TX_VERSION: "v1.6.10"
+
+jobs:
+  push-source:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Get Transifex Client
+        run: |
+          curl -o- https://raw.githubusercontent.com/transifex/cli/master/install.sh | bash -s -- $TX_VERSION
+      - name: Update & Push Translation Sources
+        env:
+          TX_TOKEN: ${{ secrets.TX_TOKEN }}
+        run: |
+          ./tx push -s --use-git-timestamps

.gitignore

@@ -0,0 +1,111 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+#*.pot
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Environments
+.env
+.venv
+env*/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# macOS
+.DS_Store
+
+# Visual Studio Code
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+!.vscode/*.code-snippets
+
+# Local History for Visual Studio Code
+.history/
+
+# Built Visual Studio Code Extensions
+*.vsix
+
+# dupeGuru Specific
+/qt/*_rc.py
+/help/*/conf.py
+/help/*/changelog.rst
+cocoa/autogen
+/cocoa/*/Info.plist
+/cocoa/*/build
+
+*.waf*
+.lock-waf*
+/tags

.pre-commit-config.yaml

@@ -0,0 +1,24 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
+    hooks:
+      - id: check-yaml
+      - id: check-toml
+      - id: end-of-file-fixer
+        exclude: ".*.json"
+      - id: trailing-whitespace
+  - repo: https://github.com/psf/black
+    rev: 24.2.0
+    hooks:
+      - id: black
+  - repo: https://github.com/PyCQA/flake8
+    rev: 7.0.0
+    hooks:
+      - id: flake8
+        exclude: ^(.tox|env|build|dist|help|qt/dg_rc.py|pkg).*
+  - repo: https://github.com/alessandrojcm/commitlint-pre-commit-hook
+    rev: v9.11.0
+    hooks:
+      - id: commitlint
+        stages: [commit-msg]
+        additional_dependencies: ["@commitlint/config-conventional"]

.sonarcloud.properties

@@ -0,0 +1 @@
+sonar.python.version=3.7, 3.8, 3.9, 3.10, 3.11

.tx/config

@@ -0,0 +1,20 @@
+[main]
+host = https://www.transifex.com
+
+[o:voltaicideas:p:dupeguru-1:r:columns]
+file_filter = locale/<lang>/LC_MESSAGES/columns.po
+source_file = locale/columns.pot
+source_lang = en
+type        = PO
+
+[o:voltaicideas:p:dupeguru-1:r:core]
+file_filter = locale/<lang>/LC_MESSAGES/core.po
+source_file = locale/core.pot
+source_lang = en
+type        = PO
+
+[o:voltaicideas:p:dupeguru-1:r:ui]
+file_filter = locale/<lang>/LC_MESSAGES/ui.po
+source_file = locale/ui.pot
+source_lang = en
+type        = PO

.vscode/extensions.json

@@ -0,0 +1,12 @@
+{
+    // List of extensions which should be recommended for users of this workspace.
+    "recommendations": [
+        "redhat.vscode-yaml",
+        "ms-python.vscode-pylance",
+        "ms-python.python",
+        "ms-python.black-formatter",
+    ],
+    // List of extensions recommended by VS Code that should not be recommended for
+    // users of this workspace.
+    "unwantedRecommendations": []
+}

.vscode/launch.json

@@ -0,0 +1,17 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "DupuGuru",
+            "type": "debugpy",
+            "request": "launch",
+            "program": "run.py",
+            "console": "integratedTerminal",
+            "subProcess": true,
+            "justMyCode": false
+        },
+    ]
+}

.vscode/settings.json

@@ -0,0 +1,17 @@
+{
+    "cSpell.words": [
+        "Dupras",
+        "hscommon"
+    ],
+    "editor.rulers": [
+        88,
+        120
+    ],
+    "python.languageServer": "Pylance",
+    "yaml.schemaStore.enable": true,
+    "[python]": {
+        "editor.formatOnSave": true,
+        "editor.defaultFormatter": "ms-python.black-formatter"
+    },
+    "python.testing.pytestEnabled": true
+}

CONTRIBUTING.md

@@ -0,0 +1,88 @@
+# Contributing to dupeGuru
+
+The following is a set of guidelines and information for contributing to dupeGuru.
+
+#### Table of Contents
+
+[Things to Know Before Starting](#things-to-know-before-starting)
+
+[Ways to Contribute](#ways-to-contribute)
+  * [Reporting Bugs](#reporting-bugs)
+  * [Suggesting Enhancements](#suggesting-enhancements)
+  * [Localization](#localization)
+  * [Code Contribution](#code-contribution)
+  * [Pull Requests](#pull-requests)
+
+[Style Guides](#style-guides)
+  * [Git Commit Messages](#git-commit-messages)
+  * [Python Style Guide](#python-style-guide)
+  * [Documentation Style Guide](#documentation-style-guide)
+
+[Additional Notes](#additional-notes)
+  * [Issue and Pull Request Labels](#issue-and-pull-request-labels)
+
+## Things to Know Before Starting
+**TODO**
+## Ways to contribute
+### Reporting Bugs
+**TODO**
+### Suggesting Enhancements
+**TODO**
+### Localization
+**TODO**
+### Code Contribution
+**TODO**
+### Pull Requests
+Please follow these steps to have your contribution considered by the maintainers:
+
+1. Keep Pull Request specific to one feature or bug.
+2. Follow the [style guides](#style-guides)
+3. After you submit your pull request, verify that all [status checks](https://help.github.com/articles/about-status-checks/) are passing <details><summary>What if the status checks are failing?</summary>If a status check is failing, and you believe that the failure is unrelated to your change, please leave a comment on the pull request explaining why you believe the failure is unrelated. A maintainer will re-run the status check for you. If we conclude that the failure was a false positive, then we will open an issue to track that problem with our status check suite.</details>
+
+While the prerequisites above must be satisfied prior to having your pull request reviewed, the reviewer(s) may ask you to complete additional design work, tests, or other changes before your pull request can be ultimately accepted.
+
+## Style Guides
+### Git Commit Messages
+- Use the present tense ("Add feature" not "Added feature")
+- Use the imperative mood ("Move cursor to..." not "Moves cursor to...")
+- Limit the first line to 72 characters or less
+- Reference issues and pull requests liberally after the first line
+
+### Python Style Guide
+- All files are formatted with [Black](https://github.com/psf/black)
+- Follow [PEP 8](https://peps.python.org/pep-0008/) as much as practical
+- Pass [flake8](https://flake8.pycqa.org/en/latest/) linting
+- Include [PEP 484](https://peps.python.org/pep-0484/) type hints (new code)
+
+### Documentation Style Guide
+**TODO**
+
+## Additional Notes
+### Issue and Pull Request Labels
+This section lists and describes the various labels used with issues and pull requests.  Each of the labels is listed with a search link as well.
+
+#### Issue Type and Status
+| Label name | Search | Description |
+|------------|--------|-------------|
+| `enhancement` | [search](https://github.com/arsenetar/dupeguru/issues?q=is%3Aopen+is%3Aissue+label%3Aenhancement) | Feature requests and enhancements. |
+| `bug` | [search](https://github.com/arsenetar/dupeguru/issues?q=is%3Aopen+is%3Aissue+label%3Abug) | Bug reports. |
+| `duplicate` | [search](https://github.com/arsenetar/dupeguru/issues?q=is%3Aopen+is%3Aissue+label%3Aduplicate) | Issue is a duplicate of existing issue. |
+| `needs-reproduction` | [search](https://github.com/arsenetar/dupeguru/issues?q=is%3Aopen+is%3Aissue+label%3Aneeds-reproduction) | A bug that has not been able to be reproduced. |
+| `needs-information` | [search](https://github.com/arsenetar/dupeguru/issues?q=is%3Aopen+is%3Aissue+label%3Aneeds-information) | More information needs to be collected about these problems or feature requests (e.g. steps to reproduce). |
+| `blocked` | [search](https://github.com/arsenetar/dupeguru/issues?q=is%3Aopen+is%3Aissue+label%3Ablocked) | Issue blocked by other issues. |
+| `beginner` | [search](https://github.com/arsenetar/dupeguru/issues?q=is%3Aopen+is%3Aissue+label%3Abeginner) | Less complex issues for users who want to start contributing. |
+
+#### Category Labels
+| Label name | Search | Description |
+|------------|--------|-------------|
+| `3rd party` | [search](https://github.com/arsenetar/dupeguru/issues?q=is%3Aopen+is%3Aissue+label%3A%223rd%20party%22)  | Related to a 3rd party dependency. |
+| `crash` | [search](https://github.com/arsenetar/dupeguru/issues?q=is%3Aopen+is%3Aissue+label%3Acrash) | Related to crashes (complete, or unhandled). |
+| `documentation` | [search](https://github.com/arsenetar/dupeguru/issues?q=is%3Aopen+is%3Aissue+label%3Adocumentation) | Related to any documentation. |
+| `linux` | [search](https://github.com/arsenetar/dupeguru/issues?q=is%3Aopen+is%3Aissue+label%3linux) | Related to running on Linux. |
+| `mac` | [search](https://github.com/arsenetar/dupeguru/issues?q=is%3Aopen+is%3Aissue+label%3Amac) | Related to running on macOS. |
+| `performance` | [search](https://github.com/arsenetar/dupeguru/issues?q=is%3Aopen+is%3Aissue+label%3Aperformance) | Related to the performance. |
+| `ui` | [search](https://github.com/arsenetar/dupeguru/issues?q=is%3Aopen+is%3Aissue+label%3Aui)| Related to the visual design. |
+| `windows` | [search](https://github.com/arsenetar/dupeguru/issues?q=is%3Aopen+is%3Aissue+label%3Awindows) | Related to running on Windows. |
+
+#### Pull Request Labels
+None at this time, if the volume of Pull Requests increase labels may be added to manage.

CREDITS

@@ -0,0 +1,21 @@
+To know who contributed to dupeGuru, you can look at the commit log, but not all contributions
+result in a commit. This file lists contributors who don't necessarily appear in the commit log.
+
+* Jason Cho, Exchange icon
+* schollidesign (https://findicons.com/pack/1035/human_o2), Zoom-in, Zoom-out, Zoom-best-fit, Zoom-original icons
+* Jérôme Cantin, Main icon
+* Gregor Tätzner, German localization
+* Frank Weber, German localization
+* Eric Dee, Chinese localization
+* Aleš Nehyba, Czech localization
+* Paolo Rossi, Italian localization
+* Hrant Ohanyan, Armenian localization
+* Igor Pavlov, Russian localization
+* Kyrill Detinov, Russian localization
+* Yuri Petrashko, Ukrainian localization
+* Nickolas Pohilets, Ukrainian localization
+* Victor Figueiredo, Brazilian localization
+* Phan Anh, Vietnamese localization
+* Gabriel Koutilellis, Greek localization
+
+Thanks!

LICENSE

@@ -0,0 +1,621 @@
+                    GNU GENERAL PUBLIC LICENSE
+                       Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The GNU General Public License is a free, copyleft license for
+software and other kinds of works.
+
+  The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works.  By contrast,
+the GNU General Public License is intended to guarantee your freedom to
+share and change all versions of a program--to make sure it remains free
+software for all its users.  We, the Free Software Foundation, use the
+GNU General Public License for most of our software; it applies also to
+any other work released this way by its authors.  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
+
+  To protect your rights, we need to prevent others from denying you
+these rights or asking you to surrender the rights.  Therefore, you have
+certain responsibilities if you distribute copies of the software, or if
+you modify it: responsibilities to respect the freedom of others.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must pass on to the recipients the same
+freedoms that you received.  You must make sure that they, too, receive
+or can get the source code.  And you must show them these terms so they
+know their rights.
+
+  Developers that use the GNU GPL protect your rights with two steps:
+(1) assert copyright on the software, and (2) offer you this License
+giving you legal permission to copy, distribute and/or modify it.
+
+  For the developers' and authors' protection, the GPL clearly explains
+that there is no warranty for this free software.  For both users' and
+authors' sake, the GPL requires that modified versions be marked as
+changed, so that their problems will not be attributed erroneously to
+authors of previous versions.
+
+  Some devices are designed to deny users access to install or run
+modified versions of the software inside them, although the manufacturer
+can do so.  This is fundamentally incompatible with the aim of
+protecting users' freedom to change the software.  The systematic
+pattern of such abuse occurs in the area of products for individuals to
+use, which is precisely where it is most unacceptable.  Therefore, we
+have designed this version of the GPL to prohibit the practice for those
+products.  If such problems arise substantially in other domains, we
+stand ready to extend this provision to those domains in future versions
+of the GPL, as needed to protect the freedom of users.
+
+  Finally, every program is threatened constantly by software patents.
+States should not allow patents to restrict development and use of
+software on general-purpose computers, but in those that do, we wish to
+avoid the special danger that patents applied to a free program could
+make it effectively proprietary.  To prevent this, the GPL assures that
+patents cannot be used to render the program non-free.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                       TERMS AND CONDITIONS
+
+  0. Definitions.
+
+  "This License" refers to version 3 of the GNU General Public License.
+
+  "Copyright" also means copyright-like laws that apply to other kinds of
+works, such as semiconductor masks.
+
+  "The Program" refers to any copyrightable work licensed under this
+License.  Each licensee is addressed as "you".  "Licensees" and
+"recipients" may be individuals or organizations.
+
+  To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of an
+exact copy.  The resulting work is called a "modified version" of the
+earlier work or a work "based on" the earlier work.
+
+  A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+  To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy.  Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
+
+  To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies.  Mere interaction with a user through
+a computer network, with no transfer of a copy, is not conveying.
+
+  An interactive user interface displays "Appropriate Legal Notices"
+to the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License.  If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+  1. Source Code.
+
+  The "source code" for a work means the preferred form of the work
+for making modifications to it.  "Object code" means any non-source
+form of a work.
+
+  A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+  The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form.  A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+  The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities.  However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work.  For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+  The Corresponding Source need not include anything that users
+can regenerate automatically from other parts of the Corresponding
+Source.
+
+  The Corresponding Source for a work in source code form is that
+same work.
+
+  2. Basic Permissions.
+
+  All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met.  This License explicitly affirms your unlimited
+permission to run the unmodified Program.  The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work.  This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
+
+  You may make, run and propagate covered works that you do not
+convey, without conditions so long as your license otherwise remains
+in force.  You may convey covered works to others for the sole purpose
+of having them make modifications exclusively for you, or provide you
+with facilities for running those works, provided that you comply with
+the terms of this License in conveying all material for which you do
+not control copyright.  Those thus making or running the covered works
+for you must do so exclusively on your behalf, under your direction
+and control, on terms that prohibit them from making any copies of
+your copyrighted material outside their relationship with you.
+
+  Conveying under any other circumstances is permitted solely under
+the conditions stated below.  Sublicensing is not allowed; section 10
+makes it unnecessary.
+
+  3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+  No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+  When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such circumvention
+is effected by exercising rights under this License with respect to
+the covered work, and you disclaim any intention to limit operation or
+modification of the work as a means of enforcing, against the work's
+users, your or third parties' legal rights to forbid circumvention of
+technological measures.
+
+  4. Conveying Verbatim Copies.
+
+  You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+  You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+  5. Conveying Modified Source Versions.
+
+  You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these conditions:
+
+    a) The work must carry prominent notices stating that you modified
+    it, and giving a relevant date.
+
+    b) The work must carry prominent notices stating that it is
+    released under this License and any conditions added under section
+    7.  This requirement modifies the requirement in section 4 to
+    "keep intact all notices".
+
+    c) You must license the entire work, as a whole, under this
+    License to anyone who comes into possession of a copy.  This
+    License will therefore apply, along with any applicable section 7
+    additional terms, to the whole of the work, and all its parts,
+    regardless of how they are packaged.  This License gives no
+    permission to license the work in any other way, but it does not
+    invalidate such permission if you have separately received it.
+
+    d) If the work has interactive user interfaces, each must display
+    Appropriate Legal Notices; however, if the Program has interactive
+    interfaces that do not display Appropriate Legal Notices, your
+    work need not make them do so.
+
+  A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit.  Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.
+
+  6. Conveying Non-Source Forms.
+
+  You may convey a covered work in object code form under the terms
+of sections 4 and 5, provided that you also convey the
+machine-readable Corresponding Source under the terms of this License,
+in one of these ways:
+
+    a) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by the
+    Corresponding Source fixed on a durable physical medium
+    customarily used for software interchange.
+
+    b) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by a
+    written offer, valid for at least three years and valid for as
+    long as you offer spare parts or customer support for that product
+    model, to give anyone who possesses the object code either (1) a
+    copy of the Corresponding Source for all the software in the
+    product that is covered by this License, on a durable physical
+    medium customarily used for software interchange, for a price no
+    more than your reasonable cost of physically performing this
+    conveying of source, or (2) access to copy the
+    Corresponding Source from a network server at no charge.
+
+    c) Convey individual copies of the object code with a copy of the
+    written offer to provide the Corresponding Source.  This
+    alternative is allowed only occasionally and noncommercially, and
+    only if you received the object code with such an offer, in accord
+    with subsection 6b.
+
+    d) Convey the object code by offering access from a designated
+    place (gratis or for a charge), and offer equivalent access to the
+    Corresponding Source in the same way through the same place at no
+    further charge.  You need not require recipients to copy the
+    Corresponding Source along with the object code.  If the place to
+    copy the object code is a network server, the Corresponding Source
+    may be on a different server (operated by you or a third party)
+    that supports equivalent copying facilities, provided you maintain
+    clear directions next to the object code saying where to find the
+    Corresponding Source.  Regardless of what server hosts the
+    Corresponding Source, you remain obligated to ensure that it is
+    available for as long as needed to satisfy these requirements.
+
+    e) Convey the object code using peer-to-peer transmission, provided
+    you inform other peers where the object code and Corresponding
+    Source of the work are being offered to the general public at no
+    charge under subsection 6d.
+
+  A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+  A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal, family,
+or household purposes, or (2) anything designed or sold for incorporation
+into a dwelling.  In determining whether a product is a consumer product,
+doubtful cases shall be resolved in favor of coverage.  For a particular
+product received by a particular user, "normally used" refers to a
+typical or common use of that class of product, regardless of the status
+of the particular user or of the way in which the particular user
+actually uses, or expects or is expected to use, the product.  A product
+is a consumer product regardless of whether the product has substantial
+commercial, industrial or non-consumer uses, unless such uses represent
+the only significant mode of use of the product.
+
+  "Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to install
+and execute modified versions of a covered work in that User Product from
+a modified version of its Corresponding Source.  The information must
+suffice to ensure that the continued functioning of the modified object
+code is in no case prevented or interfered with solely because
+modification has been made.
+
+  If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information.  But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+  The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or updates
+for a work that has been modified or installed by the recipient, or for
+the User Product in which it has been modified or installed.  Access to a
+network may be denied when the modification itself materially and
+adversely affects the operation of the network or violates the rules and
+protocols for communication across the network.
+
+  Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+  7. Additional Terms.
+
+  "Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law.  If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.
+
+  When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it.  (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.)  You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+  Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders of
+that material) supplement the terms of this License with terms:
+
+    a) Disclaiming warranty or limiting liability differently from the
+    terms of sections 15 and 16 of this License; or
+
+    b) Requiring preservation of specified reasonable legal notices or
+    author attributions in that material or in the Appropriate Legal
+    Notices displayed by works containing it; or
+
+    c) Prohibiting misrepresentation of the origin of that material, or
+    requiring that modified versions of such material be marked in
+    reasonable ways as different from the original version; or
+
+    d) Limiting the use for publicity purposes of names of licensors or
+    authors of the material; or
+
+    e) Declining to grant rights under trademark law for use of some
+    trade names, trademarks, or service marks; or
+
+    f) Requiring indemnification of licensors and authors of that
+    material by anyone who conveys the material (or modified versions of
+    it) with contractual assumptions of liability to the recipient, for
+    any liability that these contractual assumptions directly impose on
+    those licensors and authors.
+
+  All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10.  If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term.  If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+  If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+  Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions;
+the above requirements apply either way.
+
+  8. Termination.
+
+  You may not propagate or modify a covered work except as expressly
+provided under this License.  Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+  However, if you cease all violation of this License, then your
+license from a particular copyright holder is reinstated (a)
+provisionally, unless and until the copyright holder explicitly and
+finally terminates your license, and (b) permanently, if the copyright
+holder fails to notify you of the violation by some reasonable means
+prior to 60 days after the cessation.
+
+  Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+  Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License.  If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+  9. Acceptance Not Required for Having Copies.
+
+  You are not required to accept this License in order to receive or
+run a copy of the Program.  Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance.  However,
+nothing other than this License grants you permission to propagate or
+modify any covered work.  These actions infringe copyright if you do
+not accept this License.  Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+  10. Automatic Licensing of Downstream Recipients.
+
+  Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License.  You are not responsible
+for enforcing compliance by third parties with this License.
+
+  An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations.  If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+  You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License.  For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.
+
+  11. Patents.
+
+  A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based.  The
+work thus licensed is called the contributor's "contributor version".
+
+  A contributor's "essential patent claims" are all patent claims
+owned or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version.  For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+  Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.
+
+  In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement).  To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+  If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients.  "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.
+
+  If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.
+
+  A patent license is "discriminatory" if it does not include within
+the scope of its coverage, prohibits the exercise of, or is
+conditioned on the non-exercise of one or more of the rights that are
+specifically granted under this License.  You may not convey a covered
+work if you are a party to an arrangement with a third party that is
+in the business of distributing software, under which you make payment
+to the third party based on the extent of your activity of conveying
+the work, and under which the third party grants, to any of the
+parties who would receive the covered work from you, a discriminatory
+patent license (a) in connection with copies of the covered work
+conveyed by you (or copies made from those copies), or (b) primarily
+for and in connection with specific products or compilations that
+contain the covered work, unless you entered into that arrangement,
+or that patent license was granted, prior to 28 March 2007.
+
+  Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.
+
+  12. No Surrender of Others' Freedom.
+
+  If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you may
+not convey it at all.  For example, if you agree to terms that obligate you
+to collect a royalty for further conveying from those to whom you convey
+the Program, the only way you could satisfy both those terms and this
+License would be to refrain entirely from conveying the Program.
+
+  13. Use with the GNU Affero General Public License.
+
+  Notwithstanding any other provision of this License, you have
+permission to link or combine any covered work with a work licensed
+under version 3 of the GNU Affero General Public License into a single
+combined work, and to convey the resulting work.  The terms of this
+License will continue to apply to the part which is the covered work,
+but the special requirements of the GNU Affero General Public License,
+section 13, concerning interaction through a network will apply to the
+combination as such.
+
+  14. Revised Versions of this License.
+
+  The Free Software Foundation may publish revised and/or new versions of
+the GNU General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+  Each version is given a distinguishing version number.  If the
+Program specifies that a certain numbered version of the GNU General
+Public License "or any later version" applies to it, you have the
+option of following the terms and conditions either of that numbered
+version or of any later version published by the Free Software
+Foundation.  If the Program does not specify a version number of the
+GNU General Public License, you may choose any version ever published
+by the Free Software Foundation.
+
+  If the Program specifies that a proxy can decide which future
+versions of the GNU General Public License can be used, that proxy's
+public statement of acceptance of a version permanently authorizes you
+to choose that version for the Program.
+
+  Later license versions may give you additional or different
+permissions.  However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.
+
+  15. Disclaimer of Warranty.
+
+  THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
+OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
+IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+  16. Limitation of Liability.
+
+  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGES.
+
+  17. Interpretation of Sections 15 and 16.
+
+  If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+                     END OF TERMS AND CONDITIONS

MANIFEST.in

@@ -0,0 +1,5 @@
+recursive-include core *.h
+recursive-include core *.m
+include run.py
+graft locale
+graft help

Makefile

@@ -0,0 +1,123 @@
+PYTHON ?= python3
+PYTHON_VERSION_MINOR := $(shell ${PYTHON} -c "import sys; print(sys.version_info.minor)")
+PYRCC5 ?= pyrcc5
+REQ_MINOR_VERSION = 7
+PREFIX ?= /usr/local
+
+# Window compatability via Msys2
+# - venv creates Scripts instead of bin
+# - compile generates .pyd instead of .so
+# - venv with --sytem-site-packages has issues on windows as well...
+
+ifeq ($(shell ${PYTHON} -c "import platform; print(platform.system())"), Windows)
+	BIN = Scripts
+	SO = *.pyd
+	VENV_OPTIONS =
+else
+	BIN = bin
+	SO = *.so
+	VENV_OPTIONS = --system-site-packages
+endif
+
+# Set this variable if all dependencies are already met on the system. We will then avoid the
+# whole vitualenv creation and pip install dance.
+NO_VENV ?=
+
+ifdef NO_VENV
+	VENV_PYTHON = $(PYTHON)
+else
+	VENV_PYTHON = ./env/$(BIN)/python
+endif
+
+# If you're installing into a path that is not going to be the final path prefix (such as a
+# sandbox), set DESTDIR to that path.
+
+# Our build scripts are not very "make like" yet and perform their task in a bundle. For now, we
+# use one of each file to act as a representative, a target, of these groups.
+
+packages = hscommon core qt
+localedirs = $(wildcard locale/*/LC_MESSAGES)
+pofiles = $(wildcard locale/*/LC_MESSAGES/*.po)
+mofiles = $(patsubst %.po,%.mo,$(pofiles))
+
+vpath %.po $(localedirs)
+vpath %.mo $(localedirs)
+
+all: | env i18n modules qt/dg_rc.py
+	@echo "Build complete! You can run dupeGuru with 'make run'"
+
+run:
+	$(VENV_PYTHON) run.py
+
+pyc: | env
+	${VENV_PYTHON} -m compileall ${packages}
+
+reqs:
+ifneq ($(shell test $(PYTHON_VERSION_MINOR) -ge $(REQ_MINOR_VERSION); echo $$?),0)
+	$(error "Python 3.${REQ_MINOR_VERSION}+ required. Aborting.")
+endif
+ifndef NO_VENV
+	@${PYTHON} -m venv -h > /dev/null || \
+		echo "Creation of our virtualenv failed. If you're on Ubuntu, you probably need python3-venv."
+endif
+	@${PYTHON} -c 'import PyQt5' >/dev/null 2>&1 || \
+		{ echo "PyQt 5.4+ required. Install it and try again. Aborting"; exit 1; }
+
+env: | reqs
+ifndef NO_VENV
+	@echo "Creating our virtualenv"
+	${PYTHON} -m venv env
+	$(VENV_PYTHON) -m pip install -r requirements.txt
+# We can't use the "--system-site-packages" flag on creation because otherwise we end up with
+# the system's pip and that messes up things in some cases (notably in Gentoo).
+	${PYTHON} -m venv --upgrade ${VENV_OPTIONS} env
+endif
+
+build/help: | env
+	$(VENV_PYTHON) build.py --doc
+
+qt/dg_rc.py: qt/dg.qrc
+	$(PYRCC5) qt/dg.qrc > qt/dg_rc.py
+
+i18n: $(mofiles)
+
+%.mo: %.po
+	msgfmt -o $@ $<
+
+modules: | env
+	$(VENV_PYTHON) build.py --modules
+
+mergepot: | env
+	$(VENV_PYTHON) build.py --mergepot
+
+normpo: | env
+	$(VENV_PYTHON) build.py --normpo
+
+install: all pyc
+	mkdir -p ${DESTDIR}${PREFIX}/share/dupeguru
+	cp -rf ${packages} locale ${DESTDIR}${PREFIX}/share/dupeguru
+	cp -f run.py ${DESTDIR}${PREFIX}/share/dupeguru/run.py
+	chmod 755 ${DESTDIR}${PREFIX}/share/dupeguru/run.py
+	mkdir -p ${DESTDIR}${PREFIX}/bin
+	ln -sf ${PREFIX}/share/dupeguru/run.py ${DESTDIR}${PREFIX}/bin/dupeguru
+	mkdir -p ${DESTDIR}${PREFIX}/share/applications
+	cp -f pkg/dupeguru.desktop ${DESTDIR}${PREFIX}/share/applications
+	mkdir -p ${DESTDIR}${PREFIX}/share/pixmaps
+	cp -f images/dgse_logo_128.png ${DESTDIR}${PREFIX}/share/pixmaps/dupeguru.png
+
+installdocs: build/help
+	mkdir -p ${DESTDIR}${PREFIX}/share/dupeguru
+	cp -rf build/help ${DESTDIR}${PREFIX}/share/dupeguru
+
+uninstall:
+	rm -rf "${DESTDIR}${PREFIX}/share/dupeguru"
+	rm -f "${DESTDIR}${PREFIX}/bin/dupeguru"
+	rm -f "${DESTDIR}${PREFIX}/share/applications/dupeguru.desktop"
+	rm -f "${DESTDIR}${PREFIX}/share/pixmaps/dupeguru.png"
+
+clean:
+	-rm -rf build
+	-rm locale/*/LC_MESSAGES/*.mo
+	-rm core/pe/*.$(SO) qt/pe/*.$(SO)
+
+.PHONY: clean normpo mergepot modules i18n reqs run pyc install uninstall all

README.md

@@ -0,0 +1,97 @@
+# dupeGuru
+
+[dupeGuru][dupeguru] is a cross-platform (Linux, OS X, Windows) GUI tool to find duplicate files in
+a system. It is written mostly in Python 3 and uses [qt](https://www.qt.io/) for the UI.
+
+## Current status
+Still looking for additional help especially with regards to:
+* OSX maintenance: reproducing bugs, packaging verification.
+* Linux maintenance: reproducing bugs, maintaining PPA repository, Debian package, rpm package.
+* Translations: updating missing strings, transifex project at https://www.transifex.com/voltaicideas/dupeguru-1
+* Documentation: keeping it up-to-date.
+
+## Contents of this folder
+
+This folder contains the source for dupeGuru. Its documentation is in `help`, but is also
+[available online][documentation] in its built form. Here's how this source tree is organized:
+
+* core: Contains the core logic code for dupeGuru. It's Python code.
+* qt: UI code for the Qt toolkit. It's written in Python and uses PyQt.
+* images: Images used by the different UI codebases.
+* pkg: Skeleton files required to create different packages
+* help: Help document, written for Sphinx.
+* locale: .po files for localization.
+* hscommon: A collection of helpers used across HS applications.
+
+## How to build dupeGuru from source
+
+### Windows & macOS specific additional instructions
+For windows instructions see the [Windows Instructions](Windows.md).
+
+For macos instructions (qt version) see the [macOS Instructions](macos.md).
+
+### Prerequisites
+* [Python 3.7+][python]
+* PyQt5
+
+### System Setup
+When running in a linux based environment the following system packages or equivalents are needed to build:
+* python3-pyqt5
+* pyqt5-dev-tools (on some systems, see note)
+* python3-venv (only if using a virtual environment)
+* python3-dev
+* build-essential
+
+Note: On some linux systems pyrcc5 is not put on the path when installing python3-pyqt5, this will cause some issues with the resource files (and icons). These systems should have a respective pyqt5-dev-tools package, which should also be installed. The presence of pyrcc5 can be checked with `which pyrcc5`.  Debian based systems need the extra package, and Arch does not.
+
+To create packages the following are also needed:
+* python3-setuptools
+* debhelper
+
+### Building with Make
+dupeGuru comes with a makefile that can be used to build and run:
+
+    $ make && make run
+
+### Building without Make
+
+    $ cd <dupeGuru directory>
+    $ python3 -m venv --system-site-packages ./env
+    $ source ./env/bin/activate
+    $ pip install -r requirements.txt
+    $ python build.py
+    $ python run.py
+
+### Generating Debian/Ubuntu package
+To generate packages the extra requirements in requirements-extra.txt must be installed, the
+steps are as follows:
+
+    $ cd <dupeGuru directory>
+    $ python3 -m venv --system-site-packages ./env
+    $ source ./env/bin/activate
+    $ pip install -r requirements.txt -r requirements-extra.txt
+    $ python build.py --clean
+    $ python package.py
+
+This can be made a one-liner (once in the directory) as:
+
+    $ bash -c "python3 -m venv --system-site-packages env && source env/bin/activate && pip install -r requirements.txt -r requirements-extra.txt && python build.py --clean && python package.py"
+
+## Running tests
+
+The complete test suite is run with [Tox 1.7+][tox]. If you have it installed system-wide, you
+don't even need to set up a virtualenv. Just `cd` into the root project folder and run `tox`.
+
+If you don't have Tox system-wide, install it in your virtualenv with `pip install tox` and then
+run `tox`.
+
+You can also run automated tests without Tox. Extra requirements for running tests are in
+`requirements-extra.txt`. So, you can do `pip install -r requirements-extra.txt` inside your
+virtualenv and then `py.test core hscommon`
+
+[dupeguru]: https://dupeguru.voltaicideas.net/
+[cross-toolkit]: http://www.hardcoded.net/articles/cross-toolkit-software
+[documentation]: http://dupeguru.voltaicideas.net/help/en/
+[python]: http://www.python.org/
+[pyqt]: http://www.riverbankcomputing.com
+[tox]: https://tox.readthedocs.org/en/latest/

Windows.md

@@ -0,0 +1,55 @@
+## How to build dupeGuru for Windows
+
+### Prerequisites
+
+- [Python 3.7+][python]
+- [Visual Studio 2019][vs] or [Visual Studio Build Tools 2019][vsBuildTools] with the Windows 10 SDK
+- [nsis][nsis] (for installer creation)
+- [msys2][msys2] (for using makefile method)
+
+NOTE: When installing Visual Studio or the Visual Studio Build Tools with the Windows 10 SDK on versions of Windows below 10 be sure to make sure that the Universal CRT is installed before installing Visual studio as noted in the [Windows 10 SDK Notes][win10sdk] and found at [KB2999226][KB2999226].
+
+After installing python it is recommended to update setuptools before compiling packages.  To update run (example is for python launcher and 3.8):
+
+    $ py -3.8 -m pip install --upgrade setuptools
+
+More details on setting up python for compiling packages on windows can be found on the [python wiki][pythonWindowsCompilers] Take note of the required vc++ versions.
+
+### With build.py (preferred)
+To build with a different python version 3.7 vs 3.8 or 32 bit vs 64 bit specify that version instead of -3.8 to the `py` command below.  If you want to build additional versions while keeping all virtual environments setup use a different location for each virtual environment.
+
+    $ cd <dupeGuru directory>
+    $ py -3.8 -m venv .\env
+    $ .\env\Scripts\activate
+    $ pip install -r requirements.txt
+    $ python build.py
+    $ python run.py
+
+### With makefile
+It is possible to build dupeGuru with the makefile on windows using a compatable POSIX environment.  The following steps have been tested using [msys2][msys2]. Before running make:
+1. Install msys2 or other POSIX environment
+2. Install PyQt5 globally via pip
+3. Use the respective console for msys2 it is `msys2 msys`
+
+Then the following execution of the makefile should work.  Pass the correct value for PYTHON to the makefile if not on the path as python3.
+
+    $ cd <dupeGuru directory>
+    $ make PYTHON='py -3.8'
+    $ make run
+
+### Generate Windows Installer Packages
+You need to use the respective x86 or x64 version of python to build the 32 bit and 64 bit versions.  The build scripts will automatically detect the python architecture for you. When using build.py make sure the resulting python works before continuing to package.py.  NOTE: package.py looks for the 'makensis' executable in the default location for a 64 bit windows system.  The extra requirements need to be installed to run packaging: `pip install -r requirements-extra.txt`. Run the following in the respective virtual environment.
+
+    $ python package.py
+
+### Running tests
+The complete test suite can be run with tox just like on linux. NOTE: The extra requirements need to be installed to run unit tests: `pip install -r requirements-extra.txt`.
+
+[python]: http://www.python.org/
+[nsis]: http://nsis.sourceforge.net/Main_Page
+[vs]: https://www.visualstudio.com/downloads/#visual-studio-community-2019
+[vsBuildTools]: https://www.visualstudio.com/downloads/#build-tools-for-visual-studio-2019
+[win10sdk]: https://developer.microsoft.com/en-us/windows/downloads/windows-10-sdk
+[KB2999226]: https://support.microsoft.com/en-us/help/2999226/update-for-universal-c-runtime-in-windows
+[pythonWindowsCompilers]: https://wiki.python.org/moin/WindowsCompilers
+[msys2]: http://www.msys2.org/

base/cocoa/AppDelegate.h

@@ -1,25 +0,0 @@
-/* 
-Copyright 2009 Hardcoded Software (http://www.hardcoded.net)
-
-This software is licensed under the "HS" License as described in the "LICENSE" file, 
-which should be included with this package. The terms are also available at 
-http://www.hardcoded.net/licenses/hs_license
-*/
-
-#import <Cocoa/Cocoa.h>
-#import "RecentDirectories.h"
-#import "PyDupeGuru.h"
-
-@interface AppDelegateBase : NSObject
-{
-    IBOutlet PyDupeGuruBase *py;
-    IBOutlet RecentDirectories *recentDirectories;
-    IBOutlet NSMenuItem *unlockMenuItem;
-    
-    NSString *_appName;
-}
-- (IBAction)unlockApp:(id)sender;
-
-- (PyDupeGuruBase *)py;
-- (RecentDirectories *)recentDirectories;
-@end

base/cocoa/AppDelegate.m

@@ -1,38 +0,0 @@
-/* 
-Copyright 2009 Hardcoded Software (http://www.hardcoded.net)
-
-This software is licensed under the "HS" License as described in the "LICENSE" file, 
-which should be included with this package. The terms are also available at 
-http://www.hardcoded.net/licenses/hs_license
-*/
-
-#import "AppDelegate.h"
-#import "ProgressController.h"
-#import "RegistrationInterface.h"
-#import "Utils.h"
-#import "Consts.h"
-
-@implementation AppDelegateBase
-- (id)init
-{
-    self = [super init];
-    _appName = @"";
-    return self;
-}
-
-- (IBAction)unlockApp:(id)sender
-{
-    if ([[self py] isRegistered])
-        return;
-    RegistrationInterface *ri = [[RegistrationInterface alloc] initWithApp:[self py] name:_appName limitDescription:LIMIT_DESC];
-    if ([ri enterCode] == NSOKButton)
-    {
-        NSString *menuTitle = [NSString stringWithFormat:@"Thanks for buying %@",_appName];
-        [unlockMenuItem setTitle:menuTitle];
-    }
-    [ri release];
-}
-
-- (PyDupeGuruBase *)py { return py; }
-- (RecentDirectories *)recentDirectories { return recentDirectories; }
-@end

base/cocoa/Consts.h

@@ -1,28 +0,0 @@
-/* 
-Copyright 2009 Hardcoded Software (http://www.hardcoded.net)
-
-This software is licensed under the "HS" License as described in the "LICENSE" file, 
-which should be included with this package. The terms are also available at 
-http://www.hardcoded.net/licenses/hs_license
-*/
-
-#import <Cocoa/Cocoa.h>
-
-#define DuplicateSelectionChangedNotification @"DuplicateSelectionChangedNotification"
-/* ResultsChangedNotification happens on major changes, which requires a complete reload of the data*/
-#define ResultsChangedNotification @"ResultsChangedNotification"
-/* ResultsChangedNotification happens on minor changes, which requires buffer flush*/
-#define ResultsUpdatedNotification @"ResultsUpdatedNotification"
-#define ResultsMarkingChangedNotification @"ResultsMarkingChangedNotification"
-#define RegistrationRequired @"RegistrationRequired"
-#define JobStarted @"JobStarted"
-#define JobInProgress @"JobInProgress"
-
-#define jobLoad @"job_load"
-#define jobScan @"job_scan"
-#define jobCopy @"job_copy"
-#define jobMove @"job_move"
-#define jobDelete @"job_delete"
-
-#define DEMO_MAX_ACTION_COUNT 10
-#define LIMIT_DESC @"In the demo version, only 10 duplicates per session can be sent to Trash, moved or copied."

base/cocoa/DetailsPanel.h

@@ -1,24 +0,0 @@
-/* 
-Copyright 2009 Hardcoded Software (http://www.hardcoded.net)
-
-This software is licensed under the "HS" License as described in the "LICENSE" file, 
-which should be included with this package. The terms are also available at 
-http://www.hardcoded.net/licenses/hs_license
-*/
-
-#import <Cocoa/Cocoa.h>
-#import "PyApp.h"
-#import "Table.h"
-
-
-@interface DetailsPanelBase : NSWindowController
-{
-    IBOutlet TableView *detailsTable;
-}
-- (id)initWithPy:(PyApp *)aPy;
-
-- (void)refresh;
-
-/* Notifications */
-- (void)duplicateSelectionChanged:(NSNotification *)aNotification;
-@end

base/cocoa/DetailsPanel.m

@@ -1,33 +0,0 @@
-/* 
-Copyright 2009 Hardcoded Software (http://www.hardcoded.net)
-
-This software is licensed under the "HS" License as described in the "LICENSE" file, 
-which should be included with this package. The terms are also available at 
-http://www.hardcoded.net/licenses/hs_license
-*/
-
-#import "DetailsPanel.h"
-#import "Consts.h"
-
-@implementation DetailsPanelBase
-- (id)initWithPy:(PyApp *)aPy
-{
-    self = [super initWithWindowNibName:@"Details"];
-    [self window]; //So the detailsTable is initialized.
-    [detailsTable setPy:aPy];
-	[[NSNotificationCenter defaultCenter] addObserver:self selector:@selector(duplicateSelectionChanged:) name:DuplicateSelectionChangedNotification object:nil];
-    return self;
-}
-
-- (void)refresh
-{
-    [detailsTable reloadData];
-}
-
-/* Notifications */
-- (void)duplicateSelectionChanged:(NSNotification *)aNotification
-{
-    if ([[self window] isVisible])
-        [self refresh];
-}
-@end

base/cocoa/DirectoryPanel.h

@@ -1,33 +0,0 @@
-/* 
-Copyright 2009 Hardcoded Software (http://www.hardcoded.net)
-
-This software is licensed under the "HS" License as described in the "LICENSE" file, 
-which should be included with this package. The terms are also available at 
-http://www.hardcoded.net/licenses/hs_license
-*/
-
-#import <Cocoa/Cocoa.h>
-#import "RecentDirectories.h"
-#import "Outline.h"
-#import "PyDupeGuru.h"
-
-@interface DirectoryPanelBase : NSWindowController
-{
-    IBOutlet NSPopUpButton *addButtonPopUp;
-    IBOutlet OutlineView *directories;
-    IBOutlet NSButton *removeButton;
-    
-    PyDupeGuruBase *_py;
-    RecentDirectories *_recentDirectories;
-}
-- (id)initWithParentApp:(id)aParentApp;
-
-- (IBAction)askForDirectory:(id)sender;
-- (IBAction)changeDirectoryState:(id)sender;
-- (IBAction)popupAddDirectoryMenu:(id)sender;
-- (IBAction)removeSelectedDirectory:(id)sender;
-- (IBAction)toggleVisible:(id)sender;
-
-- (void)addDirectory:(NSString *)directory;
-- (void)refreshRemoveButtonText;
-@end

base/cocoa/DirectoryPanel.m

@@ -1,187 +0,0 @@
-/* 
-Copyright 2009 Hardcoded Software (http://www.hardcoded.net)
-
-This software is licensed under the "HS" License as described in the "LICENSE" file, 
-which should be included with this package. The terms are also available at 
-http://www.hardcoded.net/licenses/hs_license
-*/
-
-#import "DirectoryPanel.h"
-#import "Dialogs.h"
-#import "Utils.h"
-#import "AppDelegate.h"
-
-@implementation DirectoryPanelBase
-- (id)initWithParentApp:(id)aParentApp
-{
-    self = [super initWithWindowNibName:@"Directories"];
-    [self window];
-    AppDelegateBase *app = aParentApp;
-    _py = [app py];
-    _recentDirectories = [app recentDirectories];
-    [directories setPy:_py];
-    NSPopUpButtonCell *cell = [[directories tableColumnWithIdentifier:@"1"] dataCell];
-    [cell addItemWithTitle:@"Normal"];
-    [cell addItemWithTitle:@"Reference"];
-    [cell addItemWithTitle:@"Excluded"];
-    for (int i=0;i<[[cell itemArray] count];i++)
-    {
-        NSMenuItem *mi = [[cell itemArray] objectAtIndex:i];
-        [mi setTarget:self];
-        [mi setAction:@selector(changeDirectoryState:)];
-        [mi setTag:i];
-    }
-    [self refreshRemoveButtonText];
-    [[NSNotificationCenter defaultCenter] addObserver:self selector:@selector(directorySelectionChanged:) name:NSOutlineViewSelectionDidChangeNotification object:directories];
-    return self;
-}
-
-/* Actions */
-
-- (IBAction)askForDirectory:(id)sender
-{
-    NSOpenPanel *op = [NSOpenPanel openPanel];
-    [op setCanChooseFiles:YES];
-    [op setCanChooseDirectories:YES];
-    [op setAllowsMultipleSelection:NO];
-    [op setTitle:@"Select a directory to add to the scanning list"];
-    [op setDelegate:self];
-    if ([op runModalForTypes:nil] == NSOKButton)
-    {
-        NSString *directory = [[op filenames] objectAtIndex:0];
-        [self addDirectory:directory];
-    }
-}
-
-- (IBAction)changeDirectoryState:(id)sender
-{
-    OVNode *node = [directories itemAtRow:[directories clickedRow]];
-    [_py setDirectory:p2a([node indexPath]) state:i2n([sender tag])]; 
-    [node resetAllBuffers];
-    [directories reloadItem:node reloadChildren:YES];
-    [directories display];
-}
-
-- (IBAction)popupAddDirectoryMenu:(id)sender
-{
-    if ([[_recentDirectories directories] count] == 0)
-    {
-        [self askForDirectory:sender];
-        return;
-    }
-    NSMenu *m = [addButtonPopUp menu];
-    while ([m numberOfItems] > 0)
-        [m removeItemAtIndex:0];
-    NSMenuItem *mi = [m addItemWithTitle:@"Add New Directory..." action:@selector(askForDirectory:) keyEquivalent:@""];
-    [mi setTarget:self];
-    [m addItem:[NSMenuItem separatorItem]];
-    [_recentDirectories fillMenu:m];
-    [addButtonPopUp selectItem:nil];
-    [[addButtonPopUp cell] performClickWithFrame:[sender frame] inView:[sender superview]];
-}
-
-- (IBAction)removeSelectedDirectory:(id)sender
-{
-    [[self window] makeKeyAndOrderFront:nil];
-    if ([directories selectedRow] < 0)
-        return;
-    OVNode *node = [directories itemAtRow:[directories selectedRow]];
-    if ([node level] == 1)
-    {
-        [_py removeDirectory:i2n([node index])];
-        [directories reloadData];
-    }
-    else
-    {
-        int state = n2i([[node buffer] objectAtIndex:1]);
-        int newState = state == 2 ? 0 : 2; // If excluded, put it back
-        [_py setDirectory:p2a([node indexPath]) state:i2n(newState)];
-        [node resetAllBuffers];
-        [directories display];
-    }
-    [self refreshRemoveButtonText];
-}
-
-- (IBAction)toggleVisible:(id)sender
-{
-    if ([[self window] isVisible])
-        [[self window] close];
-    else
-        [[self window] makeKeyAndOrderFront:nil];
-}
-
-/* Public */
-
-- (void)addDirectory:(NSString *)directory
-{
-    int r = [[_py addDirectory:directory] intValue];
-    if (r)
-    {
-        NSString *m;
-        switch (r)
-        {
-            case 1:
-            {
-                m = @"This directory already is in the list.";
-                break;
-            }
-            case 2:
-            {
-                m = @"This directory does not exist.";
-                break;
-            }
-        }
-        [Dialogs showMessage:m];
-    }
-    [directories reloadData];
-    [_recentDirectories addDirectory:directory];
-    [[self window] makeKeyAndOrderFront:nil];
-}
-
-- (void)refreshRemoveButtonText
-{
-    if ([directories selectedRow] < 0)
-    {
-        [removeButton setEnabled:NO];
-        return;
-    }
-    [removeButton setEnabled:YES];
-    OVNode *node = [directories itemAtRow:[directories selectedRow]];
-    int state = n2i([[node buffer] objectAtIndex:1]);
-    NSString *buttonText = state == 2 ? @"Put Back" : @"Remove";
-    [removeButton setTitle:buttonText];
-}
-
-/* Delegate */
-
-- (void)outlineView:(NSOutlineView *)outlineView willDisplayCell:(id)cell forTableColumn:(NSTableColumn *)tableColumn item:(id)item
-{ 
-    OVNode *node = item;
-    int state = n2i([[node buffer] objectAtIndex:1]);
-    if ([cell isKindOfClass:[NSTextFieldCell class]])
-    {
-        NSTextFieldCell *textCell = cell;
-        if (state == 1)
-            [textCell setTextColor:[NSColor blueColor]];
-        else if (state == 2)
-            [textCell setTextColor:[NSColor redColor]];
-        else
-            [textCell setTextColor:[NSColor blackColor]];
-    }
-}
-
-- (BOOL)panel:(id)sender shouldShowFilename:(NSString *)path
-{
-    BOOL isdir;
-    [[NSFileManager defaultManager] fileExistsAtPath:path isDirectory:&isdir];
-    return isdir;
-}
-
-/* Notifications */
-
-- (void)directorySelectionChanged:(NSNotification *)aNotification
-{
-    [self refreshRemoveButtonText];
-}
-
-@end

base/cocoa/PyDupeGuru.h

@@ -1,66 +0,0 @@
-/* 
-Copyright 2009 Hardcoded Software (http://www.hardcoded.net)
-
-This software is licensed under the "HS" License as described in the "LICENSE" file, 
-which should be included with this package. The terms are also available at 
-http://www.hardcoded.net/licenses/hs_license
-*/
-
-#import <Cocoa/Cocoa.h>
-#import "PyApp.h"
-
-@interface PyDupeGuruBase : PyApp
-//Actions
-- (NSNumber *)addDirectory:(NSString *)name;
-- (void)removeDirectory:(NSNumber *)index;
-- (void)setDirectory:(NSArray *)indexPath state:(NSNumber *)state;
-- (void)loadResults;
-- (void)saveResults;
-- (void)loadIgnoreList;
-- (void)saveIgnoreList;
-- (void)clearIgnoreList;
-- (void)purgeIgnoreList;
-- (NSString *)exportToXHTMLwithColumns:(NSArray *)aColIds;
-
-- (NSNumber *)doScan;
-
-- (NSArray *)selectedPowerMarkerNodePaths;
-- (void)selectPowerMarkerNodePaths:(NSArray *)aIndexPaths;
-- (NSArray *)selectedResultNodePaths;
-- (void)selectResultNodePaths:(NSArray *)aIndexPaths;
-
-- (void)toggleSelectedMark;
-- (void)markAll;
-- (void)markInvert;
-- (void)markNone;
-
-- (void)addSelectedToIgnoreList;
-- (void)refreshDetailsWithSelected;
-- (void)removeSelected;
-- (void)openSelected;
-- (NSNumber *)renameSelected:(NSString *)aNewName;
-- (void)revealSelected;
-- (void)makeSelectedReference;
-- (void)applyFilter:(NSString *)filter;
-
-- (void)sortGroupsBy:(NSNumber *)aIdentifier ascending:(NSNumber *)aAscending;
-- (void)sortDupesBy:(NSNumber *)aIdentifier ascending:(NSNumber *)aAscending;
-
-- (void)copyOrMove:(NSNumber *)aCopy markedTo:(NSString *)destination recreatePath:(NSNumber *)aRecreateType;
-- (void)deleteMarked;
-- (void)removeMarked;
-
-//Data
-- (NSNumber *)getIgnoreListCount;
-- (NSNumber *)getMarkCount;
-- (NSString *)getStatLine;
-- (NSNumber *)getOperationalErrorCount;
-
-//Scanning options
-- (void)setMinMatchPercentage:(NSNumber *)percentage;
-- (void)setMixFileKind:(NSNumber *)mix_file_kind;
-- (void)setDisplayDeltaValues:(NSNumber *)display_delta_values;
-- (void)setEscapeFilterRegexp:(NSNumber *)escape_filter_regexp;
-- (void)setRemoveEmptyFolders:(NSNumber *)remove_empty_folders;
-- (void)setSizeThreshold:(int)size_threshold;
-@end

base/cocoa/ResultWindow.h

@@ -1,60 +0,0 @@
-/* 
-Copyright 2009 Hardcoded Software (http://www.hardcoded.net)
-
-This software is licensed under the "HS" License as described in the "LICENSE" file, 
-which should be included with this package. The terms are also available at 
-http://www.hardcoded.net/licenses/hs_license
-*/
-
-#import <Cocoa/Cocoa.h>
-#import "Outline.h"
-#import "DirectoryPanel.h"
-#import "PyDupeGuru.h"
-
-@interface MatchesView : OutlineView
-- (void)keyDown:(NSEvent *)theEvent;
-@end
-
-@interface ResultWindowBase : NSWindowController
-{
-@protected
-    IBOutlet PyDupeGuruBase *py;
-    IBOutlet id app;
-    IBOutlet NSView *actionMenuView;
-    IBOutlet NSSegmentedControl *deltaSwitch;
-    IBOutlet NSView *deltaSwitchView;
-    IBOutlet NSView *filterFieldView;
-    IBOutlet MatchesView *matches;
-	IBOutlet NSSegmentedControl *pmSwitch;
-    IBOutlet NSView *pmSwitchView;
-	IBOutlet NSTextField *stats;
-    
-    BOOL _powerMode;
-    BOOL _displayDelta;
-}
-/* Override */
-- (NSString *)logoImageName;
-
-/* Helpers */
-- (NSArray *)getColumnsOrder;
-- (NSDictionary *)getColumnsWidth;
-- (NSArray *)getSelected:(BOOL)aDupesOnly;
-- (NSArray *)getSelectedPaths:(BOOL)aDupesOnly;
-- (void)updatePySelection;
-- (void)performPySelection:(NSArray *)aIndexPaths;
-- (void)refreshStats;
-
-/* Actions */
-- (IBAction)changeDelta:(id)sender;
-- (IBAction)changePowerMarker:(id)sender;
-- (IBAction)copyMarked:(id)sender;
-- (IBAction)deleteMarked:(id)sender;
-- (IBAction)expandAll:(id)sender;
-- (IBAction)exportToXHTML:(id)sender;
-- (IBAction)moveMarked:(id)sender;
-- (IBAction)switchSelected:(id)sender;
-- (IBAction)togglePowerMarker:(id)sender;
-
-/* Notifications */
-- (void)jobCompleted:(NSNotification *)aNotification;
-@end

base/cocoa/ResultWindow.m

@@ -1,468 +0,0 @@
-/* 
-Copyright 2009 Hardcoded Software (http://www.hardcoded.net)
-
-This software is licensed under the "HS" License as described in the "LICENSE" file, 
-which should be included with this package. The terms are also available at 
-http://www.hardcoded.net/licenses/hs_license
-*/
-
-#import "ResultWindow.h"
-#import "Dialogs.h"
-#import "ProgressController.h"
-#import "Utils.h"
-#import "RegistrationInterface.h"
-#import "AppDelegate.h"
-#import "Consts.h"
-
-#define tbbDirectories @"tbbDirectories"
-#define tbbDetails @"tbbDetail"
-#define tbbPreferences @"tbbPreferences"
-#define tbbPowerMarker @"tbbPowerMarker"
-#define tbbScan @"tbbScan"
-#define tbbAction @"tbbAction"
-#define tbbDelta @"tbbDelta"
-#define tbbFilter @"tbbFilter"
-
-@implementation MatchesView
-- (void)keyDown:(NSEvent *)theEvent
-{
-    unichar key = [[theEvent charactersIgnoringModifiers] characterAtIndex:0];
-    // get flags and strip the lower 16 (device dependant) bits
-    unsigned int flags = ( [theEvent modifierFlags] & 0x00FF );
-    if (((key == NSDeleteFunctionKey) || (key == NSDeleteCharacter)) && (flags == 0))
-        [self sendAction:@selector(removeSelected:) to:[self delegate]];
-    else
-    if ((key == 0x20) && (flags == 0)) // Space
-        [self sendAction:@selector(markSelected:) to:[self delegate]];
-    else
-        [super keyDown:theEvent];
-}
-
-- (void)outlineView:(NSOutlineView *)outlineView setObjectValue:(id)object forTableColumn:(NSTableColumn *)tableColumn byItem:(id)item
-{
-    if (![[tableColumn identifier] isEqual:@"0"])
-        return; //We only want to cover renames.
-    OVNode *node = item;
-    NSString *oldName = [[node buffer] objectAtIndex:0];
-    NSString *newName = object;
-    if (![newName isEqual:oldName])
-    {
-        BOOL renamed = n2b([(PyDupeGuruBase *)py renameSelected:newName]);
-        if (renamed)
-            [[NSNotificationCenter defaultCenter] postNotificationName:ResultsChangedNotification object:self];
-        else
-            [Dialogs showMessage:[NSString stringWithFormat:@"The name '%@' already exists.",newName]];
-    }
-}
-@end
-
-@implementation ResultWindowBase
-- (void)awakeFromNib
-{
-    [self window];
-    [[NSNotificationCenter defaultCenter] addObserver:self selector:@selector(registrationRequired:) name:RegistrationRequired object:nil];
-	[[NSNotificationCenter defaultCenter] addObserver:self selector:@selector(jobCompleted:) name:JobCompletedNotification object:nil];
-    [[NSNotificationCenter defaultCenter] addObserver:self selector:@selector(jobStarted:) name:JobStarted object:nil];
-    [[NSNotificationCenter defaultCenter] addObserver:self selector:@selector(jobInProgress:) name:JobInProgress object:nil];
-	[[NSNotificationCenter defaultCenter] addObserver:self selector:@selector(resultsChanged:) name:ResultsChangedNotification object:nil];
-	[[NSNotificationCenter defaultCenter] addObserver:self selector:@selector(resultsUpdated:) name:ResultsUpdatedNotification object:nil];
-}
-
-/* Virtual */
-- (NSString *)logoImageName
-{
-    return @"dg_logo32";
-}
-
-/* Helpers */
-//Returns an array of identifiers, in order.
-- (NSArray *)getColumnsOrder
-{
-    NSTableColumn *col;
-    NSString *colId;
-    NSMutableArray *result = [NSMutableArray array];
-    NSEnumerator *e = [[matches tableColumns] objectEnumerator];
-    while (col = [e nextObject])
-    {
-        colId = [col identifier];
-        [result addObject:colId];
-    }
-    return result;
-}
-
-- (NSDictionary *)getColumnsWidth
-{
-    NSMutableDictionary *result = [NSMutableDictionary dictionary];
-    NSTableColumn *col;
-    NSString *colId;
-    NSNumber *width;
-    NSEnumerator *e = [[matches tableColumns] objectEnumerator];
-    while (col = [e nextObject])
-    {
-        colId = [col identifier];
-        width = [NSNumber numberWithFloat:[col width]];
-        [result setObject:width forKey:colId];
-    }
-    return result;
-}
-
-- (NSArray *)getSelected:(BOOL)aDupesOnly
-{
-    if (_powerMode)
-        aDupesOnly = NO;
-    NSIndexSet *indexes = [matches selectedRowIndexes];
-    NSMutableArray *nodeList = [NSMutableArray array];
-    OVNode *node;
-    int i = [indexes firstIndex];
-    while (i != NSNotFound)
-    {
-        node = [matches itemAtRow:i];
-        if (!aDupesOnly || ([node level] > 1))
-            [nodeList addObject:node];
-        i = [indexes indexGreaterThanIndex:i];
-    }
-    return nodeList;
-}
-
-- (NSArray *)getSelectedPaths:(BOOL)aDupesOnly
-{
-    NSMutableArray *r = [NSMutableArray array];
-    NSArray *selected = [self getSelected:aDupesOnly];
-    NSEnumerator *e = [selected objectEnumerator];
-    OVNode *node;
-    while (node = [e nextObject])
-        [r addObject:p2a([node indexPath])];
-    return r;
-}
-
-- (void)updatePySelection
-{
-	NSArray *selection;
-    if (_powerMode)
-		selection = [py selectedPowerMarkerNodePaths];
-    else
-		selection = [py selectedResultNodePaths];
-	[matches selectNodePaths:selection];
-}
-
-- (void)performPySelection:(NSArray *)aIndexPaths
-{
-    if (_powerMode)
-        [py selectPowerMarkerNodePaths:aIndexPaths];
-    else
-        [py selectResultNodePaths:aIndexPaths];
-}
-
-- (void)refreshStats
-{
-    [stats setStringValue:[py getStatLine]];
-}
-
-/* Actions */
-- (IBAction)changeDelta:(id)sender
-{
-    _displayDelta = [deltaSwitch selectedSegment] == 1;
-    [py setDisplayDeltaValues:b2n(_displayDelta)];
-    [matches reloadData];
-    [self expandAll:nil];
-}
-
-- (IBAction)changePowerMarker:(id)sender
-{
-    _powerMode = [pmSwitch selectedSegment] == 1;
-    if (_powerMode)
-        [matches setTag:2];
-    else
-        [matches setTag:0];
-    [self expandAll:nil];
-    [self outlineView:matches didClickTableColumn:nil];
-	[self updatePySelection];
-}
-
-- (IBAction)copyMarked:(id)sender
-{
-    int mark_count = [[py getMarkCount] intValue];
-    if (!mark_count)
-        return;
-    NSOpenPanel *op = [NSOpenPanel openPanel];
-    [op setCanChooseFiles:NO];
-    [op setCanChooseDirectories:YES];
-    [op setCanCreateDirectories:YES];
-    [op setAllowsMultipleSelection:NO];
-    [op setTitle:@"Select a directory to copy marked files to"];
-    if ([op runModalForTypes:nil] == NSOKButton)
-    {
-        NSString *directory = [[op filenames] objectAtIndex:0];
-        NSUserDefaults *ud = [NSUserDefaults standardUserDefaults];
-        [py copyOrMove:b2n(YES) markedTo:directory recreatePath:[ud objectForKey:@"recreatePathType"]];
-    }
-}
-
-- (IBAction)deleteMarked:(id)sender
-{
-    int mark_count = [[py getMarkCount] intValue];
-    if (!mark_count)
-        return;
-    if ([Dialogs askYesNo:[NSString stringWithFormat:@"You are about to send %d files to Trash. Continue?",mark_count]] == NSAlertSecondButtonReturn) // NO
-        return;
-    NSUserDefaults *ud = [NSUserDefaults standardUserDefaults];
-    [py setRemoveEmptyFolders:[ud objectForKey:@"removeEmptyFolders"]];
-    [py deleteMarked];
-}
-
-- (IBAction)expandAll:(id)sender
-{
-    for (int i=0;i < [matches numberOfRows];i++)
-        [matches expandItem:[matches itemAtRow:i]];
-}
-
-- (IBAction)exportToXHTML:(id)sender
-{
-    NSString *exported = [py exportToXHTMLwithColumns:[self getColumnsOrder]];
-    [[NSWorkspace sharedWorkspace] openFile:exported];
-}
-
-- (IBAction)moveMarked:(id)sender
-{
-    int mark_count = [[py getMarkCount] intValue];
-    if (!mark_count)
-        return;
-    NSOpenPanel *op = [NSOpenPanel openPanel];
-    [op setCanChooseFiles:NO];
-    [op setCanChooseDirectories:YES];
-    [op setCanCreateDirectories:YES];
-    [op setAllowsMultipleSelection:NO];
-    [op setTitle:@"Select a directory to move marked files to"];
-    if ([op runModalForTypes:nil] == NSOKButton)
-    {
-        NSString *directory = [[op filenames] objectAtIndex:0];
-        NSUserDefaults *ud = [NSUserDefaults standardUserDefaults];
-        [py setRemoveEmptyFolders:[ud objectForKey:@"removeEmptyFolders"]];
-        [py copyOrMove:b2n(NO) markedTo:directory recreatePath:[ud objectForKey:@"recreatePathType"]];
-    }
-}
-
-- (IBAction)switchSelected:(id)sender
-{
-    // It might look like a complicated way to get the length of the current dupe list on the py side
-    // but after a lot of fussing around, believe it or not, it actually is.
-    int matchesTag = _powerMode ? 2 : 0;
-    int startLen = [[py getOutlineView:matchesTag childCountsForPath:[NSArray array]] count];
-    [self performPySelection:[self getSelectedPaths:YES]];
-    [py makeSelectedReference];
-    // In some cases (when in a filtered view in Power Marker mode, it's possible that the demoted
-    // ref is not a part of the filter, making the table smaller. In those cases, we want to do a
-    // complete reload of the table to avoid a crash.
-    if ([[py getOutlineView:matchesTag childCountsForPath:[NSArray array]] count] == startLen)
-        [[NSNotificationCenter defaultCenter] postNotificationName:ResultsUpdatedNotification object:self];
-    else
-        [[NSNotificationCenter defaultCenter] postNotificationName:ResultsChangedNotification object:self];
-}
-
-- (IBAction)togglePowerMarker:(id)sender
-{
-    if ([pmSwitch selectedSegment] == 1)
-        [pmSwitch setSelectedSegment:0];
-    else
-        [pmSwitch setSelectedSegment:1];
-    [self changePowerMarker:sender];
-}
-
-/* Delegate */
-
-- (void)outlineView:(NSOutlineView *)outlineView didClickTableColumn:(NSTableColumn *)tableColumn
-{
-    if ([[outlineView sortDescriptors] count] < 1)
-        return;
-    NSSortDescriptor *sd = [[outlineView sortDescriptors] objectAtIndex:0];
-    if (_powerMode)
-        [py sortDupesBy:i2n([[sd key] intValue]) ascending:b2n([sd ascending])];
-    else
-        [py sortGroupsBy:i2n([[sd key] intValue]) ascending:b2n([sd ascending])];
-    [matches reloadData];
-    [self expandAll:nil];
-}
-
-/* Notifications */
-- (void)windowWillClose:(NSNotification *)aNotification
-{
-    [NSApp hide:NSApp];
-}
-
-- (void)jobCompleted:(NSNotification *)aNotification
-{
-    [[NSNotificationCenter defaultCenter] postNotificationName:ResultsChangedNotification object:self];
-    int r = n2i([py getOperationalErrorCount]);
-    id lastAction = [[ProgressController mainProgressController] jobId];
-    if ([lastAction isEqualTo:jobCopy])
-    {
-        if (r > 0)
-            [Dialogs showMessage:[NSString stringWithFormat:@"%d file(s) couldn't be copied.",r]];
-        else
-            [Dialogs showMessage:@"All marked files were copied sucessfully."];
-    }
-    if ([lastAction isEqualTo:jobMove])
-    {
-        if (r > 0)
-            [Dialogs showMessage:[NSString stringWithFormat:@"%d file(s) couldn't be moved. They were kept in the results, and still are marked.",r]];
-        else
-            [Dialogs showMessage:@"All marked files were moved sucessfully."];
-    }
-    if ([lastAction isEqualTo:jobDelete])
-    {
-        if (r > 0)
-            [Dialogs showMessage:[NSString stringWithFormat:@"%d file(s) couldn't be sent to Trash. They were kept in the results, and still are marked.",r]];
-        else
-            [Dialogs showMessage:@"All marked files were sucessfully sent to Trash."];
-    }
-    // Re-activate toolbar items right after the progress bar stops showing instead of waiting until
-    // a mouse-over is performed
-    [[[self window] toolbar] validateVisibleItems];
-}
-
-- (void)jobInProgress:(NSNotification *)aNotification
-{
-    [Dialogs showMessage:@"A previous action is still hanging in there. You can't start a new one yet. Wait a few seconds, then try again."];
-}
-
-- (void)jobStarted:(NSNotification *)aNotification
-{
-    NSDictionary *ui = [aNotification userInfo];
-    NSString *desc = [ui valueForKey:@"desc"];
-    [[ProgressController mainProgressController] setJobDesc:desc];
-    NSString *jobid = [ui valueForKey:@"jobid"];
-    // NSLog(jobid);
-    [[ProgressController mainProgressController] setJobId:jobid];
-    [[ProgressController mainProgressController] showSheetForParent:[self window]];
-}
-
-- (void)registrationRequired:(NSNotification *)aNotification
-{
-    NSString *msg = @"This is a demo version, which only allows you 10 delete/copy/move actions per session. You cannot continue.";
-    [Dialogs showMessage:msg];
-}
-
-- (void)resultsChanged:(NSNotification *)aNotification
-{
-    [matches reloadData];
-    [self expandAll:nil];
-    [self outlineViewSelectionDidChange:nil];
-    [self refreshStats];
-}
-
-- (void)resultsUpdated:(NSNotification *)aNotification
-{
-	[matches invalidateBuffers];
-}
-
-/* Toolbar */
-- (NSToolbarItem *)toolbar:(NSToolbar *)toolbar itemForItemIdentifier:(NSString *)itemIdentifier willBeInsertedIntoToolbar:(BOOL)flag
-{
-    NSToolbarItem *tbi = [[[NSToolbarItem alloc] initWithItemIdentifier:itemIdentifier] autorelease];
-    if ([itemIdentifier isEqualTo:tbbDirectories])
-    {
-        [tbi setLabel: @"Directories"];
-        [tbi setToolTip: @"Show/Hide the directories panel."];
-        [tbi setImage: [NSImage imageNamed: @"folder32"]];
-        [tbi setTarget: app];
-        [tbi setAction: @selector(toggleDirectories:)];
-    }
-    else if ([itemIdentifier isEqualTo:tbbDetails])
-    {
-        [tbi setLabel: @"Details"];
-        [tbi setToolTip: @"Show/Hide the details panel."];
-        [tbi setImage: [NSImage imageNamed: @"details32"]];
-        [tbi setTarget: self];
-        [tbi setAction: @selector(toggleDetailsPanel:)];
-    }
-    else if ([itemIdentifier isEqualTo:tbbPreferences])
-    {
-        [tbi setLabel: @"Preferences"];
-        [tbi setToolTip: @"Show the preferences panel."];
-        [tbi setImage: [NSImage imageNamed: @"preferences32"]];
-        [tbi setTarget: self];
-        [tbi setAction: @selector(showPreferencesPanel:)];
-    }
-    else if ([itemIdentifier isEqualTo:tbbPowerMarker])
-    {
-        [tbi setLabel: @"Power Marker"];
-        [tbi setToolTip: @"When enabled, only the duplicates are shown, not the references."];
-        [tbi setView:pmSwitchView];
-        [tbi setMinSize:[pmSwitchView frame].size];
-        [tbi setMaxSize:[pmSwitchView frame].size];        
-    }
-    else if ([itemIdentifier isEqualTo:tbbScan])
-    {
-        [tbi setLabel: @"Start Scanning"];
-        [tbi setToolTip: @"Start scanning for duplicates in the selected directories."];
-        [tbi setImage: [NSImage imageNamed:[self logoImageName]]];
-        [tbi setTarget: self];
-        [tbi setAction: @selector(startDuplicateScan:)];
-    }
-    else if ([itemIdentifier isEqualTo:tbbAction])
-    {
-        [tbi setLabel: @"Action"];
-        [tbi setView:actionMenuView];
-        [tbi setMinSize:[actionMenuView frame].size];
-        [tbi setMaxSize:[actionMenuView frame].size];
-    }
-    else if ([itemIdentifier isEqualTo:tbbDelta])
-    {
-        [tbi setLabel: @"Delta Values"];
-        [tbi setToolTip: @"When enabled, this option makes dupeGuru display, where applicable, delta values instead of absolute values."];
-        [tbi setView:deltaSwitchView];
-        [tbi setMinSize:[deltaSwitchView frame].size];
-        [tbi setMaxSize:[deltaSwitchView frame].size];
-    }
-    else if ([itemIdentifier isEqualTo:tbbFilter])
-    {
-        [tbi setLabel: @"Filter"];
-        [tbi setToolTip: @"Filters the results using regular expression."];
-        [tbi setView:filterFieldView];
-        [tbi setMinSize:[filterFieldView frame].size];
-        [tbi setMaxSize:NSMakeSize(1000, [filterFieldView frame].size.height)];
-    }
-    [tbi setPaletteLabel: [tbi label]];
-    return tbi;
-}
-
-- (NSArray *)toolbarAllowedItemIdentifiers:(NSToolbar *)toolbar
-{
-    return [NSArray arrayWithObjects:
-        tbbDirectories,
-        tbbDetails,
-        tbbPreferences,
-        tbbPowerMarker,
-        tbbScan,
-        tbbAction,
-        tbbDelta,
-        tbbFilter,
-        NSToolbarSeparatorItemIdentifier,
-        NSToolbarSpaceItemIdentifier, 
-        NSToolbarFlexibleSpaceItemIdentifier,
-        nil];
-}
-
-- (NSArray *)toolbarDefaultItemIdentifiers:(NSToolbar *)toolbar
-{
-    return [NSArray arrayWithObjects:
-        tbbScan,
-        tbbAction,
-        tbbDirectories,
-        tbbDetails,
-        tbbPowerMarker,
-        tbbDelta,
-        tbbFilter,
-        nil];
-}
-
-- (BOOL)validateToolbarItem:(NSToolbarItem *)theItem
-{
-    return ![[ProgressController mainProgressController] isShown];
-}
-
-- (BOOL)validateMenuItem:(NSMenuItem *)item
-{
-    return ![[ProgressController mainProgressController] isShown];
-}
-@end

base/cocoa/dsa_pub.pem

@@ -1,20 +0,0 @@
------BEGIN PUBLIC KEY-----
-MIIDOjCCAi0GByqGSM44BAEwggIgAoIBAQDSurIL+HKbw+jsppG6tp3+WOcA4W71
-nhwR/DD2Se076AtCXJcssAhuCDUm+AVkQ3l34D++aYWtLR575rCrwU4lZXfQe+b9
-plHK02oOuqAY8lO5y02xoHEh7XeGunZ0u8wOVZw8MI999vIJ8rtCdvIF3r26wkjx
-9sieSxVpzJHDV5JHVdK3ObkXp/ts99dOD5B3CWGS8UiroMgS0FmRl7uPuADRRn2G
-srHTBYMwJvq8HFzQmDxcLldGQMAKvRKchtH+nH6ci1unSnpDUyrsCd+7qv1cSTse
-qc4OgXBDQ94MfVEh6Bs0S9stYfJf8cp6iV18J0sqMb9rbP4qC56iBsXfAhUAj6tx
-gwima7VaNI4YiC69jpLod3MCggEAYx+/mbU8P/xGooV9MgA3nI2v2vVNkwZVFcPa
-ROLQHg+R7bAftF3+1M9AnSP2O+PnXL65DwyTOab/Z/zM/vof3LLCGLYCmzPL+xvB
-6PxlqO374kFsKHEaaw66nnFWzPSdks/il0rauAiEbO8Gn/a8F2HFdA/OCCzq83l6
-cOhya7kGXZxdjeIfpfiNjDqZXi+8VRNDcDXx5u/T4vpkliQ+4O8ZXjwE4z2dPHfu
-Bw/N7DUalkzhZygYqcgx3tUxu3x/Pso+inmIBbk/As0uZv2nEll2CkEI6CSJIpfn
-pLKNQb4E4G7h+u+8kfHcwQ59RU1uGh0PU5uM+DOPg6HsC41RwgOCAQUAAoIBABLY
-T8gN8KdxWheESorvgksdG+Fizhkafpac08MCwJFF24v5a8AvZbhcCMLhChrloKcQ
-19qHshRIuWbSma/OqCmQKH752PTOKxRKsmqAfO0Rej2aDJrd0s7YBMY72DqeSYPP
-peLlwv0gkgRW7/EbDvBI18iTbrQLZtdqs9Xajc3dyIG5wrMtAf/Gta2oWChHlBLZ
-S45++Y9ou+LtW7dMc7c+aTxbzeLG36S57kAenRzjfP8zOi3P+Cc+5b9+SZgqfFrz
-/ch/HjB2zYAKq9AZSmgp9qIlOIuXnctJUD9hHivuEXFDr6xi1cxj7Q8WnX4+C58/
-QyGS4lebbLQ35x6fTQ8=
------END PUBLIC KEY-----

base/py/__init__.py

@@ -1 +0,0 @@
-

base/py/app.py

@@ -1,251 +0,0 @@
-#!/usr/bin/env python
-# Created By: Virgil Dupras
-# Created On: 2006/11/11
-# $Id$
-# Copyright 2009 Hardcoded Software (http://www.hardcoded.net)
-# 
-# This software is licensed under the "HS" License as described in the "LICENSE" file, 
-# which should be included with this package. The terms are also available at 
-# http://www.hardcoded.net/licenses/hs_license
-
-from __future__ import unicode_literals
-
-import os
-import os.path as op
-import logging
-
-from hsutil import io, files
-from hsutil.path import Path
-from hsutil.reg import RegistrableApplication, RegistrationRequired
-from hsutil.misc import flatten, first
-from hsutil.str import escape
-
-from . import directories, results, scanner, export, fs
-
-JOB_SCAN = 'job_scan'
-JOB_LOAD = 'job_load'
-JOB_MOVE = 'job_move'
-JOB_COPY = 'job_copy'
-JOB_DELETE = 'job_delete'
-
-class NoScannableFileError(Exception):
-    pass
-
-class AllFilesAreRefError(Exception):
-    pass
-
-class DupeGuru(RegistrableApplication):
-    def __init__(self, data_module, appdata, appid):
-        RegistrableApplication.__init__(self, appid)
-        self.appdata = appdata
-        if not op.exists(self.appdata):
-            os.makedirs(self.appdata)
-        self.data = data_module
-        self.directories = directories.Directories()
-        self.results = results.Results(data_module)
-        self.scanner = scanner.Scanner()
-        self.action_count = 0
-        self.last_op_error_count = 0
-        self.options = {
-            'escape_filter_regexp': True,
-            'clean_empty_dirs': False,
-        }
-    
-    def _demo_check(self):
-        if self.registered:
-            return
-        count = self.results.mark_count
-        if count + self.action_count > 10:
-            raise RegistrationRequired()
-        else:
-            self.action_count += count
-    
-    def _do_delete(self, j):
-        def op(dupe):
-            j.add_progress()
-            return self._do_delete_dupe(dupe)
-        
-        j.start_job(self.results.mark_count)
-        self.last_op_error_count = self.results.perform_on_marked(op, True)
-    
-    def _do_delete_dupe(self, dupe):
-        if not io.exists(dupe.path):
-            dupe.parent = None
-            return True
-        self._recycle_dupe(dupe)
-        self.clean_empty_dirs(dupe.path[:-1])
-        if not io.exists(dupe.path):
-            dupe.parent = None
-            return True
-        logging.warning("Could not send {0} to trash.".format(unicode(dupe.path)))
-        return False
-    
-    def _do_load(self, j):
-        self.directories.load_from_file(op.join(self.appdata, 'last_directories.xml'))
-        j = j.start_subjob([1, 9])
-        self.results.load_from_xml(op.join(self.appdata, 'last_results.xml'), self._get_file, j)
-        files = flatten(g[:] for g in self.results.groups)
-        for file in j.iter_with_progress(files, 'Reading metadata %d/%d'):
-            file._read_all_info(attrnames=self.data.METADATA_TO_READ)
-    
-    def _get_display_info(self, dupe, group, delta=False):
-        if (dupe is None) or (group is None):
-            return ['---'] * len(self.data.COLUMNS)
-        try:
-            return self.data.GetDisplayInfo(dupe, group, delta)
-        except Exception as e:
-            logging.warning("Exception on GetDisplayInfo for %s: %s", unicode(dupe.path), unicode(e))
-            return ['---'] * len(self.data.COLUMNS)
-    
-    def _get_file(self, str_path):
-        path = Path(str_path)
-        return fs.get_file(path, self.directories.fileclasses)    
-    
-    @staticmethod
-    def _recycle_dupe(dupe):
-        raise NotImplementedError()
-    
-    def _start_job(self, jobid, func):
-        # func(j)
-        raise NotImplementedError()
-    
-    def add_directory(self, d):
-        try:
-            self.directories.add_path(Path(d))
-            return 0
-        except directories.AlreadyThereError:
-            return 1
-        except directories.InvalidPathError:
-            return 2
-    
-    def add_to_ignore_list(self, dupe):
-        g = self.results.get_group_of_duplicate(dupe)
-        for other in g:
-            if other is not dupe:
-                self.scanner.ignore_list.Ignore(unicode(other.path), unicode(dupe.path))
-    
-    def apply_filter(self, filter):
-        self.results.apply_filter(None)
-        if self.options['escape_filter_regexp']:
-            filter = escape(filter, '()[]\\.|+?^')
-            filter = escape(filter, '*', '.')
-        self.results.apply_filter(filter)
-    
-    def clean_empty_dirs(self, path):
-        if self.options['clean_empty_dirs']:
-            while files.delete_if_empty(path, ['.DS_Store']):
-                path = path[:-1]
-    
-    def copy_or_move(self, dupe, copy, destination, dest_type):
-        """
-            copy: True = Copy False = Move
-            destination: string.
-            dest_type: 0 = right in destination.
-                       1 = relative re-creation.
-                       2 = absolute re-creation.
-        """
-        source_path = dupe.path
-        location_path = first(p for p in self.directories if dupe.path in p)
-        dest_path = Path(destination)
-        if dest_type == 2:
-            dest_path = dest_path + source_path[1:-1] #Remove drive letter and filename
-        elif dest_type == 1:
-            dest_path = dest_path + source_path[location_path:-1]
-        try:
-            if not io.exists(dest_path):
-                io.makedirs(dest_path)
-            if copy:
-                files.copy(source_path, dest_path)
-            else:
-                files.move(source_path, dest_path)
-                self.clean_empty_dirs(source_path[:-1])
-        except EnvironmentError as e:
-            operation = 'Copy' if copy else 'Move'
-            logging.warning('%s operation failed on %s. Error: %s' % (operation, unicode(dupe.path), unicode(e)))
-            return False
-        return True
-    
-    def copy_or_move_marked(self, copy, destination, recreate_path):
-        def do(j):
-            def op(dupe):
-                j.add_progress()
-                return self.copy_or_move(dupe, copy, destination, recreate_path)
-            
-            j.start_job(self.results.mark_count)
-            self.last_op_error_count = self.results.perform_on_marked(op, not copy)
-        
-        self._demo_check()
-        jobid = JOB_COPY if copy else JOB_MOVE
-        self._start_job(jobid, do)
-    
-    def delete_marked(self):
-        self._demo_check()
-        self._start_job(JOB_DELETE, self._do_delete)
-    
-    def export_to_xhtml(self, column_ids):
-        column_ids = [colid for colid in column_ids if colid.isdigit()]
-        column_ids = map(int, column_ids)
-        column_ids.sort()
-        colnames = [col['display'] for i, col in enumerate(self.data.COLUMNS) if i in column_ids]
-        rows = []
-        for group in self.results.groups:
-            for dupe in group:
-                data = self._get_display_info(dupe, group)
-                row = [data[colid] for colid in column_ids]
-                row.insert(0, dupe is not group.ref)
-                rows.append(row)
-        return export.export_to_xhtml(colnames, rows)
-    
-    def load(self):
-        self._start_job(JOB_LOAD, self._do_load)
-        self.load_ignore_list()
-    
-    def load_ignore_list(self):
-        p = op.join(self.appdata, 'ignore_list.xml')
-        self.scanner.ignore_list.load_from_xml(p)
-    
-    def make_reference(self, duplicates):
-        changed_groups = set()
-        for dupe in duplicates:
-            g = self.results.get_group_of_duplicate(dupe)
-            if g not in changed_groups:
-                self.results.make_ref(dupe)
-                changed_groups.add(g)
-    
-    def save(self):
-        try:
-            self.directories.save_to_file(op.join(self.appdata, 'last_directories.xml'))
-            self.results.save_to_xml(op.join(self.appdata, 'last_results.xml'))
-        except LookupError:
-            # This is that weird issue from #39 that sometimes happens when auto-updating with
-            # Sparkle. Just ignore it.
-            pass
-    
-    def save_ignore_list(self):
-        p = op.join(self.appdata, 'ignore_list.xml')
-        self.scanner.ignore_list.save_to_xml(p)
-    
-    def start_scanning(self):
-        def do(j):
-            j.set_progress(0, 'Collecting files to scan')
-            files = list(self.directories.get_files())
-            logging.info('Scanning %d files' % len(files))
-            self.results.groups = self.scanner.GetDupeGroups(files, j)
-        
-        files = self.directories.get_files()
-        first_file = first(files)
-        if first_file is None:
-            raise NoScannableFileError()
-        if first_file.is_ref and all(f.is_ref for f in files):
-            raise AllFilesAreRefError()
-        self.results.groups = []
-        self._start_job(JOB_SCAN, do)
-    
-    #--- Properties
-    @property
-    def stat_line(self):
-        result = self.results.stat_line
-        if self.scanner.discarded_file_count:
-            result = '%s (%d discarded)' % (result, self.scanner.discarded_file_count)
-        return result
-    

base/py/app_cocoa.py

@@ -1,310 +0,0 @@
-# Created By: Virgil Dupras
-# Created On: 2006/11/11
-# $Id$
-# Copyright 2009 Hardcoded Software (http://www.hardcoded.net)
-# 
-# This software is licensed under the "HS" License as described in the "LICENSE" file, 
-# which should be included with this package. The terms are also available at 
-# http://www.hardcoded.net/licenses/hs_license
-
-from Foundation import *
-from AppKit import *
-import logging
-import os.path as op
-
-from hsutil import io, cocoa, job
-from hsutil.cocoa import install_exception_hook
-from hsutil.misc import stripnone
-from hsutil.reg import RegistrationRequired
-
-from . import app, fs
-
-JOBID2TITLE = {
-    app.JOB_SCAN: "Scanning for duplicates",
-    app.JOB_LOAD: "Loading",
-    app.JOB_MOVE: "Moving",
-    app.JOB_COPY: "Copying",
-    app.JOB_DELETE: "Sending to Trash",
-}
-
-def demo_method(method):
-    def wrapper(self, *args, **kwargs):
-        try:
-            return method(self, *args, **kwargs)
-        except RegistrationRequired:
-            NSNotificationCenter.defaultCenter().postNotificationName_object_('RegistrationRequired', self)
-    
-    return wrapper
-
-class DupeGuru(app.DupeGuru):
-    def __init__(self, data_module, appdata_subdir, appid):
-        LOGGING_LEVEL = logging.DEBUG if NSUserDefaults.standardUserDefaults().boolForKey_('debug') else logging.WARNING
-        logging.basicConfig(level=LOGGING_LEVEL, format='%(levelname)s %(message)s')
-        logging.debug('started in debug mode')
-        install_exception_hook()
-        appsupport = NSSearchPathForDirectoriesInDomains(NSApplicationSupportDirectory, NSUserDomainMask, True)[0]
-        appdata = op.join(appsupport, appdata_subdir)
-        app.DupeGuru.__init__(self, data_module, appdata, appid)
-        self.progress = cocoa.ThreadedJobPerformer()
-        self.display_delta_values = False
-        self.selected_dupes = []
-        self.RefreshDetailsTable(None,None)
-    
-    #--- Override
-    @staticmethod
-    def _recycle_dupe(dupe):
-        if not io.exists(dupe.path):
-            dupe.parent = None
-            return True
-        directory = unicode(dupe.parent.path)
-        filename = dupe.name
-        result, tag = NSWorkspace.sharedWorkspace().performFileOperation_source_destination_files_tag_(
-            NSWorkspaceRecycleOperation, directory, '', [filename])
-        if not io.exists(dupe.path):
-            dupe.parent = None
-            return True
-        logging.warning('Could not send %s to trash. tag: %d' % (unicode(dupe.path), tag))
-        return False
-    
-    def _start_job(self, jobid, func):
-        try:
-            j = self.progress.create_job()
-            self.progress.run_threaded(func, args=(j, ))
-        except job.JobInProgressError:
-            NSNotificationCenter.defaultCenter().postNotificationName_object_('JobInProgress', self)
-        else:
-            ud = {'desc': JOBID2TITLE[jobid], 'jobid':jobid}
-            NSNotificationCenter.defaultCenter().postNotificationName_object_userInfo_('JobStarted', self, ud)
-    
-    #---Helpers
-    def GetObjects(self,node_path):
-        #returns a tuple g,d
-        try:
-            g = self.results.groups[node_path[0]]
-            if len(node_path) == 2:
-                return (g,self.results.groups[node_path[0]].dupes[node_path[1]])
-            else:
-                return (g,None)
-        except IndexError:
-            return (None,None)
-    
-    def get_folder_path(self, node_path, curr_path=None):
-        if not node_path:
-            return curr_path
-        current_index = node_path[0]
-        if curr_path is None:
-            curr_path = self.directories[current_index]
-        else:
-            curr_path = self.directories.get_subfolders(curr_path)[current_index]
-        return self.get_folder_path(node_path[1:], curr_path)
-    
-    def RefreshDetailsTable(self,dupe,group):
-        l1 = self._get_display_info(dupe, group, False)
-        # we don't want the two sides of the table to display the stats for the same file
-        ref = group.ref if group is not None and group.ref is not dupe else None
-        l2 = self._get_display_info(ref, group, False)
-        names = [c['display'] for c in self.data.COLUMNS]
-        self.details_table = zip(names,l1,l2)
-    
-    #---Public
-    def AddSelectedToIgnoreList(self):
-        for dupe in self.selected_dupes:
-            self.add_to_ignore_list(dupe)
-    
-    copy_or_move_marked = demo_method(app.DupeGuru.copy_or_move_marked)
-    delete_marked = demo_method(app.DupeGuru.delete_marked)
-
-    def MakeSelectedReference(self):
-        self.make_reference(self.selected_dupes)
-    
-    def OpenSelected(self):
-        if self.selected_dupes:
-            path = unicode(self.selected_dupes[0].path)
-            NSWorkspace.sharedWorkspace().openFile_(path)
-    
-    def PurgeIgnoreList(self):
-        self.scanner.ignore_list.Filter(lambda f,s:op.exists(f) and op.exists(s))
-    
-    def RefreshDetailsWithSelected(self):
-        if self.selected_dupes:
-            self.RefreshDetailsTable(
-                self.selected_dupes[0],
-                self.results.get_group_of_duplicate(self.selected_dupes[0])
-            )
-        else:
-            self.RefreshDetailsTable(None,None)
-    
-    def RemoveDirectory(self,index):
-        try:
-            del self.directories[index]
-        except IndexError:
-            pass
-    
-    def RemoveSelected(self):
-        self.results.remove_duplicates(self.selected_dupes)
-    
-    def RenameSelected(self, newname):
-        try:
-            d = self.selected_dupes[0]
-            d.rename(newname)
-            return True
-        except (IndexError, fs.FSError) as e:
-            logging.warning("dupeGuru Warning: %s" % unicode(e))
-        return False
-    
-    def RevealSelected(self):
-        if self.selected_dupes:
-            path = unicode(self.selected_dupes[0].path)
-            NSWorkspace.sharedWorkspace().selectFile_inFileViewerRootedAtPath_(path,'')
-    
-    def start_scanning(self):
-        self.RefreshDetailsTable(None, None)
-        try:
-            app.DupeGuru.start_scanning(self)
-            return 0
-        except app.NoScannableFileError:
-            return 3
-        except app.AllFilesAreRefError:
-            return 1
-    
-    def selected_result_node_paths(self):
-        def get_path(dupe):
-            try:
-                group = self.results.get_group_of_duplicate(dupe)
-                groupindex = self.results.groups.index(group)
-                if dupe is group.ref:
-                    return [groupindex]
-                dupeindex = group.dupes.index(dupe)
-                return [groupindex, dupeindex]
-            except ValueError: # dupe not in there
-                return None
-        
-        dupes = self.selected_dupes
-        return stripnone(get_path(dupe) for dupe in dupes)
-    
-    def selected_powermarker_node_paths(self):
-        def get_path(dupe):
-            try:
-                dupeindex = self.results.dupes.index(dupe)
-                return [dupeindex]
-            except ValueError: # dupe not in there
-                return None
-        
-        dupes = self.selected_dupes
-        return stripnone(get_path(dupe) for dupe in dupes)
-    
-    def SelectResultNodePaths(self,node_paths):
-        def extract_dupe(t):
-            g,d = t
-            if d is not None:
-                return d
-            else:
-                if g is not None:
-                    return g.ref
-        
-        selected = [extract_dupe(self.GetObjects(p)) for p in node_paths]
-        self.selected_dupes = [dupe for dupe in selected if dupe is not None]
-    
-    def SelectPowerMarkerNodePaths(self,node_paths):
-        rows = [p[0] for p in node_paths]
-        self.selected_dupes = [
-            self.results.dupes[row] for row in rows if row in xrange(len(self.results.dupes))
-        ]
-    
-    def SetDirectoryState(self, node_path, state):
-        p = self.get_folder_path(node_path)
-        self.directories.set_state(p, state)
-    
-    def sort_dupes(self,key,asc):
-        self.results.sort_dupes(key,asc,self.display_delta_values)
-    
-    def sort_groups(self,key,asc):
-        self.results.sort_groups(key,asc)
-    
-    def ToggleSelectedMarkState(self):
-        for dupe in self.selected_dupes:
-            self.results.mark_toggle(dupe)
-    
-    #---Data
-    def GetOutlineViewMaxLevel(self, tag):
-        if tag == 0:
-            return 2
-        elif tag == 1:
-            return 0
-        elif tag == 2:
-            return 1
-    
-    def GetOutlineViewChildCounts(self, tag, node_path):
-        if self.progress._job_running:
-            return []
-        if tag == 0: #Normal results
-            assert not node_path # no other value is possible
-            return [len(g.dupes) for g in self.results.groups]
-        elif tag == 1: #Directories
-            try:
-                if node_path:
-                    path = self.get_folder_path(node_path)
-                    subfolders = self.directories.get_subfolders(path)
-                else:
-                    subfolders = self.directories
-                return [len(self.directories.get_subfolders(path)) for path in subfolders]
-            except IndexError: # node_path out of range
-                return []
-        else: #Power Marker
-            assert not node_path # no other value is possible
-            return [0 for d in self.results.dupes]
-    
-    def GetOutlineViewValues(self, tag, node_path):
-        if self.progress._job_running:
-            return
-        if not node_path:
-            return
-        if tag in (0,2): #Normal results / Power Marker
-            if tag == 0:
-                g, d = self.GetObjects(node_path)
-                if d is None:
-                    d = g.ref
-            else:
-                d = self.results.dupes[node_path[0]]
-                g = self.results.get_group_of_duplicate(d)
-            result = self._get_display_info(d, g, self.display_delta_values)
-            return result
-        elif tag == 1: #Directories
-            try:
-                path = self.get_folder_path(node_path)
-                name = unicode(path) if len(node_path) == 1 else path[-1]
-                return [name, self.directories.get_state(path)]
-            except IndexError: # node_path out of range
-                return []
-    
-    def GetOutlineViewMarked(self, tag, node_path):
-        # 0=unmarked 1=marked 2=unmarkable
-        if self.progress._job_running:
-            return
-        if not node_path:
-            return 2
-        if tag == 1: #Directories
-            return 2
-        if tag == 0: #Normal results
-            g, d = self.GetObjects(node_path)
-        else: #Power Marker
-            d = self.results.dupes[node_path[0]]
-        if (d is None) or (not self.results.is_markable(d)):
-            return 2
-        elif self.results.is_marked(d):
-            return 1
-        else:
-            return 0
-    
-    def GetTableViewCount(self, tag):
-        if self.progress._job_running:
-            return 0
-        return len(self.details_table)
-    
-    def GetTableViewMarkedIndexes(self,tag):
-        return []
-    
-    def GetTableViewValues(self,tag,row):
-        return self.details_table[row]
-    
-

base/py/data.py

@@ -1,42 +0,0 @@
-# Created By: Virgil Dupras
-# Created On: 2006/03/15
-# $Id$
-# Copyright 2009 Hardcoded Software (http://www.hardcoded.net)
-# 
-# This software is licensed under the "HS" License as described in the "LICENSE" file, 
-# which should be included with this package. The terms are also available at 
-# http://www.hardcoded.net/licenses/hs_license
-
-from hsutil.str import format_time, FT_DECIMAL, format_size
-
-import time
-
-def format_path(p):
-    return unicode(p[:-1])
-
-def format_timestamp(t, delta):
-    if delta:
-        return format_time(t, FT_DECIMAL)
-    else:
-        if t > 0:
-            return time.strftime('%Y/%m/%d %H:%M:%S', time.localtime(t))
-        else:
-            return '---'
-
-def format_words(w):
-    def do_format(w):
-        if isinstance(w, list):
-            return '(%s)' % ', '.join(do_format(item) for item in w)
-        else:
-            return w.replace('\n', ' ')
-    
-    return ', '.join(do_format(item) for item in w)
-
-def format_perc(p):
-    return "%0.0f" % p
-
-def format_dupe_count(c):
-    return str(c) if c else '---'
-
-def cmp_value(value):
-    return value.lower() if isinstance(value, basestring) else value

base/py/directories.py

@@ -1,173 +0,0 @@
-# Created By: Virgil Dupras
-# Created On: 2006/02/27
-# $Id$
-# Copyright 2009 Hardcoded Software (http://www.hardcoded.net)
-# 
-# This software is licensed under the "HS" License as described in the "LICENSE" file, 
-# which should be included with this package. The terms are also available at 
-# http://www.hardcoded.net/licenses/hs_license
-
-import xml.dom.minidom
-
-from hsutil import io
-from hsutil.files import FileOrPath
-from hsutil.path import Path
-
-from . import fs
-
-(STATE_NORMAL,
-STATE_REFERENCE,
-STATE_EXCLUDED) = range(3)
-
-class AlreadyThereError(Exception):
-    """The path being added is already in the directory list"""
-
-class InvalidPathError(Exception):
-    """The path being added is invalid"""
-
-class Directories(object):
-    #---Override
-    def __init__(self, fileclasses=[fs.File]):
-        self._dirs = []
-        self.states = {}
-        self.fileclasses = fileclasses
-    
-    def __contains__(self, path):
-        for p in self._dirs:
-            if path in p:
-                return True
-        return False
-    
-    def __delitem__(self,key):
-        self._dirs.__delitem__(key)
-    
-    def __getitem__(self,key):
-        return self._dirs.__getitem__(key)
-    
-    def __len__(self):
-        return len(self._dirs)
-    
-    #---Private
-    def _default_state_for_path(self, path):
-        # Override this in subclasses to specify the state of some special folders.
-        if path[-1].startswith('.'): # hidden
-            return STATE_EXCLUDED
-    
-    def _get_files(self, from_path):
-        state = self.get_state(from_path)
-        if state == STATE_EXCLUDED:
-            # Recursively get files from folders with lots of subfolder is expensive. However, there
-            # might be a subfolder in this path that is not excluded. What we want to do is to skim
-            # through self.states and see if we must continue, or we can stop right here to save time
-            if not any(p[:len(from_path)] == from_path for p in self.states):
-                return
-        try:
-            filepaths = set()
-            if state != STATE_EXCLUDED:
-                for file in fs.get_files(from_path, fileclasses=self.fileclasses):
-                    file.is_ref = state == STATE_REFERENCE
-                    filepaths.add(file.path)
-                    yield file
-            subpaths = [from_path + name for name in io.listdir(from_path)]
-            # it's possible that a folder (bundle) gets into the file list. in that case, we don't want to recurse into it
-            subfolders = [p for p in subpaths if not io.islink(p) and io.isdir(p) and p not in filepaths]
-            for subfolder in subfolders:
-                for file in self._get_files(subfolder):
-                    yield file
-        except (EnvironmentError, fs.InvalidPath):
-            pass
-    
-    #---Public
-    def add_path(self, path):
-        """Adds 'path' to self, if not already there.
-        
-        Raises AlreadyThereError if 'path' is already in self. If path is a directory containing
-        some of the directories already present in self, 'path' will be added, but all directories
-        under it will be removed. Can also raise InvalidPathError if 'path' does not exist.
-        """
-        if path in self:
-            raise AlreadyThereError()
-        if not io.exists(path):
-            raise InvalidPathError()
-        self._dirs = [p for p in self._dirs if p not in path]
-        self._dirs.append(path)
-    
-    @staticmethod
-    def get_subfolders(path):
-        """returns a sorted list of paths corresponding to subfolders in `path`"""
-        try:
-            names = [name for name in io.listdir(path) if io.isdir(path + name)]
-            names.sort(key=lambda x:x.lower())
-            return [path + name for name in names]
-        except EnvironmentError:
-            return []
-    
-    def get_files(self):
-        """Returns a list of all files that are not excluded.
-        
-        Returned files also have their 'is_ref' attr set.
-        """
-        for path in self._dirs:
-            for file in self._get_files(path):
-                yield file
-    
-    def get_state(self, path):
-        """Returns the state of 'path' (One of the STATE_* const.)
-        """
-        if path in self.states:
-            return self.states[path]
-        default_state = self._default_state_for_path(path)
-        if default_state is not None:
-            return default_state
-        parent = path[:-1]
-        if parent in self:
-            return self.get_state(parent)
-        else:
-            return STATE_NORMAL
-    
-    def load_from_file(self, infile):
-        try:
-            doc = xml.dom.minidom.parse(infile)
-        except:
-            return
-        root_path_nodes = doc.getElementsByTagName('root_directory')
-        for rdn in root_path_nodes:
-            if not rdn.getAttributeNode('path'):
-                continue
-            path = rdn.getAttributeNode('path').nodeValue
-            try:
-                self.add_path(Path(path))
-            except (AlreadyThereError, InvalidPathError):
-                pass
-        state_nodes = doc.getElementsByTagName('state')
-        for sn in state_nodes:
-            if not (sn.getAttributeNode('path') and sn.getAttributeNode('value')):
-                continue
-            path = sn.getAttributeNode('path').nodeValue
-            state = sn.getAttributeNode('value').nodeValue
-            self.set_state(Path(path), int(state))
-    
-    def save_to_file(self,outfile):
-        with FileOrPath(outfile, 'wb') as fp:
-            doc = xml.dom.minidom.Document()
-            root = doc.appendChild(doc.createElement('directories'))
-            for root_path in self:
-                root_path_node = root.appendChild(doc.createElement('root_directory'))
-                root_path_node.setAttribute('path', unicode(root_path).encode('utf-8'))
-            for path, state in self.states.iteritems():
-                state_node = root.appendChild(doc.createElement('state'))
-                state_node.setAttribute('path', unicode(path).encode('utf-8'))
-                state_node.setAttribute('value', str(state))
-            doc.writexml(fp, '\t', '\t', '\n', encoding='utf-8')
-    
-    def set_state(self, path, state):
-        if self.get_state(path) == state:
-            return
-        # we don't want to needlessly fill self.states. if get_state returns the same thing
-        # without an explicit entry, remove that entry
-        if path in self.states:
-            del self.states[path]
-            if self.get_state(path) == state: # no need for an entry
-                return
-        self.states[path] = state
-    

base/py/engine.py

@@ -1,387 +0,0 @@
-# Created By: Virgil Dupras
-# Created On: 2006/01/29
-# $Id$
-# Copyright 2009 Hardcoded Software (http://www.hardcoded.net)
-# 
-# This software is licensed under the "HS" License as described in the "LICENSE" file, 
-# which should be included with this package. The terms are also available at 
-# http://www.hardcoded.net/licenses/hs_license
-
-from __future__ import division
-import difflib
-import itertools
-import logging
-import string
-from collections import defaultdict, namedtuple
-from unicodedata import normalize
-
-from hsutil.misc import flatten
-from hsutil.str import multi_replace
-from hsutil import job
-
-(WEIGHT_WORDS,
-MATCH_SIMILAR_WORDS,
-NO_FIELD_ORDER) = range(3)
-
-JOB_REFRESH_RATE = 100
-
-def getwords(s):
-    if isinstance(s, unicode):
-        s = normalize('NFD', s)
-    s = multi_replace(s, "-_&+():;\\[]{}.,<>/?~!@#$*", ' ').lower()
-    s = ''.join(c for c in s if c in string.ascii_letters + string.digits + string.whitespace)
-    return filter(None, s.split(' ')) # filter() is to remove empty elements
-
-def getfields(s):
-    fields = [getwords(field) for field in s.split(' - ')]
-    return filter(None, fields)
-
-def unpack_fields(fields):
-    result = []
-    for field in fields:
-        if isinstance(field, list):
-            result += field
-        else:
-            result.append(field)
-    return result
-
-def compare(first, second, flags=()):
-    """Returns the % of words that match between first and second
-    
-    The result is a int in the range 0..100.
-    First and second can be either a string or a list.
-    """
-    if not (first and second):
-        return 0
-    if any(isinstance(element, list) for element in first):
-        return compare_fields(first, second, flags)
-    second = second[:] #We must use a copy of second because we remove items from it        
-    match_similar = MATCH_SIMILAR_WORDS in flags
-    weight_words = WEIGHT_WORDS in flags
-    joined = first + second
-    total_count = (sum(len(word) for word in joined) if weight_words else len(joined))
-    match_count = 0
-    in_order = True
-    for word in first:
-        if match_similar and (word not in second):
-            similar = difflib.get_close_matches(word, second, 1, 0.8)
-            if similar:
-                word = similar[0]
-        if word in second:
-            if second[0] != word:
-                in_order = False
-            second.remove(word)
-            match_count += (len(word) if weight_words else 1)
-    result = round(((match_count * 2) / total_count) * 100)
-    if (result == 100) and (not in_order):
-        result = 99 # We cannot consider a match exact unless the ordering is the same
-    return result
-
-def compare_fields(first, second, flags=()):
-    """Returns the score for the lowest matching fields.
-    
-    first and second must be lists of lists of string.
-    """
-    if len(first) != len(second):
-        return 0
-    if NO_FIELD_ORDER in flags:
-        results = []
-        #We don't want to remove field directly in the list. We must work on a copy.
-        second = second[:]
-        for field1 in first:
-            max = 0
-            matched_field = None
-            for field2 in second:
-                r = compare(field1, field2, flags)
-                if r > max:
-                    max = r
-                    matched_field = field2
-            results.append(max)
-            if matched_field:
-                second.remove(matched_field)
-    else:
-        results = [compare(word1, word2, flags) for word1, word2 in zip(first, second)]
-    return min(results) if results else 0
-
-def build_word_dict(objects, j=job.nulljob):
-    """Returns a dict of objects mapped by their words.
-    
-    objects must have a 'words' attribute being a list of strings or a list of lists of strings.
-    
-    The result will be a dict with words as keys, lists of objects as values.
-    """
-    result = defaultdict(set)
-    for object in j.iter_with_progress(objects, 'Prepared %d/%d files', JOB_REFRESH_RATE):
-        for word in unpack_fields(object.words):
-            result[word].add(object)
-    return result
-
-def merge_similar_words(word_dict):
-    """Take all keys in word_dict that are similar, and merge them together.
-    """
-    keys = word_dict.keys()
-    keys.sort(key=len)# we want the shortest word to stay
-    while keys:
-        key = keys.pop(0)
-        similars = difflib.get_close_matches(key, keys, 100, 0.8)
-        if not similars:
-            continue
-        objects = word_dict[key]
-        for similar in similars:
-            objects |= word_dict[similar]
-            del word_dict[similar]
-            keys.remove(similar)
-
-def reduce_common_words(word_dict, threshold):
-    """Remove all objects from word_dict values where the object count >= threshold
-    
-    The exception to this removal are the objects where all the words of the object are common.
-    Because if we remove them, we will miss some duplicates!
-    """
-    uncommon_words = set(word for word, objects in word_dict.items() if len(objects) < threshold)
-    for word, objects in word_dict.items():
-        if len(objects) < threshold:
-            continue
-        reduced = set()
-        for o in objects:
-            if not any(w in uncommon_words for w in unpack_fields(o.words)):
-                reduced.add(o)
-        if reduced:
-            word_dict[word] = reduced
-        else:
-            del word_dict[word]
-
-Match = namedtuple('Match', 'first second percentage')
-def get_match(first, second, flags=()):
-    #it is assumed here that first and second both have a "words" attribute
-    percentage = compare(first.words, second.words, flags)
-    return Match(first, second, percentage)
-
-def getmatches(objects, min_match_percentage=0, match_similar_words=False, weight_words=False, 
-    no_field_order=False, j=job.nulljob):
-    COMMON_WORD_THRESHOLD = 50
-    LIMIT = 5000000
-    j = j.start_subjob(2)
-    sj = j.start_subjob(2)
-    for o in objects:
-        if not hasattr(o, 'words'):
-            o.words = getwords(o.name)
-    word_dict = build_word_dict(objects, sj)
-    reduce_common_words(word_dict, COMMON_WORD_THRESHOLD)
-    if match_similar_words:
-        merge_similar_words(word_dict)
-    match_flags = []
-    if weight_words:
-        match_flags.append(WEIGHT_WORDS)
-    if match_similar_words:
-        match_flags.append(MATCH_SIMILAR_WORDS)
-    if no_field_order:
-        match_flags.append(NO_FIELD_ORDER)
-    j.start_job(len(word_dict), '0 matches found')
-    compared = defaultdict(set)
-    result = []
-    try:
-        # This whole 'popping' thing is there to avoid taking too much memory at the same time.
-        while word_dict:
-            items = word_dict.popitem()[1]
-            while items:
-                ref = items.pop()
-                compared_already = compared[ref]
-                to_compare = items - compared_already
-                compared_already |= to_compare
-                for other in to_compare:
-                    m = get_match(ref, other, match_flags)
-                    if m.percentage >= min_match_percentage:
-                        result.append(m)
-                        if len(result) >= LIMIT:
-                            return result
-            j.add_progress(desc='%d matches found' % len(result))
-    except MemoryError:
-        # This is the place where the memory usage is at its peak during the scan.
-        # Just continue the process with an incomplete list of matches.
-        del compared # This should give us enough room to call logging.
-        logging.warning('Memory Overflow. Matches: %d. Word dict: %d' % (len(result), len(word_dict)))
-        return result
-    return result
-
-def getmatches_by_contents(files, sizeattr='size', partial=False, j=job.nulljob):
-    j = j.start_subjob([2, 8])
-    size2files = defaultdict(set)
-    for file in j.iter_with_progress(files, 'Read size of %d/%d files'):
-        size2files[getattr(file, sizeattr)].add(file)
-    possible_matches = [files for files in size2files.values() if len(files) > 1]
-    del size2files
-    result = []
-    j.start_job(len(possible_matches), '0 matches found')
-    for group in possible_matches:
-        for first, second in itertools.combinations(group, 2):
-            if first.md5partial == second.md5partial:
-                if partial or first.md5 == second.md5:
-                    result.append(Match(first, second, 100))
-        j.add_progress(desc='%d matches found' % len(result))
-    return result
-
-class Group(object):
-    #---Override
-    def __init__(self):
-        self._clear()
-    
-    def __contains__(self, item):
-        return item in self.unordered
-    
-    def __getitem__(self, key):
-        return self.ordered.__getitem__(key)
-    
-    def __iter__(self):
-        return iter(self.ordered)
-    
-    def __len__(self):
-        return len(self.ordered)
-    
-    #---Private
-    def _clear(self):
-        self._percentage = None
-        self._matches_for_ref = None
-        self.matches = set()
-        self.candidates = defaultdict(set)
-        self.ordered = []
-        self.unordered = set()
-    
-    def _get_matches_for_ref(self):
-        if self._matches_for_ref is None:
-            ref = self.ref
-            self._matches_for_ref = [match for match in self.matches if ref in match]
-        return self._matches_for_ref
-    
-    #---Public
-    def add_match(self, match):
-        def add_candidate(item, match):
-            matches = self.candidates[item]
-            matches.add(match)
-            if self.unordered <= matches:
-                self.ordered.append(item)
-                self.unordered.add(item)
-        
-        if match in self.matches:
-            return
-        self.matches.add(match)
-        first, second, _ = match
-        if first not in self.unordered:
-            add_candidate(first, second)
-        if second not in self.unordered:
-            add_candidate(second, first)
-        self._percentage = None
-        self._matches_for_ref = None
-    
-    def discard_matches(self):
-        discarded = set(m for m in self.matches if not all(obj in self.unordered for obj in [m.first, m.second]))
-        self.matches -= discarded
-        self.candidates = defaultdict(set)
-        return discarded
-    
-    def get_match_of(self, item):
-        if item is self.ref:
-            return
-        for m in self._get_matches_for_ref():
-            if item in m:
-                return m
-    
-    def prioritize(self, key_func, tie_breaker=None):
-        # tie_breaker(ref, dupe) --> True if dupe should be ref
-        self.ordered.sort(key=key_func)
-        if tie_breaker is None:
-            return
-        ref = self.ref
-        key_value = key_func(ref)
-        for dupe in self.dupes:
-            if key_func(dupe) != key_value:
-                break
-            if tie_breaker(ref, dupe):
-                ref = dupe
-        if ref is not self.ref:
-            self.switch_ref(ref)
-    
-    def remove_dupe(self, item, discard_matches=True):
-        try:
-            self.ordered.remove(item)
-            self.unordered.remove(item)
-            self._percentage = None
-            self._matches_for_ref = None
-            if (len(self) > 1) and any(not getattr(item, 'is_ref', False) for item in self):
-                if discard_matches:
-                    self.matches = set(m for m in self.matches if item not in m)
-            else:
-                self._clear()
-        except ValueError:
-            pass
-    
-    def switch_ref(self, with_dupe):
-        try:
-            self.ordered.remove(with_dupe)
-            self.ordered.insert(0, with_dupe)
-            self._percentage = None
-            self._matches_for_ref = None
-        except ValueError:
-            pass
-    
-    dupes = property(lambda self: self[1:])
-    
-    @property
-    def percentage(self):
-        if self._percentage is None:
-            if self.dupes:
-                matches = self._get_matches_for_ref()
-                self._percentage = sum(match.percentage for match in matches) // len(matches)
-            else:
-                self._percentage = 0
-        return self._percentage
-    
-    @property
-    def ref(self):
-        if self:
-            return self[0]
-    
-
-def get_groups(matches, j=job.nulljob):
-    matches.sort(key=lambda match: -match.percentage)
-    dupe2group = {}
-    groups = []
-    try:
-        for match in j.iter_with_progress(matches, 'Grouped %d/%d matches', JOB_REFRESH_RATE):
-            first, second, _ = match
-            first_group = dupe2group.get(first)
-            second_group = dupe2group.get(second)
-            if first_group:
-                if second_group:
-                    if first_group is second_group:
-                        target_group = first_group
-                    else:
-                        continue
-                else:
-                    target_group = first_group
-                    dupe2group[second] = target_group
-            else:
-                if second_group:
-                    target_group = second_group
-                    dupe2group[first] = target_group
-                else:
-                    target_group = Group()
-                    groups.append(target_group)
-                    dupe2group[first] = target_group
-                    dupe2group[second] = target_group
-            target_group.add_match(match)
-    except MemoryError:
-        del dupe2group
-        del matches
-        # should free enough memory to continue
-        logging.warning('Memory Overflow. Groups: {0}'.format(len(groups)))
-    # Now that we have a group, we have to discard groups' matches and see if there're any "orphan"
-    # matches, that is, matches that were candidate in a group but that none of their 2 files were
-    # accepted in the group. With these orphan groups, it's safe to build additional groups
-    matched_files = set(flatten(groups))
-    orphan_matches = []
-    for group in groups:
-        orphan_matches += set(m for m in group.discard_matches() if not any(obj in matched_files for obj in [m.first, m.second]))
-    if groups and orphan_matches:
-        groups += get_groups(orphan_matches) # no job, as it isn't supposed to take a long time
-    return groups

base/py/export.py

@@ -1,138 +0,0 @@
-# Created By: Virgil Dupras
-# Created On: 2006/09/16
-# $Id$
-# Copyright 2009 Hardcoded Software (http://www.hardcoded.net)
-# 
-# This software is licensed under the "HS" License as described in the "LICENSE" file, 
-# which should be included with this package. The terms are also available at 
-# http://www.hardcoded.net/licenses/hs_license
-
-import tempfile
-import os.path as op
-from tempfile import mkdtemp
-
-# Yes, this is a very low-tech solution, but at least it doesn't have all these annoying dependency
-# and resource problems.
-
-MAIN_TEMPLATE = u"""
-<?xml version="1.0" encoding="utf-8"?>
-<!DOCTYPE html PUBLIC '-//W3C//DTD XHTML 1.0 Strict//EN' 'http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd'>
-<html xmlns="http://www.w3.org/1999/xhtml">
-<head>
-    <meta content="text/html; charset=utf-8" http-equiv="Content-Type"/>
-	<title>dupeGuru Results</title>
-	<style type="text/css">
-BODY
-{
-	background-color:white;
-}
-
-BODY,A,P,UL,TABLE,TR,TD
-{
-	font-family:Tahoma,Arial,sans-serif;
-	font-size:10pt;
-	color: #4477AA;
-}
-
-TABLE
-{
-	background-color: #225588;
-	margin-left: auto;
-  	margin-right: auto;
-	width: 90%;
-}
-
-TR 
-{
-    background-color: white;
-}
-
-TH 
-{ 
-	font-weight: bold; 
-	color: black;
-	background-color: #C8D6E5;
-}
-
-TH TD 
-{
-    color:black;
-}
-
-TD 
-{
-    padding-left: 2pt;
-}
-
-TD.rightelem
-{
-	text-align:right;
-	/*padding-left:0pt;*/
-	padding-right: 2pt;
-	width: 17%;
-}
-
-TD.indented
-{
-    padding-left: 12pt;
-}
-
-H1
-{
-	font-family:&quot;Courier New&quot;,monospace;
-	color:#6699CC;
-    font-size:18pt; 
-	color:#6da500;
-	border-color: #70A0CF;
-	border-width: 1pt;
-	border-style: solid;
-	margin-top:   16pt;
-	margin-left:  5%;
-	margin-right: 5%;
-	padding-top:  2pt;
-	padding-bottom:2pt;
-	text-align:   center;
-}
-</style>
-</head>
-<body>
-<h1>dupeGuru Results</h1>
-<table>
-<tr>$colheaders</tr>
-$rows
-</table>
-</body>
-</html>
-"""
-
-COLHEADERS_TEMPLATE = u"<th>{name}</th>"
-
-ROW_TEMPLATE = u"""
-<tr>
-    <td class="{indented}">{filename}</td>{cells}
-</tr>
-"""
-
-CELL_TEMPLATE = u"""<td>{value}</td>"""
-
-def export_to_xhtml(colnames, rows):
-    # a row is a list of values with the first value being a flag indicating if the row should be indented
-    if rows:
-        assert len(rows[0]) == len(colnames) + 1 # + 1 is for the "indented" flag
-    colheaders = u''.join(COLHEADERS_TEMPLATE.format(name=name) for name in colnames)
-    rendered_rows = []
-    for row in rows:
-        # [2:] is to remove the indented flag + filename
-        indented = u'indented' if row[0] else u''
-        filename = row[1]
-        cells = u''.join(CELL_TEMPLATE.format(value=value) for value in row[2:])
-        rendered_rows.append(ROW_TEMPLATE.format(indented=indented, filename=filename, cells=cells))
-    rendered_rows = u''.join(rendered_rows)
-    # The main template can't use format because the css code uses {}
-    content = MAIN_TEMPLATE.replace('$colheaders', colheaders).replace('$rows', rendered_rows)
-    folder = mkdtemp()
-    destpath = op.join(folder, u'export.htm')
-    fp = open(destpath, 'w')
-    fp.write(content.encode('utf-8'))
-    fp.close()
-    return destpath

base/py/fs.py

@@ -1,178 +0,0 @@
-# -*- coding: utf-8 -*-
-# Created By: Virgil Dupras
-# Created On: 2009-10-22
-# $Id$
-# Copyright 2009 Hardcoded Software (http://www.hardcoded.net)
-# 
-# This software is licensed under the "HS" License as described in the "LICENSE" file, 
-# which should be included with this package. The terms are also available at 
-# http://www.hardcoded.net/licenses/hs_license
-
-# This is a fork from hsfs. The reason for this fork is that hsfs has been designed for musicGuru
-# and was re-used for dupeGuru. The problem is that hsfs is way over-engineered for dupeGuru,
-# resulting needless complexity and memory usage. It's been a while since I wanted to do that fork,
-# and I'm doing it now.
-
-from __future__ import unicode_literals
-
-import hashlib
-import logging
-
-from hsutil import io
-from hsutil.misc import nonone, flatten
-from hsutil.str import get_file_ext
-
-class FSError(Exception):
-    cls_message = "An error has occured on '{name}' in '{parent}'"
-    def __init__(self, fsobject, parent=None):
-        message = self.cls_message
-        if isinstance(fsobject, basestring):
-            name = fsobject
-        elif isinstance(fsobject, File):
-            name = fsobject.name
-        else:
-            name = ''
-        parentname = unicode(parent) if parent is not None else ''
-        Exception.__init__(self, message.format(name=name, parent=parentname))
-    
-
-class AlreadyExistsError(FSError):
-    "The directory or file name we're trying to add already exists"
-    cls_message = "'{name}' already exists in '{parent}'"
-
-class InvalidPath(FSError):
-    "The path of self is invalid, and cannot be worked with."
-    cls_message = "'{name}' is invalid."
-
-class InvalidDestinationError(FSError):
-    """A copy/move operation has been called, but the destination is invalid."""
-    cls_message = "'{name}' is an invalid destination for this operation."
-
-class OperationError(FSError):
-    """A copy/move/delete operation has been called, but the checkup after the 
-    operation shows that it didn't work."""
-    cls_message = "Operation on '{name}' failed."
-
-class File(object):
-    INITIAL_INFO = {
-        'size': 0,
-        'ctime': 0,
-        'mtime': 0,
-        'md5': '',
-        'md5partial': '',
-    }
-    
-    def __init__(self, path):
-        self.path = path
-        #This offset is where we should start reading the file to get a partial md5
-        #For audio file, it should be where audio data starts
-        self._md5partial_offset = 0x4000 #16Kb
-        self._md5partial_size   = 0x4000 #16Kb
-    
-    def __getattr__(self, attrname):
-        # Only called when attr is not there
-        if attrname in self.INITIAL_INFO:
-            try:
-                self._read_info(attrname)
-            except Exception as e:
-                logging.warning("An error '%s' was raised while decoding '%s'", e, repr(self.path))
-            try:
-                return self.__dict__[attrname]
-            except KeyError:
-                return self.INITIAL_INFO[attrname]
-        raise AttributeError()
-    
-    def _read_info(self, field):
-        if field in ('size', 'ctime', 'mtime'):
-            stats = io.stat(self.path)
-            self.size = nonone(stats.st_size, 0)
-            self.ctime = nonone(stats.st_ctime, 0)
-            self.mtime = nonone(stats.st_mtime, 0)
-        elif field == 'md5partial':
-            try:
-                fp = io.open(self.path, 'rb')
-                offset = self._md5partial_offset
-                size = self._md5partial_size
-                fp.seek(offset)
-                partialdata = fp.read(size)
-                md5 = hashlib.md5(partialdata)
-                self.md5partial = md5.digest()
-                fp.close()
-            except Exception:
-                pass
-        elif field == 'md5':
-            try:
-                fp = io.open(self.path, 'rb')
-                filedata = fp.read()
-                md5 = hashlib.md5(filedata)
-                self.md5 = md5.digest()
-                fp.close()
-            except Exception:
-                pass
-    
-    def _read_all_info(self, attrnames=None):
-        """Cache all possible info.
-        
-        If `attrnames` is not None, caches only attrnames.
-        """
-        if attrnames is None:
-            attrnames = self.INITIAL_INFO.keys()
-        for attrname in attrnames:
-            if attrname not in self.__dict__:
-                self._read_info(attrname)
-    
-    #--- Public
-    @classmethod
-    def can_handle(cls, path):
-        return not io.islink(path) and io.isfile(path)
-    
-    def rename(self, newname):
-        if newname == self.name:
-            return
-        destpath = self.path[:-1] + newname
-        if io.exists(destpath):
-            raise AlreadyExistsError(newname, self.path[:-1])
-        try:
-            io.rename(self.path, destpath)
-        except EnvironmentError:
-            raise OperationError(self)
-        if not io.exists(destpath):
-            raise OperationError(self)
-        self.path = destpath
-    
-    #--- Properties
-    @property
-    def extension(self):
-        return get_file_ext(self.name)
-    
-    @property
-    def name(self):
-        return self.path[-1]
-    
-
-def get_file(path, fileclasses=[File]):
-    for fileclass in fileclasses:
-        if fileclass.can_handle(path):
-            return fileclass(path)
-
-def get_files(path, fileclasses=[File]):
-    assert all(issubclass(fileclass, File) for fileclass in fileclasses)
-    try:
-        paths = [path + name for name in io.listdir(path)]
-        result = []
-        for path in paths:
-            file = get_file(path, fileclasses=fileclasses)
-            if file is not None:
-                result.append(file)
-        return result
-    except EnvironmentError:
-        raise InvalidPath(path)
-
-def get_all_files(path, fileclasses=[File]):
-    files = get_files(path, fileclasses=fileclasses)
-    filepaths = set(f.path for f in files)
-    subpaths = [path + name for name in io.listdir(path)]
-    # it's possible that a folder (bundle) gets into the file list. in that case, we don't want to recurse into it
-    subfolders = [p for p in subpaths if not io.islink(p) and io.isdir(p) and p not in filepaths]
-    subfiles = flatten(get_all_files(subpath, fileclasses=fileclasses) for subpath in subfolders)
-    return subfiles + files

base/py/ignore.py

@@ -1,116 +0,0 @@
-# Created By: Virgil Dupras
-# Created On: 2006/05/02
-# $Id$
-# Copyright 2009 Hardcoded Software (http://www.hardcoded.net)
-# 
-# This software is licensed under the "HS" License as described in the "LICENSE" file, 
-# which should be included with this package. The terms are also available at 
-# http://www.hardcoded.net/licenses/hs_license
-
-from hsutil.files import FileOrPath
-
-import xml.dom.minidom
-
-class IgnoreList(object):
-    """An ignore list implementation that is iterable, filterable and exportable to XML.
-    
-    Call Ignore to add an ignore list entry, and AreIgnore to check if 2 items are in the list.
-    When iterated, 2 sized tuples will be returned, the tuples containing 2 items ignored together.
-    """
-    #---Override
-    def __init__(self):
-        self._ignored = {}
-        self._count = 0
-    
-    def __iter__(self):
-        for first,seconds in self._ignored.iteritems():
-            for second in seconds:
-                yield (first,second)
-    
-    def __len__(self):
-        return self._count
-    
-    #---Public
-    def AreIgnored(self,first,second):
-        def do_check(first,second):
-            try:
-                matches = self._ignored[first]
-                return second in matches
-            except KeyError:
-                return False
-        
-        return do_check(first,second) or do_check(second,first)
-    
-    def Clear(self):
-        self._ignored = {}
-        self._count = 0
-    
-    def Filter(self,func):
-        """Applies a filter on all ignored items, and remove all matches where func(first,second)
-        doesn't return True.
-        """
-        filtered = IgnoreList()
-        for first,second in self:
-            if func(first,second):
-                filtered.Ignore(first,second)
-        self._ignored = filtered._ignored
-        self._count = filtered._count
-    
-    def Ignore(self,first,second):
-        if self.AreIgnored(first,second):
-            return
-        try:
-            matches = self._ignored[first]
-            matches.add(second)
-        except KeyError:
-            try:
-                matches = self._ignored[second]
-                matches.add(first)
-            except KeyError:
-                matches = set()
-                matches.add(second)
-                self._ignored[first] = matches
-        self._count += 1
-    
-    def load_from_xml(self,infile):
-        """Loads the ignore list from a XML created with save_to_xml.
-        
-        infile can be a file object or a filename.
-        """
-        try:
-            doc = xml.dom.minidom.parse(infile)
-        except Exception:
-            return
-        file_nodes = doc.getElementsByTagName('file')
-        for fn in file_nodes:
-            if not fn.getAttributeNode('path'):
-                continue
-            file_path = fn.getAttributeNode('path').nodeValue
-            subfile_nodes = fn.getElementsByTagName('file')
-            for sfn in subfile_nodes:
-                if not sfn.getAttributeNode('path'):
-                    continue
-                subfile_path = sfn.getAttributeNode('path').nodeValue
-                self.Ignore(file_path,subfile_path)
-    
-    def save_to_xml(self,outfile):
-        """Create a XML file that can be used by load_from_xml.
-        
-        outfile can be a file object or a filename.
-        """
-        doc = xml.dom.minidom.Document()
-        root = doc.appendChild(doc.createElement('ignore_list'))
-        for file,subfiles in self._ignored.items():
-            file_node = root.appendChild(doc.createElement('file'))
-            if isinstance(file,unicode):
-                file = file.encode('utf-8')
-            file_node.setAttribute('path',file)
-            for subfile in subfiles:
-                subfile_node = file_node.appendChild(doc.createElement('file'))
-                if isinstance(subfile,unicode):
-                    subfile = subfile.encode('utf-8')
-                subfile_node.setAttribute('path',subfile)
-        with FileOrPath(outfile, 'wb') as fp:
-            doc.writexml(fp,'\t','\t','\n',encoding='utf-8')
-    
-

base/py/results.py

@@ -1,370 +0,0 @@
-# Created By: Virgil Dupras
-# Created On: 2006/02/23
-# $Id$
-# Copyright 2009 Hardcoded Software (http://www.hardcoded.net)
-# 
-# This software is licensed under the "HS" License as described in the "LICENSE" file, 
-# which should be included with this package. The terms are also available at 
-# http://www.hardcoded.net/licenses/hs_license
-
-import logging
-import re
-from xml.sax import handler, make_parser, SAXException
-from xml.sax.saxutils import XMLGenerator
-from xml.sax.xmlreader import AttributesImpl
-
-from . import engine
-from hsutil.job import nulljob
-from hsutil.markable import Markable
-from hsutil.misc import flatten, cond, nonone
-from hsutil.str import format_size
-from hsutil.files import open_if_filename
-
-class Results(Markable):
-    #---Override
-    def __init__(self, data_module):
-        super(Results, self).__init__()
-        self.__groups = []
-        self.__group_of_duplicate = {}
-        self.__groups_sort_descriptor = None # This is a tuple (key, asc)
-        self.__dupes = None
-        self.__dupes_sort_descriptor = None # This is a tuple (key, asc, delta)
-        self.__filters = None
-        self.__filtered_dupes = None
-        self.__filtered_groups = None
-        self.__recalculate_stats()
-        self.__marked_size = 0
-        self.data = data_module
-    
-    def _did_mark(self, dupe):
-        self.__marked_size += dupe.size
-    
-    def _did_unmark(self, dupe):
-        self.__marked_size -= dupe.size
-    
-    def _get_markable_count(self):
-        return self.__total_count
-    
-    def _is_markable(self, dupe):
-        if dupe.is_ref:
-            return False
-        g = self.get_group_of_duplicate(dupe)
-        if not g:
-            return False
-        if dupe is g.ref:
-            return False
-        if self.__filtered_dupes and dupe not in self.__filtered_dupes:
-            return False
-        return True
-    
-    #---Private
-    def __get_dupe_list(self):
-        if self.__dupes is None:
-            self.__dupes = flatten(group.dupes for group in self.groups)
-            if None in self.__dupes:
-                # This is debug logging to try to figure out #44
-                logging.warning("There is a None value in the Results' dupe list. dupes: %r groups: %r", self.__dupes, self.groups)
-            if self.__filtered_dupes:
-                self.__dupes = [dupe for dupe in self.__dupes if dupe in self.__filtered_dupes]
-            sd = self.__dupes_sort_descriptor
-            if sd:
-                self.sort_dupes(sd[0], sd[1], sd[2])
-        return self.__dupes
-    
-    def __get_groups(self):
-        if self.__filtered_groups is None:
-            return self.__groups
-        else:
-            return self.__filtered_groups
-    
-    def __get_stat_line(self):
-        if self.__filtered_dupes is None:
-            mark_count = self.mark_count
-            marked_size = self.__marked_size
-            total_count = self.__total_count
-            total_size = self.__total_size
-        else:
-            mark_count = len([dupe for dupe in self.__filtered_dupes if self.is_marked(dupe)])
-            marked_size = sum(dupe.size for dupe in self.__filtered_dupes if self.is_marked(dupe))
-            total_count = len([dupe for dupe in self.__filtered_dupes if self.is_markable(dupe)])
-            total_size = sum(dupe.size for dupe in self.__filtered_dupes if self.is_markable(dupe))
-        if self.mark_inverted:
-            marked_size = self.__total_size - marked_size
-        result = '%d / %d (%s / %s) duplicates marked.' % (
-            mark_count,
-            total_count,
-            format_size(marked_size, 2),
-            format_size(total_size, 2),
-        )
-        if self.__filters:
-            result += ' filter: %s' % ' --> '.join(self.__filters)
-        return result
-    
-    def __recalculate_stats(self):
-        self.__total_size = 0
-        self.__total_count = 0
-        for group in self.groups:
-            markable = [dupe for dupe in group.dupes if self._is_markable(dupe)]
-            self.__total_count += len(markable)
-            self.__total_size += sum(dupe.size for dupe in markable)
-    
-    def __set_groups(self, new_groups):
-        self.mark_none()
-        self.__groups = new_groups
-        self.__group_of_duplicate = {}
-        for g in self.__groups:
-            for dupe in g:
-                self.__group_of_duplicate[dupe] = g
-                if not hasattr(dupe, 'is_ref'):
-                    dupe.is_ref = False
-        old_filters = nonone(self.__filters, [])
-        self.apply_filter(None)
-        for filter_str in old_filters:
-            self.apply_filter(filter_str)
-    
-    #---Public
-    def apply_filter(self, filter_str):
-        ''' Applies a filter 'filter_str' to self.groups
-        
-            When you apply the filter, only  dupes with the filename matching 'filter_str' will be in
-            in the results. To cancel the filter, just call apply_filter with 'filter_str' to None, 
-            and the results will go back to normal.
-            
-            If call apply_filter on a filtered results, the filter will be applied 
-            *on the filtered results*.
-            
-            'filter_str' is a string containing a regexp to filter dupes with.
-        '''
-        if not filter_str:
-            self.__filtered_dupes = None
-            self.__filtered_groups = None
-            self.__filters = None
-        else:
-            if not self.__filters:
-                self.__filters = []
-            try:
-                filter_re = re.compile(filter_str, re.IGNORECASE)
-            except re.error:
-                return # don't apply this filter.
-            self.__filters.append(filter_str)
-            if self.__filtered_dupes is None:
-                self.__filtered_dupes = flatten(g[:] for g in self.groups)
-            self.__filtered_dupes = set(dupe for dupe in self.__filtered_dupes if filter_re.search(dupe.name))
-            filtered_groups = set()
-            for dupe in self.__filtered_dupes:
-                filtered_groups.add(self.get_group_of_duplicate(dupe))
-            self.__filtered_groups = list(filtered_groups)
-        self.__recalculate_stats()
-        sd = self.__groups_sort_descriptor
-        if sd:
-            self.sort_groups(sd[0], sd[1])
-        self.__dupes = None
-    
-    def get_group_of_duplicate(self, dupe):
-        try:
-            return self.__group_of_duplicate[dupe]
-        except (TypeError, KeyError):
-            return None
-    
-    is_markable = _is_markable
-    
-    def load_from_xml(self, infile, get_file, j=nulljob):
-        self.apply_filter(None)
-        handler = _ResultsHandler(get_file)
-        try:
-            parser = make_parser()
-        except Exception as e:
-            # This special handling is to try to figure out the cause of #47
-            # We don't silently return, because we want the user to send error report.
-            logging.exception(e)
-            try:
-                import xml.parsers.expat
-                logging.warning('importing xml.parsers.expat went ok, WTF?')
-            except Exception as e:
-                # This log should give a little more details about the cause of this all
-                logging.exception(e)
-                raise
-            raise
-        parser.setContentHandler(handler)
-        try:
-            infile, must_close = open_if_filename(infile)
-        except IOError:
-            return
-        BUFSIZE = 1024 * 1024 # 1mb buffer
-        infile.seek(0, 2)
-        j.start_job(infile.tell() // BUFSIZE)
-        infile.seek(0, 0)
-        try:
-            while True:
-                data = infile.read(BUFSIZE)
-                if not data:
-                    break
-                parser.feed(data)
-                j.add_progress()
-        except SAXException:
-            return
-        self.groups = handler.groups
-        for dupe_file in handler.marked:
-            self.mark(dupe_file)
-    
-    def make_ref(self, dupe):
-        g = self.get_group_of_duplicate(dupe)
-        r = g.ref
-        self._remove_mark_flag(dupe)
-        g.switch_ref(dupe);
-        if not r.is_ref:
-            self.__total_count += 1
-            self.__total_size += r.size
-        if not dupe.is_ref:
-            self.__total_count -= 1
-            self.__total_size -= dupe.size
-        self.__dupes = None
-    
-    def perform_on_marked(self, func, remove_from_results):
-        problems = []
-        for d in self.dupes:
-            if self.is_marked(d) and (not func(d)):
-                problems.append(d)
-        if remove_from_results:
-            to_remove = [d for d in self.dupes if self.is_marked(d) and (d not in problems)]
-            self.remove_duplicates(to_remove)
-            self.mark_none()
-            for d in problems:
-                self.mark(d)
-        return len(problems)
-    
-    def remove_duplicates(self, dupes):
-        '''Remove 'dupes' from their respective group, and remove the group is it ends up empty.
-        '''
-        affected_groups = set()
-        for dupe in dupes:
-            group = self.get_group_of_duplicate(dupe)
-            if dupe not in group.dupes:
-                return
-            group.remove_dupe(dupe, False)
-            self._remove_mark_flag(dupe)
-            self.__total_count -= 1
-            self.__total_size -= dupe.size
-            if not group:
-                self.__groups.remove(group)
-                if self.__filtered_groups:
-                    self.__filtered_groups.remove(group)
-            else:
-                affected_groups.add(group)
-        for group in affected_groups:
-            group.discard_matches()
-        self.__dupes = None
-    
-    def save_to_xml(self, outfile):
-        self.apply_filter(None)
-        outfile, must_close = open_if_filename(outfile, 'wb')
-        writer = XMLGenerator(outfile, 'utf-8')
-        writer.startDocument()
-        empty_attrs = AttributesImpl({})
-        writer.startElement('results', empty_attrs)
-        for g in self.groups:
-            writer.startElement('group', empty_attrs)
-            dupe2index = {}
-            for index, d in enumerate(g):
-                dupe2index[d] = index
-                try:
-                    words = engine.unpack_fields(d.words)
-                except AttributeError:
-                    words = ()
-                attrs = AttributesImpl({
-                    'path': unicode(d.path),
-                    'is_ref': cond(d.is_ref, 'y', 'n'),
-                    'words': ','.join(words),
-                    'marked': cond(self.is_marked(d), 'y', 'n')
-                })
-                writer.startElement('file', attrs)
-                writer.endElement('file')
-            for match in g.matches:
-                attrs = AttributesImpl({
-                    'first': str(dupe2index[match.first]),
-                    'second': str(dupe2index[match.second]),
-                    'percentage': str(int(match.percentage)),
-                })
-                writer.startElement('match', attrs)
-                writer.endElement('match')
-            writer.endElement('group')
-        writer.endElement('results')
-        writer.endDocument()
-        if must_close:
-            outfile.close()
-    
-    def sort_dupes(self, key, asc=True, delta=False):
-        if not self.__dupes:
-            self.__get_dupe_list()
-        self.__dupes.sort(key=lambda d: self.data.GetDupeSortKey(d, lambda: self.get_group_of_duplicate(d), key, delta))
-        if not asc:
-            self.__dupes.reverse()
-        self.__dupes_sort_descriptor = (key,asc,delta)
-    
-    def sort_groups(self,key,asc=True):
-        self.groups.sort(key=lambda g: self.data.GetGroupSortKey(g, key))
-        if not asc:
-            self.groups.reverse()
-        self.__groups_sort_descriptor = (key,asc)
-    
-    #---Properties
-    dupes     = property(__get_dupe_list)
-    groups    = property(__get_groups, __set_groups)
-    stat_line = property(__get_stat_line)
-
-class _ResultsHandler(handler.ContentHandler):
-    def __init__(self, get_file):
-        self.group = None
-        self.dupes = None
-        self.marked = set()
-        self.groups = []
-        self.get_file = get_file
-    
-    def startElement(self, name, attrs):
-        if name == 'group':
-            self.group = engine.Group()
-            self.dupes = []
-            return
-        if (name == 'file') and (self.group is not None):
-            if not (('path' in attrs) and ('words' in attrs)):
-                return
-            path = attrs['path']
-            file = self.get_file(path)
-            if file is None:
-                return
-            file.words = attrs['words'].split(',')
-            file.is_ref = attrs.get('is_ref') == 'y'
-            self.dupes.append(file)
-            if attrs.get('marked') == 'y':
-                self.marked.add(file)
-        if (name == 'match') and (self.group is not None):
-            try:
-                first_file = self.dupes[int(attrs['first'])]
-                second_file = self.dupes[int(attrs['second'])]
-                percentage = int(attrs['percentage'])
-                self.group.add_match(engine.Match(first_file, second_file, percentage))
-            except (IndexError, KeyError, ValueError): # Covers missing attr, non-int values and indexes out of bounds
-                pass
-    
-    def endElement(self, name):
-        def do_match(ref_file, other_files, group):
-            if not other_files:
-                return
-            for other_file in other_files:
-                group.add_match(engine.get_match(ref_file, other_file))
-            do_match(other_files[0], other_files[1:], group)
-        
-        if name == 'group':
-            group = self.group
-            self.group = None
-            dupes = self.dupes
-            self.dupes = []
-            if group is None:
-                return
-            if len(dupes) < 2:
-                return
-            if not group.matches: # <match> elements not present, do it manually, without %
-                do_match(dupes[0], dupes[1:], group)
-            group.prioritize(lambda x: dupes.index(x))
-            self.groups.append(group)
-    

base/py/scanner.py

@@ -1,108 +0,0 @@
-# Created By: Virgil Dupras
-# Created On: 2006/03/03
-# $Id$
-# Copyright 2009 Hardcoded Software (http://www.hardcoded.net)
-# 
-# This software is licensed under the "HS" License as described in the "LICENSE" file, 
-# which should be included with this package. The terms are also available at 
-# http://www.hardcoded.net/licenses/hs_license
-
-import logging
-
-
-from hsutil import job
-from hsutil.misc import dedupe
-from hsutil.str import get_file_ext, rem_file_ext
-
-from . import engine
-from .ignore import IgnoreList
-
-(SCAN_TYPE_FILENAME,
-SCAN_TYPE_FIELDS,
-SCAN_TYPE_FIELDS_NO_ORDER,
-SCAN_TYPE_TAG,
-UNUSED, # Must not be removed. Constants here are what scan_type in the prefs are.
-SCAN_TYPE_CONTENT,
-SCAN_TYPE_CONTENT_AUDIO) = range(7)
-
-SCANNABLE_TAGS = ['track', 'artist', 'album', 'title', 'genre', 'year']
-
-class Scanner(object):
-    def __init__(self):
-        self.ignore_list = IgnoreList()
-        self.discarded_file_count = 0
-    
-    def _getmatches(self, files, j):
-        if self.size_threshold:
-            j = j.start_subjob([2, 8])
-            for f in j.iter_with_progress(files, 'Read size of %d/%d files'):
-                f.size # pre-read, makes a smoother progress if read here (especially for bundles)
-            files = [f for f in files if f.size >= self.size_threshold]
-        if self.scan_type in (SCAN_TYPE_CONTENT, SCAN_TYPE_CONTENT_AUDIO):
-            sizeattr = 'size' if self.scan_type == SCAN_TYPE_CONTENT else 'audiosize'
-            return engine.getmatches_by_contents(files, sizeattr, partial=self.scan_type==SCAN_TYPE_CONTENT_AUDIO, j=j)
-        else:
-            j = j.start_subjob([2, 8])
-            kw = {}
-            kw['match_similar_words'] = self.match_similar_words
-            kw['weight_words'] = self.word_weighting
-            kw['min_match_percentage'] = self.min_match_percentage
-            if self.scan_type == SCAN_TYPE_FIELDS_NO_ORDER:
-                self.scan_type = SCAN_TYPE_FIELDS
-                kw['no_field_order'] = True
-            func = {
-                SCAN_TYPE_FILENAME: lambda f: engine.getwords(rem_file_ext(f.name)),
-                SCAN_TYPE_FIELDS: lambda f: engine.getfields(rem_file_ext(f.name)),
-                SCAN_TYPE_TAG: lambda f: [engine.getwords(unicode(getattr(f, attrname))) for attrname in SCANNABLE_TAGS if attrname in self.scanned_tags],
-            }[self.scan_type]
-            for f in j.iter_with_progress(files, 'Read metadata of %d/%d files'):
-                f.words = func(f)
-            return engine.getmatches(files, j=j, **kw)
-    
-    @staticmethod
-    def _key_func(dupe):
-        return (not dupe.is_ref, -dupe.size)
-    
-    @staticmethod
-    def _tie_breaker(ref, dupe):
-        refname = rem_file_ext(ref.name).lower()
-        dupename = rem_file_ext(dupe.name).lower()
-        if 'copy' in refname and 'copy' not in dupename:
-            return True
-        if refname.startswith(dupename) and (refname[len(dupename):].strip().isdigit()):
-            return True
-        return len(dupe.path) > len(ref.path)
-    
-    def GetDupeGroups(self, files, j=job.nulljob):
-        j = j.start_subjob([8, 2])
-        for f in [f for f in files if not hasattr(f, 'is_ref')]:
-            f.is_ref = False
-        logging.info('Getting matches')
-        matches = self._getmatches(files, j)
-        logging.info('Found %d matches' % len(matches))
-        if not self.mix_file_kind:
-            j.set_progress(100, 'Removing false matches')
-            matches = [m for m in matches if get_file_ext(m.first.name) == get_file_ext(m.second.name)]
-        if self.ignore_list:
-            j = j.start_subjob(2)
-            iter_matches = j.iter_with_progress(matches, 'Processed %d/%d matches against the ignore list')
-            matches = [m for m in iter_matches 
-                if not self.ignore_list.AreIgnored(unicode(m.first.path), unicode(m.second.path))]
-        logging.info('Grouping matches')
-        groups = engine.get_groups(matches, j)
-        matched_files = dedupe([m.first for m in matches] + [m.second for m in matches])
-        self.discarded_file_count = len(matched_files) - sum(len(g) for g in groups)
-        groups = [g for g in groups if any(not f.is_ref for f in g)]
-        logging.info('Created %d groups' % len(groups))
-        j.set_progress(100, 'Doing group prioritization')
-        for g in groups:
-            g.prioritize(self._key_func, self._tie_breaker)
-        return groups
-    
-    match_similar_words  = False
-    min_match_percentage = 80
-    mix_file_kind        = True
-    scan_type            = SCAN_TYPE_FILENAME
-    scanned_tags         = set(['artist', 'title'])
-    size_threshold       = 0
-    word_weighting       = False

base/py/tests/app_cocoa_test.py

@@ -1,366 +0,0 @@
-# Created By: Virgil Dupras
-# Created On: 2006/11/11
-# $Id$
-# Copyright 2009 Hardcoded Software (http://www.hardcoded.net)
-# 
-# This software is licensed under the "HS" License as described in the "LICENSE" file, 
-# which should be included with this package. The terms are also available at 
-# http://www.hardcoded.net/licenses/hs_license
-
-import tempfile
-import shutil
-import logging
-import os.path as op
-
-from nose.tools import eq_
-
-from hsutil.path import Path
-from hsutil.testcase import TestCase
-from hsutil.decorators import log_calls
-from hsutil import io
-
-from . import data
-from .results_test import GetTestGroups
-from .. import engine, fs
-try:
-    from ..app_cocoa import DupeGuru as DupeGuruBase
-except ImportError:
-    from nose.plugins.skip import SkipTest
-    raise SkipTest("These tests can only be run on OS X")
-
-class DupeGuru(DupeGuruBase):
-    def __init__(self):
-        DupeGuruBase.__init__(self, data, '/tmp', appid=4)
-    
-    def _start_job(self, jobid, func):
-        func(nulljob)
-    
-def r2np(rows):
-    #Transforms a list of rows [1,2,3] into a list of node paths [[1],[2],[3]]
-    return [[i] for i in rows]
-
-class TCDupeGuru(TestCase):
-    def setUp(self):
-        self.app = DupeGuru()
-        self.objects,self.matches,self.groups = GetTestGroups()
-        self.app.results.groups = self.groups
-        tmppath = self.tmppath()
-        io.mkdir(tmppath + 'foo')
-        io.mkdir(tmppath + 'bar')
-        self.app.directories.add_path(tmppath)
-    
-    def test_GetObjects(self):
-        app = self.app
-        objects = self.objects
-        groups = self.groups
-        g,d = app.GetObjects([0])
-        self.assert_(g is groups[0])
-        self.assert_(d is None)
-        g,d = app.GetObjects([0,0])
-        self.assert_(g is groups[0])
-        self.assert_(d is objects[1])
-        g,d = app.GetObjects([1,0])
-        self.assert_(g is groups[1])
-        self.assert_(d is objects[4])
-    
-    def test_GetObjects_after_sort(self):
-        app = self.app
-        objects = self.objects
-        groups = self.groups[:] #To keep the old order in memory
-        app.sort_groups(0,False) #0 = Filename
-        #Now, the group order is supposed to be reversed
-        g,d = app.GetObjects([0,0])
-        self.assert_(g is groups[1])
-        self.assert_(d is objects[4])
-    
-    def test_GetObjects_out_of_range(self):
-        app = self.app
-        self.assertEqual((None,None),app.GetObjects([2]))
-        self.assertEqual((None,None),app.GetObjects([]))
-        self.assertEqual((None,None),app.GetObjects([1,2]))
-    
-    def test_selected_result_node_paths(self):
-        # app.selected_dupes is correctly converted into node paths
-        app = self.app
-        objects = self.objects
-        paths = [[0, 0], [0, 1], [1]]
-        app.SelectResultNodePaths(paths)
-        eq_(app.selected_result_node_paths(), paths)
-    
-    def test_selected_result_node_paths_after_deletion(self):
-        # cases where the selected dupes aren't there are correctly handled
-        app = self.app
-        objects = self.objects
-        paths = [[0, 0], [0, 1], [1]]
-        app.SelectResultNodePaths(paths)
-        app.RemoveSelected()
-        # The first 2 dupes have been removed. The 3rd one is a ref. it stays there, in first pos.
-        eq_(app.selected_result_node_paths(), [[0]]) # no exception
-    
-    def test_selectResultNodePaths(self):
-        app = self.app
-        objects = self.objects
-        app.SelectResultNodePaths([[0,0],[0,1]])
-        self.assertEqual(2,len(app.selected_dupes))
-        self.assert_(app.selected_dupes[0] is objects[1])
-        self.assert_(app.selected_dupes[1] is objects[2])
-    
-    def test_selectResultNodePaths_with_ref(self):
-        app = self.app
-        objects = self.objects
-        app.SelectResultNodePaths([[0,0],[0,1],[1]])
-        self.assertEqual(3,len(app.selected_dupes))
-        self.assert_(app.selected_dupes[0] is objects[1])
-        self.assert_(app.selected_dupes[1] is objects[2])
-        self.assert_(app.selected_dupes[2] is self.groups[1].ref)
-    
-    def test_selectResultNodePaths_empty(self):
-        self.app.SelectResultNodePaths([])
-        self.assertEqual(0,len(self.app.selected_dupes))        
-    
-    def test_selectResultNodePaths_after_sort(self):
-        app = self.app
-        objects = self.objects
-        groups = self.groups[:] #To keep the old order in memory
-        app.sort_groups(0,False) #0 = Filename
-        #Now, the group order is supposed to be reversed
-        app.SelectResultNodePaths([[0,0],[1],[1,0]])
-        self.assertEqual(3,len(app.selected_dupes))
-        self.assert_(app.selected_dupes[0] is objects[4])
-        self.assert_(app.selected_dupes[1] is groups[0].ref)
-        self.assert_(app.selected_dupes[2] is objects[1])
-    
-    def test_selectResultNodePaths_out_of_range(self):
-        app = self.app
-        app.SelectResultNodePaths([[0,0],[0,1],[1],[1,1],[2]])
-        self.assertEqual(3,len(app.selected_dupes))
-    
-    def test_selected_powermarker_node_paths(self):
-        # app.selected_dupes is correctly converted into paths
-        app = self.app
-        objects = self.objects
-        paths = r2np([0, 1, 2])
-        app.SelectPowerMarkerNodePaths(paths)
-        eq_(app.selected_powermarker_node_paths(), paths)
-    
-    def test_selected_powermarker_node_paths_after_deletion(self):
-        # cases where the selected dupes aren't there are correctly handled
-        app = self.app
-        objects = self.objects
-        paths = r2np([0, 1, 2])
-        app.SelectPowerMarkerNodePaths(paths)
-        app.RemoveSelected()
-        eq_(app.selected_powermarker_node_paths(), []) # no exception
-    
-    def test_selectPowerMarkerRows(self):
-        app = self.app
-        objects = self.objects
-        app.SelectPowerMarkerNodePaths(r2np([0,1,2]))
-        self.assertEqual(3,len(app.selected_dupes))
-        self.assert_(app.selected_dupes[0] is objects[1])
-        self.assert_(app.selected_dupes[1] is objects[2])
-        self.assert_(app.selected_dupes[2] is objects[4])
-    
-    def test_selectPowerMarkerRows_empty(self):
-        self.app.SelectPowerMarkerNodePaths([])
-        self.assertEqual(0,len(self.app.selected_dupes))
-    
-    def test_selectPowerMarkerRows_after_sort(self):
-        app = self.app
-        objects = self.objects
-        app.sort_dupes(0,False) #0 = Filename
-        app.SelectPowerMarkerNodePaths(r2np([0,1,2]))
-        self.assertEqual(3,len(app.selected_dupes))
-        self.assert_(app.selected_dupes[0] is objects[4])
-        self.assert_(app.selected_dupes[1] is objects[2])
-        self.assert_(app.selected_dupes[2] is objects[1])
-    
-    def test_selectPowerMarkerRows_out_of_range(self):
-        app = self.app
-        app.SelectPowerMarkerNodePaths(r2np([0,1,2,3]))
-        self.assertEqual(3,len(app.selected_dupes))
-    
-    def test_toggleSelectedMark(self):
-        app = self.app
-        objects = self.objects
-        app.ToggleSelectedMarkState()
-        self.assertEqual(0,app.results.mark_count)
-        app.SelectPowerMarkerNodePaths(r2np([0,2]))
-        app.ToggleSelectedMarkState()
-        self.assertEqual(2,app.results.mark_count)
-        self.assert_(not app.results.is_marked(objects[0]))
-        self.assert_(app.results.is_marked(objects[1]))
-        self.assert_(not app.results.is_marked(objects[2]))
-        self.assert_(not app.results.is_marked(objects[3]))
-        self.assert_(app.results.is_marked(objects[4]))
-    
-    def test_refreshDetailsWithSelected(self):
-        def mock_refresh(dupe,group):
-            self.called = True
-            if self.app.selected_dupes:
-                self.assert_(dupe is self.app.selected_dupes[0])
-                self.assert_(group is self.app.results.get_group_of_duplicate(dupe))
-            else:
-                self.assert_(dupe is None)
-                self.assert_(group is None)
-        
-        self.app.RefreshDetailsTable = mock_refresh
-        self.called = False
-        self.app.SelectPowerMarkerNodePaths(r2np([0,2]))
-        self.app.RefreshDetailsWithSelected()
-        self.assert_(self.called)
-        self.called = False
-        self.app.SelectPowerMarkerNodePaths([])
-        self.app.RefreshDetailsWithSelected()
-        self.assert_(self.called)
-    
-    def test_makeSelectedReference(self):
-        app = self.app
-        objects = self.objects
-        groups = self.groups
-        app.SelectPowerMarkerNodePaths(r2np([0,2]))
-        app.MakeSelectedReference()
-        self.assert_(groups[0].ref is objects[1])
-        self.assert_(groups[1].ref is objects[4])
-    
-    def test_makeSelectedReference_by_selecting_two_dupes_in_the_same_group(self):
-        app = self.app
-        objects = self.objects
-        groups = self.groups
-        app.SelectPowerMarkerNodePaths(r2np([0,1,2]))
-        #Only 0 and 2 must go ref, not 1 because it is a part of the same group
-        app.MakeSelectedReference()
-        self.assert_(groups[0].ref is objects[1])
-        self.assert_(groups[1].ref is objects[4])
-    
-    def test_removeSelected(self):
-        app = self.app
-        app.SelectPowerMarkerNodePaths(r2np([0,2]))
-        app.RemoveSelected()
-        self.assertEqual(1,len(app.results.dupes))
-        app.RemoveSelected()
-        self.assertEqual(1,len(app.results.dupes))
-        app.SelectPowerMarkerNodePaths(r2np([0,2]))
-        app.RemoveSelected()
-        self.assertEqual(0,len(app.results.dupes))
-    
-    def test_addDirectory_simple(self):
-        # There's already a directory in self.app, so adding another once makes 2 of em
-        app = self.app
-        eq_(app.add_directory(self.datadirpath()), 0)
-        eq_(len(app.directories), 2)
-    
-    def test_addDirectory_already_there(self):
-        app = self.app
-        self.assertEqual(0,app.add_directory(self.datadirpath()))
-        self.assertEqual(1,app.add_directory(self.datadirpath()))
-    
-    def test_addDirectory_does_not_exist(self):
-        app = self.app
-        self.assertEqual(2,app.add_directory('/does_not_exist'))
-    
-    def test_ignore(self):
-        app = self.app
-        app.SelectPowerMarkerNodePaths(r2np([2])) #The dupe of the second, 2 sized group
-        app.AddSelectedToIgnoreList()
-        self.assertEqual(1,len(app.scanner.ignore_list))
-        app.SelectPowerMarkerNodePaths(r2np([0])) #first dupe of the 3 dupes group
-        app.AddSelectedToIgnoreList()
-        #BOTH the ref and the other dupe should have been added
-        self.assertEqual(3,len(app.scanner.ignore_list))
-    
-    def test_purgeIgnoreList(self):
-        app = self.app
-        p1 = self.filepath('zerofile')
-        p2 = self.filepath('zerofill')
-        dne = '/does_not_exist'
-        app.scanner.ignore_list.Ignore(dne,p1)
-        app.scanner.ignore_list.Ignore(p2,dne)
-        app.scanner.ignore_list.Ignore(p1,p2)
-        app.PurgeIgnoreList()
-        self.assertEqual(1,len(app.scanner.ignore_list))
-        self.assert_(app.scanner.ignore_list.AreIgnored(p1,p2))
-        self.assert_(not app.scanner.ignore_list.AreIgnored(dne,p1))
-    
-    def test_only_unicode_is_added_to_ignore_list(self):
-        def FakeIgnore(first,second):
-            if not isinstance(first,unicode):
-                self.fail()
-            if not isinstance(second,unicode):
-                self.fail()
-        
-        app = self.app
-        app.scanner.ignore_list.Ignore = FakeIgnore
-        app.SelectPowerMarkerNodePaths(r2np([2])) #The dupe of the second, 2 sized group
-        app.AddSelectedToIgnoreList()
-    
-    def test_GetOutlineViewChildCounts_out_of_range(self):
-        # Out of range requests don't crash and return an empty value
-        app = self.app
-        # [0, 2] is out of range
-        eq_(app.GetOutlineViewChildCounts(1, [0, 2]), []) # no crash
-    
-    def test_GetOutlineViewValues_out_of_range(self):
-        # Out of range requests don't crash and return an empty value
-        app = self.app
-        # [0, 2] is out of range
-        eq_(app.GetOutlineViewValues(1, [0, 2]), []) # no crash
-    
-
-class TCDupeGuru_renameSelected(TestCase):
-    def setUp(self):
-        p = self.tmppath()
-        fp = open(unicode(p + 'foo bar 1'),mode='w')
-        fp.close()
-        fp = open(unicode(p + 'foo bar 2'),mode='w')
-        fp.close()
-        fp = open(unicode(p + 'foo bar 3'),mode='w')
-        fp.close()
-        files = fs.get_files(p)
-        matches = engine.getmatches(files)
-        groups = engine.get_groups(matches)
-        g = groups[0]
-        g.prioritize(lambda x:x.name)
-        app = DupeGuru()
-        app.results.groups = groups
-        self.app = app
-        self.groups = groups
-        self.p = p
-        self.files = files
-    
-    def test_simple(self):
-        app = self.app
-        g = self.groups[0]
-        app.SelectPowerMarkerNodePaths(r2np([0]))
-        assert app.RenameSelected('renamed')
-        names = io.listdir(self.p)
-        assert 'renamed' in names
-        assert 'foo bar 2' not in names
-        eq_(g.dupes[0].name, 'renamed')
-    
-    def test_none_selected(self):
-        app = self.app
-        g = self.groups[0]
-        app.SelectPowerMarkerNodePaths([])
-        self.mock(logging, 'warning', log_calls(lambda msg: None))
-        assert not app.RenameSelected('renamed')
-        msg = logging.warning.calls[0]['msg']
-        eq_('dupeGuru Warning: list index out of range', msg)
-        names = io.listdir(self.p)
-        assert 'renamed' not in names
-        assert 'foo bar 2' in names
-        eq_(g.dupes[0].name, 'foo bar 2')
-    
-    def test_name_already_exists(self):
-        app = self.app
-        g = self.groups[0]
-        app.SelectPowerMarkerNodePaths(r2np([0]))
-        self.mock(logging, 'warning', log_calls(lambda msg: None))
-        assert not app.RenameSelected('foo bar 1')
-        msg = logging.warning.calls[0]['msg']
-        assert msg.startswith('dupeGuru Warning: \'foo bar 1\' already exists in')
-        names = io.listdir(self.p)
-        assert 'foo bar 1' in names
-        assert 'foo bar 2' in names
-        eq_(g.dupes[0].name, 'foo bar 2')
-    

base/py/tests/app_test.py

@@ -1,136 +0,0 @@
-# Created By: Virgil Dupras
-# Created On: 2007-06-23
-# $Id$
-# Copyright 2009 Hardcoded Software (http://www.hardcoded.net)
-# 
-# This software is licensed under the "HS" License as described in the "LICENSE" file, 
-# which should be included with this package. The terms are also available at 
-# http://www.hardcoded.net/licenses/hs_license
-
-import os
-
-from hsutil.testcase import TestCase
-from hsutil import io
-from hsutil.path import Path
-from hsutil.decorators import log_calls
-import hsutil.files
-from hsutil.job import nulljob
-
-from . import data
-from .. import app, fs
-from ..app import DupeGuru as DupeGuruBase
-
-class DupeGuru(DupeGuruBase):
-    def __init__(self):
-        DupeGuruBase.__init__(self, data, '/tmp', appid=4)
-    
-    def _start_job(self, jobid, func):
-        func(nulljob)
-    
-
-class TCDupeGuru(TestCase):
-    cls_tested_module = app
-    def test_apply_filter_calls_results_apply_filter(self):
-        app = DupeGuru()
-        self.mock(app.results, 'apply_filter', log_calls(app.results.apply_filter))
-        app.apply_filter('foo')
-        self.assertEqual(2, len(app.results.apply_filter.calls))
-        call = app.results.apply_filter.calls[0]
-        self.assert_(call['filter_str'] is None)
-        call = app.results.apply_filter.calls[1]
-        self.assertEqual('foo', call['filter_str'])
-    
-    def test_apply_filter_escapes_regexp(self):
-        app = DupeGuru()
-        self.mock(app.results, 'apply_filter', log_calls(app.results.apply_filter))
-        app.apply_filter('()[]\\.|+?^abc')
-        call = app.results.apply_filter.calls[1]
-        self.assertEqual('\\(\\)\\[\\]\\\\\\.\\|\\+\\?\\^abc', call['filter_str'])
-        app.apply_filter('(*)') # In "simple mode", we want the * to behave as a wilcard
-        call = app.results.apply_filter.calls[3]
-        self.assertEqual('\(.*\)', call['filter_str'])
-        app.options['escape_filter_regexp'] = False
-        app.apply_filter('(abc)')
-        call = app.results.apply_filter.calls[5]
-        self.assertEqual('(abc)', call['filter_str'])
-    
-    def test_copy_or_move(self):
-        # The goal here is just to have a test for a previous blowup I had. I know my test coverage
-        # for this unit is pathetic. What's done is done. My approach now is to add tests for
-        # every change I want to make. The blowup was caused by a missing import.
-        p = self.tmppath()
-        io.open(p + 'foo', 'w').close()
-        self.mock(hsutil.files, 'copy', log_calls(lambda source_path, dest_path: None))
-        self.mock(os, 'makedirs', lambda path: None) # We don't want the test to create that fake directory
-        app = DupeGuru()
-        app.directories.add_path(p)
-        [f] = app.directories.get_files()
-        app.copy_or_move(f, True, 'some_destination', 0)
-        self.assertEqual(1, len(hsutil.files.copy.calls))
-        call = hsutil.files.copy.calls[0]
-        self.assertEqual('some_destination', call['dest_path'])
-        self.assertEqual(f.path, call['source_path'])
-    
-    def test_copy_or_move_clean_empty_dirs(self):
-        tmppath = Path(self.tmpdir())
-        sourcepath = tmppath + 'source'
-        io.mkdir(sourcepath)
-        io.open(sourcepath + 'myfile', 'w')
-        app = DupeGuru()
-        app.directories.add_path(tmppath)
-        [myfile] = app.directories.get_files()
-        self.mock(app, 'clean_empty_dirs', log_calls(lambda path: None))
-        app.copy_or_move(myfile, False, tmppath + 'dest', 0)
-        calls = app.clean_empty_dirs.calls
-        self.assertEqual(1, len(calls))
-        self.assertEqual(sourcepath, calls[0]['path'])
-    
-    def test_Scan_with_objects_evaluating_to_false(self):
-        class FakeFile(fs.File):
-            def __nonzero__(self):
-                return False
-            
-        
-        # At some point, any() was used in a wrong way that made Scan() wrongly return 1
-        app = DupeGuru()
-        f1, f2 = [FakeFile('foo') for i in range(2)]
-        f1.is_ref, f2.is_ref = (False, False)
-        assert not (bool(f1) and bool(f2))
-        app.directories.get_files = lambda: [f1, f2]
-        app.directories._dirs.append('this is just so Scan() doesnt return 3')
-        app.start_scanning() # no exception
-    
-
-class TCDupeGuru_clean_empty_dirs(TestCase):
-    cls_tested_module = app
-    def setUp(self):
-        self.mock(hsutil.files, 'delete_if_empty', log_calls(lambda path, files_to_delete=[]: None))
-        self.app = DupeGuru()
-    
-    def test_option_off(self):
-        self.app.clean_empty_dirs(Path('/foo/bar'))
-        self.assertEqual(0, len(hsutil.files.delete_if_empty.calls))
-    
-    def test_option_on(self):
-        self.app.options['clean_empty_dirs'] = True
-        self.app.clean_empty_dirs(Path('/foo/bar'))
-        calls = hsutil.files.delete_if_empty.calls
-        self.assertEqual(1, len(calls))
-        self.assertEqual(Path('/foo/bar'), calls[0]['path'])
-        self.assertEqual(['.DS_Store'], calls[0]['files_to_delete'])
-    
-    def test_recurse_up(self):
-        # delete_if_empty must be recursively called up in the path until it returns False
-        @log_calls
-        def mock_delete_if_empty(path, files_to_delete=[]):
-            return len(path) > 1
-        
-        self.mock(hsutil.files, 'delete_if_empty', mock_delete_if_empty)
-        self.app.options['clean_empty_dirs'] = True
-        self.app.clean_empty_dirs(Path('not-empty/empty/empty'))
-        calls = hsutil.files.delete_if_empty.calls
-        self.assertEqual(3, len(calls))
-        self.assertEqual(Path('not-empty/empty/empty'), calls[0]['path'])
-        self.assertEqual(Path('not-empty/empty'), calls[1]['path'])
-        self.assertEqual(Path('not-empty'), calls[2]['path'])
-    

base/py/tests/data.py

@@ -1,45 +0,0 @@
-# -*- coding: utf-8 -*-
-# Created By: Virgil Dupras
-# Created On: 2009-10-23
-# $Id$
-# Copyright 2009 Hardcoded Software (http://www.hardcoded.net)
-# 
-# This software is licensed under the "HS" License as described in the "LICENSE" file, 
-# which should be included with this package. The terms are also available at 
-# http://www.hardcoded.net/licenses/hs_license
-
-# data module for tests
-
-from hsutil.str import format_size
-from dupeguru.data import format_path, cmp_value
-
-COLUMNS = [
-    {'attr':'name','display':'Filename'},
-    {'attr':'path','display':'Directory'},
-    {'attr':'size','display':'Size (KB)'},
-    {'attr':'extension','display':'Kind'},
-]
-
-METADATA_TO_READ = ['size']
-
-def GetDisplayInfo(dupe, group, delta):
-    size = dupe.size
-    m = group.get_match_of(dupe)
-    if m and delta:
-        r = group.ref
-        size -= r.size
-    return [
-        dupe.name,
-        format_path(dupe.path),
-        format_size(size, 0, 1, False),
-        dupe.extension,
-    ]
-
-def GetDupeSortKey(dupe, get_group, key, delta):
-    r = cmp_value(getattr(dupe, COLUMNS[key]['attr']))
-    if delta and (key == 2):
-        r -= cmp_value(getattr(get_group().ref, COLUMNS[key]['attr']))
-    return r
-
-def GetGroupSortKey(group, key):
-    return cmp_value(getattr(group.ref, COLUMNS[key]['attr']))

base/py/tests/directories_test.py

@@ -1,279 +0,0 @@
-# Created By: Virgil Dupras
-# Created On: 2006/02/27
-# $Id$
-# Copyright 2009 Hardcoded Software (http://www.hardcoded.net)
-# 
-# This software is licensed under the "HS" License as described in the "LICENSE" file, 
-# which should be included with this package. The terms are also available at 
-# http://www.hardcoded.net/licenses/hs_license
-
-import os.path as op
-import os
-import time
-
-from nose.tools import eq_
-
-from hsutil import io
-from hsutil.path import Path
-from hsutil.testcase import TestCase
-
-from ..directories import *
-
-testpath = Path(TestCase.datadirpath())
-
-def create_fake_fs(rootpath):
-    rootpath = rootpath + 'fs'
-    io.mkdir(rootpath)
-    io.mkdir(rootpath + 'dir1')
-    io.mkdir(rootpath + 'dir2')
-    io.mkdir(rootpath + 'dir3')
-    fp = io.open(rootpath + 'file1.test', 'w')
-    fp.write('1')
-    fp.close()
-    fp = io.open(rootpath + 'file2.test', 'w')
-    fp.write('12')
-    fp.close()
-    fp = io.open(rootpath + 'file3.test', 'w')
-    fp.write('123')
-    fp.close()
-    fp = io.open(rootpath + ('dir1', 'file1.test'), 'w')
-    fp.write('1')
-    fp.close()
-    fp = io.open(rootpath + ('dir2', 'file2.test'), 'w')
-    fp.write('12')
-    fp.close()
-    fp = io.open(rootpath + ('dir3', 'file3.test'), 'w')
-    fp.write('123')
-    fp.close()
-    return rootpath
-
-class TCDirectories(TestCase):
-    def test_empty(self):
-        d = Directories()
-        self.assertEqual(0,len(d))
-        self.assert_('foobar' not in d)
-    
-    def test_add_path(self):
-        d = Directories()
-        p = testpath + 'utils'
-        d.add_path(p)
-        self.assertEqual(1,len(d))
-        self.assert_(p in d)
-        self.assert_((p + 'foobar') in d)
-        self.assert_(p[:-1] not in d)
-        p = self.tmppath()
-        d.add_path(p)
-        self.assertEqual(2,len(d))
-        self.assert_(p in d)
-    
-    def test_AddPath_when_path_is_already_there(self):
-        d = Directories()
-        p = testpath + 'utils'
-        d.add_path(p)
-        self.assertRaises(AlreadyThereError, d.add_path, p)
-        self.assertRaises(AlreadyThereError, d.add_path, p + 'foobar')
-        self.assertEqual(1, len(d))
-    
-    def test_add_path_containing_paths_already_there(self):
-        d = Directories()
-        d.add_path(testpath + 'utils')
-        self.assertEqual(1, len(d))
-        d.add_path(testpath)
-        eq_(len(d), 1)
-        eq_(d[0], testpath)
-    
-    def test_AddPath_non_latin(self):
-    	p = Path(self.tmpdir())
-    	to_add = p + u'unicode\u201a'
-    	os.mkdir(unicode(to_add))
-    	d = Directories()
-    	try:
-    		d.add_path(to_add)
-    	except UnicodeDecodeError:
-    		self.fail()
-    
-    def test_del(self):
-        d = Directories()
-        d.add_path(testpath + 'utils')
-        try:
-            del d[1]
-            self.fail()
-        except IndexError:
-            pass
-        d.add_path(self.tmppath())
-        del d[1]
-        self.assertEqual(1, len(d))
-    
-    def test_states(self):
-        d = Directories()
-        p = testpath + 'utils'
-        d.add_path(p)
-        self.assertEqual(STATE_NORMAL,d.get_state(p))
-        d.set_state(p,STATE_REFERENCE)
-        self.assertEqual(STATE_REFERENCE,d.get_state(p))
-        self.assertEqual(STATE_REFERENCE,d.get_state(p + 'dir1'))
-        self.assertEqual(1,len(d.states))
-        self.assertEqual(p,d.states.keys()[0])
-        self.assertEqual(STATE_REFERENCE,d.states[p])
-    
-    def test_get_state_with_path_not_there(self):
-        # When the path's not there, just return STATE_NORMAL
-        d = Directories()
-        d.add_path(testpath + 'utils')
-        eq_(d.get_state(testpath), STATE_NORMAL)
-    
-    def test_states_remain_when_larger_directory_eat_smaller_ones(self):
-        d = Directories()
-        p = testpath + 'utils'
-        d.add_path(p)
-        d.set_state(p,STATE_EXCLUDED)
-        d.add_path(testpath)
-        d.set_state(testpath,STATE_REFERENCE)
-        self.assertEqual(STATE_EXCLUDED,d.get_state(p))
-        self.assertEqual(STATE_EXCLUDED,d.get_state(p + 'dir1'))
-        self.assertEqual(STATE_REFERENCE,d.get_state(testpath))
-    
-    def test_set_state_keep_state_dict_size_to_minimum(self):
-        d = Directories()
-        p = create_fake_fs(self.tmppath())
-        d.add_path(p)
-        d.set_state(p,STATE_REFERENCE)
-        d.set_state(p + 'dir1',STATE_REFERENCE)
-        self.assertEqual(1,len(d.states))
-        self.assertEqual(STATE_REFERENCE,d.get_state(p + 'dir1'))
-        d.set_state(p + 'dir1',STATE_NORMAL)
-        self.assertEqual(2,len(d.states))
-        self.assertEqual(STATE_NORMAL,d.get_state(p + 'dir1'))
-        d.set_state(p + 'dir1',STATE_REFERENCE)
-        self.assertEqual(1,len(d.states))
-        self.assertEqual(STATE_REFERENCE,d.get_state(p + 'dir1'))
-    
-    def test_get_files(self):
-        d = Directories()
-        p = create_fake_fs(self.tmppath())
-        d.add_path(p)
-        d.set_state(p + 'dir1',STATE_REFERENCE)
-        d.set_state(p + 'dir2',STATE_EXCLUDED)
-        files = d.get_files()
-        self.assertEqual(5, len(list(files)))
-        for f in files:
-            if f.parent.path == p + 'dir1':
-                self.assert_(f.is_ref)
-            else:
-                self.assert_(not f.is_ref)
-    
-    def test_get_files_with_inherited_exclusion(self):
-        d = Directories()
-        p = testpath + 'utils'
-        d.add_path(p)
-        d.set_state(p,STATE_EXCLUDED)
-        self.assertEqual([], list(d.get_files()))
-    
-    def test_save_and_load(self):
-        d1 = Directories()
-        d2 = Directories()
-        p1 = self.tmppath()
-        p2 = self.tmppath()
-        d1.add_path(p1)
-        d1.add_path(p2)
-        d1.set_state(p1, STATE_REFERENCE)
-        d1.set_state(p1 + 'dir1',STATE_EXCLUDED)
-        tmpxml = op.join(self.tmpdir(), 'directories_testunit.xml')
-        d1.save_to_file(tmpxml)
-        d2.load_from_file(tmpxml)
-        self.assertEqual(2, len(d2))
-        self.assertEqual(STATE_REFERENCE,d2.get_state(p1))
-        self.assertEqual(STATE_EXCLUDED,d2.get_state(p1 + 'dir1'))
-    
-    def test_invalid_path(self):
-        d = Directories()
-        p = Path('does_not_exist')
-        self.assertRaises(InvalidPathError, d.add_path, p)
-        self.assertEqual(0, len(d))
-    
-    def test_set_state_on_invalid_path(self):
-        d = Directories()
-        try:
-            d.set_state(Path('foobar',),STATE_NORMAL)
-        except LookupError:
-            self.fail()
-    
-    def test_load_from_file_with_invalid_path(self):
-        #This test simulates a load from file resulting in a
-        #InvalidPath raise. Other directories must be loaded.
-        d1 = Directories()
-        d1.add_path(testpath + 'utils')
-        #Will raise InvalidPath upon loading
-        p = self.tmppath()
-        d1.add_path(p)
-        io.rmdir(p)
-        tmpxml = op.join(self.tmpdir(), 'directories_testunit.xml')
-        d1.save_to_file(tmpxml)
-        d2 = Directories()
-        d2.load_from_file(tmpxml)
-        self.assertEqual(1, len(d2))
-    
-    def test_unicode_save(self):
-        d = Directories()
-        p1 = self.tmppath() + u'hello\xe9'
-        io.mkdir(p1)
-        io.mkdir(p1 + u'foo\xe9')
-        d.add_path(p1)
-        d.set_state(p1 + u'foo\xe9', STATE_EXCLUDED)
-        tmpxml = op.join(self.tmpdir(), 'directories_testunit.xml')
-        try:
-            d.save_to_file(tmpxml)
-        except UnicodeDecodeError:
-            self.fail()
-    
-    def test_get_files_refreshes_its_directories(self):
-        d = Directories()
-        p = create_fake_fs(self.tmppath())
-        d.add_path(p)
-        files = d.get_files()
-        self.assertEqual(6, len(list(files)))
-        time.sleep(1)
-        os.remove(str(p + ('dir1','file1.test')))
-        files = d.get_files()
-        self.assertEqual(5, len(list(files)))
-    
-    def test_get_files_does_not_choke_on_non_existing_directories(self):
-        d = Directories()
-        p = Path(self.tmpdir())
-        d.add_path(p)
-        io.rmtree(p)
-        self.assertEqual([], list(d.get_files()))
-    
-    def test_get_state_returns_excluded_by_default_for_hidden_directories(self):
-        d = Directories()
-        p = Path(self.tmpdir())
-        hidden_dir_path = p + '.foo'
-        io.mkdir(p + '.foo')
-        d.add_path(p)
-        self.assertEqual(d.get_state(hidden_dir_path), STATE_EXCLUDED)
-        # But it can be overriden
-        d.set_state(hidden_dir_path, STATE_NORMAL)
-        self.assertEqual(d.get_state(hidden_dir_path), STATE_NORMAL)
-    
-    def test_default_path_state_override(self):
-        # It's possible for a subclass to override the default state of a path
-        class MyDirectories(Directories):
-            def _default_state_for_path(self, path):
-                if 'foobar' in path:
-                    return STATE_EXCLUDED
-        
-        d = MyDirectories()
-        p1 = self.tmppath()
-        io.mkdir(p1 + 'foobar')
-        io.open(p1 + 'foobar/somefile', 'w').close()
-        io.mkdir(p1 + 'foobaz')
-        io.open(p1 + 'foobaz/somefile', 'w').close()
-        d.add_path(p1)
-        eq_(d.get_state(p1 + 'foobaz'), STATE_NORMAL)
-        eq_(d.get_state(p1 + 'foobar'), STATE_EXCLUDED)
-        eq_(len(list(d.get_files())), 1) # only the 'foobaz' file is there
-        # However, the default state can be changed
-        d.set_state(p1 + 'foobar', STATE_NORMAL)
-        eq_(d.get_state(p1 + 'foobar'), STATE_NORMAL)
-        eq_(len(list(d.get_files())), 2)
-    

base/py/tests/engine_test.py

@@ -1,804 +0,0 @@
-# Created By: Virgil Dupras
-# Created On: 2006/01/29
-# $Id$
-# Copyright 2009 Hardcoded Software (http://www.hardcoded.net)
-# 
-# This software is licensed under the "HS" License as described in the "LICENSE" file, 
-# which should be included with this package. The terms are also available at 
-# http://www.hardcoded.net/licenses/hs_license
-
-import sys
-
-from nose.tools import eq_
-
-from hsutil import job
-from hsutil.decorators import log_calls
-from hsutil.testcase import TestCase
-
-from .. import engine
-from ..engine import *
-
-class NamedObject(object):
-    def __init__(self, name="foobar", with_words=False):
-        self.name = name
-        if with_words:
-            self.words = getwords(name)
-    
-
-def get_match_triangle():
-    o1 = NamedObject(with_words=True)
-    o2 = NamedObject(with_words=True)
-    o3 = NamedObject(with_words=True)
-    m1 = get_match(o1,o2)
-    m2 = get_match(o1,o3)
-    m3 = get_match(o2,o3)
-    return [m1, m2, m3]
-
-def get_test_group():
-    m1, m2, m3 = get_match_triangle()
-    result = Group()
-    result.add_match(m1)
-    result.add_match(m2)
-    result.add_match(m3)
-    return result
-
-class TCgetwords(TestCase):
-    def test_spaces(self):
-        self.assertEqual(['a', 'b', 'c', 'd'], getwords("a b c d"))
-        self.assertEqual(['a', 'b', 'c', 'd'], getwords(" a  b  c d "))
-    
-    def test_splitter_chars(self):
-        self.assertEqual(
-            [chr(i) for i in xrange(ord('a'),ord('z')+1)],
-            getwords("a-b_c&d+e(f)g;h\\i[j]k{l}m:n.o,p<q>r/s?t~u!v@w#x$y*z")
-        )
-    
-    def test_joiner_chars(self):
-        self.assertEqual(["aec"], getwords(u"a'e\u0301c"))
-    
-    def test_empty(self):
-        self.assertEqual([], getwords(''))
-        
-    def test_returns_lowercase(self):
-        self.assertEqual(['foo', 'bar'], getwords('FOO BAR'))
-    
-    def test_decompose_unicode(self):
-        self.assertEqual(getwords(u'foo\xe9bar'), ['fooebar'])
-    
-
-class TCgetfields(TestCase):
-    def test_simple(self):
-        self.assertEqual([['a', 'b'], ['c', 'd', 'e']], getfields('a b - c d e'))
-    
-    def test_empty(self):
-        self.assertEqual([], getfields(''))
-        
-    def test_cleans_empty_fields(self):
-        expected = [['a', 'bc', 'def']]
-        actual = getfields(' - a bc def')
-        self.assertEqual(expected, actual)
-        expected = [['bc', 'def']]
-    
-
-class TCunpack_fields(TestCase):
-    def test_with_fields(self):
-        expected = ['a', 'b', 'c', 'd', 'e', 'f']
-        actual = unpack_fields([['a'], ['b', 'c'], ['d', 'e', 'f']])
-        self.assertEqual(expected, actual)
-    
-    def test_without_fields(self):
-        expected = ['a', 'b', 'c', 'd', 'e', 'f']
-        actual = unpack_fields(['a', 'b', 'c', 'd', 'e', 'f'])
-        self.assertEqual(expected, actual)
-    
-    def test_empty(self):
-        self.assertEqual([], unpack_fields([]))
-    
-
-class TCWordCompare(TestCase):
-    def test_list(self):
-        self.assertEqual(100, compare(['a', 'b', 'c', 'd'],['a', 'b', 'c', 'd']))
-        self.assertEqual(86, compare(['a', 'b', 'c', 'd'],['a', 'b', 'c']))
-    
-    def test_unordered(self):
-        #Sometimes, users don't want fuzzy matching too much When they set the slider
-        #to 100, they don't expect a filename with the same words, but not the same order, to match.
-        #Thus, we want to return 99 in that case.
-        self.assertEqual(99, compare(['a', 'b', 'c', 'd'], ['d', 'b', 'c', 'a']))
-    
-    def test_word_occurs_twice(self):
-        #if a word occurs twice in first, but once in second, we want the word to be only counted once
-        self.assertEqual(89, compare(['a', 'b', 'c', 'd', 'a'], ['d', 'b', 'c', 'a']))
-    
-    def test_uses_copy_of_lists(self):
-        first = ['foo', 'bar']
-        second = ['bar', 'bleh']
-        compare(first, second)
-        self.assertEqual(['foo', 'bar'], first)
-        self.assertEqual(['bar', 'bleh'], second)
-    
-    def test_word_weight(self):
-        self.assertEqual(int((6.0 / 13.0) * 100), compare(['foo', 'bar'], ['bar', 'bleh'], (WEIGHT_WORDS, )))
-    
-    def test_similar_words(self):
-        self.assertEqual(100, compare(['the', 'white', 'stripes'],['the', 'whites', 'stripe'], (MATCH_SIMILAR_WORDS, )))
-    
-    def test_empty(self):
-        self.assertEqual(0, compare([], []))
-    
-    def test_with_fields(self):
-        self.assertEqual(67, compare([['a', 'b'], ['c', 'd', 'e']], [['a', 'b'], ['c', 'd', 'f']]))
-    
-    def test_propagate_flags_with_fields(self):
-        def mock_compare(first, second, flags):
-            self.assertEqual((0, 1, 2, 3, 5), flags)
-        
-        self.mock(engine, 'compare_fields', mock_compare)
-        compare([['a']], [['a']], (0, 1, 2, 3, 5))
-    
-
-class TCWordCompareWithFields(TestCase):
-    def test_simple(self):
-        self.assertEqual(67, compare_fields([['a', 'b'], ['c', 'd', 'e']], [['a', 'b'], ['c', 'd', 'f']]))
-    
-    def test_empty(self):
-        self.assertEqual(0, compare_fields([], []))
-    
-    def test_different_length(self):
-        self.assertEqual(0, compare_fields([['a'], ['b']], [['a'], ['b'], ['c']]))
-    
-    def test_propagates_flags(self):
-        def mock_compare(first, second, flags):
-            self.assertEqual((0, 1, 2, 3, 5), flags)
-        
-        self.mock(engine, 'compare_fields', mock_compare)
-        compare_fields([['a']], [['a']],(0, 1, 2, 3, 5))
-    
-    def test_order(self):
-        first = [['a', 'b'], ['c', 'd', 'e']]
-        second = [['c', 'd', 'f'], ['a', 'b']]
-        self.assertEqual(0, compare_fields(first, second))
-    
-    def test_no_order(self):
-        first = [['a','b'],['c','d','e']]
-        second = [['c','d','f'],['a','b']]
-        self.assertEqual(67, compare_fields(first, second, (NO_FIELD_ORDER, )))
-        first = [['a','b'],['a','b']] #a field can only be matched once.
-        second = [['c','d','f'],['a','b']]
-        self.assertEqual(0, compare_fields(first, second, (NO_FIELD_ORDER, )))
-        first = [['a','b'],['a','b','c']] 
-        second = [['c','d','f'],['a','b']]
-        self.assertEqual(33, compare_fields(first, second, (NO_FIELD_ORDER, )))
-    
-    def test_compare_fields_without_order_doesnt_alter_fields(self):
-        #The NO_ORDER comp type altered the fields!
-        first = [['a','b'],['c','d','e']]
-        second = [['c','d','f'],['a','b']]
-        self.assertEqual(67, compare_fields(first, second, (NO_FIELD_ORDER, )))
-        self.assertEqual([['a','b'],['c','d','e']],first)
-        self.assertEqual([['c','d','f'],['a','b']],second)
-    
-
-class TCbuild_word_dict(TestCase):
-    def test_with_standard_words(self):
-        l = [NamedObject('foo bar',True)]
-        l.append(NamedObject('bar baz',True))
-        l.append(NamedObject('baz bleh foo',True))
-        d = build_word_dict(l)
-        self.assertEqual(4,len(d))
-        self.assertEqual(2,len(d['foo']))
-        self.assert_(l[0] in d['foo'])
-        self.assert_(l[2] in d['foo'])
-        self.assertEqual(2,len(d['bar']))
-        self.assert_(l[0] in d['bar'])
-        self.assert_(l[1] in d['bar'])
-        self.assertEqual(2,len(d['baz']))
-        self.assert_(l[1] in d['baz'])
-        self.assert_(l[2] in d['baz'])
-        self.assertEqual(1,len(d['bleh']))
-        self.assert_(l[2] in d['bleh'])
-    
-    def test_unpack_fields(self):
-        o = NamedObject('')
-        o.words = [['foo','bar'],['baz']]
-        d = build_word_dict([o])
-        self.assertEqual(3,len(d))
-        self.assertEqual(1,len(d['foo']))
-    
-    def test_words_are_unaltered(self):
-        o = NamedObject('')
-        o.words = [['foo','bar'],['baz']]
-        d = build_word_dict([o])
-        self.assertEqual([['foo','bar'],['baz']],o.words)
-    
-    def test_object_instances_can_only_be_once_in_words_object_list(self):
-        o = NamedObject('foo foo',True)
-        d = build_word_dict([o])
-        self.assertEqual(1,len(d['foo']))
-    
-    def test_job(self):
-        def do_progress(p,d=''):
-            self.log.append(p)
-            return True
-        
-        j = job.Job(1,do_progress)
-        self.log = []
-        s = "foo bar"
-        build_word_dict([NamedObject(s, True), NamedObject(s, True), NamedObject(s, True)], j)
-        self.assertEqual(0,self.log[0])
-        self.assertEqual(33,self.log[1])
-        self.assertEqual(66,self.log[2])
-        self.assertEqual(100,self.log[3])
-    
-
-class TCmerge_similar_words(TestCase):
-    def test_some_similar_words(self):
-        d = {
-            'foobar':set([1]),
-            'foobar1':set([2]),
-            'foobar2':set([3]),
-        }
-        merge_similar_words(d)
-        self.assertEqual(1,len(d))
-        self.assertEqual(3,len(d['foobar']))
-    
-    
-
-class TCreduce_common_words(TestCase):
-    def test_typical(self):
-        d = {
-            'foo': set([NamedObject('foo bar',True) for i in range(50)]),
-            'bar': set([NamedObject('foo bar',True) for i in range(49)])
-        }
-        reduce_common_words(d, 50)
-        self.assert_('foo' not in d)
-        self.assertEqual(49,len(d['bar']))
-    
-    def test_dont_remove_objects_with_only_common_words(self):
-        d = {
-            'common': set([NamedObject("common uncommon",True) for i in range(50)] + [NamedObject("common",True)]),
-            'uncommon': set([NamedObject("common uncommon",True)])
-        }
-        reduce_common_words(d, 50)
-        self.assertEqual(1,len(d['common']))
-        self.assertEqual(1,len(d['uncommon']))
-    
-    def test_values_still_are_set_instances(self):
-        d = {
-            'common': set([NamedObject("common uncommon",True) for i in range(50)] + [NamedObject("common",True)]),
-            'uncommon': set([NamedObject("common uncommon",True)])
-        }
-        reduce_common_words(d, 50)
-        self.assert_(isinstance(d['common'],set))
-        self.assert_(isinstance(d['uncommon'],set))
-    
-    def test_dont_raise_KeyError_when_a_word_has_been_removed(self):
-        #If a word has been removed by the reduce, an object in a subsequent common word that
-        #contains the word that has been removed would cause a KeyError.
-        d = {
-            'foo': set([NamedObject('foo bar baz',True) for i in range(50)]),
-            'bar': set([NamedObject('foo bar baz',True) for i in range(50)]),
-            'baz': set([NamedObject('foo bar baz',True) for i in range(49)])
-        }
-        try:
-            reduce_common_words(d, 50)
-        except KeyError:
-            self.fail()
-    
-    def test_unpack_fields(self):
-        #object.words may be fields.
-        def create_it():
-            o = NamedObject('')
-            o.words = [['foo','bar'],['baz']]
-            return o
-        
-        d = {
-            'foo': set([create_it() for i in range(50)])
-        }
-        try:
-            reduce_common_words(d, 50)
-        except TypeError:
-            self.fail("must support fields.")
-    
-    def test_consider_a_reduced_common_word_common_even_after_reduction(self):
-        #There was a bug in the code that causeda word that has already been reduced not to
-        #be counted as a common word for subsequent words. For example, if 'foo' is processed
-        #as a common word, keeping a "foo bar" file in it, and the 'bar' is processed, "foo bar"
-        #would not stay in 'bar' because 'foo' is not a common word anymore.
-        only_common = NamedObject('foo bar',True)
-        d = {
-            'foo': set([NamedObject('foo bar baz',True) for i in range(49)] + [only_common]),
-            'bar': set([NamedObject('foo bar baz',True) for i in range(49)] + [only_common]),
-            'baz': set([NamedObject('foo bar baz',True) for i in range(49)])
-        }
-        reduce_common_words(d, 50)
-        self.assertEqual(1,len(d['foo']))
-        self.assertEqual(1,len(d['bar']))
-        self.assertEqual(49,len(d['baz']))
-    
-
-class TCget_match(TestCase):
-    def test_simple(self):
-        o1 = NamedObject("foo bar",True)
-        o2 = NamedObject("bar bleh",True)
-        m = get_match(o1,o2)
-        self.assertEqual(50,m.percentage)
-        self.assertEqual(['foo','bar'],m.first.words)
-        self.assertEqual(['bar','bleh'],m.second.words)
-        self.assert_(m.first is o1)
-        self.assert_(m.second is o2)
-    
-    def test_in(self):
-        o1 = NamedObject("foo",True)
-        o2 = NamedObject("bar",True)
-        m = get_match(o1,o2)
-        self.assert_(o1 in m)
-        self.assert_(o2 in m)
-        self.assert_(object() not in m)
-    
-    def test_word_weight(self):
-        self.assertEqual(int((6.0 / 13.0) * 100),get_match(NamedObject("foo bar",True),NamedObject("bar bleh",True),(WEIGHT_WORDS,)).percentage)
-    
-
-class GetMatches(TestCase):
-    def test_empty(self):
-        eq_(getmatches([]), [])
-    
-    def test_simple(self):
-        l = [NamedObject("foo bar"),NamedObject("bar bleh"),NamedObject("a b c foo")]
-        r = getmatches(l)
-        self.assertEqual(2,len(r))
-        seek = [m for m in r if m.percentage == 50] #"foo bar" and "bar bleh"
-        m = seek[0]
-        self.assertEqual(['foo','bar'],m.first.words)
-        self.assertEqual(['bar','bleh'],m.second.words)
-        seek = [m for m in r if m.percentage == 33] #"foo bar" and "a b c foo"
-        m = seek[0]
-        self.assertEqual(['foo','bar'],m.first.words)
-        self.assertEqual(['a','b','c','foo'],m.second.words)
-    
-    def test_null_and_unrelated_objects(self):
-        l = [NamedObject("foo bar"),NamedObject("bar bleh"),NamedObject(""),NamedObject("unrelated object")]
-        r = getmatches(l)
-        self.assertEqual(1,len(r))
-        m = r[0]
-        self.assertEqual(50,m.percentage)
-        self.assertEqual(['foo','bar'],m.first.words)
-        self.assertEqual(['bar','bleh'],m.second.words)
-    
-    def test_twice_the_same_word(self):
-        l = [NamedObject("foo foo bar"),NamedObject("bar bleh")]
-        r = getmatches(l)
-        self.assertEqual(1,len(r))
-    
-    def test_twice_the_same_word_when_preworded(self):
-        l = [NamedObject("foo foo bar",True),NamedObject("bar bleh",True)]
-        r = getmatches(l)
-        self.assertEqual(1,len(r))
-    
-    def test_two_words_match(self):
-        l = [NamedObject("foo bar"),NamedObject("foo bar bleh")]
-        r = getmatches(l)
-        self.assertEqual(1,len(r))
-    
-    def test_match_files_with_only_common_words(self):
-        #If a word occurs more than 50 times, it is excluded from the matching process
-        #The problem with the common_word_threshold is that the files containing only common
-        #words will never be matched together. We *should* match them.
-        # This test assumes that the common word threashold const is 50
-        l = [NamedObject("foo") for i in range(50)]
-        r = getmatches(l)
-        self.assertEqual(1225,len(r))
-    
-    def test_use_words_already_there_if_there(self):
-        o1 = NamedObject('foo')
-        o2 = NamedObject('bar')
-        o2.words = ['foo']
-        eq_(1, len(getmatches([o1,o2])))
-    
-    def test_job(self):
-        def do_progress(p,d=''):
-            self.log.append(p)
-            return True
-        
-        j = job.Job(1,do_progress)
-        self.log = []
-        s = "foo bar"
-        getmatches([NamedObject(s), NamedObject(s), NamedObject(s)], j=j)
-        self.assert_(len(self.log) > 2)
-        self.assertEqual(0,self.log[0])
-        self.assertEqual(100,self.log[-1])
-    
-    def test_weight_words(self):
-        l = [NamedObject("foo bar"),NamedObject("bar bleh")]
-        m = getmatches(l, weight_words=True)[0]
-        self.assertEqual(int((6.0 / 13.0) * 100),m.percentage)
-    
-    def test_similar_word(self):
-        l = [NamedObject("foobar"),NamedObject("foobars")]
-        eq_(len(getmatches(l, match_similar_words=True)), 1)
-        eq_(getmatches(l, match_similar_words=True)[0].percentage, 100)
-        l = [NamedObject("foobar"),NamedObject("foo")]
-        eq_(len(getmatches(l, match_similar_words=True)), 0) #too far
-        l = [NamedObject("bizkit"),NamedObject("bizket")]
-        eq_(len(getmatches(l, match_similar_words=True)), 1)
-        l = [NamedObject("foobar"),NamedObject("foosbar")]
-        eq_(len(getmatches(l, match_similar_words=True)), 1)
-    
-    def test_single_object_with_similar_words(self):
-        l = [NamedObject("foo foos")]
-        eq_(len(getmatches(l, match_similar_words=True)), 0)
-    
-    def test_double_words_get_counted_only_once(self):
-        l = [NamedObject("foo bar foo bleh"),NamedObject("foo bar bleh bar")]
-        m = getmatches(l)[0]
-        self.assertEqual(75,m.percentage)
-    
-    def test_with_fields(self):
-        o1 = NamedObject("foo bar - foo bleh")
-        o2 = NamedObject("foo bar - bleh bar")
-        o1.words = getfields(o1.name)
-        o2.words = getfields(o2.name)
-        m = getmatches([o1, o2])[0]
-        self.assertEqual(50, m.percentage)
-    
-    def test_with_fields_no_order(self):
-        o1 = NamedObject("foo bar - foo bleh")
-        o2 = NamedObject("bleh bang - foo bar")
-        o1.words = getfields(o1.name)
-        o2.words = getfields(o2.name)
-        m = getmatches([o1, o2], no_field_order=True)[0]
-        eq_(m.percentage, 50)
-    
-    def test_only_match_similar_when_the_option_is_set(self):
-        l = [NamedObject("foobar"),NamedObject("foobars")]
-        eq_(len(getmatches(l, match_similar_words=False)), 0)
-    
-    def test_dont_recurse_do_match(self):
-        # with nosetests, the stack is increased. The number has to be high enough not to be failing falsely
-        sys.setrecursionlimit(100)
-        files = [NamedObject('foo bar') for i in range(101)]
-        try:
-            getmatches(files)
-        except RuntimeError:
-            self.fail()
-        finally:
-            sys.setrecursionlimit(1000)
-    
-    def test_min_match_percentage(self):
-        l = [NamedObject("foo bar"),NamedObject("bar bleh"),NamedObject("a b c foo")]
-        r = getmatches(l, min_match_percentage=50)
-        self.assertEqual(1,len(r)) #Only "foo bar" / "bar bleh" should match
-    
-    def test_MemoryError(self):
-        @log_calls
-        def mocked_match(first, second, flags):
-            if len(mocked_match.calls) > 42:
-                raise MemoryError()
-            return Match(first, second, 0)
-        
-        objects = [NamedObject() for i in range(10)] # results in 45 matches
-        self.mock(engine, 'get_match', mocked_match)
-        try:
-            r = getmatches(objects)
-        except MemoryError:
-            self.fail('MemorryError must be handled')
-        self.assertEqual(42, len(r))
-    
-
-class TCGroup(TestCase):
-    def test_empy(self):
-        g = Group()
-        self.assertEqual(None,g.ref)
-        self.assertEqual([],g.dupes)
-        self.assertEqual(0,len(g.matches))
-    
-    def test_add_match(self):
-        g = Group()
-        m = get_match(NamedObject("foo",True),NamedObject("bar",True))
-        g.add_match(m)
-        self.assert_(g.ref is m.first)
-        self.assertEqual([m.second],g.dupes)
-        self.assertEqual(1,len(g.matches))
-        self.assert_(m in g.matches)
-    
-    def test_multiple_add_match(self):
-        g = Group()
-        o1 = NamedObject("a",True)
-        o2 = NamedObject("b",True)
-        o3 = NamedObject("c",True)
-        o4 = NamedObject("d",True)
-        g.add_match(get_match(o1,o2))
-        self.assert_(g.ref is o1)
-        self.assertEqual([o2],g.dupes)
-        self.assertEqual(1,len(g.matches))
-        g.add_match(get_match(o1,o3))
-        self.assertEqual([o2],g.dupes)
-        self.assertEqual(2,len(g.matches))
-        g.add_match(get_match(o2,o3))
-        self.assertEqual([o2,o3],g.dupes)
-        self.assertEqual(3,len(g.matches))
-        g.add_match(get_match(o1,o4))
-        self.assertEqual([o2,o3],g.dupes)
-        self.assertEqual(4,len(g.matches))
-        g.add_match(get_match(o2,o4))
-        self.assertEqual([o2,o3],g.dupes)
-        self.assertEqual(5,len(g.matches))
-        g.add_match(get_match(o3,o4))
-        self.assertEqual([o2,o3,o4],g.dupes)
-        self.assertEqual(6,len(g.matches))
-    
-    def test_len(self):
-        g = Group()
-        self.assertEqual(0,len(g))
-        g.add_match(get_match(NamedObject("foo",True),NamedObject("bar",True)))
-        self.assertEqual(2,len(g))
-    
-    def test_add_same_match_twice(self):
-        g = Group()
-        m = get_match(NamedObject("foo",True),NamedObject("foo",True))
-        g.add_match(m)
-        self.assertEqual(2,len(g))
-        self.assertEqual(1,len(g.matches))
-        g.add_match(m)
-        self.assertEqual(2,len(g))
-        self.assertEqual(1,len(g.matches))
-    
-    def test_in(self):
-        g = Group()
-        o1 = NamedObject("foo",True)
-        o2 = NamedObject("bar",True)
-        self.assert_(o1 not in g)
-        g.add_match(get_match(o1,o2))
-        self.assert_(o1 in g)
-        self.assert_(o2 in g)
-    
-    def test_remove(self):
-        g = Group()
-        o1 = NamedObject("foo",True)
-        o2 = NamedObject("bar",True)
-        o3 = NamedObject("bleh",True)
-        g.add_match(get_match(o1,o2))
-        g.add_match(get_match(o1,o3))
-        g.add_match(get_match(o2,o3))
-        self.assertEqual(3,len(g.matches))
-        self.assertEqual(3,len(g))
-        g.remove_dupe(o3)
-        self.assertEqual(1,len(g.matches))
-        self.assertEqual(2,len(g))
-        g.remove_dupe(o1)
-        self.assertEqual(0,len(g.matches))
-        self.assertEqual(0,len(g))
-    
-    def test_remove_with_ref_dupes(self):
-        g = Group()
-        o1 = NamedObject("foo",True)
-        o2 = NamedObject("bar",True)
-        o3 = NamedObject("bleh",True)
-        g.add_match(get_match(o1,o2))
-        g.add_match(get_match(o1,o3))
-        g.add_match(get_match(o2,o3))
-        o1.is_ref = True
-        o2.is_ref = True
-        g.remove_dupe(o3)
-        self.assertEqual(0,len(g))
-    
-    def test_switch_ref(self):
-        o1 = NamedObject(with_words=True)
-        o2 = NamedObject(with_words=True)
-        g = Group()
-        g.add_match(get_match(o1,o2))
-        self.assert_(o1 is g.ref)
-        g.switch_ref(o2)
-        self.assert_(o2 is g.ref)
-        self.assertEqual([o1],g.dupes)
-        g.switch_ref(o2)
-        self.assert_(o2 is g.ref)
-        g.switch_ref(NamedObject('',True))
-        self.assert_(o2 is g.ref)
-    
-    def test_get_match_of(self):
-        g = Group()
-        for m in get_match_triangle():
-            g.add_match(m)
-        o = g.dupes[0]
-        m = g.get_match_of(o)
-        self.assert_(g.ref in m)
-        self.assert_(o in m)
-        self.assert_(g.get_match_of(NamedObject('',True)) is None)
-        self.assert_(g.get_match_of(g.ref) is None)
-    
-    def test_percentage(self):
-        #percentage should return the avg percentage in relation to the ref
-        m1,m2,m3 = get_match_triangle()
-        m1 = Match(m1[0], m1[1], 100)
-        m2 = Match(m2[0], m2[1], 50)
-        m3 = Match(m3[0], m3[1], 33)
-        g = Group()
-        g.add_match(m1)
-        g.add_match(m2)
-        g.add_match(m3)
-        self.assertEqual(75,g.percentage)
-        g.switch_ref(g.dupes[0])
-        self.assertEqual(66,g.percentage)
-        g.remove_dupe(g.dupes[0])
-        self.assertEqual(33,g.percentage)
-        g.add_match(m1)
-        g.add_match(m2)
-        self.assertEqual(66,g.percentage)
-    
-    def test_percentage_on_empty_group(self):
-        g = Group()
-        self.assertEqual(0,g.percentage)
-    
-    def test_prioritize(self):
-        m1,m2,m3 = get_match_triangle()
-        o1 = m1.first
-        o2 = m1.second
-        o3 = m2.second
-        o1.name = 'c'
-        o2.name = 'b'
-        o3.name = 'a'
-        g = Group()
-        g.add_match(m1)
-        g.add_match(m2)
-        g.add_match(m3)
-        self.assert_(o1 is g.ref)
-        g.prioritize(lambda x:x.name)
-        self.assert_(o3 is g.ref)
-    
-    def test_prioritize_with_tie_breaker(self):
-        # if the ref has the same key as one or more of the dupe, run the tie_breaker func among them
-        g = get_test_group()
-        o1, o2, o3 = g.ordered
-        tie_breaker = lambda ref, dupe: dupe is o3
-        g.prioritize(lambda x:0, tie_breaker)
-        self.assertTrue(g.ref is o3)
-    
-    def test_prioritize_with_tie_breaker_runs_on_all_dupes(self):
-        # Even if a dupe is chosen to switch with ref with a tie breaker, we still run the tie breaker 
-        # with other dupes and the newly chosen ref
-        g = get_test_group()
-        o1, o2, o3 = g.ordered
-        o1.foo = 1
-        o2.foo = 2
-        o3.foo = 3
-        tie_breaker = lambda ref, dupe: dupe.foo > ref.foo
-        g.prioritize(lambda x:0, tie_breaker)
-        self.assertTrue(g.ref is o3)
-    
-    def test_prioritize_with_tie_breaker_runs_only_on_tie_dupes(self):
-        # The tie breaker only runs on dupes that had the same value for the key_func
-        g = get_test_group()
-        o1, o2, o3 = g.ordered
-        o1.foo = 2
-        o2.foo = 2
-        o3.foo = 1
-        o1.bar = 1
-        o2.bar = 2
-        o3.bar = 3
-        key_func = lambda x: -x.foo
-        tie_breaker = lambda ref, dupe: dupe.bar > ref.bar
-        g.prioritize(key_func, tie_breaker)
-        self.assertTrue(g.ref is o2)
-    
-    def test_list_like(self):
-        g = Group()
-        o1,o2 = (NamedObject("foo",True),NamedObject("bar",True))
-        g.add_match(get_match(o1,o2))
-        self.assert_(g[0] is o1)
-        self.assert_(g[1] is o2)
-    
-    def test_discard_matches(self):
-        g = Group()
-        o1,o2,o3 = (NamedObject("foo",True),NamedObject("bar",True),NamedObject("baz",True))
-        g.add_match(get_match(o1,o2))
-        g.add_match(get_match(o1,o3))
-        g.discard_matches()
-        self.assertEqual(1,len(g.matches))
-        self.assertEqual(0,len(g.candidates))
-    
-
-class TCget_groups(TestCase):
-    def test_empty(self):
-        r = get_groups([])
-        self.assertEqual([],r)
-    
-    def test_simple(self):
-        l = [NamedObject("foo bar"),NamedObject("bar bleh")]
-        matches = getmatches(l)
-        m = matches[0]
-        r = get_groups(matches)
-        self.assertEqual(1,len(r))
-        g = r[0]
-        self.assert_(g.ref is m.first)
-        self.assertEqual([m.second],g.dupes)
-    
-    def test_group_with_multiple_matches(self):
-        #This results in 3 matches
-        l = [NamedObject("foo"),NamedObject("foo"),NamedObject("foo")]
-        matches = getmatches(l)
-        r = get_groups(matches)
-        self.assertEqual(1,len(r))
-        g = r[0]
-        self.assertEqual(3,len(g))
-    
-    def test_must_choose_a_group(self):
-        l = [NamedObject("a b"),NamedObject("a b"),NamedObject("b c"),NamedObject("c d"),NamedObject("c d")]
-        #There will be 2 groups here: group "a b" and group "c d"
-        #"b c" can go either of them, but not both.
-        matches = getmatches(l)
-        r = get_groups(matches)
-        self.assertEqual(2,len(r))
-        self.assertEqual(5,len(r[0])+len(r[1]))
-    
-    def test_should_all_go_in_the_same_group(self):
-        l = [NamedObject("a b"),NamedObject("a b"),NamedObject("a b"),NamedObject("a b")]
-        #There will be 2 groups here: group "a b" and group "c d"
-        #"b c" can fit in both, but it must be in only one of them
-        matches = getmatches(l)
-        r = get_groups(matches)
-        self.assertEqual(1,len(r))
-    
-    def test_give_priority_to_matches_with_higher_percentage(self):
-        o1 = NamedObject(with_words=True)
-        o2 = NamedObject(with_words=True)
-        o3 = NamedObject(with_words=True)
-        m1 = Match(o1, o2, 1)
-        m2 = Match(o2, o3, 2)
-        r = get_groups([m1,m2])
-        self.assertEqual(1,len(r))
-        g = r[0]
-        self.assertEqual(2,len(g))
-        self.assert_(o1 not in g)
-        self.assert_(o2 in g)
-        self.assert_(o3 in g)
-    
-    def test_four_sized_group(self):
-        l = [NamedObject("foobar") for i in xrange(4)]
-        m = getmatches(l)
-        r = get_groups(m)
-        self.assertEqual(1,len(r))
-        self.assertEqual(4,len(r[0]))
-    
-    def test_referenced_by_ref2(self):
-        o1 = NamedObject(with_words=True)
-        o2 = NamedObject(with_words=True)
-        o3 = NamedObject(with_words=True)
-        m1 = get_match(o1,o2)
-        m2 = get_match(o3,o1)
-        m3 = get_match(o3,o2)
-        r = get_groups([m1,m2,m3])
-        self.assertEqual(3,len(r[0]))
-    
-    def test_job(self):
-        def do_progress(p,d=''):
-            self.log.append(p)
-            return True
-        
-        self.log = []
-        j = job.Job(1,do_progress)
-        m1,m2,m3 = get_match_triangle()
-        #101%: To make sure it is processed first so the job test works correctly
-        m4 = Match(NamedObject('a',True), NamedObject('a',True), 101)
-        get_groups([m1,m2,m3,m4],j)
-        self.assertEqual(0,self.log[0])
-        self.assertEqual(100,self.log[-1])
-    
-    def test_group_admissible_discarded_dupes(self):
-        # If, with a (A, B, C, D) set, all match with A, but C and D don't match with B and that the
-        # (A, B) match is the highest (thus resulting in an (A, B) group), still match C and D
-        # in a separate group instead of discarding them.
-        A, B, C, D = [NamedObject() for _ in range(4)]
-        m1 = Match(A, B, 90) # This is the strongest "A" match
-        m2 = Match(A, C, 80) # Because C doesn't match with B, it won't be in the group
-        m3 = Match(A, D, 80) # Same thing for D
-        m4 = Match(C, D, 70) # However, because C and D match, they should have their own group.
-        groups = get_groups([m1, m2, m3, m4])
-        eq_(len(groups), 2)
-        g1, g2 = groups
-        assert A in g1
-        assert B in g1
-        assert C in g2
-        assert D in g2
-    

base/py/tests/ignore_test.py

@@ -1,152 +0,0 @@
-# Created By: Virgil Dupras
-# Created On: 2006/05/02
-# $Id$
-# Copyright 2009 Hardcoded Software (http://www.hardcoded.net)
-# 
-# This software is licensed under the "HS" License as described in the "LICENSE" file, 
-# which should be included with this package. The terms are also available at 
-# http://www.hardcoded.net/licenses/hs_license
-
-import cStringIO
-import xml.dom.minidom
-
-from nose.tools import eq_
-
-from ..ignore import *
-
-def test_empty():
-    il = IgnoreList()
-    eq_(0,len(il))
-    assert not il.AreIgnored('foo','bar')
-
-def test_simple():
-    il = IgnoreList()
-    il.Ignore('foo','bar')
-    assert il.AreIgnored('foo','bar')
-    assert il.AreIgnored('bar','foo')
-    assert not il.AreIgnored('foo','bleh')
-    assert not il.AreIgnored('bleh','bar')
-    eq_(1,len(il))
-
-def test_multiple():
-    il = IgnoreList()
-    il.Ignore('foo','bar')
-    il.Ignore('foo','bleh')
-    il.Ignore('bleh','bar')
-    il.Ignore('aybabtu','bleh')
-    assert il.AreIgnored('foo','bar')
-    assert il.AreIgnored('bar','foo')
-    assert il.AreIgnored('foo','bleh')
-    assert il.AreIgnored('bleh','bar')
-    assert not il.AreIgnored('aybabtu','bar')
-    eq_(4,len(il))
-
-def test_clear():
-    il = IgnoreList()
-    il.Ignore('foo','bar')
-    il.Clear()
-    assert not il.AreIgnored('foo','bar')
-    assert not il.AreIgnored('bar','foo')
-    eq_(0,len(il))
-
-def test_add_same_twice():
-    il = IgnoreList()
-    il.Ignore('foo','bar')
-    il.Ignore('bar','foo')
-    eq_(1,len(il))
-
-def test_save_to_xml():
-    il = IgnoreList()
-    il.Ignore('foo','bar')
-    il.Ignore('foo','bleh')
-    il.Ignore('bleh','bar')
-    f = cStringIO.StringIO()
-    il.save_to_xml(f)
-    f.seek(0)
-    doc = xml.dom.minidom.parse(f)
-    root = doc.documentElement
-    eq_('ignore_list',root.nodeName)
-    children = [c for c in root.childNodes if c.localName]
-    eq_(2,len(children))
-    eq_(2,len([c for c in children if c.nodeName == 'file']))
-    f1,f2 = children
-    subchildren = [c for c in f1.childNodes if c.localName == 'file'] +\
-        [c for c in f2.childNodes if c.localName == 'file']
-    eq_(3,len(subchildren))
-
-def test_SaveThenLoad():
-    il = IgnoreList()
-    il.Ignore('foo','bar')
-    il.Ignore('foo','bleh')
-    il.Ignore('bleh','bar')
-    il.Ignore(u'\u00e9','bar')
-    f = cStringIO.StringIO()
-    il.save_to_xml(f)
-    f.seek(0)
-    il = IgnoreList()
-    il.load_from_xml(f)
-    eq_(4,len(il))
-    assert il.AreIgnored(u'\u00e9','bar')
-    
-def test_LoadXML_with_empty_file_tags():
-    f = cStringIO.StringIO()
-    f.write('<?xml version="1.0" encoding="utf-8"?><ignore_list><file><file/></file></ignore_list>')
-    f.seek(0)
-    il = IgnoreList()
-    il.load_from_xml(f)
-    eq_(0,len(il))
-    
-def test_AreIgnore_works_when_a_child_is_a_key_somewhere_else():
-    il = IgnoreList()
-    il.Ignore('foo','bar')
-    il.Ignore('bar','baz')
-    assert il.AreIgnored('bar','foo')
-
-
-def test_no_dupes_when_a_child_is_a_key_somewhere_else():
-    il = IgnoreList()
-    il.Ignore('foo','bar')
-    il.Ignore('bar','baz')
-    il.Ignore('bar','foo')
-    eq_(2,len(il))
-
-def test_iterate():
-    #It must be possible to iterate through ignore list
-    il = IgnoreList()
-    expected = [('foo','bar'),('bar','baz'),('foo','baz')]
-    for i in expected:
-        il.Ignore(i[0],i[1])
-    for i in il:
-        expected.remove(i) #No exception should be raised
-    assert not expected #expected should be empty
-
-def test_filter():
-    il = IgnoreList()
-    il.Ignore('foo','bar')
-    il.Ignore('bar','baz')
-    il.Ignore('foo','baz')
-    il.Filter(lambda f,s: f == 'bar')
-    eq_(1,len(il))
-    assert not il.AreIgnored('foo','bar')
-    assert il.AreIgnored('bar','baz')
-
-def test_save_with_non_ascii_non_unicode_items():
-    il = IgnoreList()
-    il.Ignore('\xac','\xbf')
-    f = cStringIO.StringIO()
-    try:
-        il.save_to_xml(f)
-    except Exception as e:
-        raise AssertionError(unicode(e))
-
-def test_len():
-    il = IgnoreList()
-    eq_(0,len(il))
-    il.Ignore('foo','bar')
-    eq_(1,len(il))
-
-def test_nonzero():
-    il = IgnoreList()
-    assert not il
-    il.Ignore('foo','bar')
-    assert il

base/py/tests/results_test.py

@@ -1,718 +0,0 @@
-# Created By: Virgil Dupras
-# Created On: 2006/02/23
-# $Id$                                  
-# Copyright 2009 Hardcoded Software (http://www.hardcoded.net)
-# 
-# This software is licensed under the "HS" License as described in the "LICENSE" file, 
-# which should be included with this package. The terms are also available at 
-# http://www.hardcoded.net/licenses/hs_license
-
-import unittest
-import StringIO
-import xml.dom.minidom
-import os.path as op
-
-from hsutil.path import Path
-from hsutil.testcase import TestCase
-from hsutil.misc import first
-
-from . import engine_test, data
-from .. import engine
-from ..results import *
-
-class NamedObject(engine_test.NamedObject):
-    size = 1
-    path = property(lambda x:Path('basepath') + x.name)
-    is_ref = False
-    
-    def __nonzero__(self):
-        return False #Make sure that operations are made correctly when the bool value of files is false.
-
-# Returns a group set that looks like that:
-# "foo bar" (1)
-#   "bar bleh" (1024)
-#   "foo bleh" (1)
-# "ibabtu" (1)
-#   "ibabtu" (1)
-def GetTestGroups():
-    objects = [NamedObject("foo bar"),NamedObject("bar bleh"),NamedObject("foo bleh"),NamedObject("ibabtu"),NamedObject("ibabtu")]
-    objects[1].size = 1024
-    matches = engine.getmatches(objects) #we should have 5 matches
-    groups = engine.get_groups(matches) #We should have 2 groups
-    for g in groups:
-        g.prioritize(lambda x:objects.index(x)) #We want the dupes to be in the same order as the list is
-    groups.sort(key=len, reverse=True) # We want the group with 3 members to be first.
-    return (objects,matches,groups)
-
-class TCResultsEmpty(TestCase):
-    def setUp(self):
-        self.results = Results(data)
-    
-    def test_apply_invalid_filter(self):
-        # If the applied filter is an invalid regexp, just ignore the filter.
-        self.results.apply_filter('[') # invalid
-        self.test_stat_line() # make sure that the stats line isn't saying we applied a '[' filter
-    
-    def test_stat_line(self):
-        self.assertEqual("0 / 0 (0.00 B / 0.00 B) duplicates marked.",self.results.stat_line)
-    
-    def test_groups(self):
-        self.assertEqual(0,len(self.results.groups))
-    
-    def test_get_group_of_duplicate(self):
-        self.assert_(self.results.get_group_of_duplicate('foo') is None)
-    
-    def test_save_to_xml(self):
-        f = StringIO.StringIO()
-        self.results.save_to_xml(f)
-        f.seek(0)
-        doc = xml.dom.minidom.parse(f)
-        root = doc.documentElement
-        self.assertEqual('results',root.nodeName)
-    
-
-class TCResultsWithSomeGroups(TestCase):
-    def setUp(self):
-        self.results = Results(data)
-        self.objects,self.matches,self.groups = GetTestGroups()
-        self.results.groups = self.groups
-    
-    def test_stat_line(self):
-        self.assertEqual("0 / 3 (0.00 B / 1.01 KB) duplicates marked.",self.results.stat_line)
-    
-    def test_groups(self):
-        self.assertEqual(2,len(self.results.groups))
-    
-    def test_get_group_of_duplicate(self):
-        for o in self.objects:
-            g = self.results.get_group_of_duplicate(o)
-            self.assert_(isinstance(g, engine.Group))
-            self.assert_(o in g)
-        self.assert_(self.results.get_group_of_duplicate(self.groups[0]) is None)
-    
-    def test_remove_duplicates(self):
-        g1,g2 = self.results.groups
-        self.results.remove_duplicates([g1.dupes[0]])
-        self.assertEqual(2,len(g1))
-        self.assert_(g1 in self.results.groups)
-        self.results.remove_duplicates([g1.ref])
-        self.assertEqual(2,len(g1))
-        self.assert_(g1 in self.results.groups)
-        self.results.remove_duplicates([g1.dupes[0]])
-        self.assertEqual(0,len(g1))
-        self.assert_(g1 not in self.results.groups)
-        self.results.remove_duplicates([g2.dupes[0]])
-        self.assertEqual(0,len(g2))
-        self.assert_(g2 not in self.results.groups)
-        self.assertEqual(0,len(self.results.groups))
-    
-    def test_remove_duplicates_with_ref_files(self):
-        g1,g2 = self.results.groups
-        self.objects[0].is_ref = True
-        self.objects[1].is_ref = True
-        self.results.remove_duplicates([self.objects[2]])
-        self.assertEqual(0,len(g1))
-        self.assert_(g1 not in self.results.groups)
-    
-    def test_make_ref(self):
-        g = self.results.groups[0]
-        d = g.dupes[0]
-        self.results.make_ref(d)
-        self.assert_(d is g.ref)
-    
-    def test_sort_groups(self):
-        self.results.make_ref(self.objects[1]) #We want to make the 1024 sized object to go ref.
-        g1,g2 = self.groups
-        self.results.sort_groups(2) #2 is the key for size
-        self.assert_(self.results.groups[0] is g2)
-        self.assert_(self.results.groups[1] is g1)
-        self.results.sort_groups(2,False)
-        self.assert_(self.results.groups[0] is g1)
-        self.assert_(self.results.groups[1] is g2)
-    
-    def test_set_groups_when_sorted(self):
-        self.results.make_ref(self.objects[1]) #We want to make the 1024 sized object to go ref.
-        self.results.sort_groups(2)
-        objects,matches,groups = GetTestGroups()
-        g1,g2 = groups
-        g1.switch_ref(objects[1])
-        self.results.groups = groups
-        self.assert_(self.results.groups[0] is g2)
-        self.assert_(self.results.groups[1] is g1)
-    
-    def test_get_dupe_list(self):
-        self.assertEqual([self.objects[1],self.objects[2],self.objects[4]],self.results.dupes)
-    
-    def test_dupe_list_is_cached(self):
-        self.assert_(self.results.dupes is self.results.dupes)
-    
-    def test_dupe_list_cache_is_invalidated_when_needed(self):
-        o1,o2,o3,o4,o5 = self.objects
-        self.assertEqual([o2,o3,o5],self.results.dupes)
-        self.results.make_ref(o2)
-        self.assertEqual([o1,o3,o5],self.results.dupes)
-        objects,matches,groups = GetTestGroups()
-        o1,o2,o3,o4,o5 = objects
-        self.results.groups = groups
-        self.assertEqual([o2,o3,o5],self.results.dupes)
-    
-    def test_dupe_list_sort(self):
-        o1,o2,o3,o4,o5 = self.objects
-        o1.size = 5
-        o2.size = 4
-        o3.size = 3
-        o4.size = 2
-        o5.size = 1
-        self.results.sort_dupes(2)
-        self.assertEqual([o5,o3,o2],self.results.dupes)
-        self.results.sort_dupes(2,False)
-        self.assertEqual([o2,o3,o5],self.results.dupes)
-    
-    def test_dupe_list_remember_sort(self):
-        o1,o2,o3,o4,o5 = self.objects
-        o1.size = 5
-        o2.size = 4
-        o3.size = 3
-        o4.size = 2
-        o5.size = 1
-        self.results.sort_dupes(2)
-        self.results.make_ref(o2)
-        self.assertEqual([o5,o3,o1],self.results.dupes)
-    
-    def test_dupe_list_sort_delta_values(self):
-        o1,o2,o3,o4,o5 = self.objects
-        o1.size = 10
-        o2.size = 2 #-8
-        o3.size = 3 #-7
-        o4.size = 20
-        o5.size = 1 #-19
-        self.results.sort_dupes(2,delta=True)
-        self.assertEqual([o5,o2,o3],self.results.dupes)
-    
-    def test_sort_empty_list(self):
-        #There was an infinite loop when sorting an empty list.
-        r = Results(data)
-        r.sort_dupes(0)
-        self.assertEqual([],r.dupes)
-    
-    def test_dupe_list_update_on_remove_duplicates(self):
-        o1,o2,o3,o4,o5 = self.objects
-        self.assertEqual(3,len(self.results.dupes))
-        self.results.remove_duplicates([o2])
-        self.assertEqual(2,len(self.results.dupes))
-    
-
-class TCResultsMarkings(TestCase):
-    def setUp(self):
-        self.results = Results(data)
-        self.objects,self.matches,self.groups = GetTestGroups()
-        self.results.groups = self.groups
-    
-    def test_stat_line(self):
-        self.assertEqual("0 / 3 (0.00 B / 1.01 KB) duplicates marked.",self.results.stat_line)
-        self.results.mark(self.objects[1])
-        self.assertEqual("1 / 3 (1.00 KB / 1.01 KB) duplicates marked.",self.results.stat_line)
-        self.results.mark_invert()
-        self.assertEqual("2 / 3 (2.00 B / 1.01 KB) duplicates marked.",self.results.stat_line)
-        self.results.mark_invert()
-        self.results.unmark(self.objects[1])
-        self.results.mark(self.objects[2])
-        self.results.mark(self.objects[4])
-        self.assertEqual("2 / 3 (2.00 B / 1.01 KB) duplicates marked.",self.results.stat_line)
-        self.results.mark(self.objects[0]) #this is a ref, it can't be counted
-        self.assertEqual("2 / 3 (2.00 B / 1.01 KB) duplicates marked.",self.results.stat_line)
-        self.results.groups = self.groups
-        self.assertEqual("0 / 3 (0.00 B / 1.01 KB) duplicates marked.",self.results.stat_line)
-    
-    def test_with_ref_duplicate(self):
-        self.objects[1].is_ref = True
-        self.results.groups = self.groups
-        self.assert_(not self.results.mark(self.objects[1]))
-        self.results.mark(self.objects[2])
-        self.assertEqual("1 / 2 (1.00 B / 2.00 B) duplicates marked.",self.results.stat_line)
-    
-    def test_perform_on_marked(self):
-        def log_object(o):
-            log.append(o)
-            return True
-        
-        log = []
-        self.results.mark_all()
-        self.results.perform_on_marked(log_object,False)
-        self.assert_(self.objects[1] in log)
-        self.assert_(self.objects[2] in log)
-        self.assert_(self.objects[4] in log)
-        self.assertEqual(3,len(log))
-        log = []
-        self.results.mark_none()
-        self.results.mark(self.objects[4])
-        self.results.perform_on_marked(log_object,True)
-        self.assertEqual(1,len(log))
-        self.assert_(self.objects[4] in log)
-        self.assertEqual(1,len(self.results.groups))
-    
-    def test_perform_on_marked_with_problems(self):
-        def log_object(o):
-            log.append(o)
-            return o is not self.objects[1]
-        
-        log = []
-        self.results.mark_all()
-        self.assert_(self.results.is_marked(self.objects[1]))
-        self.assertEqual(1,self.results.perform_on_marked(log_object, True))
-        self.assertEqual(3,len(log))
-        self.assertEqual(1,len(self.results.groups))
-        self.assertEqual(2,len(self.results.groups[0]))
-        self.assert_(self.objects[1] in self.results.groups[0])
-        self.assert_(not self.results.is_marked(self.objects[2]))
-        self.assert_(self.results.is_marked(self.objects[1]))
-    
-    def test_perform_on_marked_with_ref(self):
-        def log_object(o):
-            log.append(o)
-            return True
-        
-        log = []
-        self.objects[0].is_ref = True
-        self.objects[1].is_ref = True
-        self.results.mark_all()
-        self.results.perform_on_marked(log_object,True)
-        self.assert_(self.objects[1] not in log)
-        self.assert_(self.objects[2] in log)
-        self.assert_(self.objects[4] in log)
-        self.assertEqual(2,len(log))
-        self.assertEqual(0,len(self.results.groups))
-    
-    def test_perform_on_marked_remove_objects_only_at_the_end(self):
-        def check_groups(o):
-            self.assertEqual(3,len(g1))
-            self.assertEqual(2,len(g2))
-            return True
-        
-        g1,g2 = self.results.groups
-        self.results.mark_all()
-        self.results.perform_on_marked(check_groups,True)
-        self.assertEqual(0,len(g1))
-        self.assertEqual(0,len(g2))
-        self.assertEqual(0,len(self.results.groups))
-    
-    def test_remove_duplicates(self):
-        g1 = self.results.groups[0]
-        g2 = self.results.groups[1]
-        self.results.mark(g1.dupes[0])
-        self.assertEqual("1 / 3 (1.00 KB / 1.01 KB) duplicates marked.",self.results.stat_line)
-        self.results.remove_duplicates([g1.dupes[1]])
-        self.assertEqual("1 / 2 (1.00 KB / 1.01 KB) duplicates marked.",self.results.stat_line)
-        self.results.remove_duplicates([g1.dupes[0]])
-        self.assertEqual("0 / 1 (0.00 B / 1.00 B) duplicates marked.",self.results.stat_line)
-    
-    def test_make_ref(self):
-        g = self.results.groups[0]
-        d = g.dupes[0]
-        self.results.mark(d)
-        self.assertEqual("1 / 3 (1.00 KB / 1.01 KB) duplicates marked.",self.results.stat_line)
-        self.results.make_ref(d)
-        self.assertEqual("0 / 3 (0.00 B / 3.00 B) duplicates marked.",self.results.stat_line)
-        self.results.make_ref(d)
-        self.assertEqual("0 / 3 (0.00 B / 3.00 B) duplicates marked.",self.results.stat_line)
-    
-    def test_SaveXML(self):
-        self.results.mark(self.objects[1])
-        self.results.mark_invert()
-        f = StringIO.StringIO()
-        self.results.save_to_xml(f)
-        f.seek(0)
-        doc = xml.dom.minidom.parse(f)
-        root = doc.documentElement
-        g1,g2 = root.getElementsByTagName('group')
-        d1,d2,d3 = g1.getElementsByTagName('file')
-        self.assertEqual('n',d1.getAttributeNode('marked').nodeValue)
-        self.assertEqual('n',d2.getAttributeNode('marked').nodeValue)
-        self.assertEqual('y',d3.getAttributeNode('marked').nodeValue)
-        d1,d2 = g2.getElementsByTagName('file')
-        self.assertEqual('n',d1.getAttributeNode('marked').nodeValue)
-        self.assertEqual('y',d2.getAttributeNode('marked').nodeValue)
-    
-    def test_LoadXML(self):
-        def get_file(path):
-            return [f for f in self.objects if str(f.path) == path][0]
-        
-        self.objects[4].name = 'ibabtu 2' #we can't have 2 files with the same path
-        self.results.mark(self.objects[1])
-        self.results.mark_invert()
-        f = StringIO.StringIO()
-        self.results.save_to_xml(f)
-        f.seek(0)
-        r = Results(data)
-        r.load_from_xml(f,get_file)
-        self.assert_(not r.is_marked(self.objects[0]))
-        self.assert_(not r.is_marked(self.objects[1]))
-        self.assert_(r.is_marked(self.objects[2]))
-        self.assert_(not r.is_marked(self.objects[3]))
-        self.assert_(r.is_marked(self.objects[4]))
-    
-
-class TCResultsXML(TestCase):
-    def setUp(self):
-        self.results = Results(data)
-        self.objects, self.matches, self.groups = GetTestGroups()
-        self.results.groups = self.groups
-    
-    def get_file(self, path): # use this as a callback for load_from_xml
-        return [o for o in self.objects if o.path == path][0]
-    
-    def test_save_to_xml(self):
-        self.objects[0].is_ref = True
-        self.objects[0].words = [['foo','bar']]
-        f = StringIO.StringIO()
-        self.results.save_to_xml(f)
-        f.seek(0)
-        doc = xml.dom.minidom.parse(f)
-        root = doc.documentElement
-        self.assertEqual('results',root.nodeName)
-        children = [c for c in root.childNodes if c.localName]
-        self.assertEqual(2,len(children))
-        self.assertEqual(2,len([c for c in children if c.nodeName == 'group']))
-        g1,g2 = children
-        children = [c for c in g1.childNodes if c.localName]
-        self.assertEqual(6,len(children))
-        self.assertEqual(3,len([c for c in children if c.nodeName == 'file']))
-        self.assertEqual(3,len([c for c in children if c.nodeName == 'match']))
-        d1,d2,d3 = [c for c in children if c.nodeName == 'file']
-        self.assertEqual(op.join('basepath','foo bar'),d1.getAttributeNode('path').nodeValue)
-        self.assertEqual(op.join('basepath','bar bleh'),d2.getAttributeNode('path').nodeValue)
-        self.assertEqual(op.join('basepath','foo bleh'),d3.getAttributeNode('path').nodeValue)
-        self.assertEqual('y',d1.getAttributeNode('is_ref').nodeValue)
-        self.assertEqual('n',d2.getAttributeNode('is_ref').nodeValue)
-        self.assertEqual('n',d3.getAttributeNode('is_ref').nodeValue)
-        self.assertEqual('foo,bar',d1.getAttributeNode('words').nodeValue)
-        self.assertEqual('bar,bleh',d2.getAttributeNode('words').nodeValue)
-        self.assertEqual('foo,bleh',d3.getAttributeNode('words').nodeValue)
-        children = [c for c in g2.childNodes if c.localName]
-        self.assertEqual(3,len(children))
-        self.assertEqual(2,len([c for c in children if c.nodeName == 'file']))
-        self.assertEqual(1,len([c for c in children if c.nodeName == 'match']))
-        d1,d2 = [c for c in children if c.nodeName == 'file']
-        self.assertEqual(op.join('basepath','ibabtu'),d1.getAttributeNode('path').nodeValue)
-        self.assertEqual(op.join('basepath','ibabtu'),d2.getAttributeNode('path').nodeValue)
-        self.assertEqual('n',d1.getAttributeNode('is_ref').nodeValue)
-        self.assertEqual('n',d2.getAttributeNode('is_ref').nodeValue)
-        self.assertEqual('ibabtu',d1.getAttributeNode('words').nodeValue)
-        self.assertEqual('ibabtu',d2.getAttributeNode('words').nodeValue)
-    
-    def test_LoadXML(self):
-        def get_file(path):
-            return [f for f in self.objects if str(f.path) == path][0]
-        
-        self.objects[0].is_ref = True
-        self.objects[4].name = 'ibabtu 2' #we can't have 2 files with the same path
-        f = StringIO.StringIO()
-        self.results.save_to_xml(f)
-        f.seek(0)
-        r = Results(data)
-        r.load_from_xml(f,get_file)
-        self.assertEqual(2,len(r.groups))
-        g1,g2 = r.groups
-        self.assertEqual(3,len(g1))
-        self.assert_(g1[0].is_ref)
-        self.assert_(not g1[1].is_ref)
-        self.assert_(not g1[2].is_ref)
-        self.assert_(g1[0] is self.objects[0])
-        self.assert_(g1[1] is self.objects[1])
-        self.assert_(g1[2] is self.objects[2])
-        self.assertEqual(['foo','bar'],g1[0].words)
-        self.assertEqual(['bar','bleh'],g1[1].words)
-        self.assertEqual(['foo','bleh'],g1[2].words)
-        self.assertEqual(2,len(g2))
-        self.assert_(not g2[0].is_ref)
-        self.assert_(not g2[1].is_ref)
-        self.assert_(g2[0] is self.objects[3])
-        self.assert_(g2[1] is self.objects[4])
-        self.assertEqual(['ibabtu'],g2[0].words)
-        self.assertEqual(['ibabtu'],g2[1].words)
-    
-    def test_LoadXML_with_filename(self):
-        def get_file(path):
-            return [f for f in self.objects if str(f.path) == path][0]
-        
-        filename = op.join(self.tmpdir(), 'dupeguru_results.xml')
-        self.objects[4].name = 'ibabtu 2' #we can't have 2 files with the same path
-        self.results.save_to_xml(filename)
-        r = Results(data)
-        r.load_from_xml(filename,get_file)
-        self.assertEqual(2,len(r.groups))
-    
-    def test_LoadXML_with_some_files_that_dont_exist_anymore(self):
-        def get_file(path):
-            if path.endswith('ibabtu 2'):
-                return None
-            return [f for f in self.objects if str(f.path) == path][0]
-        
-        self.objects[4].name = 'ibabtu 2' #we can't have 2 files with the same path
-        f = StringIO.StringIO()
-        self.results.save_to_xml(f)
-        f.seek(0)
-        r = Results(data)
-        r.load_from_xml(f,get_file)
-        self.assertEqual(1,len(r.groups))
-        self.assertEqual(3,len(r.groups[0]))
-    
-    def test_LoadXML_missing_attributes_and_bogus_elements(self):
-        def get_file(path):
-            return [f for f in self.objects if str(f.path) == path][0]
-        
-        doc = xml.dom.minidom.Document()
-        root = doc.appendChild(doc.createElement('foobar')) #The root element shouldn't matter, really.
-        group_node = root.appendChild(doc.createElement('group'))
-        dupe_node = group_node.appendChild(doc.createElement('file')) #Perfectly correct file
-        dupe_node.setAttribute('path',op.join('basepath','foo bar'))
-        dupe_node.setAttribute('is_ref','y')
-        dupe_node.setAttribute('words','foo,bar')
-        dupe_node = group_node.appendChild(doc.createElement('file')) #is_ref missing, default to 'n'
-        dupe_node.setAttribute('path',op.join('basepath','foo bleh'))
-        dupe_node.setAttribute('words','foo,bleh')
-        dupe_node = group_node.appendChild(doc.createElement('file')) #words are missing, invalid.
-        dupe_node.setAttribute('path',op.join('basepath','bar bleh'))
-        dupe_node = group_node.appendChild(doc.createElement('file')) #path is missing, invalid.
-        dupe_node.setAttribute('words','foo,bleh')
-        dupe_node = group_node.appendChild(doc.createElement('foobar')) #Invalid element name
-        dupe_node.setAttribute('path',op.join('basepath','bar bleh'))
-        dupe_node.setAttribute('is_ref','y')
-        dupe_node.setAttribute('words','bar,bleh')
-        match_node = group_node.appendChild(doc.createElement('match')) # match pointing to a bad index
-        match_node.setAttribute('first', '42')
-        match_node.setAttribute('second', '45')
-        match_node = group_node.appendChild(doc.createElement('match')) # match with missing attrs
-        match_node = group_node.appendChild(doc.createElement('match')) # match with non-int values
-        match_node.setAttribute('first', 'foo')
-        match_node.setAttribute('second', 'bar')
-        match_node.setAttribute('percentage', 'baz')
-        group_node = root.appendChild(doc.createElement('foobar')) #invalid group
-        group_node = root.appendChild(doc.createElement('group')) #empty group
-        f = StringIO.StringIO()
-        doc.writexml(f,'\t','\t','\n',encoding='utf-8')
-        f.seek(0)
-        r = Results(data)
-        r.load_from_xml(f,get_file)
-        self.assertEqual(1,len(r.groups))
-        self.assertEqual(2,len(r.groups[0]))
-    
-    def test_xml_non_ascii(self):
-        def get_file(path):
-            if path == op.join('basepath',u'\xe9foo bar'):
-                return objects[0]
-            if path == op.join('basepath',u'bar bleh'):
-                return objects[1]
-        
-        objects = [NamedObject(u"\xe9foo bar",True),NamedObject("bar bleh",True)]
-        matches = engine.getmatches(objects) #we should have 5 matches
-        groups = engine.get_groups(matches) #We should have 2 groups
-        for g in groups:
-            g.prioritize(lambda x:objects.index(x)) #We want the dupes to be in the same order as the list is
-        results = Results(data)
-        results.groups = groups
-        f = StringIO.StringIO()
-        results.save_to_xml(f)
-        f.seek(0)
-        r = Results(data)
-        r.load_from_xml(f,get_file)
-        g = r.groups[0]
-        self.assertEqual(u"\xe9foo bar",g[0].name)
-        self.assertEqual(['efoo','bar'],g[0].words)
-    
-    def test_load_invalid_xml(self):
-        f = StringIO.StringIO()
-        f.write('<this is invalid')
-        f.seek(0)
-        r = Results(data)
-        r.load_from_xml(f,None)
-        self.assertEqual(0,len(r.groups))
-    
-    def test_load_non_existant_xml(self):
-        r = Results(data)
-        try:
-            r.load_from_xml('does_not_exist.xml', None)
-        except IOError:
-            self.fail()
-        self.assertEqual(0,len(r.groups))
-    
-    def test_remember_match_percentage(self):
-        group = self.groups[0]
-        d1, d2, d3 = group
-        fake_matches = set()
-        fake_matches.add(engine.Match(d1, d2, 42))
-        fake_matches.add(engine.Match(d1, d3, 43))
-        fake_matches.add(engine.Match(d2, d3, 46))
-        group.matches = fake_matches
-        f = StringIO.StringIO()
-        results = self.results
-        results.save_to_xml(f)
-        f.seek(0)
-        results = Results(data)
-        results.load_from_xml(f, self.get_file)
-        group = results.groups[0]
-        d1, d2, d3 = group
-        match = group.get_match_of(d2) #d1 - d2
-        self.assertEqual(42, match[2])
-        match = group.get_match_of(d3) #d1 - d3
-        self.assertEqual(43, match[2])
-        group.switch_ref(d2)
-        match = group.get_match_of(d3) #d2 - d3
-        self.assertEqual(46, match[2])
-    
-    def test_save_and_load(self):
-        # previously, when reloading matches, they wouldn't be reloaded as namedtuples
-        f = StringIO.StringIO()
-        self.results.save_to_xml(f)
-        f.seek(0)
-        self.results.load_from_xml(f, self.get_file)
-        first(self.results.groups[0].matches).percentage
-    
-
-class TCResultsFilter(TestCase):
-    def setUp(self):
-        self.results = Results(data)
-        self.objects, self.matches, self.groups = GetTestGroups()
-        self.results.groups = self.groups
-        self.results.apply_filter(r'foo')
-    
-    def test_groups(self):
-        self.assertEqual(1, len(self.results.groups))
-        self.assert_(self.results.groups[0] is self.groups[0])
-    
-    def test_dupes(self):
-        # There are 2 objects matching. The first one is ref. Only the 3rd one is supposed to be in dupes.
-        self.assertEqual(1, len(self.results.dupes))
-        self.assert_(self.results.dupes[0] is self.objects[2])
-    
-    def test_cancel_filter(self):
-        self.results.apply_filter(None)
-        self.assertEqual(3, len(self.results.dupes))
-        self.assertEqual(2, len(self.results.groups))
-    
-    def test_dupes_reconstructed_filtered(self):
-        # make_ref resets self.__dupes to None. When it's reconstructed, we want it filtered
-        dupe = self.results.dupes[0] #3rd object
-        self.results.make_ref(dupe)
-        self.assertEqual(1, len(self.results.dupes))
-        self.assert_(self.results.dupes[0] is self.objects[0])
-    
-    def test_include_ref_dupes_in_filter(self):
-        # When only the ref of a group match the filter, include it in the group
-        self.results.apply_filter(None)
-        self.results.apply_filter(r'foo bar')
-        self.assertEqual(1, len(self.results.groups))
-        self.assertEqual(0, len(self.results.dupes))
-    
-    def test_filters_build_on_one_another(self):
-        self.results.apply_filter(r'bar')
-        self.assertEqual(1, len(self.results.groups))
-        self.assertEqual(0, len(self.results.dupes))
-    
-    def test_stat_line(self):
-        expected = '0 / 1 (0.00 B / 1.00 B) duplicates marked. filter: foo'
-        self.assertEqual(expected, self.results.stat_line)
-        self.results.apply_filter(r'bar')
-        expected = '0 / 0 (0.00 B / 0.00 B) duplicates marked. filter: foo --> bar'
-        self.assertEqual(expected, self.results.stat_line)
-        self.results.apply_filter(None)
-        expected = '0 / 3 (0.00 B / 1.01 KB) duplicates marked.'
-        self.assertEqual(expected, self.results.stat_line)
-    
-    def test_mark_count_is_filtered_as_well(self):
-        self.results.apply_filter(None)
-        # We don't want to perform mark_all() because we want the mark list to contain objects
-        for dupe in self.results.dupes:
-            self.results.mark(dupe)
-        self.results.apply_filter(r'foo')
-        expected = '1 / 1 (1.00 B / 1.00 B) duplicates marked. filter: foo'
-        self.assertEqual(expected, self.results.stat_line)
-    
-    def test_sort_groups(self):
-        self.results.apply_filter(None)
-        self.results.make_ref(self.objects[1]) # to have the 1024 b obkect as ref
-        g1,g2 = self.groups
-        self.results.apply_filter('a') # Matches both group
-        self.results.sort_groups(2) #2 is the key for size
-        self.assert_(self.results.groups[0] is g2)
-        self.assert_(self.results.groups[1] is g1)
-        self.results.apply_filter(None)
-        self.assert_(self.results.groups[0] is g2)
-        self.assert_(self.results.groups[1] is g1)
-        self.results.sort_groups(2, False)
-        self.results.apply_filter('a')
-        self.assert_(self.results.groups[1] is g2)
-        self.assert_(self.results.groups[0] is g1)
-    
-    def test_set_group(self):
-        #We want the new group to be filtered
-        self.objects, self.matches, self.groups = GetTestGroups()
-        self.results.groups = self.groups
-        self.assertEqual(1, len(self.results.groups))
-        self.assert_(self.results.groups[0] is self.groups[0])
-    
-    def test_load_cancels_filter(self):
-        def get_file(path):
-            return [f for f in self.objects if str(f.path) == path][0]
-        
-        filename = op.join(self.tmpdir(), 'dupeguru_results.xml')
-        self.objects[4].name = 'ibabtu 2' #we can't have 2 files with the same path
-        self.results.save_to_xml(filename)
-        r = Results(data)
-        r.apply_filter('foo')
-        r.load_from_xml(filename,get_file)
-        self.assertEqual(2,len(r.groups))
-    
-    def test_remove_dupe(self):
-        self.results.remove_duplicates([self.results.dupes[0]])
-        self.results.apply_filter(None)
-        self.assertEqual(2,len(self.results.groups))
-        self.assertEqual(2,len(self.results.dupes))
-        self.results.apply_filter('ibabtu')
-        self.results.remove_duplicates([self.results.dupes[0]])
-        self.results.apply_filter(None)
-        self.assertEqual(1,len(self.results.groups))
-        self.assertEqual(1,len(self.results.dupes))
-    
-    def test_filter_is_case_insensitive(self):
-        self.results.apply_filter(None)
-        self.results.apply_filter('FOO')
-        self.assertEqual(1, len(self.results.dupes))
-    
-    def test_make_ref_on_filtered_out_doesnt_mess_stats(self):
-        # When filtered, a group containing filtered out dupes will display them as being reference.
-        # When calling make_ref on such a dupe, the total size and dupecount stats gets messed up
-        # because they are *not* counted in the stats in the first place.
-        g1, g2 = self.groups
-        bar_bleh = g1[1] # The "bar bleh" dupe is filtered out
-        self.results.make_ref(bar_bleh)
-        # Now the stats should display *2* markable dupes (instead of 1)
-        expected = '0 / 2 (0.00 B / 2.00 B) duplicates marked. filter: foo'
-        self.assertEqual(expected, self.results.stat_line)
-        self.results.apply_filter(None) # Now let's make sure our unfiltered results aren't fucked up
-        expected = '0 / 3 (0.00 B / 3.00 B) duplicates marked.'
-        self.assertEqual(expected, self.results.stat_line)
-    
-
-class TCResultsRefFile(TestCase):
-    def setUp(self):
-        self.results = Results(data)
-        self.objects, self.matches, self.groups = GetTestGroups()
-        self.objects[0].is_ref = True
-        self.objects[1].is_ref = True
-        self.results.groups = self.groups
-    
-    def test_stat_line(self):
-        expected = '0 / 2 (0.00 B / 2.00 B) duplicates marked.'
-        self.assertEqual(expected, self.results.stat_line)
-    
-    def test_make_ref(self):
-        d = self.results.groups[0].dupes[1] #non-ref
-        r = self.results.groups[0].ref
-        self.results.make_ref(d)
-        expected = '0 / 1 (0.00 B / 1.00 B) duplicates marked.'
-        self.assertEqual(expected, self.results.stat_line)
-        self.results.make_ref(r)
-        expected = '0 / 2 (0.00 B / 2.00 B) duplicates marked.'
-        self.assertEqual(expected, self.results.stat_line)
-    

base/py/tests/scanner_test.py

@@ -1,438 +0,0 @@
-# Created By: Virgil Dupras
-# Created On: 2006/03/03
-# $Id$
-# Copyright 2009 Hardcoded Software (http://www.hardcoded.net)
-# 
-# This software is licensed under the "HS" License as described in the "LICENSE" file, 
-# which should be included with this package. The terms are also available at 
-# http://www.hardcoded.net/licenses/hs_license
-
-from nose.tools import eq_
-
-from hsutil import job
-from hsutil.path import Path
-
-from ..engine import getwords, Match
-from ..ignore import IgnoreList
-from ..scanner import *
-
-class NamedObject(object):
-    def __init__(self, name="foobar", size=1):
-        self.name = name
-        self.size = size
-        self.path = Path('')
-        self.words = getwords(name)
-    
-
-no = NamedObject
-
-#--- Scanner
-def test_empty():
-    s = Scanner()
-    r = s.GetDupeGroups([])
-    eq_(r, [])
-
-def test_default_settings():
-    s = Scanner()
-    eq_(s.min_match_percentage, 80)
-    eq_(s.scan_type, SCAN_TYPE_FILENAME)
-    eq_(s.mix_file_kind, True)
-    eq_(s.word_weighting, False)
-    eq_(s.match_similar_words, False)
-    assert isinstance(s.ignore_list, IgnoreList)
-
-def test_simple_with_default_settings():
-    s = Scanner()
-    f = [no('foo bar'), no('foo bar'), no('foo bleh')]
-    r = s.GetDupeGroups(f)
-    eq_(len(r), 1)
-    g = r[0]
-    #'foo bleh' cannot be in the group because the default min match % is 80
-    eq_(len(g), 2)
-    assert g.ref in f[:2]
-    assert g.dupes[0] in f[:2]
-
-def test_simple_with_lower_min_match():
-    s = Scanner()
-    s.min_match_percentage = 50
-    f = [no('foo bar'), no('foo bar'), no('foo bleh')]
-    r = s.GetDupeGroups(f)
-    eq_(len(r), 1)
-    g = r[0]
-    eq_(len(g), 3)
-
-def test_trim_all_ref_groups():
-    # When all files of a group are ref, don't include that group in the results, but also don't
-    # count the files from that group as discarded.
-    s = Scanner()
-    f = [no('foo'), no('foo'), no('bar'), no('bar')]
-    f[2].is_ref = True
-    f[3].is_ref = True
-    r = s.GetDupeGroups(f)
-    eq_(len(r), 1)
-    eq_(s.discarded_file_count, 0)
-
-def test_priorize():
-    s = Scanner()
-    f = [no('foo'), no('foo'), no('bar'), no('bar')]
-    f[1].size = 2
-    f[2].size = 3
-    f[3].is_ref = True
-    r = s.GetDupeGroups(f)
-    g1, g2 = r
-    assert f[1] in (g1.ref,g2.ref)
-    assert f[0] in (g1.dupes[0],g2.dupes[0])
-    assert f[3] in (g1.ref,g2.ref)
-    assert f[2] in (g1.dupes[0],g2.dupes[0])
-
-def test_content_scan():
-    s = Scanner()
-    s.scan_type = SCAN_TYPE_CONTENT
-    f = [no('foo'), no('bar'), no('bleh')]
-    f[0].md5 = f[0].md5partial = 'foobar'
-    f[1].md5 = f[1].md5partial = 'foobar'
-    f[2].md5 = f[2].md5partial = 'bleh'
-    r = s.GetDupeGroups(f)
-    eq_(len(r), 1)
-    eq_(len(r[0]), 2)
-    eq_(s.discarded_file_count, 0) # don't count the different md5 as discarded!
-
-def test_content_scan_compare_sizes_first():
-    class MyFile(no):
-        @property
-        def md5(file):
-            raise AssertionError()
-    
-    s = Scanner()
-    s.scan_type = SCAN_TYPE_CONTENT
-    f = [MyFile('foo', 1), MyFile('bar', 2)]
-    eq_(len(s.GetDupeGroups(f)), 0)
-
-def test_min_match_perc_doesnt_matter_for_content_scan():
-    s = Scanner()
-    s.scan_type = SCAN_TYPE_CONTENT
-    f = [no('foo'), no('bar'), no('bleh')]
-    f[0].md5 = f[0].md5partial = 'foobar'
-    f[1].md5 = f[1].md5partial = 'foobar'
-    f[2].md5 = f[2].md5partial = 'bleh'
-    s.min_match_percentage = 101
-    r = s.GetDupeGroups(f)
-    eq_(len(r), 1)
-    eq_(len(r[0]), 2)
-    s.min_match_percentage = 0
-    r = s.GetDupeGroups(f)
-    eq_(len(r), 1)
-    eq_(len(r[0]), 2)
-
-def test_content_scan_doesnt_put_md5_in_words_at_the_end():
-    s = Scanner()
-    s.scan_type = SCAN_TYPE_CONTENT
-    f = [no('foo'),no('bar')]
-    f[0].md5 = f[0].md5partial = '\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f'
-    f[1].md5 = f[1].md5partial = '\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f'
-    r = s.GetDupeGroups(f)
-    g = r[0]
-
-def test_extension_is_not_counted_in_filename_scan():
-    s = Scanner()
-    s.min_match_percentage = 100
-    f = [no('foo.bar'), no('foo.bleh')]
-    r = s.GetDupeGroups(f)
-    eq_(len(r), 1)
-    eq_(len(r[0]), 2)
-
-def test_job():
-    def do_progress(progress, desc=''):
-        log.append(progress)
-        return True
-    
-    s = Scanner()
-    log = []
-    f = [no('foo bar'), no('foo bar'), no('foo bleh')]
-    r = s.GetDupeGroups(f, job.Job(1, do_progress))
-    eq_(log[0], 0)
-    eq_(log[-1], 100)
-
-def test_mix_file_kind():
-    s = Scanner()
-    s.mix_file_kind = False
-    f = [no('foo.1'), no('foo.2')]
-    r = s.GetDupeGroups(f)
-    eq_(len(r), 0)
-
-def test_word_weighting():
-    s = Scanner()
-    s.min_match_percentage = 75
-    s.word_weighting = True
-    f = [no('foo bar'), no('foo bar bleh')]
-    r = s.GetDupeGroups(f)
-    eq_(len(r), 1)
-    g = r[0]
-    m = g.get_match_of(g.dupes[0])
-    eq_(m.percentage, 75) # 16 letters, 12 matching
-
-def test_similar_words():
-    s = Scanner()
-    s.match_similar_words = True
-    f = [no('The White Stripes'), no('The Whites Stripe'), no('Limp Bizkit'), no('Limp Bizkitt')]
-    r = s.GetDupeGroups(f)
-    eq_(len(r), 2)
-
-def test_fields():
-    s = Scanner()
-    s.scan_type = SCAN_TYPE_FIELDS
-    f = [no('The White Stripes - Little Ghost'), no('The White Stripes - Little Acorn')]
-    r = s.GetDupeGroups(f)
-    eq_(len(r), 0)
-
-def test_fields_no_order():
-    s = Scanner()
-    s.scan_type = SCAN_TYPE_FIELDS_NO_ORDER
-    f = [no('The White Stripes - Little Ghost'), no('Little Ghost - The White Stripes')]
-    r = s.GetDupeGroups(f)
-    eq_(len(r), 1)
-
-def test_tag_scan():
-    s = Scanner()
-    s.scan_type = SCAN_TYPE_TAG
-    o1 = no('foo')
-    o2 = no('bar')
-    o1.artist = 'The White Stripes'
-    o1.title = 'The Air Near My Fingers'
-    o2.artist = 'The White Stripes'
-    o2.title = 'The Air Near My Fingers'
-    r = s.GetDupeGroups([o1,o2])
-    eq_(len(r), 1)
-
-def test_tag_with_album_scan():
-    s = Scanner()
-    s.scan_type = SCAN_TYPE_TAG
-    s.scanned_tags = set(['artist', 'album', 'title'])
-    o1 = no('foo')
-    o2 = no('bar')
-    o3 = no('bleh')
-    o1.artist = 'The White Stripes'
-    o1.title = 'The Air Near My Fingers'
-    o1.album = 'Elephant'
-    o2.artist = 'The White Stripes'
-    o2.title = 'The Air Near My Fingers'
-    o2.album = 'Elephant'
-    o3.artist = 'The White Stripes'
-    o3.title = 'The Air Near My Fingers'
-    o3.album = 'foobar'
-    r = s.GetDupeGroups([o1,o2,o3])
-    eq_(len(r), 1)
-
-def test_that_dash_in_tags_dont_create_new_fields():
-    s = Scanner()
-    s.scan_type = SCAN_TYPE_TAG
-    s.scanned_tags = set(['artist', 'album', 'title'])
-    s.min_match_percentage = 50
-    o1 = no('foo')
-    o2 = no('bar')
-    o1.artist = 'The White Stripes - a'
-    o1.title = 'The Air Near My Fingers - a'
-    o1.album = 'Elephant - a'
-    o2.artist = 'The White Stripes - b'
-    o2.title = 'The Air Near My Fingers - b'
-    o2.album = 'Elephant - b'
-    r = s.GetDupeGroups([o1,o2])
-    eq_(len(r), 1)
-
-def test_tag_scan_with_different_scanned():
-    s = Scanner()
-    s.scan_type = SCAN_TYPE_TAG
-    s.scanned_tags = set(['track', 'year'])
-    o1 = no('foo')
-    o2 = no('bar')
-    o1.artist = 'The White Stripes'
-    o1.title = 'some title'
-    o1.track = 'foo'
-    o1.year = 'bar'
-    o2.artist = 'The White Stripes'
-    o2.title = 'another title'
-    o2.track = 'foo'
-    o2.year = 'bar'
-    r = s.GetDupeGroups([o1, o2])
-    eq_(len(r), 1)
-
-def test_tag_scan_only_scans_existing_tags():
-    s = Scanner()
-    s.scan_type = SCAN_TYPE_TAG
-    s.scanned_tags = set(['artist', 'foo'])
-    o1 = no('foo')
-    o2 = no('bar')
-    o1.artist = 'The White Stripes'
-    o1.foo = 'foo'
-    o2.artist = 'The White Stripes'
-    o2.foo = 'bar'
-    r = s.GetDupeGroups([o1, o2])
-    eq_(len(r), 1) # Because 'foo' is not scanned, they match
-
-def test_tag_scan_converts_to_str():
-    s = Scanner()
-    s.scan_type = SCAN_TYPE_TAG
-    s.scanned_tags = set(['track'])
-    o1 = no('foo')
-    o2 = no('bar')
-    o1.track = 42
-    o2.track = 42
-    try:
-        r = s.GetDupeGroups([o1, o2])
-    except TypeError:
-        raise AssertionError()
-    eq_(len(r), 1)
-
-def test_tag_scan_non_ascii():
-    s = Scanner()
-    s.scan_type = SCAN_TYPE_TAG
-    s.scanned_tags = set(['title'])
-    o1 = no('foo')
-    o2 = no('bar')
-    o1.title = u'foobar\u00e9'
-    o2.title = u'foobar\u00e9'
-    try:
-        r = s.GetDupeGroups([o1, o2])
-    except UnicodeEncodeError:
-        raise AssertionError()
-    eq_(len(r), 1)
-
-def test_audio_content_scan():
-    s = Scanner()
-    s.scan_type = SCAN_TYPE_CONTENT_AUDIO
-    f = [no('foo'), no('bar'), no('bleh')]
-    f[0].md5 = 'foo'
-    f[1].md5 = 'bar'
-    f[2].md5 = 'bleh'
-    f[0].md5partial = 'foo'
-    f[1].md5partial = 'foo'
-    f[2].md5partial = 'bleh'
-    f[0].audiosize = 1
-    f[1].audiosize = 1
-    f[2].audiosize = 1
-    r = s.GetDupeGroups(f)
-    eq_(len(r), 1)
-    eq_(len(r[0]), 2)
-    
-def test_audio_content_scan_compare_sizes_first():
-    class MyFile(no):
-        @property
-        def md5partial(file):
-            raise AssertionError()
-    
-    s = Scanner()
-    s.scan_type = SCAN_TYPE_CONTENT_AUDIO
-    f = [MyFile('foo'), MyFile('bar')]
-    f[0].audiosize = 1
-    f[1].audiosize = 2
-    eq_(len(s.GetDupeGroups(f)), 0)
-
-def test_ignore_list():
-    s = Scanner()
-    f1 = no('foobar')
-    f2 = no('foobar')
-    f3 = no('foobar')
-    f1.path = Path('dir1/foobar')
-    f2.path = Path('dir2/foobar')
-    f3.path = Path('dir3/foobar')
-    s.ignore_list.Ignore(str(f1.path),str(f2.path))
-    s.ignore_list.Ignore(str(f1.path),str(f3.path))
-    r = s.GetDupeGroups([f1,f2,f3])
-    eq_(len(r), 1)
-    g = r[0]
-    eq_(len(g.dupes), 1)
-    assert f1 not in g
-    assert f2 in g
-    assert f3 in g
-    # Ignored matches are not counted as discarded
-    eq_(s.discarded_file_count, 0)
-
-def test_ignore_list_checks_for_unicode():
-    #scanner was calling path_str for ignore list checks. Since the Path changes, it must
-    #be unicode(path)
-    s = Scanner()
-    f1 = no('foobar')
-    f2 = no('foobar')
-    f3 = no('foobar')
-    f1.path = Path(u'foo1\u00e9')
-    f2.path = Path(u'foo2\u00e9')
-    f3.path = Path(u'foo3\u00e9')
-    s.ignore_list.Ignore(unicode(f1.path),unicode(f2.path))
-    s.ignore_list.Ignore(unicode(f1.path),unicode(f3.path))
-    r = s.GetDupeGroups([f1,f2,f3])
-    eq_(len(r), 1)
-    g = r[0]
-    eq_(len(g.dupes), 1)
-    assert f1 not in g
-    assert f2 in g
-    assert f3 in g
-
-def test_file_evaluates_to_false():
-    # A very wrong way to use any() was added at some point, causing resulting group list
-    # to be empty.
-    class FalseNamedObject(NamedObject):
-        def __nonzero__(self):
-            return False
-        
-    
-    s = Scanner()
-    f1 = FalseNamedObject('foobar')
-    f2 = FalseNamedObject('foobar')
-    r = s.GetDupeGroups([f1, f2])
-    eq_(len(r), 1)
-
-def test_size_threshold():
-    # Only file equal or higher than the size_threshold in size are scanned
-    s = Scanner()
-    f1 = no('foo', 1)
-    f2 = no('foo', 2)
-    f3 = no('foo', 3)
-    s.size_threshold = 2
-    groups = s.GetDupeGroups([f1,f2,f3])
-    eq_(len(groups), 1)
-    [group] = groups
-    eq_(len(group), 2)
-    assert f1 not in group
-    assert f2 in group
-    assert f3 in group
-
-def test_tie_breaker_path_deepness():
-    # If there is a tie in prioritization, path deepness is used as a tie breaker
-    s = Scanner()
-    o1, o2 = no('foo'), no('foo')
-    o1.path = Path('foo')
-    o2.path = Path('foo/bar')
-    [group] = s.GetDupeGroups([o1, o2])
-    assert group.ref is o2
-
-def test_tie_breaker_copy():
-    # if copy is in the words used (even if it has a deeper path), it becomes a dupe
-    s = Scanner()
-    o1, o2 = no('foo bar Copy'), no('foo bar')
-    o1.path = Path('deeper/path')
-    o2.path = Path('foo')
-    [group] = s.GetDupeGroups([o1, o2])
-    assert group.ref is o2
-
-def test_tie_breaker_same_name_plus_digit():
-    # if ref has the same words as dupe, but has some just one extra word which is a digit, it
-    # becomes a dupe
-    s = Scanner()
-    o1, o2 = no('foo bar 42'), no('foo bar')
-    o1.path = Path('deeper/path')
-    o2.path = Path('foo')
-    [group] = s.GetDupeGroups([o1, o2])
-    assert group.ref is o2
-
-def test_partial_group_match():
-    # Count the number od discarded matches (when a file doesn't match all other dupes of the 
-    # group) in Scanner.discarded_file_count
-    s = Scanner()
-    o1, o2, o3 = no('a b'), no('a'), no('b')
-    s.min_match_percentage = 50
-    [group] = s.GetDupeGroups([o1, o2, o3])
-    eq_(len(group), 2)
-    assert o1 in group
-    assert o2 in group
-    assert o3 not in group
-    eq_(s.discarded_file_count, 1)

base/qt/LICENSE

@@ -1,11 +0,0 @@
-Copyright 2009 Hardcoded Software Inc. (http://www.hardcoded.net)
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
-
-    * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
-    * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
-    * Neither the name of Hardcoded Software Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
-    * If the source code has been published less than two years ago, any redistribution, in whole or in part, must retain full licensing functionality, without any attempt to change, obscure or in other ways circumvent its intent.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

base/qt/WARNING

@@ -1,11 +0,0 @@
-WARNING ABOUT THE HS LICENSE AND PyQt
-
-Although Qt is now LGPL licensed, PyQt still is dual licensed. Until Nokia buys Riverbank and
-releases PyQt as LGPL, users of this part of the code (The PyQt-based GUI code) have to use the
-GPL version of PyQt, unless they possess a commercial license to it.
-
-There is no problem to this AS LONG AS YOU DON'T REDISTRIBUTE HS LICENSED CODE. The GPL license, from the point of view of the user, is very permissive. You can do WHATEVER you want with the GPLed version of PyQt, as long as you don't redistribute any of the code, or code dependent on it. When you do, the code you distribute has to be GPL compliant. The HS license is NOT, I repeat, NOT compliant with the GPL.
-
-So, what does it all mean? You have no restriction on the usage of the PyQt-dependent-HS-licensed code, but unless you possess a commercial PyQt license, Hardcoded Software (or anyone) cannot accept any contribution from you for this part of the code.
-
-Note that this only affects the PyQt dependent code, and not any other part of HS licensed code (if it has "import PyQt4" in it, it's PyQt dependent code). For the rest of the code, the only restrictions that apply are the ones from the HS license.

base/qt/about_box.py

@@ -1,37 +0,0 @@
-# Created By: Virgil Dupras
-# Created On: 2009-05-09
-# $Id$
-# Copyright 2009 Hardcoded Software (http://www.hardcoded.net)
-# 
-# This software is licensed under the "HS" License as described in the "LICENSE" file, 
-# which should be included with this package. The terms are also available at 
-# http://www.hardcoded.net/licenses/hs_license
-
-from PyQt4.QtCore import Qt, QCoreApplication, SIGNAL
-from PyQt4.QtGui import QDialog, QDialogButtonBox, QPixmap
-
-from about_box_ui import Ui_AboutBox
-
-class AboutBox(QDialog, Ui_AboutBox):
-    def __init__(self, parent, app):
-        flags = Qt.CustomizeWindowHint | Qt.WindowTitleHint | Qt.WindowSystemMenuHint | Qt.MSWindowsFixedSizeDialogHint
-        QDialog.__init__(self, parent, flags)
-        self.app = app
-        self._setupUi()
-        
-        self.connect(self.buttonBox, SIGNAL('clicked(QAbstractButton*)'), self.buttonClicked)
-    
-    def _setupUi(self):
-        self.setupUi(self)
-        # Stuff that can't be done in the Designer
-        self.setWindowTitle(u"About %s" % QCoreApplication.instance().applicationName())
-        self.nameLabel.setText(QCoreApplication.instance().applicationName())
-        self.versionLabel.setText('Version ' + QCoreApplication.instance().applicationVersion())
-        self.logoLabel.setPixmap(QPixmap(':/%s_big' % self.app.LOGO_NAME))
-        self.registerButton = self.buttonBox.addButton("Register", QDialogButtonBox.ActionRole)
-        
-    #--- Events
-    def buttonClicked(self, button):
-        if button is self.registerButton:
-            self.app.ask_for_reg_code()
-    

base/qt/about_box.ui

@@ -1,133 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<ui version="4.0">
- <class>AboutBox</class>
- <widget class="QDialog" name="AboutBox">
-  <property name="geometry">
-   <rect>
-    <x>0</x>
-    <y>0</y>
-    <width>400</width>
-    <height>190</height>
-   </rect>
-  </property>
-  <property name="sizePolicy">
-   <sizepolicy hsizetype="Fixed" vsizetype="Fixed">
-    <horstretch>0</horstretch>
-    <verstretch>0</verstretch>
-   </sizepolicy>
-  </property>
-  <property name="windowTitle">
-   <string>About dupeGuru</string>
-  </property>
-  <layout class="QHBoxLayout" name="horizontalLayout">
-   <item>
-    <widget class="QLabel" name="logoLabel">
-     <property name="text">
-      <string/>
-     </property>
-     <property name="pixmap">
-      <pixmap resource="dg.qrc">:/logo_me_big</pixmap>
-     </property>
-    </widget>
-   </item>
-   <item>
-    <layout class="QVBoxLayout" name="verticalLayout">
-     <item>
-      <widget class="QLabel" name="nameLabel">
-       <property name="font">
-        <font>
-         <weight>75</weight>
-         <bold>true</bold>
-        </font>
-       </property>
-       <property name="text">
-        <string>dupeGuru</string>
-       </property>
-      </widget>
-     </item>
-     <item>
-      <widget class="QLabel" name="versionLabel">
-       <property name="text">
-        <string>Version</string>
-       </property>
-      </widget>
-     </item>
-     <item>
-      <widget class="QLabel" name="label_3">
-       <property name="text">
-        <string>Copyright Hardcoded Software 2009</string>
-       </property>
-      </widget>
-     </item>
-     <item>
-      <widget class="QLabel" name="label">
-       <property name="font">
-        <font>
-         <weight>75</weight>
-         <bold>true</bold>
-        </font>
-       </property>
-       <property name="text">
-        <string>Registered To:</string>
-       </property>
-      </widget>
-     </item>
-     <item>
-      <widget class="QLabel" name="registeredEmailLabel">
-       <property name="text">
-        <string>UNREGISTERED</string>
-       </property>
-      </widget>
-     </item>
-     <item>
-      <widget class="QDialogButtonBox" name="buttonBox">
-       <property name="orientation">
-        <enum>Qt::Horizontal</enum>
-       </property>
-       <property name="standardButtons">
-        <set>QDialogButtonBox::Ok</set>
-       </property>
-      </widget>
-     </item>
-    </layout>
-   </item>
-  </layout>
- </widget>
- <resources>
-  <include location="dg.qrc"/>
- </resources>
- <connections>
-  <connection>
-   <sender>buttonBox</sender>
-   <signal>accepted()</signal>
-   <receiver>AboutBox</receiver>
-   <slot>accept()</slot>
-   <hints>
-    <hint type="sourcelabel">
-     <x>248</x>
-     <y>254</y>
-    </hint>
-    <hint type="destinationlabel">
-     <x>157</x>
-     <y>274</y>
-    </hint>
-   </hints>
-  </connection>
-  <connection>
-   <sender>buttonBox</sender>
-   <signal>rejected()</signal>
-   <receiver>AboutBox</receiver>
-   <slot>reject()</slot>
-   <hints>
-    <hint type="sourcelabel">
-     <x>316</x>
-     <y>260</y>
-    </hint>
-    <hint type="destinationlabel">
-     <x>286</x>
-     <y>274</y>
-    </hint>
-   </hints>
-  </connection>
- </connections>
-</ui>

base/qt/app.py

@@ -1,259 +0,0 @@
-# Created By: Virgil Dupras
-# Created On: 2009-04-25
-# $Id$
-# Copyright 2009 Hardcoded Software (http://www.hardcoded.net)
-# 
-# This software is licensed under the "HS" License as described in the "LICENSE" file, 
-# which should be included with this package. The terms are also available at 
-# http://www.hardcoded.net/licenses/hs_license
-
-from __future__ import unicode_literals
-
-import logging
-import os
-import os.path as op
-
-from PyQt4.QtCore import Qt, QTimer, QObject, QCoreApplication, QUrl, SIGNAL
-from PyQt4.QtGui import QProgressDialog, QDesktopServices, QFileDialog, QDialog, QMessageBox
-
-from hsutil import job
-from hsutil.reg import RegistrationRequired
-
-from dupeguru import fs
-from dupeguru.app import (DupeGuru as DupeGuruBase, JOB_SCAN, JOB_LOAD, JOB_MOVE, JOB_COPY, 
-    JOB_DELETE)
-    
-from qtlib.progress import Progress
-
-from . import platform
-
-from .main_window import MainWindow
-from .directories_dialog import DirectoriesDialog
-from .about_box import AboutBox
-from .reg import Registration
-
-JOBID2TITLE = {
-    JOB_SCAN: "Scanning for duplicates",
-    JOB_LOAD: "Loading",
-    JOB_MOVE: "Moving",
-    JOB_COPY: "Copying",
-    JOB_DELETE: "Sending files to the recycle bin",
-}
-
-def demo_method(method):
-    def wrapper(self, *args, **kwargs):
-        try:
-            return method(self, *args, **kwargs)
-        except RegistrationRequired:
-            msg = "The demo version of dupeGuru only allows 10 actions (delete/move/copy) per session."
-            QMessageBox.information(self.main_window, 'Demo', msg)
-    
-    return wrapper
-
-class DupeGuru(DupeGuruBase, QObject):
-    LOGO_NAME = '<replace this>'
-    NAME = '<replace this>'
-    DELTA_COLUMNS = frozenset()
-    
-    def __init__(self, data_module, appid):
-        appdata = unicode(QDesktopServices.storageLocation(QDesktopServices.DataLocation))
-        if not op.exists(appdata):
-            os.makedirs(appdata)
-        # For basicConfig() to work, we have to be sure that no logging has taken place before this call.
-        logging.basicConfig(filename=op.join(appdata, 'debug.log'), level=logging.WARNING)
-        DupeGuruBase.__init__(self, data_module, appdata, appid)
-        QObject.__init__(self)
-        self._setup()
-    
-    #--- Private
-    def _setup(self):
-        self.selected_dupe = None
-        self.prefs = self._create_preferences()
-        self.prefs.load()
-        self._update_options()
-        self.main_window = self._create_main_window()
-        self._progress = Progress(self.main_window)
-        self.directories_dialog = DirectoriesDialog(self.main_window, self)
-        self.details_dialog = self._create_details_dialog(self.main_window)
-        self.preferences_dialog = self._create_preferences_dialog(self.main_window)
-        self.about_box = AboutBox(self.main_window, self)
-        
-        self.reg = Registration(self)
-        self.set_registration(self.prefs.registration_code, self.prefs.registration_email)
-        if not self.registered:
-            # The timer scheme is because if the nag is not shown before the application is 
-            # completely initialized, the nag will be shown before the app shows up in the task bar
-            # In some circumstances, the nag is hidden by other window, which may make the user think
-            # that the application haven't launched.
-            self._nagTimer = QTimer()
-            self.connect(self._nagTimer, SIGNAL('timeout()'), self.mustShowNag)
-            self._nagTimer.start(0)
-        self.main_window.show()
-        self.load()
-        
-        self.connect(QCoreApplication.instance(), SIGNAL('aboutToQuit()'), self.application_will_terminate)
-        self.connect(self._progress, SIGNAL('finished(QString)'), self.job_finished)
-    
-    def _setup_as_registered(self):
-        self.prefs.registration_code = self.registration_code
-        self.prefs.registration_email = self.registration_email
-        self.main_window.actionRegister.setVisible(False)
-        self.about_box.registerButton.hide()
-        self.about_box.registeredEmailLabel.setText(self.prefs.registration_email)
-    
-    def _update_options(self):
-        self.scanner.mix_file_kind = self.prefs.mix_file_kind
-        self.options['escape_filter_regexp'] = self.prefs.use_regexp
-        self.options['clean_empty_dirs'] = self.prefs.remove_empty_folders
-    
-    #--- Virtual
-    def _create_details_dialog(self, parent):
-        raise NotImplementedError()
-    
-    def _create_main_window(self):
-        return MainWindow(app=self)
-    
-    def _create_preferences(self):
-        raise NotImplementedError()
-    
-    def _create_preferences_dialog(self, parent):
-        raise NotImplementedError()
-    
-    #--- Override
-    @staticmethod
-    def _recycle_dupe(dupe):
-        platform.recycle_file(dupe.path)
-    
-    def _start_job(self, jobid, func):
-        title = JOBID2TITLE[jobid]
-        try:
-            j = self._progress.create_job()
-            self._progress.run(jobid, title, func, args=(j, ))
-        except job.JobInProgressError:
-            msg = "A previous action is still hanging in there. You can't start a new one yet. Wait a few seconds, then try again."
-            QMessageBox.information(self.main_window, 'Action in progress', msg)
-    
-    #--- Public
-    def add_dupes_to_ignore_list(self, duplicates):
-        for dupe in duplicates:
-            self.add_to_ignore_list(dupe)
-        self.remove_duplicates(duplicates)
-    
-    def apply_filter(self, filter):
-        DupeGuruBase.apply_filter(self, filter)
-        self.emit(SIGNAL('resultsChanged()'))
-    
-    def ask_for_reg_code(self):
-        if self.reg.ask_for_code():
-            #XXX bug???
-            self._setup_ui_as_registered()
-    
-    @demo_method
-    def copy_or_move_marked(self, copy):
-        opname = 'copy' if copy else 'move'
-        title = "Select a directory to {0} marked files to".format(opname)
-        flags = QFileDialog.ShowDirsOnly
-        destination = unicode(QFileDialog.getExistingDirectory(self.main_window, title, '', flags))
-        if not destination:
-            return
-        recreate_path = self.prefs.destination_type
-        DupeGuruBase.copy_or_move_marked(self, copy, destination, recreate_path)
-    
-    delete_marked = demo_method(DupeGuruBase.delete_marked)
-    
-    def make_reference(self, duplicates):
-        DupeGuruBase.make_reference(self, duplicates)
-        self.emit(SIGNAL('resultsChanged()'))
-    
-    def mark_all(self):
-        self.results.mark_all()
-        self.emit(SIGNAL('dupeMarkingChanged()'))
-    
-    def mark_invert(self):
-        self.results.mark_invert()
-        self.emit(SIGNAL('dupeMarkingChanged()'))
-    
-    def mark_none(self):
-        self.results.mark_none()
-        self.emit(SIGNAL('dupeMarkingChanged()'))
-    
-    def openDebugLog(self):
-        debugLogPath = op.join(self.appdata, 'debug.log')
-        url = QUrl.fromLocalFile(debugLogPath)
-        QDesktopServices.openUrl(url)
-    
-    def open_selected(self):
-        if self.selected_dupe is None:
-            return
-        url = QUrl.fromLocalFile(unicode(self.selected_dupe.path))
-        QDesktopServices.openUrl(url)
-    
-    def remove_duplicates(self, duplicates):
-        self.results.remove_duplicates(duplicates)
-        self.emit(SIGNAL('resultsChanged()'))
-    
-    def remove_marked_duplicates(self):
-        marked = [d for d in self.results.dupes if self.results.is_marked(d)]
-        self.remove_duplicates(marked)
-    
-    def rename_dupe(self, dupe, newname):
-        try:
-            dupe.move(dupe.parent, newname)
-            return True
-        except (IndexError, fs.FSError) as e:
-            logging.warning("dupeGuru Warning: %s" % unicode(e))
-        return False
-    
-    def reveal_selected(self):
-        if self.selected_dupe is None:
-            return
-        url = QUrl.fromLocalFile(unicode(self.selected_dupe.path[:-1]))
-        QDesktopServices.openUrl(url)
-    
-    def select_duplicate(self, dupe):
-        self.selected_dupe = dupe
-        self.emit(SIGNAL('duplicateSelected()'))
-    
-    def show_about_box(self):
-        self.about_box.show()
-    
-    def show_details(self):
-        self.details_dialog.show()
-    
-    def show_directories(self):
-        self.directories_dialog.show()
-    
-    def show_help(self):
-        url = QUrl.fromLocalFile(op.abspath('help/intro.htm'))
-        QDesktopServices.openUrl(url)
-    
-    def show_preferences(self):
-        self.preferences_dialog.load()
-        result = self.preferences_dialog.exec_()
-        if result == QDialog.Accepted:
-            self.preferences_dialog.save()
-            self.prefs.save()
-            self._update_options()
-    
-    def toggle_marking_for_dupes(self, dupes):
-        for dupe in dupes:
-            self.results.mark_toggle(dupe)
-        self.emit(SIGNAL('dupeMarkingChanged()'))
-    
-    #--- Events
-    def application_will_terminate(self):
-        self.save()
-        self.save_ignore_list()
-    
-    def mustShowNag(self):
-        self._nagTimer.stop() # must be shown only once
-        self.reg.show_nag()
-    
-    def job_finished(self, jobid):
-        self.emit(SIGNAL('resultsChanged()'))
-        if jobid == JOB_LOAD:
-            self.emit(SIGNAL('directoriesChanged()'))
-        if jobid in (JOB_MOVE, JOB_COPY, JOB_DELETE) and self.last_op_error_count > 0:
-            msg = "{0} files could not be processed.".format(self.results.mark_count)
-            QMessageBox.warning(self.main_window, 'Warning', msg)
-    

base/qt/details_table.py

@@ -1,83 +0,0 @@
-# Created By: Virgil Dupras
-# Created On: 2009-05-17
-# $Id$
-# Copyright 2009 Hardcoded Software (http://www.hardcoded.net)
-# 
-# This software is licensed under the "HS" License as described in the "LICENSE" file, 
-# which should be included with this package. The terms are also available at 
-# http://www.hardcoded.net/licenses/hs_license
-
-from PyQt4.QtCore import Qt, SIGNAL, QAbstractTableModel, QVariant
-from PyQt4.QtGui import QHeaderView, QTableView
-
-HEADER = ['Attribute', 'Selected', 'Reference']
-
-class DetailsModel(QAbstractTableModel):
-    def __init__(self, app):
-        QAbstractTableModel.__init__(self)
-        self._app = app
-        self._dupe_data = None
-        self._ref_data = None
-        self.connect(app, SIGNAL('duplicateSelected()'), self.duplicateSelected)
-    
-    def columnCount(self, parent):
-        return len(HEADER)
-    
-    def data(self, index, role):
-        if not index.isValid():
-            return QVariant()
-        if role != Qt.DisplayRole:
-            return QVariant()
-        column = index.column()
-        row = index.row()
-        if column == 0:
-            return QVariant(self._app.data.COLUMNS[row]['display'])
-        elif column == 1 and self._dupe_data:
-            return QVariant(self._dupe_data[row])
-        elif column == 2 and self._ref_data:
-            return QVariant(self._ref_data[row])
-        return QVariant()
-    
-    def headerData(self, section, orientation, role):
-        if orientation == Qt.Horizontal and role == Qt.DisplayRole and section < len(HEADER):
-            return QVariant(HEADER[section])
-        return QVariant()
-    
-    def rowCount(self, parent):
-        return len(self._app.data.COLUMNS)
-    
-    #--- Events
-    def duplicateSelected(self):
-        dupe = self._app.selected_dupe
-        if dupe is None:
-            group = None
-            ref = None
-        else:
-            group = self._app.results.get_group_of_duplicate(dupe)
-            ref = group.ref if group.ref is not dupe else None
-        self._dupe_data = self._app._get_display_info(dupe, group)
-        self._ref_data = self._app._get_display_info(ref, group)
-        self.reset()
-    
-
-class DetailsTable(QTableView):
-    def __init__(self, *args):
-        QTableView.__init__(self, *args)
-        self.setAlternatingRowColors(True)
-        self.setSelectionBehavior(QTableView.SelectRows)
-        self.setShowGrid(False)
-    
-    def setModel(self, model):
-        QTableView.setModel(self, model)
-        # The model needs to be set to set header stuff
-        hheader = self.horizontalHeader()
-        hheader.setHighlightSections(False)
-        hheader.setStretchLastSection(False)
-        hheader.resizeSection(0, 100)
-        hheader.setResizeMode(0, QHeaderView.Fixed)
-        hheader.setResizeMode(1, QHeaderView.Stretch)
-        hheader.setResizeMode(2, QHeaderView.Stretch)
-        vheader = self.verticalHeader()
-        vheader.setVisible(False)
-        vheader.setDefaultSectionSize(18)
-    

base/qt/dg.qrc

@@ -1,17 +0,0 @@
-<!DOCTYPE RCC><RCC version="1.0">
-<qresource>
-    <file alias="details">images/details32.png</file>
-    <file alias="logo_pe">images/dgpe_logo_32.png</file>
-    <file alias="logo_pe_big">images/dgpe_logo_128.png</file>
-    <file alias="logo_me">images/dgme_logo_32.png</file>
-    <file alias="logo_me_big">images/dgme_logo_128.png</file>
-    <file alias="logo_se">images/dgse_logo_32.png</file>
-    <file alias="logo_se_big">images/dgse_logo_128.png</file>
-    <file alias="folder">images/folderwin32.png</file>
-    <file alias="gear">images/gear.png</file>
-    <file alias="preferences">images/preferences32.png</file>
-    <file alias="actions">images/actions32.png</file>
-    <file alias="delta">images/delta32.png</file>
-    <file alias="power_marker">images/power_marker32.png</file>
-</qresource>
-</RCC>

base/qt/directories_dialog.py

@@ -1,85 +0,0 @@
-# Created By: Virgil Dupras
-# Created On: 2009-04-25
-# $Id$
-# Copyright 2009 Hardcoded Software (http://www.hardcoded.net)
-# 
-# This software is licensed under the "HS" License as described in the "LICENSE" file, 
-# which should be included with this package. The terms are also available at 
-# http://www.hardcoded.net/licenses/hs_license
-
-from PyQt4.QtCore import SIGNAL, Qt
-from PyQt4.QtGui import QDialog, QFileDialog, QHeaderView
-
-from . import platform
-from .directories_dialog_ui import Ui_DirectoriesDialog
-from .directories_model import DirectoriesModel, DirectoriesDelegate
-
-class DirectoriesDialog(QDialog, Ui_DirectoriesDialog):
-    def __init__(self, parent, app):
-        flags = Qt.CustomizeWindowHint | Qt.WindowTitleHint | Qt.WindowSystemMenuHint
-        QDialog.__init__(self, parent, flags)
-        self.app = app
-        self.lastAddedFolder = platform.INITIAL_FOLDER_IN_DIALOGS
-        self._setupUi()
-        self._updateRemoveButton()
-        
-        self.connect(self.doneButton, SIGNAL('clicked()'), self.doneButtonClicked)
-        self.connect(self.addButton, SIGNAL('clicked()'), self.addButtonClicked)
-        self.connect(self.removeButton, SIGNAL('clicked()'), self.removeButtonClicked)
-        self.connect(self.treeView.selectionModel(), SIGNAL('selectionChanged(QItemSelection,QItemSelection)'), self.selectionChanged)
-        self.connect(self.app, SIGNAL('directoriesChanged()'), self.directoriesChanged)
-    
-    def _setupUi(self):
-        self.setupUi(self)
-        # Stuff that can't be done in the Designer
-        self.directoriesModel = DirectoriesModel(self.app)
-        self.directoriesDelegate = DirectoriesDelegate()
-        self.treeView.setItemDelegate(self.directoriesDelegate)
-        self.treeView.setModel(self.directoriesModel)
-        
-        header = self.treeView.header()
-        header.setStretchLastSection(False)
-        header.setResizeMode(0, QHeaderView.Stretch)
-        header.setResizeMode(1, QHeaderView.Fixed)
-        header.resizeSection(1, 100)
-    
-    def _updateRemoveButton(self):
-        indexes = self.treeView.selectedIndexes()
-        if not indexes:
-            self.removeButton.setEnabled(False)
-            return
-        self.removeButton.setEnabled(True)
-        index = indexes[0]
-        node = index.internalPointer()
-        # label = 'Remove' if node.parent is None else 'Exclude'
-    
-    def addButtonClicked(self):
-        title = u"Select a directory to add to the scanning list"
-        flags = QFileDialog.ShowDirsOnly
-        dirpath = unicode(QFileDialog.getExistingDirectory(self, title, self.lastAddedFolder, flags))
-        if not dirpath:
-            return
-        self.lastAddedFolder = dirpath
-        self.app.add_directory(dirpath)
-        self.directoriesModel.reset()
-    
-    def directoriesChanged(self):
-        self.directoriesModel.reset()
-    
-    def doneButtonClicked(self):
-        self.hide()
-    
-    def removeButtonClicked(self):
-        indexes = self.treeView.selectedIndexes()
-        if not indexes:
-            return
-        index = indexes[0]
-        node = index.internalPointer()
-        if node.parent is None:
-            row = index.row()
-            del self.app.directories[row]
-            self.directoriesModel.reset()
-    
-    def selectionChanged(self, selected, deselected):
-        self._updateRemoveButton()
-    

base/qt/directories_dialog.ui

@@ -1,133 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<ui version="4.0">
- <class>DirectoriesDialog</class>
- <widget class="QDialog" name="DirectoriesDialog">
-  <property name="geometry">
-   <rect>
-    <x>0</x>
-    <y>0</y>
-    <width>420</width>
-    <height>338</height>
-   </rect>
-  </property>
-  <property name="windowTitle">
-   <string>Directories</string>
-  </property>
-  <layout class="QVBoxLayout" name="verticalLayout">
-   <item>
-    <widget class="QTreeView" name="treeView">
-     <property name="editTriggers">
-      <set>QAbstractItemView::DoubleClicked|QAbstractItemView::EditKeyPressed|QAbstractItemView::SelectedClicked</set>
-     </property>
-     <property name="uniformRowHeights">
-      <bool>true</bool>
-     </property>
-     <attribute name="headerStretchLastSection">
-      <bool>false</bool>
-     </attribute>
-    </widget>
-   </item>
-   <item>
-    <layout class="QHBoxLayout" name="horizontalLayout">
-     <item>
-      <spacer name="horizontalSpacer">
-       <property name="orientation">
-        <enum>Qt::Horizontal</enum>
-       </property>
-       <property name="sizeHint" stdset="0">
-        <size>
-         <width>40</width>
-         <height>20</height>
-        </size>
-       </property>
-      </spacer>
-     </item>
-     <item>
-      <widget class="QPushButton" name="removeButton">
-       <property name="minimumSize">
-        <size>
-         <width>91</width>
-         <height>0</height>
-        </size>
-       </property>
-       <property name="maximumSize">
-        <size>
-         <width>16777215</width>
-         <height>32</height>
-        </size>
-       </property>
-       <property name="text">
-        <string>Remove</string>
-       </property>
-      </widget>
-     </item>
-     <item>
-      <widget class="QPushButton" name="addButton">
-       <property name="minimumSize">
-        <size>
-         <width>91</width>
-         <height>0</height>
-        </size>
-       </property>
-       <property name="maximumSize">
-        <size>
-         <width>16777215</width>
-         <height>32</height>
-        </size>
-       </property>
-       <property name="text">
-        <string>Add</string>
-       </property>
-      </widget>
-     </item>
-     <item>
-      <spacer name="horizontalSpacer_2">
-       <property name="orientation">
-        <enum>Qt::Horizontal</enum>
-       </property>
-       <property name="sizeType">
-        <enum>QSizePolicy::Fixed</enum>
-       </property>
-       <property name="sizeHint" stdset="0">
-        <size>
-         <width>40</width>
-         <height>20</height>
-        </size>
-       </property>
-      </spacer>
-     </item>
-     <item>
-      <widget class="QPushButton" name="doneButton">
-       <property name="sizePolicy">
-        <sizepolicy hsizetype="Fixed" vsizetype="Fixed">
-         <horstretch>0</horstretch>
-         <verstretch>0</verstretch>
-        </sizepolicy>
-       </property>
-       <property name="minimumSize">
-        <size>
-         <width>91</width>
-         <height>0</height>
-        </size>
-       </property>
-       <property name="maximumSize">
-        <size>
-         <width>16777215</width>
-         <height>32</height>
-        </size>
-       </property>
-       <property name="text">
-        <string>Done</string>
-       </property>
-       <property name="default">
-        <bool>true</bool>
-       </property>
-      </widget>
-     </item>
-    </layout>
-   </item>
-  </layout>
- </widget>
- <resources/>
- <connections/>
-</ui>

base/qt/directories_model.py

@@ -1,113 +0,0 @@
-# Created By: Virgil Dupras
-# Created On: 2009-04-25
-# $Id$
-# Copyright 2009 Hardcoded Software (http://www.hardcoded.net)
-# 
-# This software is licensed under the "HS" License as described in the "LICENSE" file, 
-# which should be included with this package. The terms are also available at 
-# http://www.hardcoded.net/licenses/hs_license
-
-from PyQt4.QtCore import QModelIndex, Qt, QRect, QEvent, QPoint
-from PyQt4.QtGui import QComboBox, QStyledItemDelegate, QMouseEvent, QApplication, QBrush
-
-from qtlib.tree_model import TreeNode, TreeModel
-
-HEADERS = ['Name', 'State']
-STATES = ['Normal', 'Reference', 'Excluded']
-
-class DirectoriesDelegate(QStyledItemDelegate):
-    def createEditor(self, parent, option, index):
-        editor = QComboBox(parent);
-        editor.addItems(STATES)
-        return editor
-    
-    def setEditorData(self, editor, index):
-        value = index.model().data(index, Qt.EditRole)
-        editor.setCurrentIndex(value);
-        press = QMouseEvent(QEvent.MouseButtonPress, QPoint(0, 0), Qt.LeftButton, Qt.LeftButton, Qt.NoModifier)
-        release = QMouseEvent(QEvent.MouseButtonRelease, QPoint(0, 0), Qt.LeftButton, Qt.LeftButton, Qt.NoModifier)
-        QApplication.sendEvent(editor, press)
-        QApplication.sendEvent(editor, release)
-        # editor.showPopup() # this causes a weird glitch. the ugly workaround is above.
-    
-    def setModelData(self, editor, model, index):
-        value = editor.currentIndex()
-        model.setData(index, value, Qt.EditRole)
-    
-    def updateEditorGeometry(self, editor, option, index):
-        editor.setGeometry(option.rect)
-    
-
-class DirectoryNode(TreeNode):
-    def __init__(self, model, parent, ref, row):
-        TreeNode.__init__(self, model, parent, row)
-        self.ref = ref
-    
-    def _createNode(self, ref, row):
-        return DirectoryNode(self.model, self, ref, row)
-    
-    def _getChildren(self):
-        return self.model._dirs.get_subfolders(self.ref)
-    
-    @property
-    def name(self):
-        if self.parent is not None:
-            return self.ref[-1]
-        else:
-            return unicode(self.ref)
-    
-
-class DirectoriesModel(TreeModel):
-    def __init__(self, app):
-        self._dirs = app.directories
-        TreeModel.__init__(self)
-    
-    def _createNode(self, ref, row):
-        return DirectoryNode(self, None, ref, row)
-    
-    def _getChildren(self):
-        return self._dirs
-    
-    def columnCount(self, parent):
-        return 2
-    
-    def data(self, index, role):
-        if not index.isValid():
-            return None
-        node = index.internalPointer()
-        if role == Qt.DisplayRole:
-            if index.column() == 0:
-                return node.name
-            else:
-                return STATES[self._dirs.get_state(node.ref)]
-        elif role == Qt.EditRole and index.column() == 1:
-            return self._dirs.get_state(node.ref)
-        elif role == Qt.ForegroundRole:
-            state = self._dirs.get_state(node.ref)
-            if state == 1:
-                return QBrush(Qt.blue)
-            elif state == 2:
-                return QBrush(Qt.red)
-        return None
-    
-    def flags(self, index):
-        if not index.isValid():
-            return 0
-        result = Qt.ItemIsEnabled | Qt.ItemIsSelectable
-        if index.column() == 1:
-            result |= Qt.ItemIsEditable
-        return result
-    
-    def headerData(self, section, orientation, role):
-        if orientation == Qt.Horizontal:
-            if role == Qt.DisplayRole and section < len(HEADERS):
-                return HEADERS[section]
-        return None
-    
-    def setData(self, index, value, role):
-        if not index.isValid() or role != Qt.EditRole or index.column() != 1:
-            return False
-        node = index.internalPointer()
-        self._dirs.set_state(node.ref, value)
-        return True
-    

base/qt/main_window.py

@@ -1,332 +0,0 @@
-# Created By: Virgil Dupras
-# Created On: 2009-04-25
-# $Id$
-# Copyright 2009 Hardcoded Software (http://www.hardcoded.net)
-# 
-# This software is licensed under the "HS" License as described in the "LICENSE" file, 
-# which should be included with this package. The terms are also available at 
-# http://www.hardcoded.net/licenses/hs_license
-
-from PyQt4.QtCore import Qt, QCoreApplication, QProcess, SIGNAL, QUrl
-from PyQt4.QtGui import (QMainWindow, QMenu, QPixmap, QIcon, QToolButton, QLabel, QHeaderView,
-    QMessageBox, QInputDialog, QLineEdit, QItemSelectionModel, QDesktopServices)
-
-from hsutil.misc import nonone
-
-from dupeguru.app import NoScannableFileError, AllFilesAreRefError
-
-import dg_rc
-from main_window_ui import Ui_MainWindow
-from results_model import ResultsDelegate, ResultsModel
-
-class MainWindow(QMainWindow, Ui_MainWindow):
-    def __init__(self, app):
-        QMainWindow.__init__(self, None)
-        self.app = app
-        self._last_filter = None
-        self._setupUi()
-        self.resultsDelegate = ResultsDelegate() 
-        self.resultsModel = ResultsModel(self.app) 
-        self.resultsView.setModel(self.resultsModel) 
-        self.resultsView.setItemDelegate(self.resultsDelegate) 
-        self._load_columns()
-        self._update_column_actions_status()
-        self.resultsView.expandAll()
-        self._update_status_line()
-        
-        self.connect(self.app, SIGNAL('resultsChanged()'), self.resultsChanged)
-        self.connect(self.app, SIGNAL('dupeMarkingChanged()'), self.dupeMarkingChanged)
-        self.connect(self.actionQuit, SIGNAL('triggered()'), QCoreApplication.instance().quit)
-        self.connect(self.resultsView.selectionModel(), SIGNAL('selectionChanged(QItemSelection,QItemSelection)'), self.selectionChanged)
-        self.connect(self.menuColumns, SIGNAL('triggered(QAction*)'), self.columnToggled)
-        self.connect(QCoreApplication.instance(), SIGNAL('aboutToQuit()'), self.application_will_terminate)
-        self.connect(self.resultsModel, SIGNAL('modelReset()'), self.resultsReset)
-        self.connect(self.resultsView, SIGNAL('doubleClicked()'), self.resultsDoubleClicked)
-    
-    def _setupUi(self):
-        self.setupUi(self)
-        # Stuff that can't be setup in the Designer
-        h = self.resultsView.header()
-        h.setHighlightSections(False)
-        h.setMovable(True)
-        h.setStretchLastSection(False)
-        h.setDefaultAlignment(Qt.AlignLeft)
-        
-        self.setWindowTitle(QCoreApplication.instance().applicationName())
-        self.actionScan.setIcon(QIcon(QPixmap(':/%s' % self.app.LOGO_NAME)))
-        
-        # Columns menu
-        menu = self.menuColumns
-        self._column_actions = []
-        for index, column in enumerate(self.app.data.COLUMNS):
-            action = menu.addAction(column['display'])
-            action.setCheckable(True)
-            action.column_index = index
-            self._column_actions.append(action)
-        menu.addSeparator()
-        action = menu.addAction("Reset to Defaults")
-        action.column_index = -1
-        
-        # Action menu
-        actionMenu = QMenu('Actions', self.toolBar)
-        actionMenu.setIcon(QIcon(QPixmap(":/actions")))
-        actionMenu.addAction(self.actionDeleteMarked)
-        actionMenu.addAction(self.actionMoveMarked)
-        actionMenu.addAction(self.actionCopyMarked)
-        actionMenu.addAction(self.actionRemoveMarked)
-        actionMenu.addSeparator()
-        actionMenu.addAction(self.actionRemoveSelected)
-        actionMenu.addAction(self.actionIgnoreSelected)
-        actionMenu.addAction(self.actionMakeSelectedReference)
-        actionMenu.addSeparator()
-        actionMenu.addAction(self.actionOpenSelected)
-        actionMenu.addAction(self.actionRevealSelected)
-        actionMenu.addAction(self.actionRenameSelected)
-        self.actionActions.setMenu(actionMenu)
-        button = QToolButton(self.toolBar)
-        button.setDefaultAction(actionMenu.menuAction())
-        button.setToolButtonStyle(Qt.ToolButtonTextUnderIcon)
-        self.actionsButton = button
-        self.toolBar.insertWidget(self.actionActions, button) # the action is a placeholder
-        self.toolBar.removeAction(self.actionActions)
-        
-        self.statusLabel = QLabel(self)
-        self.statusbar.addPermanentWidget(self.statusLabel, 1)
-    
-    #--- Private
-    def _confirm(self, title, msg, default_button=QMessageBox.Yes):
-        buttons = QMessageBox.Yes | QMessageBox.No
-        answer = QMessageBox.question(self, title, msg, buttons, default_button)
-        return answer == QMessageBox.Yes
-    
-    def _load_columns(self):
-        h = self.resultsView.header()
-        h.setResizeMode(QHeaderView.Interactive)
-        prefs = self.app.prefs
-        attrs = zip(prefs.columns_width, prefs.columns_visible)
-        for index, (width, visible) in enumerate(attrs):
-            h.resizeSection(index, width)
-            h.setSectionHidden(index, not visible)
-        h.setResizeMode(0, QHeaderView.Stretch)
-    
-    def _redraw_results(self):
-        # HACK. this is the only way I found to update the widget without reseting everything
-        self.resultsView.scroll(0, 1)
-        self.resultsView.scroll(0, -1)
-    
-    def _save_columns(self):
-        h = self.resultsView.header()
-        widths = []
-        visible = []
-        for i in range(len(self.app.data.COLUMNS)):
-            widths.append(h.sectionSize(i))
-            visible.append(not h.isSectionHidden(i))
-        prefs = self.app.prefs
-        prefs.columns_width = widths
-        prefs.columns_visible = visible
-        prefs.save()
-    
-    def _update_column_actions_status(self):
-        h = self.resultsView.header()
-        for action in self._column_actions:
-            colid = action.column_index
-            action.setChecked(not h.isSectionHidden(colid))
-    
-    def _update_status_line(self):
-        self.statusLabel.setText(self.app.stat_line)
-    
-    #--- Actions
-    def aboutTriggered(self):
-        self.app.show_about_box()
-    
-    def actionsTriggered(self):
-        self.actionsButton.showMenu()
-    
-    def addToIgnoreListTriggered(self):
-        dupes = self.resultsView.selectedDupes()
-        if not dupes:
-            return
-        title = "Add to Ignore List"
-        msg = "All selected {0} matches are going to be ignored in all subsequent scans. Continue?".format(len(dupes))
-        if self._confirm(title, msg):
-            self.app.add_dupes_to_ignore_list(dupes)
-    
-    def applyFilterTriggered(self):
-        title = "Apply Filter"
-        msg = "Type the filter you want to apply on your results. See help for details."
-        text = nonone(self._last_filter, '[*]')
-        answer, ok = QInputDialog.getText(self, title, msg, QLineEdit.Normal, text)
-        if not ok:
-            return
-        answer = unicode(answer)
-        self.app.apply_filter(answer)
-        self._last_filter = answer
-    
-    def cancelFilterTriggered(self):
-        self.app.apply_filter('')
-    
-    def checkForUpdateTriggered(self):
-        QProcess.execute('updater.exe', ['/checknow'])
-    
-    def clearIgnoreListTriggered(self):
-        title = "Clear Ignore List"
-        count = len(self.app.scanner.ignore_list)
-        if not count:
-            QMessageBox.information(self, title, "Nothing to clear.")
-            return
-        msg = "Do you really want to remove all {0} items from the ignore list?".format(count)
-        if self._confirm(title, msg, QMessageBox.No):
-            self.app.scanner.ignore_list.Clear()
-            QMessageBox.information(self, title, "Ignore list cleared.")
-    
-    def copyTriggered(self):
-        self.app.copy_or_move_marked(True)
-    
-    def deleteTriggered(self):
-        count = self.app.results.mark_count
-        if not count:
-            return
-        title = "Delete duplicates"
-        msg = "You are about to send {0} files to the recycle bin. Continue?".format(count)
-        if self._confirm(title, msg):
-            self.app.delete_marked()
-    
-    def deltaTriggered(self):
-        self.resultsModel.delta = self.actionDelta.isChecked()
-        self._redraw_results()
-    
-    def detailsTriggered(self):
-        self.app.show_details()
-    
-    def directoriesTriggered(self):
-        self.app.show_directories()
-    
-    def exportTriggered(self):
-        h = self.resultsView.header()
-        column_ids = []
-        for i in range(len(self.app.data.COLUMNS)):
-            if not h.isSectionHidden(i):
-                column_ids.append(str(i))
-        exported_path = self.app.export_to_xhtml(column_ids)
-        url = QUrl.fromLocalFile(exported_path)
-        QDesktopServices.openUrl(url)
-    
-    def makeReferenceTriggered(self):
-        self.app.make_reference(self.resultsView.selectedDupes())
-    
-    def markAllTriggered(self):
-        self.app.mark_all()
-    
-    def markInvertTriggered(self):
-        self.app.mark_invert()
-    
-    def markNoneTriggered(self):
-        self.app.mark_none()
-    
-    def markSelectedTriggered(self):
-        dupes = self.resultsView.selectedDupes()
-        self.app.toggle_marking_for_dupes(dupes)
-    
-    def moveTriggered(self):
-        self.app.copy_or_move_marked(False)
-    
-    def openDebugLogTriggered(self):
-        self.app.openDebugLog()
-    
-    def openTriggered(self):
-        self.app.open_selected()
-    
-    def powerMarkerTriggered(self):
-        self.resultsModel.power_marker = self.actionPowerMarker.isChecked()
-    
-    def preferencesTriggered(self):
-        self.app.show_preferences()
-    
-    def registerTrigerred(self):
-        self.app.ask_for_reg_code()
-    
-    def removeMarkedTriggered(self):
-        count = self.app.results.mark_count
-        if not count:
-            return
-        title = "Remove duplicates"
-        msg = "You are about to remove {0} files from results. Continue?".format(count)
-        if self._confirm(title, msg):
-            self.app.remove_marked_duplicates()
-    
-    def removeSelectedTriggered(self):
-        dupes = self.resultsView.selectedDupes()
-        if not dupes:
-            return
-        title = "Remove duplicates"
-        msg = "You are about to remove {0} files from results. Continue?".format(len(dupes))
-        if self._confirm(title, msg):
-            self.app.remove_duplicates(dupes)
-    
-    def renameTriggered(self):
-        self.resultsView.edit(self.resultsView.selectionModel().currentIndex())
-    
-    def revealTriggered(self):
-        self.app.reveal_selected()
-    
-    def scanTriggered(self):
-        title = "Start a new scan"
-        if len(self.app.results.groups) > 0:
-            msg = "Are you sure you want to start a new duplicate scan?"
-            if not self._confirm(title, msg):
-                return
-        try:
-            self.app.start_scanning()
-        except NoScannableFileError:
-            msg = "The selected directories contain no scannable file."
-            QMessageBox.warning(self, title, msg)
-            self.app.show_directories()
-        except AllFilesAreRefError:
-            msg = "You cannot make a duplicate scan with only reference directories."
-            QMessageBox.warning(self, title, msg)
-    
-    def showHelpTriggered(self):
-        self.app.show_help()
-    
-    #--- Events
-    def application_will_terminate(self):
-        self._save_columns()
-    
-    def columnToggled(self, action):
-        colid = action.column_index
-        if colid == -1:
-            self.app.prefs.reset_columns()
-            self._load_columns()
-        else:
-            h = self.resultsView.header()
-            h.setSectionHidden(colid, not h.isSectionHidden(colid))
-        self._update_column_actions_status()
-    
-    def contextMenuEvent(self, event):
-        self.actionActions.menu().exec_(event.globalPos())
-    
-    def dupeMarkingChanged(self):
-        self._redraw_results()
-        self._update_status_line()
-    
-    def resultsChanged(self):
-        self.resultsView.model().reset()
-    
-    def resultsDoubleClicked(self):
-        self.app.open_selected()
-    
-    def resultsReset(self):
-        self.resultsView.expandAll()
-        dupe = self.app.selected_dupe
-        if dupe is not None:
-            [modelIndex] = self.resultsModel.indexesForDupes([dupe])
-            if modelIndex.isValid():
-                flags = QItemSelectionModel.ClearAndSelect | QItemSelectionModel.Rows
-                self.resultsView.selectionModel().setCurrentIndex(modelIndex, flags)
-        self._update_status_line()
-    
-    def selectionChanged(self, selected, deselected):
-        index = self.resultsView.selectionModel().currentIndex()
-        dupe = index.internalPointer().dupe if index.isValid() else None
-        self.app.select_duplicate(dupe)
-    

base/qt/main_window.ui

@@ -1,957 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<ui version="4.0">
- <class>MainWindow</class>
- <widget class="QMainWindow" name="MainWindow">
-  <property name="geometry">
-   <rect>
-    <x>0</x>
-    <y>0</y>
-    <width>630</width>
-    <height>514</height>
-   </rect>
-  </property>
-  <property name="windowTitle">
-   <string>dupeGuru</string>
-  </property>
-  <widget class="QWidget" name="centralwidget">
-   <layout class="QVBoxLayout" name="verticalLayout_2">
-    <property name="margin">
-     <number>0</number>
-    </property>
-    <item>
-     <widget class="ResultsView" name="resultsView">
-      <property name="selectionMode">
-       <enum>QAbstractItemView::ExtendedSelection</enum>
-      </property>
-      <property name="selectionBehavior">
-       <enum>QAbstractItemView::SelectRows</enum>
-      </property>
-      <property name="rootIsDecorated">
-       <bool>false</bool>
-      </property>
-      <property name="uniformRowHeights">
-       <bool>true</bool>
-      </property>
-      <property name="itemsExpandable">
-       <bool>false</bool>
-      </property>
-      <property name="sortingEnabled">
-       <bool>true</bool>
-      </property>
-      <property name="expandsOnDoubleClick">
-       <bool>false</bool>
-      </property>
-      <attribute name="headerStretchLastSection">
-       <bool>false</bool>
-      </attribute>
-     </widget>
-    </item>
-   </layout>
-  </widget>
-  <widget class="QMenuBar" name="menubar">
-   <property name="geometry">
-    <rect>
-     <x>0</x>
-     <y>0</y>
-     <width>630</width>
-     <height>22</height>
-    </rect>
-   </property>
-   <widget class="QMenu" name="menuColumns">
-    <property name="title">
-     <string>Columns</string>
-    </property>
-   </widget>
-   <widget class="QMenu" name="menuActions">
-    <property name="title">
-     <string>Actions</string>
-    </property>
-    <addaction name="actionDeleteMarked"/>
-    <addaction name="actionMoveMarked"/>
-    <addaction name="actionCopyMarked"/>
-    <addaction name="actionRemoveMarked"/>
-    <addaction name="separator"/>
-    <addaction name="actionRemoveSelected"/>
-    <addaction name="actionIgnoreSelected"/>
-    <addaction name="actionMakeSelectedReference"/>
-    <addaction name="separator"/>
-    <addaction name="actionOpenSelected"/>
-    <addaction name="actionRevealSelected"/>
-    <addaction name="actionRenameSelected"/>
-    <addaction name="separator"/>
-    <addaction name="actionApplyFilter"/>
-    <addaction name="actionCancelFilter"/>
-   </widget>
-   <widget class="QMenu" name="menuMark">
-    <property name="title">
-     <string>Mark</string>
-    </property>
-    <addaction name="actionMarkAll"/>
-    <addaction name="actionMarkNone"/>
-    <addaction name="actionInvertMarking"/>
-    <addaction name="actionMarkSelected"/>
-   </widget>
-   <widget class="QMenu" name="menuModes">
-    <property name="title">
-     <string>Modes</string>
-    </property>
-    <addaction name="actionPowerMarker"/>
-    <addaction name="actionDelta"/>
-   </widget>
-   <widget class="QMenu" name="menuWindow">
-    <property name="title">
-     <string>Windows</string>
-    </property>
-    <addaction name="actionDetails"/>
-    <addaction name="actionDirectories"/>
-    <addaction name="actionPreferences"/>
-   </widget>
-   <widget class="QMenu" name="menuHelp">
-    <property name="title">
-     <string>Help</string>
-    </property>
-    <addaction name="actionShowHelp"/>
-    <addaction name="actionRegister"/>
-    <addaction name="actionCheckForUpdate"/>
-    <addaction name="actionOpenDebugLog"/>
-    <addaction name="actionAbout"/>
-   </widget>
-   <widget class="QMenu" name="menuFile">
-    <property name="title">
-     <string>File</string>
-    </property>
-    <addaction name="actionScan"/>
-    <addaction name="separator"/>
-    <addaction name="actionExport"/>
-    <addaction name="actionClearIgnoreList"/>
-    <addaction name="separator"/>
-    <addaction name="actionQuit"/>
-   </widget>
-   <addaction name="menuFile"/>
-   <addaction name="menuMark"/>
-   <addaction name="menuActions"/>
-   <addaction name="menuColumns"/>
-   <addaction name="menuModes"/>
-   <addaction name="menuWindow"/>
-   <addaction name="menuHelp"/>
-  </widget>
-  <widget class="QToolBar" name="toolBar">
-   <property name="windowTitle">
-    <string>toolBar</string>
-   </property>
-   <property name="movable">
-    <bool>false</bool>
-   </property>
-   <property name="toolButtonStyle">
-    <enum>Qt::ToolButtonTextUnderIcon</enum>
-   </property>
-   <property name="floatable">
-    <bool>false</bool>
-   </property>
-   <attribute name="toolBarArea">
-    <enum>TopToolBarArea</enum>
-   </attribute>
-   <attribute name="toolBarBreak">
-    <bool>false</bool>
-   </attribute>
-   <addaction name="actionScan"/>
-   <addaction name="actionActions"/>
-   <addaction name="actionDirectories"/>
-   <addaction name="actionDetails"/>
-   <addaction name="actionPreferences"/>
-   <addaction name="actionDelta"/>
-   <addaction name="actionPowerMarker"/>
-  </widget>
-  <widget class="QStatusBar" name="statusbar">
-   <property name="sizeGripEnabled">
-    <bool>true</bool>
-   </property>
-  </widget>
-  <action name="actionScan">
-   <property name="icon">
-    <iconset resource="dg.qrc">
-     <normaloff>:/logo_pe</normaloff>:/logo_pe</iconset>
-   </property>
-   <property name="text">
-    <string>Start Scan</string>
-   </property>
-   <property name="toolTip">
-    <string>Start scanning for duplicates</string>
-   </property>
-   <property name="shortcut">
-    <string>Ctrl+S</string>
-   </property>
-  </action>
-  <action name="actionDirectories">
-   <property name="icon">
-    <iconset resource="dg.qrc">
-     <normaloff>:/folder</normaloff>:/folder</iconset>
-   </property>
-   <property name="text">
-    <string>Directories</string>
-   </property>
-   <property name="shortcut">
-    <string>Ctrl+4</string>
-   </property>
-  </action>
-  <action name="actionDetails">
-   <property name="icon">
-    <iconset resource="dg.qrc">
-     <normaloff>:/details</normaloff>:/details</iconset>
-   </property>
-   <property name="text">
-    <string>Details</string>
-   </property>
-   <property name="shortcut">
-    <string>Ctrl+3</string>
-   </property>
-  </action>
-  <action name="actionActions">
-   <property name="icon">
-    <iconset resource="dg.qrc">
-     <normaloff>:/actions</normaloff>:/actions</iconset>
-   </property>
-   <property name="text">
-    <string>Actions</string>
-   </property>
-  </action>
-  <action name="actionPreferences">
-   <property name="icon">
-    <iconset resource="dg.qrc">
-     <normaloff>:/preferences</normaloff>:/preferences</iconset>
-   </property>
-   <property name="text">
-    <string>Preferences</string>
-   </property>
-   <property name="shortcut">
-    <string>Ctrl+5</string>
-   </property>
-  </action>
-  <action name="actionDelta">
-   <property name="checkable">
-    <bool>true</bool>
-   </property>
-   <property name="icon">
-    <iconset resource="dg.qrc">
-     <normaloff>:/delta</normaloff>:/delta</iconset>
-   </property>
-   <property name="text">
-    <string>Delta Values</string>
-   </property>
-   <property name="shortcut">
-    <string>Ctrl+2</string>
-   </property>
-  </action>
-  <action name="actionPowerMarker">
-   <property name="checkable">
-    <bool>true</bool>
-   </property>
-   <property name="icon">
-    <iconset resource="dg.qrc">
-     <normaloff>:/power_marker</normaloff>:/power_marker</iconset>
-   </property>
-   <property name="text">
-    <string>Power Marker</string>
-   </property>
-   <property name="shortcut">
-    <string>Ctrl+1</string>
-   </property>
-  </action>
-  <action name="actionDeleteMarked">
-   <property name="text">
-    <string>Send Marked to Recycle Bin</string>
-   </property>
-   <property name="shortcut">
-    <string>Ctrl+D</string>
-   </property>
-  </action>
-  <action name="actionMoveMarked">
-   <property name="text">
-    <string>Move Marked to...</string>
-   </property>
-   <property name="shortcut">
-    <string>Ctrl+M</string>
-   </property>
-  </action>
-  <action name="actionCopyMarked">
-   <property name="text">
-    <string>Copy Marked to...</string>
-   </property>
-   <property name="shortcut">
-    <string>Ctrl+Shift+M</string>
-   </property>
-  </action>
-  <action name="actionRemoveMarked">
-   <property name="text">
-    <string>Remove Marked from Results</string>
-   </property>
-   <property name="shortcut">
-    <string>Ctrl+R</string>
-   </property>
-  </action>
-  <action name="actionRemoveSelected">
-   <property name="text">
-    <string>Remove Selected from Results</string>
-   </property>
-   <property name="shortcut">
-    <string>Ctrl+Del</string>
-   </property>
-  </action>
-  <action name="actionIgnoreSelected">
-   <property name="text">
-    <string>Add Selected to Ignore List</string>
-   </property>
-   <property name="shortcut">
-    <string>Ctrl+Shift+Del</string>
-   </property>
-  </action>
-  <action name="actionMakeSelectedReference">
-   <property name="text">
-    <string>Make Selected Reference</string>
-   </property>
-   <property name="shortcut">
-    <string>Ctrl+Space</string>
-   </property>
-  </action>
-  <action name="actionOpenSelected">
-   <property name="text">
-    <string>Open Selected with Default Application</string>
-   </property>
-   <property name="shortcut">
-    <string>Ctrl+O</string>
-   </property>
-  </action>
-  <action name="actionRevealSelected">
-   <property name="text">
-    <string>Open Containing Folder of Selected</string>
-   </property>
-   <property name="shortcut">
-    <string>Ctrl+Shift+O</string>
-   </property>
-  </action>
-  <action name="actionRenameSelected">
-   <property name="text">
-    <string>Rename Selected</string>
-   </property>
-   <property name="shortcut">
-    <string>F2</string>
-   </property>
-  </action>
-  <action name="actionMarkAll">
-   <property name="text">
-    <string>Mark All</string>
-   </property>
-   <property name="shortcut">
-    <string>Ctrl+A</string>
-   </property>
-  </action>
-  <action name="actionMarkNone">
-   <property name="text">
-    <string>Mark None</string>
-   </property>
-   <property name="shortcut">
-    <string>Ctrl+Shift+A</string>
-   </property>
-  </action>
-  <action name="actionInvertMarking">
-   <property name="text">
-    <string>Invert Marking</string>
-   </property>
-   <property name="shortcut">
-    <string>Ctrl+Alt+A</string>
-   </property>
-  </action>
-  <action name="actionMarkSelected">
-   <property name="text">
-    <string>Mark Selected</string>
-   </property>
-  </action>
-  <action name="actionClearIgnoreList">
-   <property name="text">
-    <string>Clear Ignore List</string>
-   </property>
-  </action>
-  <action name="actionQuit">
-   <property name="text">
-    <string>Quit</string>
-   </property>
-   <property name="shortcut">
-    <string>Ctrl+Q</string>
-   </property>
-  </action>
-  <action name="actionApplyFilter">
-   <property name="text">
-    <string>Apply Filter</string>
-   </property>
-   <property name="shortcut">
-    <string>Ctrl+F</string>
-   </property>
-  </action>
-  <action name="actionCancelFilter">
-   <property name="text">
-    <string>Cancel Filter</string>
-   </property>
-   <property name="shortcut">
-    <string>Ctrl+Shift+F</string>
-   </property>
-  </action>
-  <action name="actionShowHelp">
-   <property name="text">
-    <string>dupeGuru Help</string>
-   </property>
-   <property name="shortcut">
-    <string>F1</string>
-   </property>
-  </action>
-  <action name="actionAbout">
-   <property name="text">
-    <string>About dupeGuru</string>
-   </property>
-  </action>
-  <action name="actionRegister">
-   <property name="text">
-    <string>Register dupeGuru</string>
-   </property>
-  </action>
-  <action name="actionCheckForUpdate">
-   <property name="text">
-    <string>Check for Update</string>
-   </property>
-  </action>
-  <action name="actionExport">
-   <property name="text">
-    <string>Export To XHTML</string>
-   </property>
-  </action>
-  <action name="actionOpenDebugLog">
-   <property name="text">
-    <string>Open Debug Log</string>
-   </property>
-  </action>
- </widget>
- <customwidgets>
-  <customwidget>
-   <class>ResultsView</class>
-   <extends>QTreeView</extends>
-   <header>results_model</header>
-  </customwidget>
- </customwidgets>
- <resources>
-  <include location="dg.qrc"/>
- </resources>
- <connections>
-  <connection>
-   <sender>actionDirectories</sender>
-   <signal>triggered()</signal>
-   <receiver>MainWindow</receiver>
-   <slot>directoriesTriggered()</slot>
-   <hints>
-    <hint type="sourcelabel">
-     <x>-1</x>
-     <y>-1</y>
-    </hint>
-    <hint type="destinationlabel">
-     <x>314</x>
-     <y>256</y>
-    </hint>
-   </hints>
-  </connection>
-  <connection>
-   <sender>actionActions</sender>
-   <signal>triggered()</signal>
-   <receiver>MainWindow</receiver>
-   <slot>actionsTriggered()</slot>
-   <hints>
-    <hint type="sourcelabel">
-     <x>-1</x>
-     <y>-1</y>
-    </hint>
-    <hint type="destinationlabel">
-     <x>314</x>
-     <y>256</y>
-    </hint>
-   </hints>
-  </connection>
-  <connection>
-   <sender>actionCopyMarked</sender>
-   <signal>triggered()</signal>
-   <receiver>MainWindow</receiver>
-   <slot>copyTriggered()</slot>
-   <hints>
-    <hint type="sourcelabel">
-     <x>-1</x>
-     <y>-1</y>
-    </hint>
-    <hint type="destinationlabel">
-     <x>314</x>
-     <y>256</y>
-    </hint>
-   </hints>
-  </connection>
-  <connection>
-   <sender>actionDeleteMarked</sender>
-   <signal>triggered()</signal>
-   <receiver>MainWindow</receiver>
-   <slot>deleteTriggered()</slot>
-   <hints>
-    <hint type="sourcelabel">
-     <x>-1</x>
-     <y>-1</y>
-    </hint>
-    <hint type="destinationlabel">
-     <x>314</x>
-     <y>256</y>
-    </hint>
-   </hints>
-  </connection>
-  <connection>
-   <sender>actionDelta</sender>
-   <signal>triggered()</signal>
-   <receiver>MainWindow</receiver>
-   <slot>deltaTriggered()</slot>
-   <hints>
-    <hint type="sourcelabel">
-     <x>-1</x>
-     <y>-1</y>
-    </hint>
-    <hint type="destinationlabel">
-     <x>314</x>
-     <y>256</y>
-    </hint>
-   </hints>
-  </connection>
-  <connection>
-   <sender>actionDetails</sender>
-   <signal>triggered()</signal>
-   <receiver>MainWindow</receiver>
-   <slot>detailsTriggered()</slot>
-   <hints>
-    <hint type="sourcelabel">
-     <x>-1</x>
-     <y>-1</y>
-    </hint>
-    <hint type="destinationlabel">
-     <x>314</x>
-     <y>256</y>
-    </hint>
-   </hints>
-  </connection>
-  <connection>
-   <sender>actionIgnoreSelected</sender>
-   <signal>triggered()</signal>
-   <receiver>MainWindow</receiver>
-   <slot>addToIgnoreListTriggered()</slot>
-   <hints>
-    <hint type="sourcelabel">
-     <x>-1</x>
-     <y>-1</y>
-    </hint>
-    <hint type="destinationlabel">
-     <x>314</x>
-     <y>256</y>
-    </hint>
-   </hints>
-  </connection>
-  <connection>
-   <sender>actionMakeSelectedReference</sender>
-   <signal>triggered()</signal>
-   <receiver>MainWindow</receiver>
-   <slot>makeReferenceTriggered()</slot>
-   <hints>
-    <hint type="sourcelabel">
-     <x>-1</x>
-     <y>-1</y>
-    </hint>
-    <hint type="destinationlabel">
-     <x>314</x>
-     <y>256</y>
-    </hint>
-   </hints>
-  </connection>
-  <connection>
-   <sender>actionMoveMarked</sender>
-   <signal>triggered()</signal>
-   <receiver>MainWindow</receiver>
-   <slot>moveTriggered()</slot>
-   <hints>
-    <hint type="sourcelabel">
-     <x>-1</x>
-     <y>-1</y>
-    </hint>
-    <hint type="destinationlabel">
-     <x>314</x>
-     <y>256</y>
-    </hint>
-   </hints>
-  </connection>
-  <connection>
-   <sender>actionOpenSelected</sender>
-   <signal>triggered()</signal>
-   <receiver>MainWindow</receiver>
-   <slot>openTriggered()</slot>
-   <hints>
-    <hint type="sourcelabel">
-     <x>-1</x>
-     <y>-1</y>
-    </hint>
-    <hint type="destinationlabel">
-     <x>314</x>
-     <y>256</y>
-    </hint>
-   </hints>
-  </connection>
-  <connection>
-   <sender>actionPowerMarker</sender>
-   <signal>triggered()</signal>
-   <receiver>MainWindow</receiver>
-   <slot>powerMarkerTriggered()</slot>
-   <hints>
-    <hint type="sourcelabel">
-     <x>-1</x>
-     <y>-1</y>
-    </hint>
-    <hint type="destinationlabel">
-     <x>314</x>
-     <y>256</y>
-    </hint>
-   </hints>
-  </connection>
-  <connection>
-   <sender>actionPreferences</sender>
-   <signal>triggered()</signal>
-   <receiver>MainWindow</receiver>
-   <slot>preferencesTriggered()</slot>
-   <hints>
-    <hint type="sourcelabel">
-     <x>-1</x>
-     <y>-1</y>
-    </hint>
-    <hint type="destinationlabel">
-     <x>314</x>
-     <y>256</y>
-    </hint>
-   </hints>
-  </connection>
-  <connection>
-   <sender>actionRemoveMarked</sender>
-   <signal>triggered()</signal>
-   <receiver>MainWindow</receiver>
-   <slot>removeMarkedTriggered()</slot>
-   <hints>
-    <hint type="sourcelabel">
-     <x>-1</x>
-     <y>-1</y>
-    </hint>
-    <hint type="destinationlabel">
-     <x>314</x>
-     <y>256</y>
-    </hint>
-   </hints>
-  </connection>
-  <connection>
-   <sender>actionRemoveSelected</sender>
-   <signal>triggered()</signal>
-   <receiver>MainWindow</receiver>
-   <slot>removeSelectedTriggered()</slot>
-   <hints>
-    <hint type="sourcelabel">
-     <x>-1</x>
-     <y>-1</y>
-    </hint>
-    <hint type="destinationlabel">
-     <x>314</x>
-     <y>256</y>
-    </hint>
-   </hints>
-  </connection>
-  <connection>
-   <sender>actionRevealSelected</sender>
-   <signal>triggered()</signal>
-   <receiver>MainWindow</receiver>
-   <slot>revealTriggered()</slot>
-   <hints>
-    <hint type="sourcelabel">
-     <x>-1</x>
-     <y>-1</y>
-    </hint>
-    <hint type="destinationlabel">
-     <x>314</x>
-     <y>256</y>
-    </hint>
-   </hints>
-  </connection>
-  <connection>
-   <sender>actionRenameSelected</sender>
-   <signal>triggered()</signal>
-   <receiver>MainWindow</receiver>
-   <slot>renameTriggered()</slot>
-   <hints>
-    <hint type="sourcelabel">
-     <x>-1</x>
-     <y>-1</y>
-    </hint>
-    <hint type="destinationlabel">
-     <x>314</x>
-     <y>256</y>
-    </hint>
-   </hints>
-  </connection>
-  <connection>
-   <sender>actionScan</sender>
-   <signal>triggered()</signal>
-   <receiver>MainWindow</receiver>
-   <slot>scanTriggered()</slot>
-   <hints>
-    <hint type="sourcelabel">
-     <x>-1</x>
-     <y>-1</y>
-    </hint>
-    <hint type="destinationlabel">
-     <x>314</x>
-     <y>256</y>
-    </hint>
-   </hints>
-  </connection>
-  <connection>
-   <sender>actionClearIgnoreList</sender>
-   <signal>triggered()</signal>
-   <receiver>MainWindow</receiver>
-   <slot>clearIgnoreListTriggered()</slot>
-   <hints>
-    <hint type="sourcelabel">
-     <x>-1</x>
-     <y>-1</y>
-    </hint>
-    <hint type="destinationlabel">
-     <x>314</x>
-     <y>256</y>
-    </hint>
-   </hints>
-  </connection>
-  <connection>
-   <sender>actionMarkAll</sender>
-   <signal>triggered()</signal>
-   <receiver>MainWindow</receiver>
-   <slot>markAllTriggered()</slot>
-   <hints>
-    <hint type="sourcelabel">
-     <x>-1</x>
-     <y>-1</y>
-    </hint>
-    <hint type="destinationlabel">
-     <x>314</x>
-     <y>256</y>
-    </hint>
-   </hints>
-  </connection>
-  <connection>
-   <sender>actionMarkNone</sender>
-   <signal>triggered()</signal>
-   <receiver>MainWindow</receiver>
-   <slot>markNoneTriggered()</slot>
-   <hints>
-    <hint type="sourcelabel">
-     <x>-1</x>
-     <y>-1</y>
-    </hint>
-    <hint type="destinationlabel">
-     <x>314</x>
-     <y>256</y>
-    </hint>
-   </hints>
-  </connection>
-  <connection>
-   <sender>actionMarkSelected</sender>
-   <signal>triggered()</signal>
-   <receiver>MainWindow</receiver>
-   <slot>markSelectedTriggered()</slot>
-   <hints>
-    <hint type="sourcelabel">
-     <x>-1</x>
-     <y>-1</y>
-    </hint>
-    <hint type="destinationlabel">
-     <x>314</x>
-     <y>256</y>
-    </hint>
-   </hints>
-  </connection>
-  <connection>
-   <sender>actionInvertMarking</sender>
-   <signal>triggered()</signal>
-   <receiver>MainWindow</receiver>
-   <slot>markInvertTriggered()</slot>
-   <hints>
-    <hint type="sourcelabel">
-     <x>-1</x>
-     <y>-1</y>
-    </hint>
-    <hint type="destinationlabel">
-     <x>314</x>
-     <y>256</y>
-    </hint>
-   </hints>
-  </connection>
-  <connection>
-   <sender>actionApplyFilter</sender>
-   <signal>triggered()</signal>
-   <receiver>MainWindow</receiver>
-   <slot>applyFilterTriggered()</slot>
-   <hints>
-    <hint type="sourcelabel">
-     <x>-1</x>
-     <y>-1</y>
-    </hint>
-    <hint type="destinationlabel">
-     <x>314</x>
-     <y>256</y>
-    </hint>
-   </hints>
-  </connection>
-  <connection>
-   <sender>actionCancelFilter</sender>
-   <signal>triggered()</signal>
-   <receiver>MainWindow</receiver>
-   <slot>cancelFilterTriggered()</slot>
-   <hints>
-    <hint type="sourcelabel">
-     <x>-1</x>
-     <y>-1</y>
-    </hint>
-    <hint type="destinationlabel">
-     <x>314</x>
-     <y>256</y>
-    </hint>
-   </hints>
-  </connection>
-  <connection>
-   <sender>actionShowHelp</sender>
-   <signal>triggered()</signal>
-   <receiver>MainWindow</receiver>
-   <slot>showHelpTriggered()</slot>
-   <hints>
-    <hint type="sourcelabel">
-     <x>-1</x>
-     <y>-1</y>
-    </hint>
-    <hint type="destinationlabel">
-     <x>314</x>
-     <y>256</y>
-    </hint>
-   </hints>
-  </connection>
-  <connection>
-   <sender>actionAbout</sender>
-   <signal>triggered()</signal>
-   <receiver>MainWindow</receiver>
-   <slot>aboutTriggered()</slot>
-   <hints>
-    <hint type="sourcelabel">
-     <x>-1</x>
-     <y>-1</y>
-    </hint>
-    <hint type="destinationlabel">
-     <x>314</x>
-     <y>256</y>
-    </hint>
-   </hints>
-  </connection>
-  <connection>
-   <sender>actionRegister</sender>
-   <signal>triggered()</signal>
-   <receiver>MainWindow</receiver>
-   <slot>registerTrigerred()</slot>
-   <hints>
-    <hint type="sourcelabel">
-     <x>-1</x>
-     <y>-1</y>
-    </hint>
-    <hint type="destinationlabel">
-     <x>314</x>
-     <y>256</y>
-    </hint>
-   </hints>
-  </connection>
-  <connection>
-   <sender>actionCheckForUpdate</sender>
-   <signal>triggered()</signal>
-   <receiver>MainWindow</receiver>
-   <slot>checkForUpdateTriggered()</slot>
-   <hints>
-    <hint type="sourcelabel">
-     <x>-1</x>
-     <y>-1</y>
-    </hint>
-    <hint type="destinationlabel">
-     <x>314</x>
-     <y>256</y>
-    </hint>
-   </hints>
-  </connection>
-  <connection>
-   <sender>actionExport</sender>
-   <signal>triggered()</signal>
-   <receiver>MainWindow</receiver>
-   <slot>exportTriggered()</slot>
-   <hints>
-    <hint type="sourcelabel">
-     <x>-1</x>
-     <y>-1</y>
-    </hint>
-    <hint type="destinationlabel">
-     <x>314</x>
-     <y>256</y>
-    </hint>
-   </hints>
-  </connection>
-  <connection>
-   <sender>actionOpenDebugLog</sender>
-   <signal>triggered()</signal>
-   <receiver>MainWindow</receiver>
-   <slot>openDebugLogTriggered()</slot>
-   <hints>
-    <hint type="sourcelabel">
-     <x>-1</x>
-     <y>-1</y>
-    </hint>
-    <hint type="destinationlabel">
-     <x>314</x>
-     <y>256</y>
-    </hint>
-   </hints>
-  </connection>
- </connections>
- <slots>
-  <slot>directoriesTriggered()</slot>
-  <slot>scanTriggered()</slot>
-  <slot>actionsTriggered()</slot>
-  <slot>detailsTriggered()</slot>
-  <slot>preferencesTriggered()</slot>
-  <slot>deltaTriggered()</slot>
-  <slot>powerMarkerTriggered()</slot>
-  <slot>deleteTriggered()</slot>
-  <slot>moveTriggered()</slot>
-  <slot>copyTriggered()</slot>
-  <slot>removeMarkedTriggered()</slot>
-  <slot>removeSelectedTriggered()</slot>
-  <slot>addToIgnoreListTriggered()</slot>
-  <slot>makeReferenceTriggered()</slot>
-  <slot>openTriggered()</slot>
-  <slot>revealTriggered()</slot>
-  <slot>renameTriggered()</slot>
-  <slot>clearIgnoreListTriggered()</slot>
-  <slot>clearPictureCacheTriggered()</slot>
-  <slot>markAllTriggered()</slot>
-  <slot>markNoneTriggered()</slot>
-  <slot>markInvertTriggered()</slot>
-  <slot>markSelectedTriggered()</slot>
-  <slot>applyFilterTriggered()</slot>
-  <slot>cancelFilterTriggered()</slot>
-  <slot>showHelpTriggered()</slot>
-  <slot>aboutTriggered()</slot>
-  <slot>registerTrigerred()</slot>
-  <slot>checkForUpdateTriggered()</slot>
-  <slot>exportTriggered()</slot>
-  <slot>openDebugLogTriggered()</slot>
- </slots>
-</ui>

base/qt/platform.py

@@ -1,19 +0,0 @@
-# -*- coding: utf-8 -*-
-# Created By: Virgil Dupras
-# Created On: 2009-09-27
-# $Id$
-# Copyright 2009 Hardcoded Software (http://www.hardcoded.net)
-# 
-# This software is licensed under the "HS" License as described in the "LICENSE" file, 
-# which should be included with this package. The terms are also available at 
-# http://www.hardcoded.net/licenses/hs_license
-
-import logging
-import sys
-
-if sys.platform == 'win32':
-    from platform_win import *
-elif sys.platform == 'darwin':
-    from platform_osx import *
-else:
-    pass # unsupported platform

base/qt/platform_osx.py

@@ -1,16 +0,0 @@
-# -*- coding: utf-8 -*-
-# Created By: Virgil Dupras
-# Created On: 2009-10-14
-# $Id$
-# Copyright 2009 Hardcoded Software (http://www.hardcoded.net)
-# 
-# This software is licensed under the "HS" License as described in the "LICENSE" file, 
-# which should be included with this package. The terms are also available at 
-# http://www.hardcoded.net/licenses/hs_license
-
-# dummy unit to allow the app to run under OSX during development
-
-INITIAL_FOLDER_IN_DIALOGS = '/'
-
-def recycle_file(path):
-    pass

base/qt/platform_win.py

@@ -1,23 +0,0 @@
-# -*- coding: utf-8 -*-
-# Created By: Virgil Dupras
-# Created On: 2009-08-31
-# $Id$
-# Copyright 2009 Hardcoded Software (http://www.hardcoded.net)
-# 
-# This software is licensed under the "HS" License as described in the "LICENSE" file, 
-# which should be included with this package. The terms are also available at 
-# http://www.hardcoded.net/licenses/hs_license
-
-from __future__ import unicode_literals
-
-import logging
-
-import winshell
-
-INITIAL_FOLDER_IN_DIALOGS = 'C:\\'
-
-def recycle_file(path):
-    try:
-        winshell.delete_file(unicode(path), no_confirm=True, silent=True)
-    except winshell.x_winshell as e:
-        logging.warning("winshell error: %s", e)

base/qt/preferences.py

@@ -1,116 +0,0 @@
-# Created By: Virgil Dupras
-# Created On: 2009-05-03
-# $Id$
-# Copyright 2009 Hardcoded Software (http://www.hardcoded.net)
-# 
-# This software is licensed under the "HS" License as described in the "LICENSE" file, 
-# which should be included with this package. The terms are also available at 
-# http://www.hardcoded.net/licenses/hs_license
-
-from PyQt4.QtCore import QSettings, QVariant
-
-from hsutil.misc import tryint
-
-def variant_to_py(v):
-    value = None
-    ok = False
-    t = v.type()
-    if t == QVariant.String:
-        value = unicode(v.toString())
-        ok = True # anyway
-        # might be bool or int, try them
-        if v == 'true':
-            value = True
-        elif value == 'false':
-            value = False
-        else:
-            value = tryint(value, value)
-    elif t == QVariant.Int:
-        value, ok = v.toInt()
-    elif t == QVariant.Bool:
-        value, ok = v.toBool(), True
-    elif t in (QVariant.List, QVariant.StringList):
-        value, ok = map(variant_to_py, v.toList()), True
-    if not ok:
-        raise TypeError(u"Can't convert {0} of type {1}".format(repr(v), v.type()))
-    return value    
-
-def py_to_variant(v):
-    if isinstance(v, (list, tuple)):
-        return QVariant(map(py_to_variant, v))
-    return QVariant(v)
-
-class Preferences(object):
-    # (width, is_visible)
-    COLUMNS_DEFAULT_ATTRS = []
-    
-    def __init__(self):
-        self.reset()
-        self.reset_columns()
-    
-    def _load_specific(self, settings, get):
-        # load prefs specific to the dg edition
-        pass
-    
-    def load(self):
-        self.reset()
-        settings = QSettings()
-        def get(name, default):
-            if settings.contains(name):
-                return variant_to_py(settings.value(name))
-            else:
-                return default
-        
-        self.filter_hardness = get('FilterHardness', self.filter_hardness)
-        self.mix_file_kind = get('MixFileKind', self.mix_file_kind)
-        self.use_regexp = get('UseRegexp', self.use_regexp)
-        self.remove_empty_folders = get('RemoveEmptyFolders', self.remove_empty_folders)
-        self.destination_type = get('DestinationType', self.destination_type)
-        widths = get('ColumnsWidth', self.columns_width)
-        # only set nonzero values
-        for index, width in enumerate(widths[:len(self.columns_width)]):
-            if width > 0:
-                self.columns_width[index] = width
-        self.columns_visible = get('ColumnsVisible', self.columns_visible)
-        self.registration_code = get('RegistrationCode', self.registration_code)
-        self.registration_email = get('RegistrationEmail', self.registration_email)
-        self._load_specific(settings, get)
-    
-    def _reset_specific(self):
-        # reset prefs specific to the dg edition
-        pass
-    
-    def reset(self):
-        self.filter_hardness = 95
-        self.mix_file_kind = True
-        self.use_regexp = False
-        self.remove_empty_folders = False
-        self.destination_type = 1
-        self.registration_code = ''
-        self.registration_email = ''
-        self._reset_specific()
-    
-    def reset_columns(self):
-        self.columns_width = [width for width, _ in self.COLUMNS_DEFAULT_ATTRS]
-        self.columns_visible = [visible for _, visible in self.COLUMNS_DEFAULT_ATTRS]
-    
-    def _save_specific(self, settings, set_):
-        # save prefs specific to the dg edition
-        pass
-    
-    def save(self):
-        settings = QSettings()
-        def set_(name, value):
-            settings.setValue(name, py_to_variant(value))
-        
-        set_('FilterHardness', self.filter_hardness)
-        set_('MixFileKind', self.mix_file_kind)
-        set_('UseRegexp', self.use_regexp)
-        set_('RemoveEmptyFolders', self.remove_empty_folders)
-        set_('DestinationType', self.destination_type)
-        set_('ColumnsWidth', self.columns_width)
-        set_('ColumnsVisible', self.columns_visible)
-        set_('RegistrationCode', self.registration_code)
-        set_('RegistrationEmail', self.registration_email)
-        self._save_specific(settings, set_)
-    

base/qt/reg.py

@@ -1,36 +0,0 @@
-# Created By: Virgil Dupras
-# Created On: 2009-05-09
-# $Id$
-# Copyright 2009 Hardcoded Software (http://www.hardcoded.net)
-# 
-# This software is licensed under the "HS" License as described in the "LICENSE" file, 
-# which should be included with this package. The terms are also available at 
-# http://www.hardcoded.net/licenses/hs_license
-
-from hashlib import md5
-
-from PyQt4.QtGui import QDialog
-
-from reg_submit_dialog import RegSubmitDialog
-from reg_demo_dialog import RegDemoDialog
-
-class Registration(object):
-    def __init__(self, app):
-        self.app = app
-    
-    def ask_for_code(self):
-        dialog = RegSubmitDialog(self.app.main_window, self.app.is_code_valid)
-        result = dialog.exec_()
-        code = unicode(dialog.codeEdit.text())
-        email = unicode(dialog.emailEdit.text())
-        dialog.setParent(None) # free it
-        if result == QDialog.Accepted and self.app.is_code_valid(code, email):
-            self.app.set_registration(code, email)
-            return True
-        return False
-    
-    def show_nag(self):
-        dialog = RegDemoDialog(self.app.main_window, self)
-        dialog.exec_()
-        dialog.setParent(None) # free it
-    

base/qt/reg_demo_dialog.py

@@ -1,47 +0,0 @@
-# Created By: Virgil Dupras
-# Created On: 2009-05-10
-# $Id$
-# Copyright 2009 Hardcoded Software (http://www.hardcoded.net)
-# 
-# This software is licensed under the "HS" License as described in the "LICENSE" file, 
-# which should be included with this package. The terms are also available at 
-# http://www.hardcoded.net/licenses/hs_license
-
-from PyQt4.QtCore import SIGNAL, Qt, QUrl, QCoreApplication
-from PyQt4.QtGui import QDialog, QMessageBox, QDesktopServices
-
-from reg_demo_dialog_ui import Ui_RegDemoDialog
-
-class RegDemoDialog(QDialog, Ui_RegDemoDialog):
-    def __init__(self, parent, reg):
-        flags = Qt.CustomizeWindowHint | Qt.WindowTitleHint | Qt.WindowSystemMenuHint
-        QDialog.__init__(self, parent, flags)
-        self.reg = reg
-        self._setupUi()
-        
-        self.connect(self.enterCodeButton, SIGNAL('clicked()'), self.enterCodeClicked)
-        self.connect(self.purchaseButton, SIGNAL('clicked()'), self.purchaseClicked)
-    
-    def _setupUi(self):
-        self.setupUi(self)
-        # Stuff that can't be setup in the Designer
-        appname = QCoreApplication.instance().applicationName()
-        title = self.windowTitle()
-        title = title.replace('$appname', appname)
-        self.setWindowTitle(title)
-        title = self.titleLabel.text()
-        title = title.replace('$appname', appname)
-        self.titleLabel.setText(title)
-        desc = self.demoDescLabel.text()
-        desc = desc.replace('$appname', appname)
-        self.demoDescLabel.setText(desc)
-    
-    #--- Events
-    def enterCodeClicked(self):
-        if self.reg.ask_for_code():
-            self.accept()
-    
-    def purchaseClicked(self):
-        url = QUrl('http://www.hardcoded.net/purchase.htm')
-        QDesktopServices.openUrl(url)
-    

base/qt/reg_demo_dialog.ui

@@ -1,140 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<ui version="4.0">
- <class>RegDemoDialog</class>
- <widget class="QDialog" name="RegDemoDialog">
-  <property name="geometry">
-   <rect>
-    <x>0</x>
-    <y>0</y>
-    <width>387</width>
-    <height>161</height>
-   </rect>
-  </property>
-  <property name="windowTitle">
-   <string>$appname Demo Version</string>
-  </property>
-  <layout class="QVBoxLayout" name="verticalLayout">
-   <item>
-    <widget class="QLabel" name="titleLabel">
-     <property name="font">
-      <font>
-       <weight>75</weight>
-       <bold>true</bold>
-      </font>
-     </property>
-     <property name="text">
-      <string>$appname Demo Version</string>
-     </property>
-    </widget>
-   </item>
-   <item>
-    <widget class="QLabel" name="demoDescLabel">
-     <property name="text">
-      <string>You are currently running a demo version of $appname. This version has limited functionalities, and you need to buy it to have access to these functionalities.</string>
-     </property>
-     <property name="wordWrap">
-      <bool>true</bool>
-     </property>
-    </widget>
-   </item>
-   <item>
-    <widget class="QLabel" name="label_3">
-     <property name="text">
-      <string>In the demo version, only 10 duplicates per session can be sent to the recycle bin, moved or copied.</string>
-     </property>
-     <property name="wordWrap">
-      <bool>true</bool>
-     </property>
-    </widget>
-   </item>
-   <item>
-    <spacer name="verticalSpacer">
-     <property name="orientation">
-      <enum>Qt::Vertical</enum>
-     </property>
-     <property name="sizeHint" stdset="0">
-      <size>
-       <width>20</width>
-       <height>40</height>
-      </size>
-     </property>
-    </spacer>
-   </item>
-   <item>
-    <layout class="QHBoxLayout" name="horizontalLayout">
-     <item>
-      <spacer name="horizontalSpacer">
-       <property name="orientation">
-        <enum>Qt::Horizontal</enum>
-       </property>
-       <property name="sizeHint" stdset="0">
-        <size>
-         <width>40</width>
-         <height>20</height>
-        </size>
-       </property>
-      </spacer>
-     </item>
-     <item>
-      <widget class="QPushButton" name="tryButton">
-       <property name="minimumSize">
-        <size>
-         <width>110</width>
-         <height>0</height>
-        </size>
-       </property>
-       <property name="text">
-        <string>Try Demo</string>
-       </property>
-      </widget>
-     </item>
-     <item>
-      <widget class="QPushButton" name="enterCodeButton">
-       <property name="minimumSize">
-        <size>
-         <width>110</width>
-         <height>0</height>
-        </size>
-       </property>
-       <property name="text">
-        <string>Enter Code</string>
-       </property>
-      </widget>
-     </item>
-     <item>
-      <widget class="QPushButton" name="purchaseButton">
-       <property name="minimumSize">
-        <size>
-         <width>110</width>
-         <height>0</height>
-        </size>
-       </property>
-       <property name="text">
-        <string>Purchase</string>
-       </property>
-      </widget>
-     </item>
-    </layout>
-   </item>
-  </layout>
- </widget>
- <resources/>
- <connections>
-  <connection>
-   <sender>tryButton</sender>
-   <signal>clicked()</signal>
-   <receiver>RegDemoDialog</receiver>
-   <slot>accept()</slot>
-   <hints>
-    <hint type="sourcelabel">
-     <x>112</x>
-     <y>161</y>
-    </hint>
-    <hint type="destinationlabel">
-     <x>201</x>
-     <y>94</y>
-    </hint>
-   </hints>
-  </connection>
- </connections>
-</ui>

base/qt/reg_submit_dialog.py

@@ -1,49 +0,0 @@
-# Created By: Virgil Dupras
-# Created On: 2009-05-09
-# $Id$
-# Copyright 2009 Hardcoded Software (http://www.hardcoded.net)
-# 
-# This software is licensed under the "HS" License as described in the "LICENSE" file, 
-# which should be included with this package. The terms are also available at 
-# http://www.hardcoded.net/licenses/hs_license
-
-from PyQt4.QtCore import SIGNAL, Qt, QUrl, QCoreApplication
-from PyQt4.QtGui import QDialog, QMessageBox, QDesktopServices
-
-from reg_submit_dialog_ui import Ui_RegSubmitDialog
-
-class RegSubmitDialog(QDialog, Ui_RegSubmitDialog):
-    def __init__(self, parent, is_valid_func):
-        flags = Qt.CustomizeWindowHint | Qt.WindowTitleHint | Qt.WindowSystemMenuHint
-        QDialog.__init__(self, parent, flags)
-        self._setupUi()
-        self.is_valid_func = is_valid_func
-        
-        self.connect(self.submitButton, SIGNAL('clicked()'), self.submitClicked)
-        self.connect(self.purchaseButton, SIGNAL('clicked()'), self.purchaseClicked)
-    
-    def _setupUi(self):
-        self.setupUi(self)
-        # Stuff that can't be setup in the Designer
-        appname = QCoreApplication.instance().applicationName()
-        prompt = self.promptLabel.text()
-        prompt = prompt.replace('$appname', appname)
-        self.promptLabel.setText(prompt)
-    
-    #--- Events
-    def purchaseClicked(self):
-        url = QUrl('http://www.hardcoded.net/purchase.htm')
-        QDesktopServices.openUrl(url)
-    
-    def submitClicked(self):
-        code = unicode(self.codeEdit.text())
-        email = unicode(self.emailEdit.text())
-        title = "Registration"
-        if self.is_valid_func(code, email):
-            msg = "This code is valid. Thanks!"
-            QMessageBox.information(self, title, msg)
-            self.accept()
-        else:
-            msg = "This code is invalid"
-            QMessageBox.warning(self, title, msg)
-    

base/qt/reg_submit_dialog.ui

@@ -1,149 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<ui version="4.0">
- <class>RegSubmitDialog</class>
- <widget class="QDialog" name="RegSubmitDialog">
-  <property name="geometry">
-   <rect>
-    <x>0</x>
-    <y>0</y>
-    <width>365</width>
-    <height>134</height>
-   </rect>
-  </property>
-  <property name="windowTitle">
-   <string>Enter your registration code</string>
-  </property>
-  <layout class="QVBoxLayout" name="verticalLayout">
-   <item>
-    <widget class="QLabel" name="promptLabel">
-     <property name="text">
-      <string>Please enter your $appname registration code and registered e-mail (the e-mail you used for the purchase), then press &quot;Submit&quot;.</string>
-     </property>
-     <property name="alignment">
-      <set>Qt::AlignLeading|Qt::AlignLeft|Qt::AlignTop</set>
-     </property>
-     <property name="wordWrap">
-      <bool>true</bool>
-     </property>
-    </widget>
-   </item>
-   <item>
-    <layout class="QFormLayout" name="formLayout">
-     <property name="sizeConstraint">
-      <enum>QLayout::SetNoConstraint</enum>
-     </property>
-     <property name="fieldGrowthPolicy">
-      <enum>QFormLayout::ExpandingFieldsGrow</enum>
-     </property>
-     <property name="labelAlignment">
-      <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
-     </property>
-     <property name="formAlignment">
-      <set>Qt::AlignLeading|Qt::AlignLeft|Qt::AlignTop</set>
-     </property>
-     <item row="0" column="0">
-      <widget class="QLabel" name="label_2">
-       <property name="text">
-        <string>Registration code:</string>
-       </property>
-      </widget>
-     </item>
-     <item row="1" column="0">
-      <widget class="QLabel" name="label_3">
-       <property name="text">
-        <string>Registered e-mail:</string>
-       </property>
-      </widget>
-     </item>
-     <item row="0" column="1">
-      <widget class="QLineEdit" name="codeEdit"/>
-     </item>
-     <item row="1" column="1">
-      <widget class="QLineEdit" name="emailEdit"/>
-     </item>
-    </layout>
-   </item>
-   <item>
-    <layout class="QHBoxLayout" name="horizontalLayout">
-     <item>
-      <widget class="QPushButton" name="purchaseButton">
-       <property name="text">
-        <string>Purchase</string>
-       </property>
-       <property name="autoDefault">
-        <bool>false</bool>
-       </property>
-      </widget>
-     </item>
-     <item>
-      <spacer name="horizontalSpacer">
-       <property name="orientation">
-        <enum>Qt::Horizontal</enum>
-       </property>
-       <property name="sizeHint" stdset="0">
-        <size>
-         <width>40</width>
-         <height>20</height>
-        </size>
-       </property>
-      </spacer>
-     </item>
-     <item>
-      <widget class="QPushButton" name="cancelButton">
-       <property name="sizePolicy">
-        <sizepolicy hsizetype="Fixed" vsizetype="Fixed">
-         <horstretch>0</horstretch>
-         <verstretch>0</verstretch>
-        </sizepolicy>
-       </property>
-       <property name="text">
-        <string>Cancel</string>
-       </property>
-       <property name="autoDefault">
-        <bool>false</bool>
-       </property>
-      </widget>
-     </item>
-     <item>
-      <widget class="QPushButton" name="submitButton">
-       <property name="sizePolicy">
-        <sizepolicy hsizetype="Fixed" vsizetype="Fixed">
-         <horstretch>0</horstretch>
-         <verstretch>0</verstretch>
-        </sizepolicy>
-       </property>
-       <property name="text">
-        <string>Submit</string>
-       </property>
-       <property name="autoDefault">
-        <bool>false</bool>
-       </property>
-       <property name="default">
-        <bool>true</bool>
-       </property>
-      </widget>
-     </item>
-    </layout>
-   </item>
-  </layout>
- </widget>
- <resources/>
- <connections>
-  <connection>
-   <sender>cancelButton</sender>
-   <signal>clicked()</signal>
-   <receiver>RegSubmitDialog</receiver>
-   <slot>reject()</slot>
-   <hints>
-    <hint type="sourcelabel">
-     <x>260</x>
-     <y>159</y>
-    </hint>
-    <hint type="destinationlabel">
-     <x>198</x>
-     <y>97</y>
-    </hint>
-   </hints>
-  </connection>
- </connections>
-</ui>

base/qt/results_model.py

@@ -1,204 +0,0 @@
-# Created By: Virgil Dupras
-# Created On: 2009-04-23
-# $Id$
-# Copyright 2009 Hardcoded Software (http://www.hardcoded.net)
-# 
-# This software is licensed under the "HS" License as described in the "LICENSE" file, 
-# which should be included with this package. The terms are also available at 
-# http://www.hardcoded.net/licenses/hs_license
-
-from PyQt4.QtCore import SIGNAL, Qt, QAbstractItemModel, QModelIndex, QRect
-from PyQt4.QtGui import QBrush, QStyledItemDelegate, QFont, QTreeView, QColor
-
-from qtlib.tree_model import TreeNode, TreeModel
-
-class ResultNode(TreeNode):
-    def __init__(self, model, parent, row, dupe, group):
-        TreeNode.__init__(self, model, parent, row)
-        self.dupe = dupe
-        self.group = group
-        self._normalData = None
-        self._deltaData = None
-    
-    def _createNode(self, ref, row):
-        return ResultNode(self.model, self, row, ref, self.group)
-    
-    def _getChildren(self):
-        return self.group.dupes if self.dupe is self.group.ref else []
-    
-    def invalidate(self):
-        self._normalData = None
-        self._deltaData = None
-        TreeNode.invalidate(self)
-    
-    @property
-    def normalData(self):
-        if self._normalData is None:
-            self._normalData = self.model._app._get_display_info(self.dupe, self.group, delta=False)
-        return self._normalData
-    
-    @property
-    def deltaData(self):
-        if self._deltaData is None:
-            self._deltaData = self.model._app._get_display_info(self.dupe, self.group, delta=True)
-        return self._deltaData
-    
-
-class ResultsDelegate(QStyledItemDelegate):
-    def initStyleOption(self, option, index):
-        QStyledItemDelegate.initStyleOption(self, option, index)
-        node = index.internalPointer()
-        if node.group.ref is node.dupe:
-            newfont = QFont(option.font)
-            newfont.setBold(True)
-            option.font = newfont
-    
-
-class ResultsModel(TreeModel):
-    def __init__(self, app):
-        self._app = app
-        self._results = app.results
-        self._data = app.data
-        self._delta_columns = app.DELTA_COLUMNS
-        self.delta = False
-        self._power_marker = False
-        TreeModel.__init__(self)
-    
-    def _createNode(self, ref, row):
-        if self.power_marker:
-            # ref is a dupe
-            group = self._results.get_group_of_duplicate(ref)
-            return ResultNode(self, None, row, ref, group)
-        else:
-            # ref is a group
-            return ResultNode(self, None, row, ref.ref, ref)
-    
-    def _getChildren(self):
-        return self._results.dupes if self.power_marker else self._results.groups
-    
-    def columnCount(self, parent):
-        return len(self._data.COLUMNS)
-    
-    def data(self, index, role):
-        if not index.isValid():
-            return None
-        node = index.internalPointer()
-        if role == Qt.DisplayRole:
-            data = node.deltaData if self.delta else node.normalData
-            return data[index.column()]
-        elif role == Qt.CheckStateRole:
-            if index.column() == 0 and node.dupe is not node.group.ref:
-                state = Qt.Checked if self._results.is_marked(node.dupe) else Qt.Unchecked
-                return state
-        elif role == Qt.ForegroundRole:
-            if node.dupe is node.group.ref or node.dupe.is_ref:
-                return QBrush(Qt.blue)
-            elif self.delta and index.column() in self._delta_columns:
-                return QBrush(QColor(255, 142, 40)) # orange
-        elif role == Qt.EditRole:
-            if index.column() == 0:
-                return node.normalData[index.column()]
-        return None
-    
-    def dupesForIndexes(self, indexes):
-        nodes = [index.internalPointer() for index in indexes]
-        return [node.dupe for node in nodes]
-    
-    def indexesForDupes(self, dupes):
-        def index(dupe):
-            try:
-                if self.power_marker:
-                    row = self._results.dupes.index(dupe)
-                    node = self.subnodes[row]
-                    assert node.dupe is dupe
-                    return self.createIndex(row, 0, node)
-                else:
-                    group = self._results.get_group_of_duplicate(dupe)
-                    row = self._results.groups.index(group)
-                    node = self.subnodes[row]
-                    if dupe is group.ref:
-                        assert node.dupe is dupe
-                        return self.createIndex(row, 0, node)
-                    subrow = group.dupes.index(dupe)
-                    subnode = node.subnodes[subrow]
-                    assert subnode.dupe is dupe
-                    return self.createIndex(subrow, 0, subnode)
-            except ValueError: # the dupe is not there anymore
-                return QModelIndex()
-        
-        return map(index, dupes)
-    
-    def flags(self, index):
-        if not index.isValid():
-            return Qt.ItemIsEnabled
-        flags = Qt.ItemIsEnabled | Qt.ItemIsSelectable
-        if index.column() == 0:
-            flags |= Qt.ItemIsUserCheckable | Qt.ItemIsEditable
-        return flags
-    
-    def headerData(self, section, orientation, role):
-        if orientation == Qt.Horizontal and role == Qt.DisplayRole and section < len(self._data.COLUMNS):
-            return self._data.COLUMNS[section]['display']
-        return None
-    
-    def setData(self, index, value, role):
-        if not index.isValid():
-            return False
-        node = index.internalPointer()
-        if role == Qt.CheckStateRole:
-            if index.column() == 0:
-                self._app.toggle_marking_for_dupes([node.dupe])
-                return True
-        if role == Qt.EditRole:
-            if index.column() == 0:
-                value = unicode(value.toString())
-                if self._app.rename_dupe(node.dupe, value):
-                    node.reset()
-                    return True
-        return False
-    
-    def sort(self, column, order):
-        if self.power_marker:
-            self._results.sort_dupes(column, order == Qt.AscendingOrder, self.delta)
-        else:
-            self._results.sort_groups(column, order == Qt.AscendingOrder)
-        self.reset()
-    
-    def toggleMarked(self, indexes):
-        assert indexes
-        dupes = self.dupesForIndexes(indexes)
-        self._app.toggle_marking_for_dupes(dupes)
-    
-    #--- Properties
-    @property
-    def power_marker(self):
-        return self._power_marker
-    
-    @power_marker.setter
-    def power_marker(self, value):
-        if value == self._power_marker:
-            return
-        self._power_marker = value
-        self.reset()
-    
-
-class ResultsView(QTreeView):
-    #--- Override
-    def keyPressEvent(self, event):
-        if event.text() == ' ':
-            self.model().toggleMarked(self.selectionModel().selectedRows())
-            return
-        QTreeView.keyPressEvent(self, event)
-    
-    def mouseDoubleClickEvent(self, event):
-        self.emit(SIGNAL('doubleClicked()'))
-        # We don't call the superclass' method because the default behavior is to rename the cell.
-    
-    def setModel(self, model):
-        assert isinstance(model, ResultsModel)
-        QTreeView.setModel(self, model)
-    
-    #--- Public
-    def selectedDupes(self):
-        return self.model().dupesForIndexes(self.selectionModel().selectedRows())
-    

build.py

@@ -0,0 +1,165 @@
+# Copyright 2017 Virgil Dupras
+#
+# This software is licensed under the "GPLv3" License as described in the "LICENSE" file,
+# which should be included with this package. The terms are also available at
+# http://www.gnu.org/licenses/gpl-3.0.html
+
+from pathlib import Path
+import sys
+from optparse import OptionParser
+import shutil
+from multiprocessing import Pool
+
+from setuptools import sandbox
+from hscommon import sphinxgen
+from hscommon.build import (
+    add_to_pythonpath,
+    print_and_do,
+    fix_qt_resource_file,
+)
+from hscommon import loc
+
+
+def parse_args():
+    usage = "usage: %prog [options]"
+    parser = OptionParser(usage=usage)
+    parser.add_option(
+        "--clean",
+        action="store_true",
+        dest="clean",
+        help="Clean build folder before building",
+    )
+    parser.add_option("--doc", action="store_true", dest="doc", help="Build only the help file (en)")
+    parser.add_option("--alldoc", action="store_true", dest="all_doc", help="Build only the help file in all languages")
+    parser.add_option("--loc", action="store_true", dest="loc", help="Build only localization")
+    parser.add_option(
+        "--updatepot",
+        action="store_true",
+        dest="updatepot",
+        help="Generate .pot files from source code.",
+    )
+    parser.add_option(
+        "--mergepot",
+        action="store_true",
+        dest="mergepot",
+        help="Update all .po files based on .pot files.",
+    )
+    parser.add_option(
+        "--normpo",
+        action="store_true",
+        dest="normpo",
+        help="Normalize all PO files (do this before commit).",
+    )
+    parser.add_option(
+        "--modules",
+        action="store_true",
+        dest="modules",
+        help="Build the python modules.",
+    )
+    (options, args) = parser.parse_args()
+    return options
+
+
+def build_one_help(language):
+    print(f"Generating Help in {language}")
+    current_path = Path(".").absolute()
+    changelog_path = current_path.joinpath("help", "changelog")
+    tixurl = "https://github.com/arsenetar/dupeguru/issues/{}"
+    changelogtmpl = current_path.joinpath("help", "changelog.tmpl")
+    conftmpl = current_path.joinpath("help", "conf.tmpl")
+    help_basepath = current_path.joinpath("help", language)
+    help_destpath = current_path.joinpath("build", "help", language)
+    confrepl = {"language": language}
+    sphinxgen.gen(
+        help_basepath,
+        help_destpath,
+        changelog_path,
+        tixurl,
+        confrepl,
+        conftmpl,
+        changelogtmpl,
+    )
+
+
+def build_help():
+    languages = ["en", "de", "fr", "hy", "ru", "uk"]
+    # Running with Pools as for some reason sphinx seems to cross contaminate the output otherwise
+    with Pool(len(languages)) as p:
+        p.map(build_one_help, languages)
+
+
+def build_localizations():
+    loc.compile_all_po("locale")
+    locale_dest = Path("build", "locale")
+    if locale_dest.exists():
+        shutil.rmtree(locale_dest)
+    shutil.copytree("locale", locale_dest, ignore=shutil.ignore_patterns("*.po", "*.pot"))
+
+
+def build_updatepot():
+    print("Building .pot files from source files")
+    print("Building core.pot")
+    loc.generate_pot(["core"], Path("locale", "core.pot"), ["tr"])
+    print("Building columns.pot")
+    loc.generate_pot(["core"], Path("locale", "columns.pot"), ["coltr"])
+    print("Building ui.pot")
+    loc.generate_pot(["qt"], Path("locale", "ui.pot"), ["tr"], merge=True)
+
+
+def build_mergepot():
+    print("Updating .po files using .pot files")
+    loc.merge_pots_into_pos("locale")
+
+
+def build_normpo():
+    loc.normalize_all_pos("locale")
+
+
+def build_pe_modules():
+    print("Building PE Modules")
+    # Leverage setup.py to build modules
+    sandbox.run_setup("setup.py", ["build_ext", "--inplace"])
+
+
+def build_normal():
+    print("Building dupeGuru with UI qt")
+    add_to_pythonpath(".")
+    print("Building dupeGuru")
+    build_pe_modules()
+    print("Building localizations")
+    build_localizations()
+    print("Building Qt stuff")
+    Path("qt", "dg_rc.py").unlink(missing_ok=True)
+    print_and_do("pyrcc5 {} > {}".format(Path("qt", "dg.qrc"), Path("qt", "dg_rc.py")))
+    fix_qt_resource_file(Path("qt", "dg_rc.py"))
+    build_help()
+
+
+def main():
+    if sys.version_info < (3, 7):
+        sys.exit("Python < 3.7 is unsupported.")
+    options = parse_args()
+    if options.clean and Path("build").exists():
+        shutil.rmtree("build")
+    if not Path("build").exists():
+        Path("build").mkdir()
+    if options.doc:
+        build_one_help("en")
+    elif options.all_doc:
+        build_help()
+    elif options.loc:
+        build_localizations()
+    elif options.updatepot:
+        build_updatepot()
+    elif options.mergepot:
+        build_mergepot()
+    elif options.normpo:
+        build_normpo()
+    elif options.modules:
+        build_pe_modules()
+    else:
+        build_normal()
+
+
+if __name__ == "__main__":
+    main()

commitlint.config.js

@@ -0,0 +1,17 @@
+const Configuration = {
+    /*
+     * Resolve and load @commitlint/config-conventional from node_modules.
+     * Referenced packages must be installed
+     */
+    extends: ['@commitlint/config-conventional'],
+    /*
+     * Any rules defined here will override rules from @commitlint/config-conventional
+     */
+    rules: {
+        'header-max-length': [2, 'always', 72],
+        'subject-case': [2, 'always', 'sentence-case'],
+        'scope-enum': [2, 'always'],
+    },
+};
+
+module.exports = Configuration;

core/__init__.py

@@ -0,0 +1,2 @@
+__version__ = "4.3.1"
+__appname__ = "dupeGuru"

core/app.py

@@ -0,0 +1,899 @@
+# Copyright 2016 Hardcoded Software (http://www.hardcoded.net)
+#
+# This software is licensed under the "GPLv3" License as described in the "LICENSE" file,
+# which should be included with this package. The terms are also available at
+# http://www.gnu.org/licenses/gpl-3.0.html
+
+import cProfile
+import datetime
+import os
+import os.path as op
+import logging
+import subprocess
+import re
+import shutil
+from pathlib import Path
+
+from send2trash import send2trash
+from hscommon.jobprogress import job
+from hscommon.notify import Broadcaster
+from hscommon.conflict import smart_move, smart_copy
+from hscommon.gui.progress_window import ProgressWindow
+from hscommon.util import delete_if_empty, first, escape, nonone, allsame
+from hscommon.trans import tr
+from hscommon import desktop
+
+from core import se, me, pe
+from core.pe.photo import get_delta_dimensions
+from core.util import cmp_value, fix_surrogate_encoding
+from core import directories, results, export, fs, prioritize
+from core.ignore import IgnoreList
+from core.exclude import ExcludeDict as ExcludeList
+from core.scanner import ScanType
+from core.gui.deletion_options import DeletionOptions
+from core.gui.details_panel import DetailsPanel
+from core.gui.directory_tree import DirectoryTree
+from core.gui.ignore_list_dialog import IgnoreListDialog
+from core.gui.exclude_list_dialog import ExcludeListDialogCore
+from core.gui.problem_dialog import ProblemDialog
+from core.gui.stats_label import StatsLabel
+
+HAD_FIRST_LAUNCH_PREFERENCE = "HadFirstLaunch"
+DEBUG_MODE_PREFERENCE = "DebugMode"
+
+MSG_NO_MARKED_DUPES = tr("There are no marked duplicates. Nothing has been done.")
+MSG_NO_SELECTED_DUPES = tr("There are no selected duplicates. Nothing has been done.")
+MSG_MANY_FILES_TO_OPEN = tr(
+    "You're about to open many files at once. Depending on what those "
+    "files are opened with, doing so can create quite a mess. Continue?"
+)
+
+
+class DestType:
+    DIRECT = 0
+    RELATIVE = 1
+    ABSOLUTE = 2
+
+
+class JobType:
+    SCAN = "job_scan"
+    LOAD = "job_load"
+    MOVE = "job_move"
+    COPY = "job_copy"
+    DELETE = "job_delete"
+
+
+class AppMode:
+    STANDARD = 0
+    MUSIC = 1
+    PICTURE = 2
+
+
+JOBID2TITLE = {
+    JobType.SCAN: tr("Scanning for duplicates"),
+    JobType.LOAD: tr("Loading"),
+    JobType.MOVE: tr("Moving"),
+    JobType.COPY: tr("Copying"),
+    JobType.DELETE: tr("Sending to Trash"),
+}
+
+
+class DupeGuru(Broadcaster):
+    """Holds everything together.
+
+    Instantiated once per running application, it holds a reference to every high-level object
+    whose reference needs to be held: :class:`~core.results.Results`,
+    :class:`~core.directories.Directories`, :mod:`core.gui` instances, etc..
+
+    It also hosts high level methods and acts as a coordinator for all those elements. This is why
+    some of its methods seem a bit shallow, like for example :meth:`mark_all` and
+    :meth:`remove_duplicates`. These methos are just proxies for a method in :attr:`results`, but
+    they are also followed by a notification call which is very important if we want GUI elements
+    to be correctly notified of a change in the data they're presenting.
+
+    .. attribute:: directories
+
+        Instance of :class:`~core.directories.Directories`. It holds the current folder selection.
+
+    .. attribute:: results
+
+        Instance of :class:`core.results.Results`. Holds the results of the latest scan.
+
+    .. attribute:: selected_dupes
+
+        List of currently selected dupes from our :attr:`results`. Whenever the user changes its
+        selection at the UI level, :attr:`result_table` takes care of updating this attribute, so
+        you can trust that it's always up-to-date.
+
+    .. attribute:: result_table
+
+        Instance of :mod:`meta-gui <core.gui>` table listing the results from :attr:`results`
+    """
+
+    # --- View interface
+    # get_default(key_name)
+    # set_default(key_name, value)
+    # show_message(msg)
+    # open_url(url)
+    # open_path(path)
+    # reveal_path(path)
+    # ask_yes_no(prompt) --> bool
+    # create_results_window()
+    # show_results_window()
+    # show_problem_dialog()
+    # select_dest_folder(prompt: str) --> str
+    # select_dest_file(prompt: str, ext: str) --> str
+
+    NAME = PROMPT_NAME = "dupeGuru"
+
+    def __init__(self, view, portable=False):
+        if view.get_default(DEBUG_MODE_PREFERENCE):
+            logging.getLogger().setLevel(logging.DEBUG)
+            logging.debug("Debug mode enabled")
+        Broadcaster.__init__(self)
+        self.view = view
+        self.appdata = desktop.special_folder_path(desktop.SpecialFolder.APPDATA, portable=portable)
+        if not op.exists(self.appdata):
+            os.makedirs(self.appdata)
+        self.app_mode = AppMode.STANDARD
+        self.discarded_file_count = 0
+        self.exclude_list = ExcludeList()
+        hash_cache_file = op.join(self.appdata, "hash_cache.db")
+        fs.filesdb.connect(hash_cache_file)
+        self.directories = directories.Directories(self.exclude_list)
+        self.results = results.Results(self)
+        self.ignore_list = IgnoreList()
+        # In addition to "app-level" options, this dictionary also holds options that will be
+        # sent to the scanner. They don't have default values because those defaults values are
+        # defined in the scanner class.
+        self.options = {
+            "escape_filter_regexp": True,
+            "clean_empty_dirs": False,
+            "ignore_hardlink_matches": False,
+            "copymove_dest_type": DestType.RELATIVE,
+            "include_exists_check": True,
+            "rehash_ignore_mtime": False,
+        }
+        self.selected_dupes = []
+        self.details_panel = DetailsPanel(self)
+        self.directory_tree = DirectoryTree(self)
+        self.problem_dialog = ProblemDialog(self)
+        self.ignore_list_dialog = IgnoreListDialog(self)
+        self.exclude_list_dialog = ExcludeListDialogCore(self)
+        self.stats_label = StatsLabel(self)
+        self.result_table = None
+        self.deletion_options = DeletionOptions()
+        self.progress_window = ProgressWindow(self._job_completed, self._job_error)
+        children = [self.directory_tree, self.stats_label, self.details_panel]
+        for child in children:
+            child.connect()
+
+    # --- Private
+    def _recreate_result_table(self):
+        if self.result_table is not None:
+            self.result_table.disconnect()
+        if self.app_mode == AppMode.PICTURE:
+            self.result_table = pe.result_table.ResultTable(self)
+        elif self.app_mode == AppMode.MUSIC:
+            self.result_table = me.result_table.ResultTable(self)
+        else:
+            self.result_table = se.result_table.ResultTable(self)
+        self.result_table.connect()
+        self.view.create_results_window()
+
+    def _get_picture_cache_path(self):
+        cache_name = "cached_pictures.db"
+        return op.join(self.appdata, cache_name)
+
+    def _get_dupe_sort_key(self, dupe, get_group, key, delta):
+        if self.app_mode in (AppMode.MUSIC, AppMode.PICTURE) and key == "folder_path":
+            dupe_folder_path = getattr(dupe, "display_folder_path", dupe.folder_path)
+            return str(dupe_folder_path).lower()
+        if self.app_mode == AppMode.PICTURE and delta and key == "dimensions":
+            r = cmp_value(dupe, key)
+            ref_value = cmp_value(get_group().ref, key)
+            return get_delta_dimensions(r, ref_value)
+        if key == "marked":
+            return self.results.is_marked(dupe)
+        if key == "percentage":
+            m = get_group().get_match_of(dupe)
+            return m.percentage
+        elif key == "dupe_count":
+            return 0
+        else:
+            result = cmp_value(dupe, key)
+        if delta:
+            refval = cmp_value(get_group().ref, key)
+            if key in self.result_table.DELTA_COLUMNS:
+                result -= refval
+            else:
+                same = cmp_value(dupe, key) == refval
+                result = (same, result)
+        return result
+
+    def _get_group_sort_key(self, group, key):
+        if self.app_mode in (AppMode.MUSIC, AppMode.PICTURE) and key == "folder_path":
+            dupe_folder_path = getattr(group.ref, "display_folder_path", group.ref.folder_path)
+            return str(dupe_folder_path).lower()
+        if key == "percentage":
+            return group.percentage
+        if key == "dupe_count":
+            return len(group)
+        if key == "marked":
+            return len([dupe for dupe in group.dupes if self.results.is_marked(dupe)])
+        return cmp_value(group.ref, key)
+
+    def _do_delete(self, j, link_deleted, use_hardlinks, direct_deletion):
+        def op(dupe):
+            j.add_progress()
+            return self._do_delete_dupe(dupe, link_deleted, use_hardlinks, direct_deletion)
+
+        j.start_job(self.results.mark_count)
+        self.results.perform_on_marked(op, True)
+
+    def _do_delete_dupe(self, dupe, link_deleted, use_hardlinks, direct_deletion):
+        if not dupe.path.exists():
+            return
+        logging.debug("Sending '%s' to trash", dupe.path)
+        str_path = str(dupe.path)
+        if direct_deletion:
+            if op.isdir(str_path):
+                shutil.rmtree(str_path)
+            else:
+                os.remove(str_path)
+        else:
+            send2trash(str_path)  # Raises OSError when there's a problem
+        if link_deleted:
+            group = self.results.get_group_of_duplicate(dupe)
+            ref = group.ref
+            linkfunc = os.link if use_hardlinks else os.symlink
+            linkfunc(str(ref.path), str_path)
+        self.clean_empty_dirs(dupe.path.parent)
+
+    def _create_file(self, path):
+        # We add fs.Folder to fileclasses in case the file we're loading contains folder paths.
+        return fs.get_file(path, self.fileclasses + [se.fs.Folder])
+
+    def _get_file(self, str_path):
+        path = Path(str_path)
+        f = self._create_file(path)
+        if f is None:
+            return None
+        try:
+            f._read_all_info(attrnames=self.METADATA_TO_READ)
+            return f
+        except OSError:
+            return None
+
+    def _get_export_data(self):
+        columns = [col for col in self.result_table._columns.ordered_columns if col.visible and col.name != "marked"]
+        colnames = [col.display for col in columns]
+        rows = []
+        for group_id, group in enumerate(self.results.groups):
+            for dupe in group:
+                data = self.get_display_info(dupe, group)
+                row = [fix_surrogate_encoding(data[col.name]) for col in columns]
+                row.insert(0, group_id)
+                rows.append(row)
+        return colnames, rows
+
+    def _results_changed(self):
+        self.selected_dupes = [d for d in self.selected_dupes if self.results.get_group_of_duplicate(d) is not None]
+        self.notify("results_changed")
+
+    def _start_job(self, jobid, func, args=()):
+        title = JOBID2TITLE[jobid]
+        try:
+            self.progress_window.run(jobid, title, func, args=args)
+        except job.JobInProgressError:
+            msg = tr(
+                "A previous action is still hanging in there. You can't start a new one yet. Wait "
+                "a few seconds, then try again."
+            )
+            self.view.show_message(msg)
+
+    def _job_completed(self, jobid):
+        if jobid == JobType.SCAN:
+            self._results_changed()
+            fs.filesdb.commit()
+            if not self.results.groups:
+                self.view.show_message(tr("No duplicates found."))
+            else:
+                self.view.show_results_window()
+        if jobid in {JobType.MOVE, JobType.DELETE}:
+            self._results_changed()
+        if jobid == JobType.LOAD:
+            self._recreate_result_table()
+            self._results_changed()
+            self.view.show_results_window()
+        if jobid in {JobType.COPY, JobType.MOVE, JobType.DELETE}:
+            if self.results.problems:
+                self.problem_dialog.refresh()
+                self.view.show_problem_dialog()
+            else:
+                if jobid == JobType.COPY:
+                    msg = tr("All marked files were copied successfully.")
+                elif jobid == JobType.MOVE:
+                    msg = tr("All marked files were moved successfully.")
+                elif jobid == JobType.DELETE and self.deletion_options.direct:
+                    msg = tr("All marked files were deleted successfully.")
+                else:
+                    msg = tr("All marked files were successfully sent to Trash.")
+                self.view.show_message(msg)
+
+    def _job_error(self, jobid, err):
+        if jobid == JobType.LOAD:
+            msg = tr("Could not load file: {}").format(err)
+            self.view.show_message(msg)
+            return False
+        else:
+            raise err
+
+    @staticmethod
+    def _remove_hardlink_dupes(files):
+        seen_inodes = set()
+        result = []
+        for file in files:
+            try:
+                inode = file.path.stat().st_ino
+            except OSError:
+                # The file was probably deleted or something
+                continue
+            if inode not in seen_inodes:
+                seen_inodes.add(inode)
+                result.append(file)
+        return result
+
+    def _select_dupes(self, dupes):
+        if dupes == self.selected_dupes:
+            return
+        self.selected_dupes = dupes
+        self.notify("dupes_selected")
+
+    # --- Protected
+    def _get_fileclasses(self):
+        if self.app_mode == AppMode.PICTURE:
+            return [pe.photo.PLAT_SPECIFIC_PHOTO_CLASS]
+        elif self.app_mode == AppMode.MUSIC:
+            return [me.fs.MusicFile]
+        else:
+            return [se.fs.File]
+
+    def _prioritization_categories(self):
+        if self.app_mode == AppMode.PICTURE:
+            return pe.prioritize.all_categories()
+        elif self.app_mode == AppMode.MUSIC:
+            return me.prioritize.all_categories()
+        else:
+            return prioritize.all_categories()
+
+    # --- Public
+    def add_directory(self, d):
+        """Adds folder ``d`` to :attr:`directories`.
+
+        Shows an error message dialog if something bad happens.
+
+        :param str d: path of folder to add
+        """
+        try:
+            self.directories.add_path(Path(d))
+            self.notify("directories_changed")
+        except directories.AlreadyThereError:
+            self.view.show_message(tr("'{}' already is in the list.").format(d))
+        except directories.InvalidPathError:
+            self.view.show_message(tr("'{}' does not exist.").format(d))
+
+    def add_selected_to_ignore_list(self):
+        """Adds :attr:`selected_dupes` to :attr:`ignore_list`."""
+        dupes = self.without_ref(self.selected_dupes)
+        if not dupes:
+            self.view.show_message(MSG_NO_SELECTED_DUPES)
+            return
+        msg = tr("All selected %d matches are going to be ignored in all subsequent scans. Continue?")
+        if not self.view.ask_yes_no(msg % len(dupes)):
+            return
+        for dupe in dupes:
+            g = self.results.get_group_of_duplicate(dupe)
+            for other in g:
+                if other is not dupe:
+                    self.ignore_list.ignore(str(other.path), str(dupe.path))
+        self.remove_duplicates(dupes)
+        self.ignore_list_dialog.refresh()
+
+    def apply_filter(self, result_filter):
+        """Apply a filter ``filter`` to the results so that it shows only dupe groups that match it.
+
+        :param str filter: filter to apply
+        """
+        self.results.apply_filter(None)
+        if self.options["escape_filter_regexp"]:
+            result_filter = escape(result_filter, set("()[]\\.|+?^"))
+            result_filter = escape(result_filter, "*", ".")
+        self.results.apply_filter(result_filter)
+        self._results_changed()
+
+    def clean_empty_dirs(self, path):
+        if self.options["clean_empty_dirs"]:
+            while delete_if_empty(path, [".DS_Store"]):
+                path = path.parent
+
+    def clear_picture_cache(self):
+        try:
+            os.remove(self._get_picture_cache_path())
+        except FileNotFoundError:
+            pass  # we don't care
+
+    def clear_hash_cache(self):
+        fs.filesdb.clear()
+
+    def copy_or_move(self, dupe, copy: bool, destination: str, dest_type: DestType):
+        source_path = dupe.path
+        location_path = first(p for p in self.directories if p in dupe.path.parents)
+        dest_path = Path(destination)
+        if dest_type in {DestType.RELATIVE, DestType.ABSOLUTE}:
+            # no filename, no windows drive letter
+            source_base = source_path.relative_to(source_path.anchor).parent
+            if dest_type == DestType.RELATIVE:
+                source_base = source_base.relative_to(location_path.relative_to(location_path.anchor))
+            dest_path = dest_path.joinpath(source_base)
+        if not dest_path.exists():
+            dest_path.mkdir(parents=True)
+        # Add filename to dest_path. For file move/copy, it's not required, but for folders, yes.
+        dest_path = dest_path.joinpath(source_path.name)
+        logging.debug("Copy/Move operation from '%s' to '%s'", source_path, dest_path)
+        # Raises an EnvironmentError if there's a problem
+        if copy:
+            smart_copy(source_path, dest_path)
+        else:
+            smart_move(source_path, dest_path)
+            self.clean_empty_dirs(source_path.parent)
+
+    def copy_or_move_marked(self, copy):
+        """Start an async move (or copy) job on marked duplicates.
+
+        :param bool copy: If True, duplicates will be copied instead of moved
+        """
+
+        def do(j):
+            def op(dupe):
+                j.add_progress()
+                self.copy_or_move(dupe, copy, destination, desttype)
+
+            j.start_job(self.results.mark_count)
+            self.results.perform_on_marked(op, not copy)
+
+        if not self.results.mark_count:
+            self.view.show_message(MSG_NO_MARKED_DUPES)
+            return
+        destination = self.view.select_dest_folder(
+            tr("Select a directory to copy marked files to")
+            if copy
+            else tr("Select a directory to move marked files to")
+        )
+        if destination:
+            desttype = self.options["copymove_dest_type"]
+            jobid = JobType.COPY if copy else JobType.MOVE
+            self._start_job(jobid, do)
+
+    def delete_marked(self):
+        """Start an async job to send marked duplicates to the trash."""
+        if not self.results.mark_count:
+            self.view.show_message(MSG_NO_MARKED_DUPES)
+            return
+        if not self.deletion_options.show(self.results.mark_count):
+            return
+        args = [
+            self.deletion_options.link_deleted,
+            self.deletion_options.use_hardlinks,
+            self.deletion_options.direct,
+        ]
+        logging.debug("Starting deletion job with args %r", args)
+        self._start_job(JobType.DELETE, self._do_delete, args=args)
+
+    def export_to_xhtml(self):
+        """Export current results to XHTML.
+
+        The configuration of the :attr:`result_table` (columns order and visibility) is used to
+        determine how the data is presented in the export. In other words, the exported table in
+        the resulting XHTML will look just like the results table.
+        """
+        colnames, rows = self._get_export_data()
+        export_path = export.export_to_xhtml(colnames, rows)
+        desktop.open_path(export_path)
+
+    def export_to_csv(self):
+        """Export current results to CSV.
+
+        The columns and their order in the resulting CSV file is determined in the same way as in
+        :meth:`export_to_xhtml`.
+        """
+        dest_file = self.view.select_dest_file(tr("Select a destination for your exported CSV"), "csv")
+        if dest_file:
+            colnames, rows = self._get_export_data()
+            try:
+                export.export_to_csv(dest_file, colnames, rows)
+            except OSError as e:
+                self.view.show_message(tr("Couldn't write to file: {}").format(str(e)))
+
+    def get_display_info(self, dupe, group, delta=False):
+        def empty_data():
+            return {c.name: "---" for c in self.result_table.COLUMNS[1:]}
+
+        if (dupe is None) or (group is None):
+            return empty_data()
+        try:
+            return dupe.get_display_info(group, delta)
+        except Exception as e:
+            logging.warning("Exception (type: %s) on GetDisplayInfo for %s: %s", type(e), str(dupe.path), str(e))
+            return empty_data()
+
+    def invoke_custom_command(self):
+        """Calls command in ``CustomCommand`` pref with ``%d`` and ``%r`` placeholders replaced.
+
+        Using the current selection, ``%d`` is replaced with the currently selected dupe and ``%r``
+        is replaced with that dupe's ref file. If there's no selection, the command is not invoked.
+        If the dupe is a ref, ``%d`` and ``%r`` will be the same.
+        """
+        cmd = self.view.get_default("CustomCommand")
+        if not cmd:
+            msg = tr("You have no custom command set up. Set it up in your preferences.")
+            self.view.show_message(msg)
+            return
+        if not self.selected_dupes:
+            return
+        dupes = self.selected_dupes
+        refs = [self.results.get_group_of_duplicate(dupe).ref for dupe in dupes]
+        for dupe, ref in zip(dupes, refs):
+            dupe_cmd = cmd.replace("%d", str(dupe.path))
+            dupe_cmd = dupe_cmd.replace("%r", str(ref.path))
+            match = re.match(r'"([^"]+)"(.*)', dupe_cmd)
+            if match is not None:
+                # This code here is because subprocess. Popen doesn't seem to accept, under Windows,
+                # executable paths with spaces in it, *even* when they're enclosed in "". So this is
+                # a workaround to make the damn thing work.
+                exepath, args = match.groups()
+                path, exename = op.split(exepath)
+                p = subprocess.Popen(
+                    exename + args, shell=True, cwd=path, stdout=subprocess.PIPE, stderr=subprocess.STDOUT
+                )
+                output = p.stdout.read()
+                logging.info("Custom command %s %s: %s", exename, args, output)
+            else:
+                p = subprocess.Popen(dupe_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+                output = p.stdout.read()
+                logging.info("Custom command %s: %s", dupe_cmd, output)
+
+    def load(self):
+        """Load directory selection and ignore list from files in appdata.
+
+        This method is called during startup so that directory selection and ignore list, which
+        is persistent data, is the same as when the last session was closed (when :meth:`save` was
+        called).
+        """
+        self.directories.load_from_file(op.join(self.appdata, "last_directories.xml"))
+        self.notify("directories_changed")
+        p = op.join(self.appdata, "ignore_list.xml")
+        self.ignore_list.load_from_xml(p)
+        self.ignore_list_dialog.refresh()
+        p = op.join(self.appdata, "exclude_list.xml")
+        self.exclude_list.load_from_xml(p)
+        self.exclude_list_dialog.refresh()
+
+    def load_directories(self, filepath):
+        # Clear out previous entries
+        self.directories.__init__()
+        self.directories.load_from_file(filepath)
+        self.notify("directories_changed")
+
+    def load_from(self, filename):
+        """Start an async job to load results from ``filename``.
+
+        :param str filename: path of the XML file (created with :meth:`save_as`) to load
+        """
+
+        def do(j):
+            self.results.load_from_xml(filename, self._get_file, j)
+
+        self._start_job(JobType.LOAD, do)
+
+    def make_selected_reference(self):
+        """Promote :attr:`selected_dupes` to reference position within their respective groups.
+
+        Each selected dupe will become the :attr:`~core.engine.Group.ref` of its group. If there's
+        more than one dupe selected for the same group, only the first (in the order currently shown
+        in :attr:`result_table`) dupe will be promoted.
+        """
+        dupes = self.without_ref(self.selected_dupes)
+        changed_groups = set()
+        for dupe in dupes:
+            g = self.results.get_group_of_duplicate(dupe)
+            if g not in changed_groups and self.results.make_ref(dupe):
+                changed_groups.add(g)
+        # It's not always obvious to users what this action does, so to make it a bit clearer,
+        # we change our selection to the ref of all changed groups. However, we also want to keep
+        # the files that were ref before and weren't changed by the action. In effect, what this
+        # does is that we keep our old selection, but remove all non-ref dupes from it.
+        # If no group was changed, however, we don't touch the selection.
+        if not self.result_table.power_marker:
+            if changed_groups:
+                self.selected_dupes = [
+                    d for d in self.selected_dupes if self.results.get_group_of_duplicate(d).ref is d
+                ]
+            self.notify("results_changed")
+        else:
+            # If we're in "Dupes Only" mode (previously called Power Marker), things are a bit
+            # different. The refs are not shown in the table, and if our operation is successful,
+            # this means that there's no way to follow our dupe selection. Then, the best thing to
+            # do is to keep our selection index-wise (different dupe selection, but same index
+            # selection).
+            self.notify("results_changed_but_keep_selection")
+
+    def mark_all(self):
+        """Set all dupes in the results as marked."""
+        self.results.mark_all()
+        self.notify("marking_changed")
+
+    def mark_none(self):
+        """Set all dupes in the results as unmarked."""
+        self.results.mark_none()
+        self.notify("marking_changed")
+
+    def mark_invert(self):
+        """Invert the marked state of all dupes in the results."""
+        self.results.mark_invert()
+        self.notify("marking_changed")
+
+    def mark_dupe(self, dupe, marked):
+        """Change marked status of ``dupe``.
+
+        :param dupe: dupe to mark/unmark
+        :type dupe: :class:`~core.fs.File`
+        :param bool marked: True = mark, False = unmark
+        """
+        if marked:
+            self.results.mark(dupe)
+        else:
+            self.results.unmark(dupe)
+        self.notify("marking_changed")
+
+    def open_selected(self):
+        """Open :attr:`selected_dupes` with their associated application."""
+        if len(self.selected_dupes) > 10 and not self.view.ask_yes_no(MSG_MANY_FILES_TO_OPEN):
+            return
+        for dupe in self.selected_dupes:
+            desktop.open_path(dupe.path)
+
+    def purge_ignore_list(self):
+        """Remove files that don't exist from :attr:`ignore_list`."""
+        self.ignore_list.filter(lambda f, s: op.exists(f) and op.exists(s))
+        self.ignore_list_dialog.refresh()
+
+    def remove_directories(self, indexes):
+        """Remove root directories at ``indexes`` from :attr:`directories`.
+
+        :param indexes: Indexes of the directories to remove.
+        :type indexes: list of int
+        """
+        try:
+            indexes = sorted(indexes, reverse=True)
+            for index in indexes:
+                del self.directories[index]
+            self.notify("directories_changed")
+        except IndexError:
+            pass
+
+    def remove_duplicates(self, duplicates):
+        """Remove ``duplicates`` from :attr:`results`.
+
+        Calls :meth:`~core.results.Results.remove_duplicates` and send appropriate notifications.
+
+        :param duplicates: duplicates to remove.
+        :type duplicates: list of :class:`~core.fs.File`
+        """
+        self.results.remove_duplicates(self.without_ref(duplicates))
+        self.notify("results_changed_but_keep_selection")
+
+    def remove_marked(self):
+        """Removed marked duplicates from the results (without touching the files themselves)."""
+        if not self.results.mark_count:
+            self.view.show_message(MSG_NO_MARKED_DUPES)
+            return
+        msg = tr("You are about to remove %d files from results. Continue?")
+        if not self.view.ask_yes_no(msg % self.results.mark_count):
+            return
+        self.results.perform_on_marked(lambda x: None, True)
+        self._results_changed()
+
+    def remove_selected(self):
+        """Removed :attr:`selected_dupes` from the results (without touching the files themselves)."""
+        dupes = self.without_ref(self.selected_dupes)
+        if not dupes:
+            self.view.show_message(MSG_NO_SELECTED_DUPES)
+            return
+        msg = tr("You are about to remove %d files from results. Continue?")
+        if not self.view.ask_yes_no(msg % len(dupes)):
+            return
+        self.remove_duplicates(dupes)
+
+    def rename_selected(self, newname):
+        """Renames the selected dupes's file to ``newname``.
+
+        If there's more than one selected dupes, the first one is used.
+
+        :param str newname: The filename to rename the dupe's file to.
+        """
+        try:
+            d = self.selected_dupes[0]
+            d.rename(newname)
+            return True
+        except (IndexError, fs.FSError) as e:
+            logging.warning("dupeGuru Warning: %s" % str(e))
+        return False
+
+    def reprioritize_groups(self, sort_key):
+        """Sort dupes in each group (in :attr:`results`) according to ``sort_key``.
+
+        Called by the re-prioritize dialog. Calls :meth:`~core.engine.Group.prioritize` and, once
+        the sorting is done, show a message that confirms the action.
+
+        :param sort_key: The key being sent to :meth:`~core.engine.Group.prioritize`
+        :type sort_key: f(dupe)
+        """
+        count = 0
+        for group in self.results.groups:
+            if group.prioritize(key_func=sort_key):
+                count += 1
+        if count:
+            self.results.refresh_required = True
+        self._results_changed()
+        msg = tr("{} duplicate groups were changed by the re-prioritization.").format(count)
+        self.view.show_message(msg)
+
+    def reveal_selected(self):
+        if self.selected_dupes:
+            desktop.reveal_path(self.selected_dupes[0].path)
+
+    def save(self):
+        if not op.exists(self.appdata):
+            os.makedirs(self.appdata)
+        self.directories.save_to_file(op.join(self.appdata, "last_directories.xml"))
+        p = op.join(self.appdata, "ignore_list.xml")
+        self.ignore_list.save_to_xml(p)
+        p = op.join(self.appdata, "exclude_list.xml")
+        self.exclude_list.save_to_xml(p)
+        self.notify("save_session")
+
+    def close(self):
+        fs.filesdb.close()
+
+    def save_as(self, filename):
+        """Save results in ``filename``.
+
+        :param str filename: path of the file to save results (as XML) to.
+        """
+        try:
+            self.results.save_to_xml(filename)
+        except OSError as e:
+            self.view.show_message(tr("Couldn't write to file: {}").format(str(e)))
+
+    def save_directories_as(self, filename):
+        """Save directories in ``filename``.
+
+        :param str filename: path of the file to save directories (as XML) to.
+        """
+        try:
+            self.directories.save_to_file(filename)
+        except OSError as e:
+            self.view.show_message(tr("Couldn't write to file: {}").format(str(e)))
+
+    def start_scanning(self, profile_scan=False):
+        """Starts an async job to scan for duplicates.
+
+        Scans folders selected in :attr:`directories` and put the results in :attr:`results`
+        """
+        scanner = self.SCANNER_CLASS()
+        fs.filesdb.ignore_mtime = self.options["rehash_ignore_mtime"] is True
+        if not self.directories.has_any_file():
+            self.view.show_message(tr("The selected directories contain no scannable file."))
+            return
+        # Send relevant options down to the scanner instance
+        for k, v in self.options.items():
+            if hasattr(scanner, k):
+                setattr(scanner, k, v)
+        if self.app_mode == AppMode.PICTURE:
+            scanner.cache_path = self._get_picture_cache_path()
+        self.results.groups = []
+        self._recreate_result_table()
+        self._results_changed()
+
+        def do(j):
+            if profile_scan:
+                pr = cProfile.Profile()
+                pr.enable()
+            j.set_progress(0, tr("Collecting files to scan"))
+            if scanner.scan_type == ScanType.FOLDERS:
+                files = list(self.directories.get_folders(folderclass=se.fs.Folder, j=j))
+            else:
+                files = list(self.directories.get_files(fileclasses=self.fileclasses, j=j))
+            if self.options["ignore_hardlink_matches"]:
+                files = self._remove_hardlink_dupes(files)
+            logging.info("Scanning %d files" % len(files))
+            self.results.groups = scanner.get_dupe_groups(files, self.ignore_list, j)
+            self.discarded_file_count = scanner.discarded_file_count
+            if profile_scan:
+                pr.disable()
+                pr.dump_stats(op.join(self.appdata, f"{datetime.datetime.now():%Y-%m-%d_%H-%M-%S}.profile"))
+
+        self._start_job(JobType.SCAN, do)
+
+    def toggle_selected_mark_state(self):
+        selected = self.without_ref(self.selected_dupes)
+        if not selected:
+            return
+        if allsame(self.results.is_marked(d) for d in selected):
+            markfunc = self.results.mark_toggle
+        else:
+            markfunc = self.results.mark
+        for dupe in selected:
+            markfunc(dupe)
+        self.notify("marking_changed")
+
+    def without_ref(self, dupes):
+        """Returns ``dupes`` with all reference elements removed."""
+        return [dupe for dupe in dupes if self.results.get_group_of_duplicate(dupe).ref is not dupe]
+
+    def get_default(self, key, fallback_value=None):
+        result = nonone(self.view.get_default(key), fallback_value)
+        if fallback_value is not None and not isinstance(result, type(fallback_value)):
+            # we don't want to end up with garbage values from the prefs
+            try:
+                result = type(fallback_value)(result)
+            except Exception:
+                result = fallback_value
+        return result
+
+    def set_default(self, key, value):
+        self.view.set_default(key, value)
+
+    # --- Properties
+    @property
+    def stat_line(self):
+        result = self.results.stat_line
+        if self.discarded_file_count:
+            result = tr("%s (%d discarded)") % (result, self.discarded_file_count)
+        return result
+
+    @property
+    def fileclasses(self):
+        return self._get_fileclasses()
+
+    @property
+    def SCANNER_CLASS(self):
+        if self.app_mode == AppMode.PICTURE:
+            return pe.scanner.ScannerPE
+        elif self.app_mode == AppMode.MUSIC:
+            return me.scanner.ScannerME
+        else:
+            return se.scanner.ScannerSE
+
+    @property
+    def METADATA_TO_READ(self):
+        if self.app_mode == AppMode.PICTURE:
+            return ["size", "mtime", "dimensions", "exif_timestamp"]
+        elif self.app_mode == AppMode.MUSIC:
+            return [
+                "size",
+                "mtime",
+                "duration",
+                "bitrate",
+                "samplerate",
+                "title",
+                "artist",
+                "album",
+                "genre",
+                "year",
+                "track",
+                "comment",
+            ]
+        else:
+            return ["size", "mtime"]

core/directories.py

@@ -0,0 +1,298 @@
+# Copyright 2017 Virgil Dupras
+#
+# This software is licensed under the "GPLv3" License as described in the "LICENSE" file,
+# which should be included with this package. The terms are also available at
+# http://www.gnu.org/licenses/gpl-3.0.html
+
+import os
+from xml.etree import ElementTree as ET
+import logging
+from pathlib import Path
+
+from hscommon.jobprogress import job
+from hscommon.util import FileOrPath
+from hscommon.trans import tr
+
+from core import fs
+
+__all__ = [
+    "Directories",
+    "DirectoryState",
+    "AlreadyThereError",
+    "InvalidPathError",
+]
+
+
+class DirectoryState:
+    """Enum describing how a folder should be considered.
+
+    * DirectoryState.Normal: Scan all files normally
+    * DirectoryState.Reference: Scan files, but make sure never to delete any of them
+    * DirectoryState.Excluded: Don't scan this folder
+    """
+
+    NORMAL = 0
+    REFERENCE = 1
+    EXCLUDED = 2
+
+
+class AlreadyThereError(Exception):
+    """The path being added is already in the directory list"""
+
+
+class InvalidPathError(Exception):
+    """The path being added is invalid"""
+
+
+class Directories:
+    """Holds user folder selection.
+
+    Manages the selection that the user make through the folder selection dialog. It also manages
+    folder states, and how recursion applies to them.
+
+    Then, when the user starts the scan, :meth:`get_files` is called to retrieve all files (wrapped
+    in :mod:`core.fs`) that have to be scanned according to the chosen folders/states.
+    """
+
+    # ---Override
+    def __init__(self, exclude_list=None):
+        self._dirs = []
+        # {path: state}
+        self.states = {}
+        self._exclude_list = exclude_list
+
+    def __contains__(self, path):
+        for p in self._dirs:
+            if path == p or p in path.parents:
+                return True
+        return False
+
+    def __delitem__(self, key):
+        self._dirs.__delitem__(key)
+
+    def __getitem__(self, key):
+        return self._dirs.__getitem__(key)
+
+    def __len__(self):
+        return len(self._dirs)
+
+    # ---Private
+    def _default_state_for_path(self, path):
+        # New logic with regex filters
+        if self._exclude_list is not None and self._exclude_list.mark_count > 0:
+            # We iterate even if we only have one item here
+            for denied_path_re in self._exclude_list.compiled:
+                if denied_path_re.match(str(path.name)):
+                    return DirectoryState.EXCLUDED
+            return DirectoryState.NORMAL
+        # Override this in subclasses to specify the state of some special folders.
+        if path.name.startswith("."):
+            return DirectoryState.EXCLUDED
+        return DirectoryState.NORMAL
+
+    def _get_files(self, from_path, fileclasses, j):
+        try:
+            with os.scandir(from_path) as iter:
+                root_path = Path(from_path)
+                state = self.get_state(root_path)
+                # if we have no un-excluded dirs under this directory skip going deeper
+                skip_dirs = state == DirectoryState.EXCLUDED and not any(
+                    p.parts[: len(root_path.parts)] == root_path.parts for p in self.states
+                )
+                count = 0
+                for item in iter:
+                    j.check_if_cancelled()
+                    try:
+                        if item.is_dir():
+                            if skip_dirs:
+                                continue
+                            yield from self._get_files(item.path, fileclasses, j)
+                            continue
+                        elif state == DirectoryState.EXCLUDED:
+                            continue
+                        # File excluding or not
+                        if (
+                            self._exclude_list is None
+                            or not self._exclude_list.mark_count
+                            or not self._exclude_list.is_excluded(str(from_path), item.name)
+                        ):
+                            file = fs.get_file(item, fileclasses=fileclasses)
+                            if file:
+                                file.is_ref = state == DirectoryState.REFERENCE
+                                count += 1
+                                yield file
+                    except (OSError, fs.InvalidPath):
+                        pass
+                logging.debug(
+                    "Collected %d files in folder %s",
+                    count,
+                    str(root_path),
+                )
+        except OSError:
+            pass
+
+    def _get_folders(self, from_folder, j):
+        j.check_if_cancelled()
+        try:
+            for subfolder in from_folder.subfolders:
+                yield from self._get_folders(subfolder, j)
+            state = self.get_state(from_folder.path)
+            if state != DirectoryState.EXCLUDED:
+                from_folder.is_ref = state == DirectoryState.REFERENCE
+                logging.debug("Yielding Folder %r state: %d", from_folder, state)
+                yield from_folder
+        except (OSError, fs.InvalidPath):
+            pass
+
+    # ---Public
+    def add_path(self, path):
+        """Adds ``path`` to self, if not already there.
+
+        Raises :exc:`AlreadyThereError` if ``path`` is already in self. If path is a directory
+        containing some of the directories already present in self, ``path`` will be added, but all
+        directories under it will be removed. Can also raise :exc:`InvalidPathError` if ``path``
+        does not exist.
+
+        :param Path path: path to add
+        """
+        if path in self:
+            raise AlreadyThereError()
+        if not path.exists():
+            raise InvalidPathError()
+        self._dirs = [p for p in self._dirs if path not in p.parents]
+        self._dirs.append(path)
+
+    @staticmethod
+    def get_subfolders(path):
+        """Returns a sorted list of paths corresponding to subfolders in ``path``.
+
+        :param Path path: get subfolders from there
+        :rtype: list of Path
+        """
+        try:
+            subpaths = [p for p in path.glob("*") if p.is_dir()]
+            subpaths.sort(key=lambda x: x.name.lower())
+            return subpaths
+        except OSError:
+            return []
+
+    def get_files(self, fileclasses=None, j=job.nulljob):
+        """Returns a list of all files that are not excluded.
+
+        Returned files also have their ``is_ref`` attr set if applicable.
+        """
+        if fileclasses is None:
+            fileclasses = [fs.File]
+        file_count = 0
+        for path in self._dirs:
+            for file in self._get_files(path, fileclasses=fileclasses, j=j):
+                file_count += 1
+                if not isinstance(j, job.NullJob):
+                    j.set_progress(-1, tr("Collected {} files to scan").format(file_count))
+                yield file
+
+    def get_folders(self, folderclass=None, j=job.nulljob):
+        """Returns a list of all folders that are not excluded.
+
+        Returned folders also have their ``is_ref`` attr set if applicable.
+        """
+        if folderclass is None:
+            folderclass = fs.Folder
+        folder_count = 0
+        for path in self._dirs:
+            from_folder = folderclass(path)
+            for folder in self._get_folders(from_folder, j):
+                folder_count += 1
+                if not isinstance(j, job.NullJob):
+                    j.set_progress(-1, tr("Collected {} folders to scan").format(folder_count))
+                yield folder
+
+    def get_state(self, path):
+        """Returns the state of ``path``.
+
+        :rtype: :class:`DirectoryState`
+        """
+        # direct match? easy result.
+        if path in self.states:
+            return self.states[path]
+        state = self._default_state_for_path(path)
+        # Save non-default states in cache, necessary for _get_files()
+        if state != DirectoryState.NORMAL:
+            self.states[path] = state
+            return state
+        # find the longest parent path that is in states and return that state if found
+        # NOTE: path.parents is ordered longest to shortest
+        for parent_path in path.parents:
+            if parent_path in self.states:
+                return self.states[parent_path]
+        return state
+
+    def has_any_file(self):
+        """Returns whether selected folders contain any file.
+
+        Because it stops at the first file it finds, it's much faster than get_files().
+
+        :rtype: bool
+        """
+        try:
+            next(self.get_files())
+            return True
+        except StopIteration:
+            return False
+
+    def load_from_file(self, infile):
+        """Load folder selection from ``infile``.
+
+        :param file infile: path or file pointer to XML generated through :meth:`save_to_file`
+        """
+        try:
+            root = ET.parse(infile).getroot()
+        except Exception:
+            return
+        for rdn in root.iter("root_directory"):
+            attrib = rdn.attrib
+            if "path" not in attrib:
+                continue
+            path = attrib["path"]
+            try:
+                self.add_path(Path(path))
+            except (AlreadyThereError, InvalidPathError):
+                pass
+        for sn in root.iter("state"):
+            attrib = sn.attrib
+            if not ("path" in attrib and "value" in attrib):
+                continue
+            path = attrib["path"]
+            state = attrib["value"]
+            self.states[Path(path)] = int(state)
+
+    def save_to_file(self, outfile):
+        """Save folder selection as XML to ``outfile``.
+
+        :param file outfile: path or file pointer to XML file to save to.
+        """
+        with FileOrPath(outfile, "wb") as fp:
+            root = ET.Element("directories")
+            for root_path in self:
+                root_path_node = ET.SubElement(root, "root_directory")
+                root_path_node.set("path", str(root_path))
+            for path, state in self.states.items():
+                state_node = ET.SubElement(root, "state")
+                state_node.set("path", str(path))
+                state_node.set("value", str(state))
+            tree = ET.ElementTree(root)
+            tree.write(fp, encoding="utf-8")
+
+    def set_state(self, path, state):
+        """Set the state of folder at ``path``.
+
+        :param Path path: path of the target folder
+        :param state: state to set folder to
+        :type state: :class:`DirectoryState`
+        """
+        if self.get_state(path) == state:
+            return
+        for iter_path in list(self.states.keys()):
+            if path in iter_path.parents:
+                del self.states[iter_path]
+        self.states[path] = state

core/engine.py

@@ -0,0 +1,545 @@
+# Created By: Virgil Dupras
+# Created On: 2006/01/29
+# Copyright 2015 Hardcoded Software (http://www.hardcoded.net)
+#
+# This software is licensed under the "GPLv3" License as described in the "LICENSE" file,
+# which should be included with this package. The terms are also available at
+# http://www.gnu.org/licenses/gpl-3.0.html
+
+import difflib
+import itertools
+import logging
+import string
+from collections import defaultdict, namedtuple
+from unicodedata import normalize
+
+from hscommon.util import flatten, multi_replace
+from hscommon.trans import tr
+from hscommon.jobprogress import job
+
+(
+    WEIGHT_WORDS,
+    MATCH_SIMILAR_WORDS,
+    NO_FIELD_ORDER,
+) = range(3)
+
+JOB_REFRESH_RATE = 100
+PROGRESS_MESSAGE = tr("%d matches found from %d groups")
+
+
+def getwords(s):
+    # We decompose the string so that ascii letters with accents can be part of the word.
+    s = normalize("NFD", s)
+    s = multi_replace(s, "-_&+():;\\[]{}.,<>/?~!@#$*", " ").lower()
+    # logging.debug(f"DEBUG chars for: {s}\n"
+    #               f"{[c for c in s if ord(c) != 32]}\n"
+    #               f"{[ord(c) for c in s if ord(c) != 32]}")
+    # HACK We shouldn't ignore non-ascii characters altogether. Any Unicode char
+    # above common european characters that cannot be "sanitized" (ie. stripped
+    # of their accents, etc.) are preserved as is. The arbitrary limit is
+    # obtained from this one: ord("\u037e") GREEK QUESTION MARK
+    s = "".join(
+        c
+        for c in s
+        if (ord(c) <= 894 and c in string.ascii_letters + string.digits + string.whitespace) or ord(c) > 894
+    )
+    return [_f for _f in s.split(" ") if _f]  # remove empty elements
+
+
+def getfields(s):
+    fields = [getwords(field) for field in s.split(" - ")]
+    return [_f for _f in fields if _f]
+
+
+def unpack_fields(fields):
+    result = []
+    for field in fields:
+        if isinstance(field, list):
+            result += field
+        else:
+            result.append(field)
+    return result
+
+
+def compare(first, second, flags=()):
+    """Returns the % of words that match between ``first`` and ``second``
+
+    The result is a ``int`` in the range 0..100.
+    ``first`` and ``second`` can be either a string or a list (of words).
+    """
+    if not (first and second):
+        return 0
+    if any(isinstance(element, list) for element in first):
+        return compare_fields(first, second, flags)
+    second = second[:]  # We must use a copy of second because we remove items from it
+    match_similar = MATCH_SIMILAR_WORDS in flags
+    weight_words = WEIGHT_WORDS in flags
+    joined = first + second
+    total_count = sum(len(word) for word in joined) if weight_words else len(joined)
+    match_count = 0
+    in_order = True
+    for word in first:
+        if match_similar and (word not in second):
+            similar = difflib.get_close_matches(word, second, 1, 0.8)
+            if similar:
+                word = similar[0]
+        if word in second:
+            if second[0] != word:
+                in_order = False
+            second.remove(word)
+            match_count += len(word) if weight_words else 1
+    result = round(((match_count * 2) / total_count) * 100)
+    if (result == 100) and (not in_order):
+        result = 99  # We cannot consider a match exact unless the ordering is the same
+    return result
+
+
+def compare_fields(first, second, flags=()):
+    """Returns the score for the lowest matching :ref:`fields`.
+
+    ``first`` and ``second`` must be lists of lists of string. Each sub-list is then compared with
+    :func:`compare`.
+    """
+    if len(first) != len(second):
+        return 0
+    if NO_FIELD_ORDER in flags:
+        results = []
+        # We don't want to remove field directly in the list. We must work on a copy.
+        second = second[:]
+        for field1 in first:
+            max_score = 0
+            matched_field = None
+            for field2 in second:
+                r = compare(field1, field2, flags)
+                if r > max_score:
+                    max_score = r
+                    matched_field = field2
+            results.append(max_score)
+            if matched_field:
+                second.remove(matched_field)
+    else:
+        results = [compare(field1, field2, flags) for field1, field2 in zip(first, second)]
+    return min(results) if results else 0
+
+
+def build_word_dict(objects, j=job.nulljob):
+    """Returns a dict of objects mapped by their words.
+
+    objects must have a ``words`` attribute being a list of strings or a list of lists of strings
+    (:ref:`fields`).
+
+    The result will be a dict with words as keys, lists of objects as values.
+    """
+    result = defaultdict(set)
+    for object in j.iter_with_progress(objects, "Prepared %d/%d files", JOB_REFRESH_RATE):
+        for word in unpack_fields(object.words):
+            result[word].add(object)
+    return result
+
+
+def merge_similar_words(word_dict):
+    """Take all keys in ``word_dict`` that are similar, and merge them together.
+
+    ``word_dict`` has been built with :func:`build_word_dict`. Similarity is computed with Python's
+    ``difflib.get_close_matches()``, which computes the number of edits that are necessary to make
+    a word equal to the other.
+    """
+    keys = list(word_dict.keys())
+    keys.sort(key=len)  # we want the shortest word to stay
+    while keys:
+        key = keys.pop(0)
+        similars = difflib.get_close_matches(key, keys, 100, 0.8)
+        if not similars:
+            continue
+        objects = word_dict[key]
+        for similar in similars:
+            objects |= word_dict[similar]
+            del word_dict[similar]
+            keys.remove(similar)
+
+
+def reduce_common_words(word_dict, threshold):
+    """Remove all objects from ``word_dict`` values where the object count >= ``threshold``
+
+    ``word_dict`` has been built with :func:`build_word_dict`.
+
+    The exception to this removal are the objects where all the words of the object are common.
+    Because if we remove them, we will miss some duplicates!
+    """
+    uncommon_words = {word for word, objects in word_dict.items() if len(objects) < threshold}
+    for word, objects in list(word_dict.items()):
+        if len(objects) < threshold:
+            continue
+        reduced = set()
+        for o in objects:
+            if not any(w in uncommon_words for w in unpack_fields(o.words)):
+                reduced.add(o)
+        if reduced:
+            word_dict[word] = reduced
+        else:
+            del word_dict[word]
+
+
+# Writing docstrings in a namedtuple is tricky. From Python 3.3, it's possible to set __doc__, but
+# some research allowed me to find a more elegant solution, which is what is done here. See
+# http://stackoverflow.com/questions/1606436/adding-docstrings-to-namedtuples-in-python
+
+
+class Match(namedtuple("Match", "first second percentage")):
+    """Represents a match between two :class:`~core.fs.File`.
+
+    Regarless of the matching method, when two files are determined to match, a Match pair is created,
+    which holds, of course, the two matched files, but also their match "level".
+
+    .. attribute:: first
+
+        first file of the pair.
+
+    .. attribute:: second
+
+        second file of the pair.
+
+    .. attribute:: percentage
+
+        their match level according to the scan method which found the match. int from 1 to 100. For
+        exact scan methods, such as Contents scans, this will always be 100.
+    """
+
+    __slots__ = ()
+
+
+def get_match(first, second, flags=()):
+    # it is assumed here that first and second both have a "words" attribute
+    percentage = compare(first.words, second.words, flags)
+    return Match(first, second, percentage)
+
+
+def getmatches(
+    objects,
+    min_match_percentage=0,
+    match_similar_words=False,
+    weight_words=False,
+    no_field_order=False,
+    j=job.nulljob,
+):
+    """Returns a list of :class:`Match` within ``objects`` after fuzzily matching their words.
+
+    :param objects: List of :class:`~core.fs.File` to match.
+    :param int min_match_percentage: minimum % of words that have to match.
+    :param bool match_similar_words: make similar words (see :func:`merge_similar_words`) match.
+    :param bool weight_words: longer words are worth more in match % computations.
+    :param bool no_field_order: match :ref:`fields` regardless of their order.
+    :param j: A :ref:`job progress instance <jobs>`.
+    """
+    COMMON_WORD_THRESHOLD = 50
+    LIMIT = 5000000
+    j = j.start_subjob(2)
+    sj = j.start_subjob(2)
+    for o in objects:
+        if not hasattr(o, "words"):
+            o.words = getwords(o.name)
+    word_dict = build_word_dict(objects, sj)
+    reduce_common_words(word_dict, COMMON_WORD_THRESHOLD)
+    if match_similar_words:
+        merge_similar_words(word_dict)
+    match_flags = []
+    if weight_words:
+        match_flags.append(WEIGHT_WORDS)
+    if match_similar_words:
+        match_flags.append(MATCH_SIMILAR_WORDS)
+    if no_field_order:
+        match_flags.append(NO_FIELD_ORDER)
+    j.start_job(len(word_dict), PROGRESS_MESSAGE % (0, 0))
+    compared = defaultdict(set)
+    result = []
+    try:
+        word_count = 0
+        # This whole 'popping' thing is there to avoid taking too much memory at the same time.
+        while word_dict:
+            items = word_dict.popitem()[1]
+            while items:
+                ref = items.pop()
+                compared_already = compared[ref]
+                to_compare = items - compared_already
+                compared_already |= to_compare
+                for other in to_compare:
+                    m = get_match(ref, other, match_flags)
+                    if m.percentage >= min_match_percentage:
+                        result.append(m)
+                        if len(result) >= LIMIT:
+                            return result
+            word_count += 1
+            j.add_progress(desc=PROGRESS_MESSAGE % (len(result), word_count))
+    except MemoryError:
+        # This is the place where the memory usage is at its peak during the scan.
+        # Just continue the process with an incomplete list of matches.
+        del compared  # This should give us enough room to call logging.
+        logging.warning("Memory Overflow. Matches: %d. Word dict: %d" % (len(result), len(word_dict)))
+        return result
+    return result
+
+
+def getmatches_by_contents(files, bigsize=0, j=job.nulljob):
+    """Returns a list of :class:`Match` within ``files`` if their contents is the same.
+
+    :param bigsize: The size in bytes over which we consider files big enough to
+                    justify taking samples of the file for hashing. If 0, compute digest as usual.
+    :param j: A :ref:`job progress instance <jobs>`.
+    """
+    size2files = defaultdict(set)
+    for f in files:
+        size2files[f.size].add(f)
+    del files
+    possible_matches = [files for files in size2files.values() if len(files) > 1]
+    del size2files
+    result = []
+    j.start_job(len(possible_matches), PROGRESS_MESSAGE % (0, 0))
+    group_count = 0
+    for group in possible_matches:
+        for first, second in itertools.combinations(group, 2):
+            if first.is_ref and second.is_ref:
+                continue  # Don't spend time comparing two ref pics together.
+            if first.size == 0 and second.size == 0:
+                # skip hashing for zero length files
+                result.append(Match(first, second, 100))
+                continue
+            # if digests are the same (and not None) then files match
+            if first.digest_partial is not None and first.digest_partial == second.digest_partial:
+                if bigsize > 0 and first.size > bigsize:
+                    if first.digest_samples is not None and first.digest_samples == second.digest_samples:
+                        result.append(Match(first, second, 100))
+                else:
+                    if first.digest is not None and first.digest == second.digest:
+                        result.append(Match(first, second, 100))
+        group_count += 1
+        j.add_progress(desc=PROGRESS_MESSAGE % (len(result), group_count))
+    return result
+
+
+class Group:
+    """A group of :class:`~core.fs.File` that match together.
+
+    This manages match pairs into groups and ensures that all files in the group match to each
+    other.
+
+    .. attribute:: ref
+
+        The "reference" file, which is the file among the group that isn't going to be deleted.
+
+    .. attribute:: ordered
+
+        Ordered list of duplicates in the group (including the :attr:`ref`).
+
+    .. attribute:: unordered
+
+        Set duplicates in the group (including the :attr:`ref`).
+
+    .. attribute:: dupes
+
+        An ordered list of the group's duplicate, without :attr:`ref`. Equivalent to
+        ``ordered[1:]``
+
+    .. attribute:: percentage
+
+        Average match percentage of match pairs containing :attr:`ref`.
+    """
+
+    # ---Override
+    def __init__(self):
+        self._clear()
+
+    def __contains__(self, item):
+        return item in self.unordered
+
+    def __getitem__(self, key):
+        return self.ordered.__getitem__(key)
+
+    def __iter__(self):
+        return iter(self.ordered)
+
+    def __len__(self):
+        return len(self.ordered)
+
+    # ---Private
+    def _clear(self):
+        self._percentage = None
+        self._matches_for_ref = None
+        self.matches = set()
+        self.candidates = defaultdict(set)
+        self.ordered = []
+        self.unordered = set()
+
+    def _get_matches_for_ref(self):
+        if self._matches_for_ref is None:
+            ref = self.ref
+            self._matches_for_ref = [match for match in self.matches if ref in match]
+        return self._matches_for_ref
+
+    # ---Public
+    def add_match(self, match):
+        """Adds ``match`` to internal match list and possibly add duplicates to the group.
+
+        A duplicate can only be considered as such if it matches all other duplicates in the group.
+        This method registers that pair (A, B) represented in ``match`` as possible candidates and,
+        if A and/or B end up matching every other duplicates in the group, add these duplicates to
+        the group.
+
+        :param tuple match: pair of :class:`~core.fs.File` to add
+        """
+
+        def add_candidate(item, match):
+            matches = self.candidates[item]
+            matches.add(match)
+            if self.unordered <= matches:
+                self.ordered.append(item)
+                self.unordered.add(item)
+
+        if match in self.matches:
+            return
+        self.matches.add(match)
+        first, second, _ = match
+        if first not in self.unordered:
+            add_candidate(first, second)
+        if second not in self.unordered:
+            add_candidate(second, first)
+        self._percentage = None
+        self._matches_for_ref = None
+
+    def discard_matches(self):
+        """Remove all recorded matches that didn't result in a duplicate being added to the group.
+
+        You can call this after the duplicate scanning process to free a bit of memory.
+        """
+        discarded = {m for m in self.matches if not all(obj in self.unordered for obj in [m.first, m.second])}
+        self.matches -= discarded
+        self.candidates = defaultdict(set)
+        return discarded
+
+    def get_match_of(self, item):
+        """Returns the match pair between ``item`` and :attr:`ref`."""
+        if item is self.ref:
+            return
+        for m in self._get_matches_for_ref():
+            if item in m:
+                return m
+
+    def prioritize(self, key_func, tie_breaker=None):
+        """Reorders :attr:`ordered` according to ``key_func``.
+
+        :param key_func: Key (f(x)) to be used for sorting
+        :param tie_breaker: function to be used to select the reference position in case the top
+                            duplicates have the same key_func() result.
+        """
+        # tie_breaker(ref, dupe) --> True if dupe should be ref
+        # Returns True if anything changed during prioritization.
+        new_order = sorted(self.ordered, key=lambda x: (-x.is_ref, key_func(x)))
+        changed = new_order != self.ordered
+        self.ordered = new_order
+        if tie_breaker is None:
+            return changed
+        ref = self.ref
+        key_value = key_func(ref)
+        for dupe in self.dupes:
+            if key_func(dupe) != key_value:
+                break
+            if tie_breaker(ref, dupe):
+                ref = dupe
+        if ref is not self.ref:
+            self.switch_ref(ref)
+            return True
+        return changed
+
+    def remove_dupe(self, item, discard_matches=True):
+        try:
+            self.ordered.remove(item)
+            self.unordered.remove(item)
+            self._percentage = None
+            self._matches_for_ref = None
+            if (len(self) > 1) and any(not getattr(item, "is_ref", False) for item in self):
+                if discard_matches:
+                    self.matches = {m for m in self.matches if item not in m}
+            else:
+                self._clear()
+        except ValueError:
+            pass
+
+    def switch_ref(self, with_dupe):
+        """Make the :attr:`ref` dupe of the group switch position with ``with_dupe``."""
+        if self.ref.is_ref:
+            return False
+        try:
+            self.ordered.remove(with_dupe)
+            self.ordered.insert(0, with_dupe)
+            self._percentage = None
+            self._matches_for_ref = None
+            return True
+        except ValueError:
+            return False
+
+    dupes = property(lambda self: self[1:])
+
+    @property
+    def percentage(self):
+        if self._percentage is None:
+            if self.dupes:
+                matches = self._get_matches_for_ref()
+                self._percentage = sum(match.percentage for match in matches) // len(matches)
+            else:
+                self._percentage = 0
+        return self._percentage
+
+    @property
+    def ref(self):
+        if self:
+            return self[0]
+
+
+def get_groups(matches):
+    """Returns a list of :class:`Group` from ``matches``.
+
+    Create groups out of match pairs in the smartest way possible.
+    """
+    matches.sort(key=lambda match: -match.percentage)
+    dupe2group = {}
+    groups = []
+    try:
+        for match in matches:
+            first, second, _ = match
+            first_group = dupe2group.get(first)
+            second_group = dupe2group.get(second)
+            if first_group:
+                if second_group:
+                    if first_group is second_group:
+                        target_group = first_group
+                    else:
+                        continue
+                else:
+                    target_group = first_group
+                    dupe2group[second] = target_group
+            else:
+                if second_group:
+                    target_group = second_group
+                    dupe2group[first] = target_group
+                else:
+                    target_group = Group()
+                    groups.append(target_group)
+                    dupe2group[first] = target_group
+                    dupe2group[second] = target_group
+            target_group.add_match(match)
+    except MemoryError:
+        del dupe2group
+        del matches
+        # should free enough memory to continue
+        logging.warning(f"Memory Overflow. Groups: {len(groups)}")
+    # Now that we have a group, we have to discard groups' matches and see if there're any "orphan"
+    # matches, that is, matches that were candidate in a group but that none of their 2 files were
+    # accepted in the group. With these orphan groups, it's safe to build additional groups
+    matched_files = set(flatten(groups))
+    orphan_matches = []
+    for group in groups:
+        orphan_matches += {
+            m for m in group.discard_matches() if not any(obj in matched_files for obj in [m.first, m.second])
+        }
+    if groups and orphan_matches:
+        groups += get_groups(orphan_matches)  # no job, as it isn't supposed to take a long time
+    return groups

core/exclude.py

@@ -0,0 +1,513 @@
+# This software is licensed under the "GPLv3" License as described in the "LICENSE" file,
+# which should be included with this package. The terms are also available at
+# http://www.gnu.org/licenses/gpl-3.0.html
+
+from core.markable import Markable
+from xml.etree import ElementTree as ET
+
+# TODO: perhaps use regex module for better Unicode support? https://pypi.org/project/regex/
+# also https://pypi.org/project/re2/
+# TODO update the Result list with newly added regexes if possible
+import re
+from os import sep
+import logging
+import functools
+from hscommon.util import FileOrPath
+from hscommon.plat import ISWINDOWS
+import time
+
+default_regexes = [
+    r"^thumbs\.db$",  # Obsolete after WindowsXP
+    r"^desktop\.ini$",  # Windows metadata
+    r"^\.DS_Store$",  # MacOS metadata
+    r"^\.Trash\-.*",  # Linux trash directories
+    r"^\$Recycle\.Bin$",  # Windows
+    r"^\..*",  # Hidden files on Unix-like
+]
+# These are too broad
+forbidden_regexes = [r".*", r"\/.*", r".*\/.*", r".*\\\\.*", r".*\..*"]
+
+
+def timer(func):
+    @functools.wraps(func)
+    def wrapper_timer(*args):
+        start = time.perf_counter_ns()
+        value = func(*args)
+        end = time.perf_counter_ns()
+        print(f"DEBUG: func {func.__name__!r} took {end - start} ns.")
+        return value
+
+    return wrapper_timer
+
+
+def memoize(func):
+    func.cache = dict()
+
+    @functools.wraps(func)
+    def _memoize(*args):
+        if args not in func.cache:
+            func.cache[args] = func(*args)
+        return func.cache[args]
+
+    return _memoize
+
+
+class AlreadyThereException(Exception):
+    """Expression already in the list"""
+
+    def __init__(self, arg="Expression is already in excluded list."):
+        super().__init__(arg)
+
+
+class ExcludeList(Markable):
+    """A list of lists holding regular expression strings and the compiled re.Pattern"""
+
+    # Used to filter out directories and files that we would rather avoid scanning.
+    # The list() class allows us to preserve item order without too much hassle.
+    # The downside is we have to compare strings every time we look for an item in the list
+    # since we use regex strings as keys.
+    # If _use_union is True, the compiled regexes will be combined into one single
+    # Pattern instead of separate Patterns which may or may not give better
+    # performance compared to looping through each Pattern individually.
+
+    # ---Override
+    def __init__(self, union_regex=True):
+        Markable.__init__(self)
+        self._use_union = union_regex
+        # list([str regex, bool iscompilable, re.error exception, Pattern compiled], ...)
+        self._excluded = []
+        self._excluded_compiled = set()
+        self._dirty = True
+
+    def __iter__(self):
+        """Iterate in order."""
+        for item in self._excluded:
+            regex = item[0]
+            yield self.is_marked(regex), regex
+
+    def __contains__(self, item):
+        return self.has_entry(item)
+
+    def __len__(self):
+        """Returns the total number of regexes regardless of mark status."""
+        return len(self._excluded)
+
+    def __getitem__(self, key):
+        """Returns the list item corresponding to key."""
+        for item in self._excluded:
+            if item[0] == key:
+                return item
+        raise KeyError(f"Key {key} is not in exclusion list.")
+
+    def __setitem__(self, key, value):
+        # TODO if necessary
+        pass
+
+    def __delitem__(self, key):
+        # TODO if necessary
+        pass
+
+    def get_compiled(self, key):
+        """Returns the (precompiled) Pattern for key"""
+        return self.__getitem__(key)[3]
+
+    def is_markable(self, regex):
+        return self._is_markable(regex)
+
+    def _is_markable(self, regex):
+        """Return the cached result of "compilable" property"""
+        for item in self._excluded:
+            if item[0] == regex:
+                return item[1]
+        return False  # should not be necessary, the regex SHOULD be in there
+
+    def _did_mark(self, regex):
+        self._add_compiled(regex)
+
+    def _did_unmark(self, regex):
+        self._remove_compiled(regex)
+
+    def _add_compiled(self, regex):
+        self._dirty = True
+        if self._use_union:
+            return
+        for item in self._excluded:
+            # FIXME probably faster to just rebuild the set from the compiled instead of comparing strings
+            if item[0] == regex:
+                # no need to test if already present since it's a set()
+                self._excluded_compiled.add(item[3])
+                break
+
+    def _remove_compiled(self, regex):
+        self._dirty = True
+        if self._use_union:
+            return
+        for item in self._excluded_compiled:
+            if regex in item.pattern:
+                self._excluded_compiled.remove(item)
+                break
+
+    # @timer
+    @memoize
+    def _do_compile(self, expr):
+        return re.compile(expr)
+
+    # @timer
+    # @memoize  # probably not worth memoizing this one if we memoize the above
+    def compile_re(self, regex):
+        compiled = None
+        try:
+            compiled = self._do_compile(regex)
+        except Exception as e:
+            return False, e, compiled
+        return True, None, compiled
+
+    def error(self, regex):
+        """Return the compilation error Exception for regex.
+        It should have a "msg" attr."""
+        for item in self._excluded:
+            if item[0] == regex:
+                return item[2]
+
+    def build_compiled_caches(self, union=False):
+        if not union:
+            self._cached_compiled_files = [x for x in self._excluded_compiled if not has_sep(x.pattern)]
+            self._cached_compiled_paths = [x for x in self._excluded_compiled if has_sep(x.pattern)]
+            self._dirty = False
+            return
+
+        marked_count = [x for marked, x in self if marked]
+        # If there is no item, the compiled Pattern will be '' and match everything!
+        if not marked_count:
+            self._cached_compiled_union_all = []
+            self._cached_compiled_union_files = []
+            self._cached_compiled_union_paths = []
+        else:
+            # HACK returned as a tuple to get a free iterator and keep interface
+            # the same regardless of whether the client asked for union or not
+            self._cached_compiled_union_all = (re.compile("|".join(marked_count)),)
+            files_marked = [x for x in marked_count if not has_sep(x)]
+            if not files_marked:
+                self._cached_compiled_union_files = tuple()
+            else:
+                self._cached_compiled_union_files = (re.compile("|".join(files_marked)),)
+            paths_marked = [x for x in marked_count if has_sep(x)]
+            if not paths_marked:
+                self._cached_compiled_union_paths = tuple()
+            else:
+                self._cached_compiled_union_paths = (re.compile("|".join(paths_marked)),)
+        self._dirty = False
+
+    @property
+    def compiled(self):
+        """Should be used by other classes to retrieve the up-to-date list of patterns."""
+        if self._use_union:
+            if self._dirty:
+                self.build_compiled_caches(self._use_union)
+            return self._cached_compiled_union_all
+        return self._excluded_compiled
+
+    @property
+    def compiled_files(self):
+        """When matching against filenames only, we probably won't be seeing any
+        directory separator, so we filter out regexes with os.sep in them.
+        The interface should be expected to be a generator, even if it returns only
+        one item (one Pattern in the union case)."""
+        if self._dirty:
+            self.build_compiled_caches(self._use_union)
+        return self._cached_compiled_union_files if self._use_union else self._cached_compiled_files
+
+    @property
+    def compiled_paths(self):
+        """Returns patterns with only separators in them, for more precise filtering."""
+        if self._dirty:
+            self.build_compiled_caches(self._use_union)
+        return self._cached_compiled_union_paths if self._use_union else self._cached_compiled_paths
+
+    # ---Public
+    def add(self, regex, forced=False):
+        """This interface should throw exceptions if there is an error during
+        regex compilation"""
+        if self.has_entry(regex):
+            # This exception should never be ignored
+            raise AlreadyThereException()
+        if regex in forbidden_regexes:
+            raise ValueError("Forbidden (dangerous) expression.")
+
+        iscompilable, exception, compiled = self.compile_re(regex)
+        if not iscompilable and not forced:
+            # This exception can be ignored, but taken into account
+            # to avoid adding to compiled set
+            raise exception
+        else:
+            self._do_add(regex, iscompilable, exception, compiled)
+
+    def _do_add(self, regex, iscompilable, exception, compiled):
+        # We need to insert at the top
+        self._excluded.insert(0, [regex, iscompilable, exception, compiled])
+
+    @property
+    def marked_count(self):
+        """Returns the number of marked regexes only."""
+        return len([x for marked, x in self if marked])
+
+    def has_entry(self, regex):
+        for item in self._excluded:
+            if regex == item[0]:
+                return True
+        return False
+
+    def is_excluded(self, dirname, filename):
+        """Return True if the file or the absolute path to file is supposed to be
+        filtered out, False otherwise."""
+        matched = False
+        for expr in self.compiled_files:
+            if expr.fullmatch(filename):
+                matched = True
+                break
+        if not matched:
+            for expr in self.compiled_paths:
+                if expr.fullmatch(dirname + sep + filename):
+                    matched = True
+                    break
+        return matched
+
+    def remove(self, regex):
+        for item in self._excluded:
+            if item[0] == regex:
+                self._excluded.remove(item)
+        self._remove_compiled(regex)
+
+    def rename(self, regex, newregex):
+        if regex == newregex:
+            return
+        found = False
+        was_marked = False
+        is_compilable = False
+        for item in self._excluded:
+            if item[0] == regex:
+                found = True
+                was_marked = self.is_marked(regex)
+                is_compilable, exception, compiled = self.compile_re(newregex)
+                # We overwrite the found entry
+                self._excluded[self._excluded.index(item)] = [newregex, is_compilable, exception, compiled]
+                self._remove_compiled(regex)
+                break
+        if not found:
+            return
+        if is_compilable:
+            self._add_compiled(newregex)
+            if was_marked:
+                # Not marked by default when added, add it back
+                self.mark(newregex)
+
+    # def change_index(self, regex, new_index):
+    # """Internal list must be a list, not dict."""
+    #     item = self._excluded.pop(regex)
+    #     self._excluded.insert(new_index, item)
+
+    def restore_defaults(self):
+        for _, regex in self:
+            if regex not in default_regexes:
+                self.unmark(regex)
+        for default_regex in default_regexes:
+            if not self.has_entry(default_regex):
+                self.add(default_regex)
+            self.mark(default_regex)
+
+    def load_from_xml(self, infile):
+        """Loads the ignore list from a XML created with save_to_xml.
+
+        infile can be a file object or a filename.
+        """
+        try:
+            root = ET.parse(infile).getroot()
+        except Exception as e:
+            logging.warning(f"Error while loading {infile}: {e}")
+            self.restore_defaults()
+            return e
+
+        marked = set()
+        exclude_elems = (e for e in root if e.tag == "exclude")
+        for exclude_item in exclude_elems:
+            regex_string = exclude_item.get("regex")
+            if not regex_string:
+                continue
+            try:
+                # "forced" avoids compilation exceptions and adds anyway
+                self.add(regex_string, forced=True)
+            except AlreadyThereException:
+                logging.error(
+                    f'Regex "{regex_string}" \
+loaded from XML was already present in the list.'
+                )
+                continue
+            if exclude_item.get("marked") == "y":
+                marked.add(regex_string)
+
+        for item in marked:
+            self.mark(item)
+
+    def save_to_xml(self, outfile):
+        """Create a XML file that can be used by load_from_xml.
+        outfile can be a file object or a filename."""
+        root = ET.Element("exclude_list")
+        # reversed in order to keep order of entries when reloading from xml later
+        for item in reversed(self._excluded):
+            exclude_node = ET.SubElement(root, "exclude")
+            exclude_node.set("regex", str(item[0]))
+            exclude_node.set("marked", ("y" if self.is_marked(item[0]) else "n"))
+        tree = ET.ElementTree(root)
+        with FileOrPath(outfile, "wb") as fp:
+            tree.write(fp, encoding="utf-8")
+
+
+class ExcludeDict(ExcludeList):
+    """Exclusion list holding a set of regular expressions as keys, the compiled
+    Pattern, compilation error and compilable boolean as values."""
+
+    # Implemntation around a dictionary instead of a list, which implies
+    # to keep the index of each string-key as its sub-element and keep it updated
+    # whenever insert/remove is done.
+
+    def __init__(self, union_regex=False):
+        Markable.__init__(self)
+        self._use_union = union_regex
+        # { "regex string":
+        #   {
+        #       "index": int,
+        #       "compilable": bool,
+        #       "error": str,
+        #       "compiled": Pattern or None
+        #   }
+        # }
+        self._excluded = {}
+        self._excluded_compiled = set()
+        self._dirty = True
+
+    def __iter__(self):
+        """Iterate in order."""
+        for regex in ordered_keys(self._excluded):
+            yield self.is_marked(regex), regex
+
+    def __getitem__(self, key):
+        """Returns the dict item correponding to key"""
+        return self._excluded.__getitem__(key)
+
+    def get_compiled(self, key):
+        """Returns the compiled item for key"""
+        return self.__getitem__(key).get("compiled")
+
+    def is_markable(self, regex):
+        return self._is_markable(regex)
+
+    def _is_markable(self, regex):
+        """Return the cached result of "compilable" property"""
+        exists = self._excluded.get(regex)
+        if exists:
+            return exists.get("compilable")
+        return False
+
+    def _add_compiled(self, regex):
+        self._dirty = True
+        if self._use_union:
+            return
+        try:
+            self._excluded_compiled.add(self._excluded.get(regex).get("compiled"))
+        except Exception as e:
+            logging.error(f"Exception while adding regex {regex} to compiled set: {e}")
+            return
+
+    def is_compilable(self, regex):
+        """Returns the cached "compilable" value"""
+        return self._excluded[regex]["compilable"]
+
+    def error(self, regex):
+        """Return the compilation error message for regex string"""
+        return self._excluded.get(regex).get("error")
+
+    # ---Public
+    def _do_add(self, regex, iscompilable, exception, compiled):
+        # We always insert at the top, so index should be 0
+        # and other indices should be pushed by one
+        for value in self._excluded.values():
+            value["index"] += 1
+        self._excluded[regex] = {"index": 0, "compilable": iscompilable, "error": exception, "compiled": compiled}
+
+    def has_entry(self, regex):
+        if regex in self._excluded.keys():
+            return True
+        return False
+
+    def remove(self, regex):
+        old_value = self._excluded.pop(regex)
+        # Bring down all indices which where above it
+        index = old_value["index"]
+        if index == len(self._excluded) - 1:  # we start at 0...
+            # Old index was at the end, no need to update other indices
+            self._remove_compiled(regex)
+            return
+
+        for value in self._excluded.values():
+            if value.get("index") > old_value["index"]:
+                value["index"] -= 1
+        self._remove_compiled(regex)
+
+    def rename(self, regex, newregex):
+        if regex == newregex or regex not in self._excluded.keys():
+            return
+        was_marked = self.is_marked(regex)
+        previous = self._excluded.pop(regex)
+        iscompilable, error, compiled = self.compile_re(newregex)
+        self._excluded[newregex] = {
+            "index": previous.get("index"),
+            "compilable": iscompilable,
+            "error": error,
+            "compiled": compiled,
+        }
+        self._remove_compiled(regex)
+        if iscompilable:
+            self._add_compiled(newregex)
+            if was_marked:
+                self.mark(newregex)
+
+    def save_to_xml(self, outfile):
+        """Create a XML file that can be used by load_from_xml.
+
+        outfile can be a file object or a filename.
+        """
+        root = ET.Element("exclude_list")
+        # reversed in order to keep order of entries when reloading from xml later
+        reversed_list = []
+        for key in ordered_keys(self._excluded):
+            reversed_list.append(key)
+        for item in reversed(reversed_list):
+            exclude_node = ET.SubElement(root, "exclude")
+            exclude_node.set("regex", str(item))
+            exclude_node.set("marked", ("y" if self.is_marked(item) else "n"))
+        tree = ET.ElementTree(root)
+        with FileOrPath(outfile, "wb") as fp:
+            tree.write(fp, encoding="utf-8")
+
+
+def ordered_keys(_dict):
+    """Returns an iterator over the keys of dictionary sorted by "index" key"""
+    if not len(_dict):
+        return
+    list_of_items = []
+    for item in _dict.items():
+        list_of_items.append(item)
+    list_of_items.sort(key=lambda x: x[1].get("index"))
+    for item in list_of_items:
+        yield item[0]
+
+
+if ISWINDOWS:
+
+    def has_sep(regexp):
+        return "\\" + sep in regexp
+
+else:
+
+    def has_sep(regexp):
+        return sep in regexp

core/export.py

@@ -0,0 +1,151 @@
+# Created By: Virgil Dupras
+# Created On: 2006/09/16
+# Copyright 2015 Hardcoded Software (http://www.hardcoded.net)
+#
+# This software is licensed under the "GPLv3" License as described in the "LICENSE" file,
+# which should be included with this package. The terms are also available at
+# http://www.gnu.org/licenses/gpl-3.0.html
+
+import os.path as op
+from tempfile import mkdtemp
+import csv
+
+# Yes, this is a very low-tech solution, but at least it doesn't have all these annoying dependency
+# and resource problems.
+
+MAIN_TEMPLATE = """
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE html PUBLIC '-//W3C//DTD XHTML 1.0 Strict//EN' 'http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd'>
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head>
+    <meta content="text/html; charset=utf-8" http-equiv="Content-Type"/>
+        <title>dupeGuru Results</title>
+        <style type="text/css">
+BODY
+{
+    background-color:white;
+}
+
+BODY,A,P,UL,TABLE,TR,TD
+{
+    font-family:Tahoma,Arial,sans-serif;
+    font-size:10pt;
+    color: #4477AA;
+}
+
+TABLE
+{
+    background-color: #225588;
+    margin-left: auto;
+    margin-right: auto;
+    width: 90%;
+}
+
+TR
+{
+    background-color: white;
+}
+
+TH
+{
+    font-weight: bold;
+    color: black;
+    background-color: #C8D6E5;
+}
+
+TH TD
+{
+    color:black;
+}
+
+TD
+{
+    padding-left: 2pt;
+}
+
+TD.rightelem
+{
+    text-align:right;
+    /*padding-left:0pt;*/
+    padding-right: 2pt;
+    width: 17%;
+}
+
+TD.indented
+{
+    padding-left: 12pt;
+}
+
+H1
+{
+    font-family:&quot;Courier New&quot;,monospace;
+    color:#6699CC;
+    font-size:18pt;
+    color:#6da500;
+    border-color: #70A0CF;
+    border-width: 1pt;
+    border-style: solid;
+    margin-top:   16pt;
+    margin-left:  5%;
+    margin-right: 5%;
+    padding-top:  2pt;
+    padding-bottom:2pt;
+    text-align:   center;
+}
+</style>
+</head>
+<body>
+<h1>dupeGuru Results</h1>
+<table>
+<tr>$colheaders</tr>
+$rows
+</table>
+</body>
+</html>
+"""
+
+COLHEADERS_TEMPLATE = "<th>{name}</th>"
+
+ROW_TEMPLATE = """
+<tr>
+    <td class="{indented}">{filename}</td>{cells}
+</tr>
+"""
+
+CELL_TEMPLATE = """<td>{value}</td>"""
+
+
+def export_to_xhtml(colnames, rows):
+    # a row is a list of values with the first value being a flag indicating if the row should be indented
+    if rows:
+        assert len(rows[0]) == len(colnames) + 1  # + 1 is for the "indented" flag
+    colheaders = "".join(COLHEADERS_TEMPLATE.format(name=name) for name in colnames)
+    rendered_rows = []
+    previous_group_id = None
+    for row in rows:
+        # [2:] is to remove the indented flag + filename
+        if row[0] != previous_group_id:
+            # We've just changed dupe group, which means that this dupe is a ref. We don't indent it.
+            indented = ""
+        else:
+            indented = "indented"
+        filename = row[1]
+        cells = "".join(CELL_TEMPLATE.format(value=value) for value in row[2:])
+        rendered_rows.append(ROW_TEMPLATE.format(indented=indented, filename=filename, cells=cells))
+        previous_group_id = row[0]
+    rendered_rows = "".join(rendered_rows)
+    # The main template can't use format because the css code uses {}
+    content = MAIN_TEMPLATE.replace("$colheaders", colheaders).replace("$rows", rendered_rows)
+    folder = mkdtemp()
+    destpath = op.join(folder, "export.htm")
+    fp = open(destpath, "wt", encoding="utf-8")
+    fp.write(content)
+    fp.close()
+    return destpath
+
+
+def export_to_csv(dest, colnames, rows):
+    writer = csv.writer(open(dest, "wt", encoding="utf-8"))
+    writer.writerow(["Group ID"] + colnames)
+    for row in rows:
+        writer.writerow(row)

core/fs.py

@@ -0,0 +1,447 @@
+# Created By: Virgil Dupras
+# Created On: 2009-10-22
+# Copyright 2015 Hardcoded Software (http://www.hardcoded.net)
+#
+# This software is licensed under the "GPLv3" License as described in the "LICENSE" file,
+# which should be included with this package. The terms are also available at
+# http://www.gnu.org/licenses/gpl-3.0.html
+
+# This is a fork from hsfs. The reason for this fork is that hsfs has been designed for musicGuru
+# and was re-used for dupeGuru. The problem is that hsfs is way over-engineered for dupeGuru,
+# resulting needless complexity and memory usage. It's been a while since I wanted to do that fork,
+# and I'm doing it now.
+
+import os
+
+from math import floor
+import logging
+import sqlite3
+from sys import platform
+from threading import Lock
+from typing import Any, AnyStr, Union, Callable
+
+from pathlib import Path
+from hscommon.util import nonone, get_file_ext
+
+hasher: Callable
+try:
+    import xxhash
+
+    hasher = xxhash.xxh128
+except ImportError:
+    import hashlib
+
+    hasher = hashlib.md5
+
+__all__ = [
+    "File",
+    "Folder",
+    "get_file",
+    "get_files",
+    "FSError",
+    "AlreadyExistsError",
+    "InvalidPath",
+    "InvalidDestinationError",
+    "OperationError",
+]
+
+NOT_SET = object()
+
+# The goal here is to not run out of memory on really big files. However, the chunk
+# size has to be large enough so that the python loop isn't too costly in terms of
+# CPU.
+CHUNK_SIZE = 1024 * 1024  # 1 MiB
+
+# Minimum size below which partial hashing is not used
+MIN_FILE_SIZE = 3 * CHUNK_SIZE  # 3MiB, because we take 3 samples
+
+# Partial hashing offset and size
+PARTIAL_OFFSET_SIZE = (0x4000, 0x4000)
+
+
+class FSError(Exception):
+    cls_message = "An error has occured on '{name}' in '{parent}'"
+
+    def __init__(self, fsobject, parent=None):
+        message = self.cls_message
+        if isinstance(fsobject, str):
+            name = fsobject
+        elif isinstance(fsobject, File):
+            name = fsobject.name
+        else:
+            name = ""
+        parentname = str(parent) if parent is not None else ""
+        Exception.__init__(self, message.format(name=name, parent=parentname))
+
+
+class AlreadyExistsError(FSError):
+    "The directory or file name we're trying to add already exists"
+    cls_message = "'{name}' already exists in '{parent}'"
+
+
+class InvalidPath(FSError):
+    "The path of self is invalid, and cannot be worked with."
+    cls_message = "'{name}' is invalid."
+
+
+class InvalidDestinationError(FSError):
+    """A copy/move operation has been called, but the destination is invalid."""
+
+    cls_message = "'{name}' is an invalid destination for this operation."
+
+
+class OperationError(FSError):
+    """A copy/move/delete operation has been called, but the checkup after the
+    operation shows that it didn't work."""
+
+    cls_message = "Operation on '{name}' failed."
+
+
+class FilesDB:
+    schema_version = 1
+    schema_version_description = "Changed from md5 to xxhash if available."
+
+    create_table_query = """CREATE TABLE IF NOT EXISTS files (path TEXT PRIMARY KEY, size INTEGER, mtime_ns INTEGER,
+        entry_dt DATETIME, digest BLOB, digest_partial BLOB, digest_samples BLOB)"""
+    drop_table_query = "DROP TABLE IF EXISTS files;"
+    select_query = "SELECT {key} FROM files WHERE path=:path AND size=:size and mtime_ns=:mtime_ns"
+    select_query_ignore_mtime = "SELECT {key} FROM files WHERE path=:path AND size=:size"
+    insert_query = """
+        INSERT INTO files (path, size, mtime_ns, entry_dt, {key})
+        VALUES (:path, :size, :mtime_ns, datetime('now'), :value)
+        ON CONFLICT(path) DO UPDATE SET size=:size, mtime_ns=:mtime_ns, entry_dt=datetime('now'), {key}=:value;
+    """
+
+    ignore_mtime = False
+
+    def __init__(self):
+        self.conn = None
+        self.lock = None
+
+    def connect(self, path: Union[AnyStr, os.PathLike]) -> None:
+        if platform.startswith("gnu0"):
+            self.conn = sqlite3.connect(path, check_same_thread=False, isolation_level=None)
+        else:
+            self.conn = sqlite3.connect(path, check_same_thread=False)
+        self.lock = Lock()
+        self._check_upgrade()
+
+    def _check_upgrade(self) -> None:
+        with self.lock, self.conn as conn:
+            has_schema = conn.execute(
+                "SELECT NAME FROM sqlite_master WHERE type='table' AND name='schema_version'"
+            ).fetchall()
+            version = None
+            if has_schema:
+                version = conn.execute("SELECT version FROM schema_version ORDER BY version DESC").fetchone()[0]
+            else:
+                conn.execute("CREATE TABLE schema_version (version int PRIMARY KEY, description TEXT)")
+            if version != self.schema_version:
+                conn.execute(self.drop_table_query)
+                conn.execute(
+                    "INSERT OR REPLACE INTO schema_version VALUES (:version, :description)",
+                    {"version": self.schema_version, "description": self.schema_version_description},
+                )
+            conn.execute(self.create_table_query)
+
+    def clear(self) -> None:
+        with self.lock, self.conn as conn:
+            conn.execute(self.drop_table_query)
+            conn.execute(self.create_table_query)
+
+    def get(self, path: Path, key: str) -> Union[bytes, None]:
+        stat = path.stat()
+        size = stat.st_size
+        mtime_ns = stat.st_mtime_ns
+        try:
+            with self.conn as conn:
+                if self.ignore_mtime:
+                    cursor = conn.execute(
+                        self.select_query_ignore_mtime.format(key=key), {"path": str(path), "size": size}
+                    )
+                else:
+                    cursor = conn.execute(
+                        self.select_query.format(key=key),
+                        {"path": str(path), "size": size, "mtime_ns": mtime_ns},
+                    )
+                result = cursor.fetchone()
+                cursor.close()
+
+            if result:
+                return result[0]
+        except Exception as ex:
+            logging.warning(f"Couldn't get {key} for {path} w/{size}, {mtime_ns}: {ex}")
+
+        return None
+
+    def put(self, path: Path, key: str, value: Any) -> None:
+        stat = path.stat()
+        size = stat.st_size
+        mtime_ns = stat.st_mtime_ns
+        try:
+            with self.lock, self.conn as conn:
+                conn.execute(
+                    self.insert_query.format(key=key),
+                    {"path": str(path), "size": size, "mtime_ns": mtime_ns, "value": value},
+                )
+        except Exception as ex:
+            logging.warning(f"Couldn't put {key} for {path} w/{size}, {mtime_ns}: {ex}")
+
+    def commit(self) -> None:
+        with self.lock:
+            self.conn.commit()
+
+    def close(self) -> None:
+        with self.lock:
+            self.conn.close()
+
+
+filesdb = FilesDB()  # Singleton
+
+
+class File:
+    """Represents a file and holds metadata to be used for scanning."""
+
+    INITIAL_INFO = {"size": 0, "mtime": 0, "digest": b"", "digest_partial": b"", "digest_samples": b""}
+    # Slots for File make us save quite a bit of memory. In a memory test I've made with a lot of
+    # files, I saved 35% memory usage with "unread" files (no _read_info() call) and gains become
+    # even greater when we take into account read attributes (70%!). Yeah, it's worth it.
+    __slots__ = ("path", "unicode_path", "is_ref", "words") + tuple(INITIAL_INFO.keys())
+
+    def __init__(self, path):
+        for attrname in self.INITIAL_INFO:
+            setattr(self, attrname, NOT_SET)
+        if type(path) is os.DirEntry:
+            self.path = Path(path.path)
+            self.size = nonone(path.stat().st_size, 0)
+            self.mtime = nonone(path.stat().st_mtime, 0)
+        else:
+            self.path = path
+        if self.path:
+            self.unicode_path = str(self.path)
+
+    def __repr__(self):
+        return f"<{self.__class__.__name__} {str(self.path)}>"
+
+    def __getattribute__(self, attrname):
+        result = object.__getattribute__(self, attrname)
+        if result is NOT_SET:
+            try:
+                self._read_info(attrname)
+            except Exception as e:
+                logging.warning("An error '%s' was raised while decoding '%s'", e, repr(self.path))
+            result = object.__getattribute__(self, attrname)
+            if result is NOT_SET:
+                result = self.INITIAL_INFO[attrname]
+        return result
+
+    def _calc_digest(self):
+        # type: () -> bytes
+
+        with self.path.open("rb") as fp:
+            file_hash = hasher()
+            # The goal here is to not run out of memory on really big files. However, the chunk
+            # size has to be large enough so that the python loop isn't too costly in terms of
+            # CPU.
+            CHUNK_SIZE = 1024 * 1024  # 1 mb
+            filedata = fp.read(CHUNK_SIZE)
+            while filedata:
+                file_hash.update(filedata)
+                filedata = fp.read(CHUNK_SIZE)
+            return file_hash.digest()
+
+    def _calc_digest_partial(self):
+        # type: () -> bytes
+        with self.path.open("rb") as fp:
+            fp.seek(PARTIAL_OFFSET_SIZE[0])
+            partial_data = fp.read(PARTIAL_OFFSET_SIZE[1])
+            return hasher(partial_data).digest()
+
+    def _calc_digest_samples(self) -> bytes:
+        size = self.size
+        with self.path.open("rb") as fp:
+            # Chunk at 25% of the file
+            fp.seek(floor(size * 25 / 100), 0)
+            file_data = fp.read(CHUNK_SIZE)
+            file_hash = hasher(file_data)
+
+            # Chunk at 60% of the file
+            fp.seek(floor(size * 60 / 100), 0)
+            file_data = fp.read(CHUNK_SIZE)
+            file_hash.update(file_data)
+
+            # Last chunk of the file
+            fp.seek(-CHUNK_SIZE, 2)
+            file_data = fp.read(CHUNK_SIZE)
+            file_hash.update(file_data)
+            return file_hash.digest()
+
+    def _read_info(self, field):
+        # print(f"_read_info({field}) for {self}")
+        if field in ("size", "mtime"):
+            stats = self.path.stat()
+            self.size = nonone(stats.st_size, 0)
+            self.mtime = nonone(stats.st_mtime, 0)
+        elif field == "digest_partial":
+            self.digest_partial = filesdb.get(self.path, "digest_partial")
+            if self.digest_partial is None:
+                # If file is smaller than partial requirements just use the full digest
+                if self.size < PARTIAL_OFFSET_SIZE[0] + PARTIAL_OFFSET_SIZE[1]:
+                    self.digest_partial = self.digest
+                else:
+                    self.digest_partial = self._calc_digest_partial()
+                filesdb.put(self.path, "digest_partial", self.digest_partial)
+        elif field == "digest":
+            self.digest = filesdb.get(self.path, "digest")
+            if self.digest is None:
+                self.digest = self._calc_digest()
+                filesdb.put(self.path, "digest", self.digest)
+        elif field == "digest_samples":
+            size = self.size
+            # Might as well hash such small files entirely.
+            if size <= MIN_FILE_SIZE:
+                self.digest_samples = self.digest
+                return
+            self.digest_samples = filesdb.get(self.path, "digest_samples")
+            if self.digest_samples is None:
+                self.digest_samples = self._calc_digest_samples()
+                filesdb.put(self.path, "digest_samples", self.digest_samples)
+
+    def _read_all_info(self, attrnames=None):
+        """Cache all possible info.
+
+        If `attrnames` is not None, caches only attrnames.
+        """
+        if attrnames is None:
+            attrnames = self.INITIAL_INFO.keys()
+        for attrname in attrnames:
+            getattr(self, attrname)
+
+    # --- Public
+    @classmethod
+    def can_handle(cls, path):
+        """Returns whether this file wrapper class can handle ``path``."""
+        return not path.is_symlink() and path.is_file()
+
+    def exists(self) -> bool:
+        """Safely check if the underlying file exists, treat error as non-existent"""
+        try:
+            return self.path.exists()
+        except OSError as ex:
+            logging.warning(f"Checking {self.path} raised: {ex}")
+            return False
+
+    def rename(self, newname):
+        if newname == self.name:
+            return
+        destpath = self.path.parent.joinpath(newname)
+        if destpath.exists():
+            raise AlreadyExistsError(newname, self.path.parent)
+        try:
+            self.path.rename(destpath)
+        except OSError:
+            raise OperationError(self)
+        if not destpath.exists():
+            raise OperationError(self)
+        self.path = destpath
+
+    def get_display_info(self, group, delta):
+        """Returns a display-ready dict of dupe's data."""
+        raise NotImplementedError()
+
+    # --- Properties
+    @property
+    def extension(self):
+        return get_file_ext(self.name)
+
+    @property
+    def name(self):
+        return self.path.name
+
+    @property
+    def folder_path(self):
+        return self.path.parent
+
+
+class Folder(File):
+    """A wrapper around a folder path.
+
+    It has the size/digest info of a File, but its value is the sum of its subitems.
+    """
+
+    __slots__ = File.__slots__ + ("_subfolders",)
+
+    def __init__(self, path):
+        File.__init__(self, path)
+        self.size = NOT_SET
+        self._subfolders = None
+
+    def _all_items(self):
+        folders = self.subfolders
+        files = get_files(self.path)
+        return folders + files
+
+    def _read_info(self, field):
+        # print(f"_read_info({field}) for Folder {self}")
+        if field in {"size", "mtime"}:
+            size = sum((f.size for f in self._all_items()), 0)
+            self.size = size
+            stats = self.path.stat()
+            self.mtime = nonone(stats.st_mtime, 0)
+        elif field in {"digest", "digest_partial", "digest_samples"}:
+            # What's sensitive here is that we must make sure that subfiles'
+            # digest are always added up in the same order, but we also want a
+            # different digest if a file gets moved in a different subdirectory.
+
+            def get_dir_digest_concat():
+                items = self._all_items()
+                items.sort(key=lambda f: f.path)
+                digests = [getattr(f, field) for f in items]
+                return b"".join(digests)
+
+            digest = hasher(get_dir_digest_concat()).digest()
+            setattr(self, field, digest)
+
+    @property
+    def subfolders(self):
+        if self._subfolders is None:
+            with os.scandir(self.path) as iter:
+                subfolders = [p for p in iter if not p.is_symlink() and p.is_dir()]
+            self._subfolders = [self.__class__(p) for p in subfolders]
+        return self._subfolders
+
+    @classmethod
+    def can_handle(cls, path):
+        return not path.is_symlink() and path.is_dir()
+
+
+def get_file(path, fileclasses=[File]):
+    """Wraps ``path`` around its appropriate :class:`File` class.
+
+    Whether a class is "appropriate" is decided by :meth:`File.can_handle`
+
+    :param Path path: path to wrap
+    :param fileclasses: List of candidate :class:`File` classes
+    """
+    for fileclass in fileclasses:
+        if fileclass.can_handle(path):
+            return fileclass(path)
+
+
+def get_files(path, fileclasses=[File]):
+    """Returns a list of :class:`File` for each file contained in ``path``.
+
+    :param Path path: path to scan
+    :param fileclasses: List of candidate :class:`File` classes
+    """
+    assert all(issubclass(fileclass, File) for fileclass in fileclasses)
+    try:
+        result = []
+        with os.scandir(path) as iter:
+            for item in iter:
+                file = get_file(item, fileclasses=fileclasses)
+                if file is not None:
+                    result.append(file)
+        return result
+    except OSError:
+        raise InvalidPath(path)

core/gui/__init__.py

@@ -0,0 +1,15 @@
+"""
+Meta GUI elements in dupeGuru
+-----------------------------
+
+dupeGuru is designed with a `cross-toolkit`_ approach in mind. It means that its core code
+(which doesn't depend on any GUI toolkit) has elements which preformat core information in a way
+that makes it easy for a UI layer to consume.
+
+For example, we have :class:`~core.gui.ResultTable` which takes information from
+:class:`~core.results.Results` and mashes it in rows and columns which are ready to be fetched by
+either Cocoa's ``NSTableView`` or Qt's ``QTableView``. It tells them which cell is supposed to be
+blue, which is supposed to be orange, does the sorting logic, holds selection, etc..
+
+.. _cross-toolkit: http://www.hardcoded.net/articles/cross-toolkit-software
+"""

core/gui/base.py

@@ -0,0 +1,35 @@
+# Created By: Virgil Dupras
+# Created On: 2010-02-06
+# Copyright 2015 Hardcoded Software (http://www.hardcoded.net)
+#
+# This software is licensed under the "GPLv3" License as described in the "LICENSE" file,
+# which should be included with this package. The terms are also available at
+# http://www.gnu.org/licenses/gpl-3.0.html
+
+from hscommon.notify import Listener
+
+
+class DupeGuruGUIObject(Listener):
+    def __init__(self, app):
+        Listener.__init__(self, app)
+        self.app = app
+
+    def directories_changed(self):
+        # Implemented in child classes
+        pass
+
+    def dupes_selected(self):
+        # Implemented in child classes
+        pass
+
+    def marking_changed(self):
+        # Implemented in child classes
+        pass
+
+    def results_changed(self):
+        # Implemented in child classes
+        pass
+
+    def results_changed_but_keep_selection(self):
+        # Implemented in child classes
+        pass

core/gui/deletion_options.py

@@ -0,0 +1,106 @@
+# Created On: 2012-05-30
+# Copyright 2015 Hardcoded Software (http://www.hardcoded.net)
+#
+# This software is licensed under the "GPLv3" License as described in the "LICENSE" file,
+# which should be included with this package. The terms are also available at
+# http://www.gnu.org/licenses/gpl-3.0.html
+
+import os
+
+from hscommon.gui.base import GUIObject
+from hscommon.trans import tr
+
+
+class DeletionOptionsView:
+    """Expected interface for :class:`DeletionOptions`'s view.
+
+    *Not actually used in the code. For documentation purposes only.*
+
+    Our view presents the user with an appropriate way (probably a mix of checkboxes and radio
+    buttons) to set the different flags in :class:`DeletionOptions`. Note that
+    :attr:`DeletionOptions.use_hardlinks` is only relevant if :attr:`DeletionOptions.link_deleted`
+    is true. This is why we toggle the "enabled" state of that flag.
+
+    We expect the view to set :attr:`DeletionOptions.link_deleted` immediately as the user changes
+    its value because it will toggle :meth:`set_hardlink_option_enabled`
+
+    Other than the flags, there's also a prompt message which has a dynamic content, defined by
+    :meth:`update_msg`.
+    """
+
+    def update_msg(self, msg: str):
+        """Update the dialog's prompt with ``str``."""
+
+    def show(self):
+        """Show the dialog in a modal fashion.
+
+        Returns whether the dialog was "accepted" (the user pressed OK).
+        """
+
+    def set_hardlink_option_enabled(self, is_enabled: bool):
+        """Enable or disable the widget controlling :attr:`DeletionOptions.use_hardlinks`."""
+
+
+class DeletionOptions(GUIObject):
+    """Present the user with deletion options before proceeding.
+
+    When the user activates "Send to trash", we present him with a couple of options that changes
+    the behavior of that deletion operation.
+    """
+
+    def __init__(self):
+        GUIObject.__init__(self)
+        #: Whether symlinks or hardlinks are used when doing :attr:`link_deleted`.
+        #: *bool*. *get/set*
+        self.use_hardlinks = False
+        #: Delete dupes directly and don't send to trash.
+        #: *bool*. *get/set*
+        self.direct = False
+
+    def show(self, mark_count):
+        """Prompt the user with a modal dialog offering our deletion options.
+
+        :param int mark_count: Number of dupes marked for deletion.
+        :rtype: bool
+        :returns: Whether the user accepted the dialog (we cancel deletion if false).
+        """
+        self._link_deleted = False
+        self.view.set_hardlink_option_enabled(False)
+        self.use_hardlinks = False
+        self.direct = False
+        msg = tr("You are sending {} file(s) to the Trash.").format(mark_count)
+        self.view.update_msg(msg)
+        return self.view.show()
+
+    def supports_links(self):
+        """Returns whether our platform supports symlinks."""
+        # When on a platform that doesn't implement it, calling os.symlink() (with the wrong number
+        # of arguments) raises NotImplementedError, which allows us to gracefully check for the
+        # feature.
+        try:
+            os.symlink()
+        except NotImplementedError:
+            # Windows XP, not supported
+            return False
+        except OSError:
+            # Vista+, symbolic link privilege not held
+            return False
+        except TypeError:
+            # wrong number of arguments
+            return True
+
+    @property
+    def link_deleted(self):
+        """Replace deleted dupes with symlinks (or hardlinks) to the dupe group reference.
+
+        *bool*. *get/set*
+
+        Whether the link is a symlink or hardlink is decided by :attr:`use_hardlinks`.
+        """
+        return self._link_deleted
+
+    @link_deleted.setter
+    def link_deleted(self, value):
+        self._link_deleted = value
+        hardlinks_enabled = value and self.supports_links()
+        self.view.set_hardlink_option_enabled(hardlinks_enabled)

core/gui/details_panel.py

@@ -0,0 +1,47 @@
+# Created By: Virgil Dupras
+# Created On: 2010-02-05
+# Copyright 2015 Hardcoded Software (http://www.hardcoded.net)
+#
+# This software is licensed under the "GPLv3" License as described in the "LICENSE" file,
+# which should be included with this package. The terms are also available at
+# http://www.gnu.org/licenses/gpl-3.0.html
+
+from hscommon.gui.base import GUIObject
+from core.gui.base import DupeGuruGUIObject
+
+
+class DetailsPanel(GUIObject, DupeGuruGUIObject):
+    def __init__(self, app):
+        GUIObject.__init__(self, multibind=True)
+        DupeGuruGUIObject.__init__(self, app)
+        self._table = []
+
+    def _view_updated(self):
+        self._refresh()
+        self.view.refresh()
+
+    # --- Private
+    def _refresh(self):
+        if self.app.selected_dupes:
+            dupe = self.app.selected_dupes[0]
+            group = self.app.results.get_group_of_duplicate(dupe)
+        else:
+            dupe = None
+            group = None
+        data1 = self.app.get_display_info(dupe, group, False)
+        # we don't want the two sides of the table to display the stats for the same file
+        ref = group.ref if group is not None and group.ref is not dupe else None
+        data2 = self.app.get_display_info(ref, group, False)
+        columns = self.app.result_table.COLUMNS[1:]  # first column is the 'marked' column
+        self._table = [(c.display, data1[c.name], data2[c.name]) for c in columns]
+
+    # --- Public
+    def row_count(self):
+        return len(self._table)
+
+    def row(self, row_index):
+        return self._table[row_index]
+
+    # --- Event Handlers
+    def dupes_selected(self):
+        self._view_updated()

core/gui/directory_tree.py

@@ -0,0 +1,106 @@
+# Created By: Virgil Dupras
+# Created On: 2010-02-06
+# Copyright 2015 Hardcoded Software (http://www.hardcoded.net)
+#
+# This software is licensed under the "GPLv3" License as described in the "LICENSE" file,
+# which should be included with this package. The terms are also available at
+# http://www.gnu.org/licenses/gpl-3.0.html
+
+from hscommon.gui.tree import Tree, Node
+
+from core.directories import DirectoryState
+from core.gui.base import DupeGuruGUIObject
+
+STATE_ORDER = [DirectoryState.NORMAL, DirectoryState.REFERENCE, DirectoryState.EXCLUDED]
+
+
+# Lazily loads children
+class DirectoryNode(Node):
+    def __init__(self, tree, path, name):
+        Node.__init__(self, name)
+        self._tree = tree
+        self._directory_path = path
+        self._loaded = False
+        self._state = STATE_ORDER.index(self._tree.app.directories.get_state(path))
+
+    def __len__(self):
+        if not self._loaded:
+            self._load()
+        return Node.__len__(self)
+
+    def _load(self):
+        self.clear()
+        subpaths = self._tree.app.directories.get_subfolders(self._directory_path)
+        for path in subpaths:
+            self.append(DirectoryNode(self._tree, path, path.name))
+        self._loaded = True
+
+    def update_all_states(self):
+        self._state = STATE_ORDER.index(self._tree.app.directories.get_state(self._directory_path))
+        for node in self:
+            node.update_all_states()
+
+    # The state propery is an index to the combobox
+    @property
+    def state(self):
+        return self._state
+
+    @state.setter
+    def state(self, value):
+        if value == self._state:
+            return
+        self._state = value
+        state = STATE_ORDER[value]
+        self._tree.app.directories.set_state(self._directory_path, state)
+        self._tree.update_all_states()
+
+
+class DirectoryTree(Tree, DupeGuruGUIObject):
+    # --- model -> view calls:
+    # refresh()
+    # refresh_states() # when only states label need to be refreshed
+    #
+    def __init__(self, app):
+        Tree.__init__(self)
+        DupeGuruGUIObject.__init__(self, app)
+
+    def _view_updated(self):
+        self._refresh()
+        self.view.refresh()
+
+    def _refresh(self):
+        self.clear()
+        for path in self.app.directories:
+            self.append(DirectoryNode(self, path, str(path)))
+
+    def add_directory(self, path):
+        self.app.add_directory(path)
+
+    def remove_selected(self):
+        selected_paths = self.selected_paths
+        if not selected_paths:
+            return
+        to_delete = [path[0] for path in selected_paths if len(path) == 1]
+        if to_delete:
+            self.app.remove_directories(to_delete)
+        else:
+            # All selected nodes or on second-or-more level, exclude them.
+            nodes = self.selected_nodes
+            newstate = DirectoryState.EXCLUDED
+            if all(node.state == DirectoryState.EXCLUDED for node in nodes):
+                newstate = DirectoryState.NORMAL
+            for node in nodes:
+                node.state = newstate
+
+    def select_all(self):
+        self.selected_nodes = list(self)
+        self.view.refresh()
+
+    def update_all_states(self):
+        for node in self:
+            node.update_all_states()
+        self.view.refresh_states()
+
+    # --- Event Handlers
+    def directories_changed(self):
+        self._view_updated()

core/gui/exclude_list_dialog.py

@@ -0,0 +1,90 @@
+# Created On: 2012/03/13
+# Copyright 2015 Hardcoded Software (http://www.hardcoded.net)
+#
+# This software is licensed under the "GPLv3" License as described in the "LICENSE" file,
+# which should be included with this package. The terms are also available at
+# http://www.gnu.org/licenses/gpl-3.0.html
+
+from core.gui.exclude_list_table import ExcludeListTable
+from core.exclude import has_sep
+from os import sep
+import logging
+
+
+class ExcludeListDialogCore:
+    def __init__(self, app):
+        self.app = app
+        self.exclude_list = self.app.exclude_list  # Markable from exclude.py
+        self.exclude_list_table = ExcludeListTable(self, app)  # GUITable, this is the "model"
+
+    def restore_defaults(self):
+        self.exclude_list.restore_defaults()
+        self.refresh()
+
+    def refresh(self):
+        self.exclude_list_table.refresh()
+
+    def remove_selected(self):
+        for row in self.exclude_list_table.selected_rows:
+            self.exclude_list_table.remove(row)
+            self.exclude_list.remove(row.regex)
+        self.refresh()
+
+    def rename_selected(self, newregex):
+        """Rename the selected regex to ``newregex``.
+        If there is more than one selected row, the first one is used.
+        :param str newregex: The regex to rename the row's regex to.
+        :return bool: true if success, false if error.
+        """
+        try:
+            r = self.exclude_list_table.selected_rows[0]
+            self.exclude_list.rename(r.regex, newregex)
+            self.refresh()
+            return True
+        except Exception as e:
+            logging.warning(f"Error while renaming regex to {newregex}: {e}")
+        return False
+
+    def add(self, regex):
+        self.exclude_list.add(regex)
+        self.exclude_list.mark(regex)
+        self.exclude_list_table.add(regex)
+
+    def test_string(self, test_string):
+        """Set the highlight property on each row when its regex matches the
+        test_string supplied. Return True if any row matched."""
+        matched = False
+        for row in self.exclude_list_table.rows:
+            compiled_regex = self.exclude_list.get_compiled(row.regex)
+
+            if self.is_match(test_string, compiled_regex):
+                row.highlight = True
+                matched = True
+            else:
+                row.highlight = False
+        return matched
+
+    def is_match(self, test_string, compiled_regex):
+        # This method is like an inverted version of ExcludeList.is_excluded()
+        if not compiled_regex:
+            return False
+        matched = False
+
+        # Test only the filename portion of the path
+        if not has_sep(compiled_regex.pattern) and sep in test_string:
+            filename = test_string.rsplit(sep, 1)[1]
+            if compiled_regex.fullmatch(filename):
+                matched = True
+            return matched
+
+        # Test the entire path + filename
+        if compiled_regex.fullmatch(test_string):
+            matched = True
+        return matched
+
+    def reset_rows_highlight(self):
+        for row in self.exclude_list_table.rows:
+            row.highlight = False
+
+    def show(self):
+        self.view.show()

core/gui/exclude_list_table.py

@@ -0,0 +1,96 @@
+# This software is licensed under the "GPLv3" License as described in the "LICENSE" file,
+# which should be included with this package. The terms are also available at
+# http://www.gnu.org/licenses/gpl-3.0.html
+
+from core.gui.base import DupeGuruGUIObject
+from hscommon.gui.table import GUITable, Row
+from hscommon.gui.column import Column, Columns
+from hscommon.trans import trget
+
+tr = trget("ui")
+
+
+class ExcludeListTable(GUITable, DupeGuruGUIObject):
+    COLUMNS = [Column("marked", ""), Column("regex", tr("Regular Expressions"))]
+
+    def __init__(self, exclude_list_dialog, app):
+        GUITable.__init__(self)
+        DupeGuruGUIObject.__init__(self, app)
+        self._columns = Columns(self)
+        self.dialog = exclude_list_dialog
+
+    def rename_selected(self, newname):
+        row = self.selected_row
+        if row is None:
+            return False
+        row._data = None
+        return self.dialog.rename_selected(newname)
+
+    # --- Virtual
+    def _do_add(self, regex):
+        """(Virtual) Creates a new row, adds it in the table.
+        Returns ``(row, insert_index)``."""
+        # Return index 0 to insert at the top
+        return ExcludeListRow(self, self.dialog.exclude_list.is_marked(regex), regex), 0
+
+    def _do_delete(self):
+        self.dialog.exclude_list.remove(self.selected_row.regex)
+
+    # --- Override
+    def add(self, regex):
+        row, insert_index = self._do_add(regex)
+        self.insert(insert_index, row)
+        self.view.refresh()
+
+    def _fill(self):
+        for enabled, regex in self.dialog.exclude_list:
+            self.append(ExcludeListRow(self, enabled, regex))
+
+    def refresh(self, refresh_view=True):
+        """Override to avoid keeping previous selection in case of multiple rows
+        selected previously."""
+        self.cancel_edits()
+        del self[:]
+        self._fill()
+        if refresh_view:
+            self.view.refresh()
+
+
+class ExcludeListRow(Row):
+    def __init__(self, table, enabled, regex):
+        Row.__init__(self, table)
+        self._app = table.app
+        self._data = None
+        self.enabled = str(enabled)
+        self.regex = str(regex)
+        self.highlight = False
+
+    @property
+    def data(self):
+        if self._data is None:
+            self._data = {"marked": self.enabled, "regex": self.regex}
+        return self._data
+
+    @property
+    def markable(self):
+        return self._app.exclude_list.is_markable(self.regex)
+
+    @property
+    def marked(self):
+        return self._app.exclude_list.is_marked(self.regex)
+
+    @marked.setter
+    def marked(self, value):
+        if value:
+            self._app.exclude_list.mark(self.regex)
+        else:
+            self._app.exclude_list.unmark(self.regex)
+
+    @property
+    def error(self):
+        # This assumes error() returns an Exception()
+        message = self._app.exclude_list.error(self.regex)
+        if hasattr(message, "msg"):
+            return self._app.exclude_list.error(self.regex).msg
+        else:
+            return message  # Exception object

core/gui/ignore_list_dialog.py

@@ -0,0 +1,39 @@
+# Created On: 2012/03/13
+# Copyright 2015 Hardcoded Software (http://www.hardcoded.net)
+#
+# This software is licensed under the "GPLv3" License as described in the "LICENSE" file,
+# which should be included with this package. The terms are also available at
+# http://www.gnu.org/licenses/gpl-3.0.html
+
+from hscommon.trans import tr
+from core.gui.ignore_list_table import IgnoreListTable
+
+
+class IgnoreListDialog:
+    # --- View interface
+    # show()
+    #
+
+    def __init__(self, app):
+        self.app = app
+        self.ignore_list = self.app.ignore_list
+        self.ignore_list_table = IgnoreListTable(self)  # GUITable
+
+    def clear(self):
+        if not self.ignore_list:
+            return
+        msg = tr("Do you really want to remove all %d items from the ignore list?") % len(self.ignore_list)
+        if self.app.view.ask_yes_no(msg):
+            self.ignore_list.clear()
+            self.refresh()
+
+    def refresh(self):
+        self.ignore_list_table.refresh()
+
+    def remove_selected(self):
+        for row in self.ignore_list_table.selected_rows:
+            self.ignore_list.remove(row.path1_original, row.path2_original)
+        self.refresh()
+
+    def show(self):
+        self.view.show()

core/gui/ignore_list_table.py

@@ -0,0 +1,41 @@
+# Created By: Virgil Dupras
+# Created On: 2012-03-13
+# Copyright 2015 Hardcoded Software (http://www.hardcoded.net)
+#
+# This software is licensed under the "GPLv3" License as described in the "LICENSE" file,
+# which should be included with this package. The terms are also available at
+# http://www.gnu.org/licenses/gpl-3.0.html
+
+from hscommon.gui.table import GUITable, Row
+from hscommon.gui.column import Column, Columns
+from hscommon.trans import trget
+
+coltr = trget("columns")
+
+
+class IgnoreListTable(GUITable):
+    COLUMNS = [
+        # the str concat below saves us needless localization.
+        Column("path1", coltr("File Path") + " 1"),
+        Column("path2", coltr("File Path") + " 2"),
+    ]
+
+    def __init__(self, ignore_list_dialog):
+        GUITable.__init__(self)
+        self._columns = Columns(self)
+        self.view = None
+        self.dialog = ignore_list_dialog
+
+    # --- Override
+    def _fill(self):
+        for path1, path2 in self.dialog.ignore_list:
+            self.append(IgnoreListRow(self, path1, path2))
+
+
+class IgnoreListRow(Row):
+    def __init__(self, table, path1, path2):
+        Row.__init__(self, table)
+        self.path1_original = path1
+        self.path2_original = path2
+        self.path1 = str(path1)
+        self.path2 = str(path2)

core/gui/prioritize_dialog.py

@@ -0,0 +1,86 @@
+# Created By: Virgil Dupras
+# Created On: 2011-09-06
+# Copyright 2015 Hardcoded Software (http://www.hardcoded.net)
+#
+# This software is licensed under the "GPLv3" License as described in the "LICENSE" file,
+# which should be included with this package. The terms are also available at
+# http://www.gnu.org/licenses/gpl-3.0.html
+
+from hscommon.gui.base import GUIObject
+from hscommon.gui.selectable_list import GUISelectableList
+
+
+class CriterionCategoryList(GUISelectableList):
+    def __init__(self, dialog):
+        self.dialog = dialog
+        GUISelectableList.__init__(self, [c.NAME for c in dialog.categories])
+
+    def _update_selection(self):
+        self.dialog.select_category(self.dialog.categories[self.selected_index])
+        GUISelectableList._update_selection(self)
+
+
+class PrioritizationList(GUISelectableList):
+    def __init__(self, dialog):
+        self.dialog = dialog
+        GUISelectableList.__init__(self)
+
+    def _refresh_contents(self):
+        self[:] = [crit.display for crit in self.dialog.prioritizations]
+
+    def move_indexes(self, indexes, dest_index):
+        indexes.sort()
+        prilist = self.dialog.prioritizations
+        selected = [prilist[i] for i in indexes]
+        for i in reversed(indexes):
+            del prilist[i]
+        prilist[dest_index:dest_index] = selected
+        self._refresh_contents()
+
+    def remove_selected(self):
+        prilist = self.dialog.prioritizations
+        for i in sorted(self.selected_indexes, reverse=True):
+            del prilist[i]
+        self._refresh_contents()
+
+
+class PrioritizeDialog(GUIObject):
+    def __init__(self, app):
+        GUIObject.__init__(self)
+        self.app = app
+        self.categories = [cat(app.results) for cat in app._prioritization_categories()]
+        self.category_list = CriterionCategoryList(self)
+        self.criteria = []
+        self.criteria_list = GUISelectableList()
+        self.prioritizations = []
+        self.prioritization_list = PrioritizationList(self)
+
+    # --- Override
+    def _view_updated(self):
+        self.category_list.select(0)
+
+    # --- Private
+    def _sort_key(self, dupe):
+        return tuple(crit.sort_key(dupe) for crit in self.prioritizations)
+
+    # --- Public
+    def select_category(self, category):
+        self.criteria = category.criteria_list()
+        self.criteria_list[:] = [c.display_value for c in self.criteria]
+
+    def add_selected(self):
+        # Add selected criteria in criteria_list to prioritization_list.
+        if self.criteria_list.selected_index is None:
+            return
+        for i in self.criteria_list.selected_indexes:
+            crit = self.criteria[i]
+            self.prioritizations.append(crit)
+            del crit
+        self.prioritization_list[:] = [crit.display for crit in self.prioritizations]
+
+    def remove_selected(self):
+        self.prioritization_list.remove_selected()
+        self.prioritization_list.select([])
+
+    def perform_reprioritization(self):
+        self.app.reprioritize_groups(self._sort_key)

core/gui/problem_dialog.py

@@ -0,0 +1,29 @@
+# Created By: Virgil Dupras
+# Created On: 2010-04-12
+# Copyright 2015 Hardcoded Software (http://www.hardcoded.net)
+#
+# This software is licensed under the "GPLv3" License as described in the "LICENSE" file,
+# which should be included with this package. The terms are also available at
+# http://www.gnu.org/licenses/gpl-3.0.html
+
+from hscommon import desktop
+
+from core.gui.problem_table import ProblemTable
+
+
+class ProblemDialog:
+    def __init__(self, app):
+        self.app = app
+        self._selected_dupe = None
+        self.problem_table = ProblemTable(self)
+
+    def refresh(self):
+        self._selected_dupe = None
+        self.problem_table.refresh()
+
+    def reveal_selected_dupe(self):
+        if self._selected_dupe is not None:
+            desktop.reveal_path(self._selected_dupe.path)
+
+    def select_dupe(self, dupe):
+        self._selected_dupe = dupe

core/gui/problem_table.py

@@ -0,0 +1,44 @@
+# Created By: Virgil Dupras
+# Created On: 2010-04-12
+# Copyright 2015 Hardcoded Software (http://www.hardcoded.net)
+#
+# This software is licensed under the "GPLv3" License as described in the "LICENSE" file,
+# which should be included with this package. The terms are also available at
+# http://www.gnu.org/licenses/gpl-3.0.html
+
+from hscommon.gui.table import GUITable, Row
+from hscommon.gui.column import Column, Columns
+from hscommon.trans import trget
+
+coltr = trget("columns")
+
+
+class ProblemTable(GUITable):
+    COLUMNS = [
+        Column("path", coltr("File Path")),
+        Column("msg", coltr("Error Message")),
+    ]
+
+    def __init__(self, problem_dialog):
+        GUITable.__init__(self)
+        self._columns = Columns(self)
+        self.dialog = problem_dialog
+
+    # --- Override
+    def _update_selection(self):
+        row = self.selected_row
+        dupe = row.dupe if row is not None else None
+        self.dialog.select_dupe(dupe)
+
+    def _fill(self):
+        problems = self.dialog.app.results.problems
+        for dupe, msg in problems:
+            self.append(ProblemRow(self, dupe, msg))
+
+
+class ProblemRow(Row):
+    def __init__(self, table, dupe, msg):
+        Row.__init__(self, table)
+        self.dupe = dupe
+        self.msg = msg
+        self.path = str(dupe.path)

core/gui/result_table.py

@@ -0,0 +1,193 @@
+# Created By: Virgil Dupras
+# Created On: 2010-02-11
+# Copyright 2015 Hardcoded Software (http://www.hardcoded.net)
+#
+# This software is licensed under the "GPLv3" License as described in the "LICENSE" file,
+# which should be included with this package. The terms are also available at
+# http://www.gnu.org/licenses/gpl-3.0.html
+
+from operator import attrgetter
+
+from hscommon.gui.table import GUITable, Row
+from hscommon.gui.column import Columns
+
+from core.gui.base import DupeGuruGUIObject
+
+
+class DupeRow(Row):
+    def __init__(self, table, group, dupe):
+        Row.__init__(self, table)
+        self._app = table.app
+        self._group = group
+        self._dupe = dupe
+        self._data = None
+        self._data_delta = None
+        self._delta_columns = None
+
+    def is_cell_delta(self, column_name):
+        """Returns whether a cell is in delta mode (orange color).
+
+        If the result table is in delta mode, returns True if the column is one of the "delta
+        columns", that is, one of the columns that display a a differential value rather than an
+        absolute value.
+
+        If not, returns True if the dupe's value is different from its ref value.
+        """
+        if not self.table.delta_values:
+            return False
+        if self.isref:
+            return False
+        if self._delta_columns is None:
+            # table.DELTA_COLUMNS are always "delta"
+            self._delta_columns = self.table.DELTA_COLUMNS.copy()
+            dupe_info = self.data
+            if self._group.ref is None:
+                return False
+            ref_info = self._group.ref.get_display_info(group=self._group, delta=False)
+            for key, value in dupe_info.items():
+                if (key not in self._delta_columns) and (ref_info[key].lower() != value.lower()):
+                    self._delta_columns.add(key)
+        return column_name in self._delta_columns
+
+    @property
+    def data(self):
+        if self._data is None:
+            self._data = self._app.get_display_info(self._dupe, self._group, False)
+        return self._data
+
+    @property
+    def data_delta(self):
+        if self._data_delta is None:
+            self._data_delta = self._app.get_display_info(self._dupe, self._group, True)
+        return self._data_delta
+
+    @property
+    def isref(self):
+        return self._dupe is self._group.ref
+
+    @property
+    def markable(self):
+        return self._app.results.is_markable(self._dupe)
+
+    @property
+    def marked(self):
+        return self._app.results.is_marked(self._dupe)
+
+    @marked.setter
+    def marked(self, value):
+        self._app.mark_dupe(self._dupe, value)
+
+
+class ResultTable(GUITable, DupeGuruGUIObject):
+    def __init__(self, app):
+        GUITable.__init__(self)
+        DupeGuruGUIObject.__init__(self, app)
+        self._columns = Columns(self, prefaccess=app, savename="ResultTable")
+        self._power_marker = False
+        self._delta_values = False
+        self._sort_descriptors = ("name", True)
+
+    # --- Override
+    def _view_updated(self):
+        self._refresh_with_view()
+
+    def _restore_selection(self, previous_selection):
+        if self.app.selected_dupes:
+            to_find = set(self.app.selected_dupes)
+            indexes = [i for i, r in enumerate(self) if r._dupe in to_find]
+            self.selected_indexes = indexes
+
+    def _update_selection(self):
+        rows = self.selected_rows
+        self.app._select_dupes(list(map(attrgetter("_dupe"), rows)))
+
+    def _fill(self):
+        if not self.power_marker:
+            for group in self.app.results.groups:
+                self.append(DupeRow(self, group, group.ref))
+                for dupe in group.dupes:
+                    self.append(DupeRow(self, group, dupe))
+        else:
+            for dupe in self.app.results.dupes:
+                group = self.app.results.get_group_of_duplicate(dupe)
+                self.append(DupeRow(self, group, dupe))
+
+    def _refresh_with_view(self):
+        self.refresh()
+        self.view.show_selected_row()
+
+    # --- Public
+    def get_row_value(self, index, column):
+        try:
+            row = self[index]
+        except IndexError:
+            return "---"
+        if self.delta_values:
+            return row.data_delta[column]
+        else:
+            return row.data[column]
+
+    def rename_selected(self, newname):
+        row = self.selected_row
+        if row is None:
+            # There's all kinds of way the current row can be swept off during rename. When it
+            # happens, selected_row will be None.
+            return False
+        row._data = None
+        row._data_delta = None
+        return self.app.rename_selected(newname)
+
+    def sort(self, key, asc):
+        if self.power_marker:
+            self.app.results.sort_dupes(key, asc, self.delta_values)
+        else:
+            self.app.results.sort_groups(key, asc)
+        self._sort_descriptors = (key, asc)
+        self._refresh_with_view()
+
+    # --- Properties
+    @property
+    def power_marker(self):
+        return self._power_marker
+
+    @power_marker.setter
+    def power_marker(self, value):
+        if value == self._power_marker:
+            return
+        self._power_marker = value
+        key, asc = self._sort_descriptors
+        self.sort(key, asc)
+        # no need to refresh, it has happened in sort()
+
+    @property
+    def delta_values(self):
+        return self._delta_values
+
+    @delta_values.setter
+    def delta_values(self, value):
+        if value == self._delta_values:
+            return
+        self._delta_values = value
+        self.refresh()
+
+    @property
+    def selected_dupe_count(self):
+        return sum(1 for row in self.selected_rows if not row.isref)
+
+    # --- Event Handlers
+    def marking_changed(self):
+        self.view.invalidate_markings()
+
+    def results_changed(self):
+        self._refresh_with_view()
+
+    def results_changed_but_keep_selection(self):
+        # What we want to to here is that instead of restoring selected *dupes* after refresh, we
+        # restore selected *paths*.
+        indexes = self.selected_indexes
+        self.refresh(refresh_view=False)
+        self.select(indexes)
+        self.view.refresh()
+
+    def save_session(self):
+        self._columns.save_columns()

core/gui/stats_label.py

@@ -0,0 +1,23 @@
+# Created By: Virgil Dupras
+# Created On: 2010-02-11
+# Copyright 2015 Hardcoded Software (http://www.hardcoded.net)
+#
+# This software is licensed under the "GPLv3" License as described in the "LICENSE" file,
+# which should be included with this package. The terms are also available at
+# http://www.gnu.org/licenses/gpl-3.0.html
+
+from core.gui.base import DupeGuruGUIObject
+
+
+class StatsLabel(DupeGuruGUIObject):
+    def _view_updated(self):
+        self.view.refresh()
+
+    @property
+    def display(self):
+        return self.app.stat_line
+
+    def results_changed(self):
+        self.view.refresh()
+
+    marking_changed = results_changed

core/ignore.py

@@ -0,0 +1,127 @@
+# Created By: Virgil Dupras
+# Created On: 2006/05/02
+# Copyright 2015 Hardcoded Software (http://www.hardcoded.net)
+#
+# This software is licensed under the "GPLv3" License as described in the "LICENSE" file,
+# which should be included with this package. The terms are also available at
+# http://www.gnu.org/licenses/gpl-3.0.html
+
+from xml.etree import ElementTree as ET
+
+from hscommon.util import FileOrPath
+
+
+class IgnoreList:
+    """An ignore list implementation that is iterable, filterable and exportable to XML.
+
+    Call Ignore to add an ignore list entry, and AreIgnore to check if 2 items are in the list.
+    When iterated, 2 sized tuples will be returned, the tuples containing 2 items ignored together.
+    """
+
+    # ---Override
+    def __init__(self):
+        self.clear()
+
+    def __iter__(self):
+        for first, seconds in self._ignored.items():
+            for second in seconds:
+                yield (first, second)
+
+    def __len__(self):
+        return self._count
+
+    # ---Public
+    def are_ignored(self, first, second):
+        def do_check(first, second):
+            try:
+                matches = self._ignored[first]
+                return second in matches
+            except KeyError:
+                return False
+
+        return do_check(first, second) or do_check(second, first)
+
+    def clear(self):
+        self._ignored = {}
+        self._count = 0
+
+    def filter(self, func):
+        """Applies a filter on all ignored items, and remove all matches where func(first,second)
+        doesn't return True.
+        """
+        filtered = IgnoreList()
+        for first, second in self:
+            if func(first, second):
+                filtered.ignore(first, second)
+        self._ignored = filtered._ignored
+        self._count = filtered._count
+
+    def ignore(self, first, second):
+        if self.are_ignored(first, second):
+            return
+        try:
+            matches = self._ignored[first]
+            matches.add(second)
+        except KeyError:
+            try:
+                matches = self._ignored[second]
+                matches.add(first)
+            except KeyError:
+                matches = set()
+                matches.add(second)
+                self._ignored[first] = matches
+        self._count += 1
+
+    def remove(self, first, second):
+        def inner(first, second):
+            try:
+                matches = self._ignored[first]
+                if second in matches:
+                    matches.discard(second)
+                    if not matches:
+                        del self._ignored[first]
+                    self._count -= 1
+                    return True
+                else:
+                    return False
+            except KeyError:
+                return False
+
+        if not inner(first, second) and not inner(second, first):
+            raise ValueError()
+
+    def load_from_xml(self, infile):
+        """Loads the ignore list from a XML created with save_to_xml.
+
+        infile can be a file object or a filename.
+        """
+        try:
+            root = ET.parse(infile).getroot()
+        except Exception:
+            return
+        file_elems = (e for e in root if e.tag == "file")
+        for fn in file_elems:
+            file_path = fn.get("path")
+            if not file_path:
+                continue
+            subfile_elems = (e for e in fn if e.tag == "file")
+            for sfn in subfile_elems:
+                subfile_path = sfn.get("path")
+                if subfile_path:
+                    self.ignore(file_path, subfile_path)
+
+    def save_to_xml(self, outfile):
+        """Create a XML file that can be used by load_from_xml.
+
+        outfile can be a file object or a filename.
+        """
+        root = ET.Element("ignore_list")
+        for filename, subfiles in self._ignored.items():
+            file_node = ET.SubElement(root, "file")
+            file_node.set("path", filename)
+            for subfilename in subfiles:
+                subfile_node = ET.SubElement(file_node, "file")
+                subfile_node.set("path", subfilename)
+        tree = ET.ElementTree(root)
+        with FileOrPath(outfile, "wb") as fp:
+            tree.write(fp, encoding="utf-8")

core/markable.py

@@ -0,0 +1,120 @@
+# Created By: Virgil Dupras
+# Created On: 2006/02/23
+# Copyright 2015 Hardcoded Software (http://www.hardcoded.net)
+
+# This software is licensed under the "GPLv3" License as described in the "LICENSE" file,
+# which should be included with this package. The terms are also available at
+# http://www.gnu.org/licenses/gpl-3.0.html
+
+
+class Markable:
+    def __init__(self):
+        self.__marked = set()
+        self.__inverted = False
+
+    # ---Virtual
+    # About did_mark and did_unmark: They only happen what an object is actually added/removed
+    # in self.__marked, and is not affected by __inverted. Thus, self.mark while __inverted
+    # is True will launch _DidUnmark.
+    def _did_mark(self, o):
+        # Implemented in child classes
+        pass
+
+    def _did_unmark(self, o):
+        # Implemented in child classes
+        pass
+
+    def _get_markable_count(self):
+        return 0
+
+    def _is_markable(self, o):
+        return True
+
+    # ---Protected
+    def _remove_mark_flag(self, o):
+        try:
+            self.__marked.remove(o)
+            self._did_unmark(o)
+        except KeyError:
+            pass
+
+    # ---Public
+    def is_marked(self, o):
+        if not self._is_markable(o):
+            return False
+        is_marked = o in self.__marked
+        if self.__inverted:
+            is_marked = not is_marked
+        return is_marked
+
+    def mark(self, o):
+        if self.is_marked(o):
+            return False
+        if not self._is_markable(o):
+            return False
+        return self.mark_toggle(o)
+
+    def mark_multiple(self, objects):
+        for o in objects:
+            self.mark(o)
+
+    def mark_all(self):
+        self.mark_none()
+        self.__inverted = True
+
+    def mark_invert(self):
+        self.__inverted = not self.__inverted
+
+    def mark_none(self):
+        for o in self.__marked:
+            self._did_unmark(o)
+        self.__marked = set()
+        self.__inverted = False
+
+    def mark_toggle(self, o):
+        try:
+            self.__marked.remove(o)
+            self._did_unmark(o)
+        except KeyError:
+            if not self._is_markable(o):
+                return False
+            self.__marked.add(o)
+            self._did_mark(o)
+        return True
+
+    def mark_toggle_multiple(self, objects):
+        for o in objects:
+            self.mark_toggle(o)
+
+    def unmark(self, o):
+        if not self.is_marked(o):
+            return False
+        return self.mark_toggle(o)
+
+    def unmark_multiple(self, objects):
+        for o in objects:
+            self.unmark(o)
+
+    # --- Properties
+    @property
+    def mark_count(self):
+        if self.__inverted:
+            return self._get_markable_count() - len(self.__marked)
+        else:
+            return len(self.__marked)
+
+    @property
+    def mark_inverted(self):
+        return self.__inverted
+
+
+class MarkableList(list, Markable):
+    def __init__(self):
+        list.__init__(self)
+        Markable.__init__(self)
+
+    def _get_markable_count(self):
+        return len(self)
+
+    def _is_markable(self, o):
+        return o in self

core/me/__init__.py

@@ -0,0 +1 @@
+from core.me import fs, prioritize, result_table, scanner  # noqa

core/me/fs.py

@@ -0,0 +1,115 @@
+# Created By: Virgil Dupras
+# Created On: 2009-10-23
+# Copyright 2015 Hardcoded Software (http://www.hardcoded.net)
+#
+# This software is licensed under the "GPLv3" License as described in the "LICENSE" file,
+# which should be included with this package. The terms are also available at
+# http://www.gnu.org/licenses/gpl-3.0.html
+
+import mutagen
+from hscommon.util import get_file_ext, format_size, format_time
+
+from core.util import format_timestamp, format_perc, format_words, format_dupe_count
+from core import fs
+
+TAG_FIELDS = {
+    "audiosize",
+    "duration",
+    "bitrate",
+    "samplerate",
+    "title",
+    "artist",
+    "album",
+    "genre",
+    "year",
+    "track",
+    "comment",
+}
+
+# This is a temporary workaround for migration from hsaudiotag for the can_handle method
+SUPPORTED_EXTS = {"mp3", "wma", "m4a", "m4p", "ogg", "flac", "aif", "aiff", "aifc"}
+
+
+class MusicFile(fs.File):
+    INITIAL_INFO = fs.File.INITIAL_INFO.copy()
+    INITIAL_INFO.update(
+        {
+            "audiosize": 0,
+            "bitrate": 0,
+            "duration": 0,
+            "samplerate": 0,
+            "artist": "",
+            "album": "",
+            "title": "",
+            "genre": "",
+            "comment": "",
+            "year": "",
+            "track": 0,
+        }
+    )
+    __slots__ = fs.File.__slots__ + tuple(INITIAL_INFO.keys())
+
+    @classmethod
+    def can_handle(cls, path):
+        if not fs.File.can_handle(path):
+            return False
+        return get_file_ext(path.name) in SUPPORTED_EXTS
+
+    def get_display_info(self, group, delta):
+        size = self.size
+        duration = self.duration
+        bitrate = self.bitrate
+        samplerate = self.samplerate
+        mtime = self.mtime
+        m = group.get_match_of(self)
+        if m:
+            percentage = m.percentage
+            dupe_count = 0
+            if delta:
+                r = group.ref
+                size -= r.size
+                duration -= r.duration
+                bitrate -= r.bitrate
+                samplerate -= r.samplerate
+                mtime -= r.mtime
+        else:
+            percentage = group.percentage
+            dupe_count = len(group.dupes)
+        dupe_folder_path = getattr(self, "display_folder_path", self.folder_path)
+        return {
+            "name": self.name,
+            "folder_path": str(dupe_folder_path),
+            "size": format_size(size, 2, 2, False),
+            "duration": format_time(duration, with_hours=False),
+            "bitrate": str(bitrate),
+            "samplerate": str(samplerate),
+            "extension": self.extension,
+            "mtime": format_timestamp(mtime, delta and m),
+            "title": self.title,
+            "artist": self.artist,
+            "album": self.album,
+            "genre": self.genre,
+            "year": self.year,
+            "track": str(self.track),
+            "comment": self.comment,
+            "percentage": format_perc(percentage),
+            "words": format_words(self.words) if hasattr(self, "words") else "",
+            "dupe_count": format_dupe_count(dupe_count),
+        }
+
+    def _read_info(self, field):
+        fs.File._read_info(self, field)
+        if field in TAG_FIELDS:
+            # The various conversions here are to make this look like the previous implementation
+            file = mutagen.File(str(self.path), easy=True)
+            self.audiosize = self.path.stat().st_size
+            self.bitrate = file.info.bitrate / 1000
+            self.duration = file.info.length
+            self.samplerate = file.info.sample_rate
+            self.artist = ", ".join(file.tags.get("artist") or [])
+            self.album = ", ".join(file.tags.get("album") or [])
+            self.title = ", ".join(file.tags.get("title") or [])
+            self.genre = ", ".join(file.tags.get("genre") or [])
+            self.comment = ", ".join(file.tags.get("comment") or [""])
+            self.year = ", ".join(file.tags.get("date") or [])
+            self.track = (file.tags.get("tracknumber") or [""])[0]