Merge remote-tracking branch 'upstream/master' into colors-bytes

.github/workflows/default.yml

@@ -9,43 +9,22 @@ on:
     branches: [master]
 
 jobs:
-  lint:
+  pre-commit:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v2
+      - uses: actions/checkout@v3
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v4
         with:
-          python-version: "3.10"
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install -r requirements.txt -r requirements-extra.txt
-      - name: Lint with flake8
-        run: |
-          flake8 .
-  format:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v2
-        with:
-          python-version: "3.10"
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install -r requirements.txt -r requirements-extra.txt
-      - name: Check format with black
-        run: |
-          black .
+          python-version: "3.11"
+      - uses: pre-commit/action@v3.0.0
   test:
-    needs: [lint, format]
+    needs: [pre-commit]
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: [3.7, 3.8, 3.9, "3.10"]
+        python-version: [3.7, 3.8, 3.9, "3.10", "3.11"]
         exclude:
           - os: macos-latest
             python-version: 3.7
@@ -53,17 +32,21 @@ jobs:
             python-version: 3.8
           - os: macos-latest
             python-version: 3.9
+          - os: macos-latest
+            python-version: "3.10"
           - os: windows-latest
             python-version: 3.7
           - os: windows-latest
             python-version: 3.8
           - os: windows-latest
             python-version: 3.9
+          - os: windows-latest
+            python-version: "3.10"
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies

.gitignore

@@ -108,4 +108,4 @@ cocoa/autogen
 
 *.waf*
 .lock-waf*
-/tags
+/tags

.pre-commit-config.yaml

@@ -0,0 +1,24 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v2.3.0
+    hooks:
+      - id: check-yaml
+      - id: check-toml
+      - id: end-of-file-fixer
+        exclude: ".*.json"
+      - id: trailing-whitespace
+  - repo: https://github.com/psf/black
+    rev: 22.10.0
+    hooks:
+      - id: black
+  - repo: https://github.com/PyCQA/flake8
+    rev: 6.0.0
+    hooks:
+      - id: flake8
+        exclude: ^(.tox|env|build|dist|help|qt/dg_rc.py|pkg).*
+  - repo: https://github.com/alessandrojcm/commitlint-pre-commit-hook
+    rev: v9.3.0
+    hooks:
+      - id: commitlint
+        stages: [commit-msg]
+        additional_dependencies: ["@commitlint/config-conventional"]

.sonarcloud.properties

@@ -1 +1 @@
-sonar.python.version=3.7, 3.8, 3.9, 3.10
+sonar.python.version=3.7, 3.8, 3.9, 3.10, 3.11

.tx/config

@@ -18,4 +18,3 @@ file_filter = locale/<lang>/LC_MESSAGES/ui.po
 source_file = locale/ui.pot
 source_lang = en
 type        = PO
-

LICENSE

@@ -619,4 +619,3 @@ Program, unless a warranty or assumption of liability accompanies a
 copy of the Program in return for a fee.
 
                      END OF TERMS AND CONDITIONS
-

Makefile

@@ -4,7 +4,7 @@ PYRCC5 ?= pyrcc5
 REQ_MINOR_VERSION = 7
 PREFIX ?= /usr/local
 
-# Window compatability via Msys2 
+# Window compatability via Msys2
 # - venv creates Scripts instead of bin
 # - compile generates .pyd instead of .so
 # - venv with --sytem-site-packages has issues on windows as well...
@@ -12,7 +12,7 @@ PREFIX ?= /usr/local
 ifeq ($(shell ${PYTHON} -c "import platform; print(platform.system())"), Windows)
 	BIN = Scripts
 	SO = *.pyd
-	VENV_OPTIONS = 
+	VENV_OPTIONS =
 else
 	BIN = bin
 	SO = *.so
@@ -43,7 +43,7 @@ mofiles = $(patsubst %.po,%.mo,$(pofiles))
 vpath %.po $(localedirs)
 vpath %.mo $(localedirs)
 
-all: | env i18n modules qt/dg_rc.py 
+all: | env i18n modules qt/dg_rc.py
 	@echo "Build complete! You can run dupeGuru with 'make run'"
 
 run:
@@ -82,7 +82,7 @@ qt/dg_rc.py: qt/dg.qrc
 i18n: $(mofiles)
 
 %.mo: %.po
-	msgfmt -o $@ $<	
+	msgfmt -o $@ $<
 
 modules: | env
 	$(VENV_PYTHON) build.py --modules

README.md

@@ -63,7 +63,7 @@ dupeGuru comes with a makefile that can be used to build and run:
     $ python run.py
 
 ### Generating Debian/Ubuntu package
-To generate packages the extra requirements in requirements-extra.txt must be installed, the 
+To generate packages the extra requirements in requirements-extra.txt must be installed, the
 steps are as follows:
 
     $ cd <dupeGuru directory>

Windows.md

@@ -29,7 +29,7 @@ To build with a different python version 3.7 vs 3.8 or 32 bit vs 64 bit specify
 It is possible to build dupeGuru with the makefile on windows using a compatable POSIX environment.  The following steps have been tested using [msys2][msys2]. Before running make:
 1. Install msys2 or other POSIX environment
 2. Install PyQt5 globally via pip
-3. Use the respective console for msys2 it is `msys2 msys` 
+3. Use the respective console for msys2 it is `msys2 msys`
 
 Then the following execution of the makefile should work.  Pass the correct value for PYTHON to the makefile if not on the path as python3.
 

commitlint.config.js

@@ -0,0 +1,17 @@
+const Configuration = {
+    /*
+     * Resolve and load @commitlint/config-conventional from node_modules.
+     * Referenced packages must be installed
+     */
+    extends: ['@commitlint/config-conventional'],
+    /*
+     * Any rules defined here will override rules from @commitlint/config-conventional
+     */
+    rules: {
+        'header-max-length': [2, 'always', 72],
+        'subject-case': [2, 'always', 'sentence-case'],
+        'scope-enum': [2, 'always'],
+    },
+};
+
+module.exports = Configuration;

core/app.py

@@ -126,8 +126,6 @@ class DupeGuru(Broadcaster):
 
     NAME = PROMPT_NAME = "dupeGuru"
 
-    PICTURE_CACHE_TYPE = "sqlite"  # set to 'shelve' for a ShelveCache
-
     def __init__(self, view, portable=False):
         if view.get_default(DEBUG_MODE_PREFERENCE):
             logging.getLogger().setLevel(logging.DEBUG)
@@ -153,7 +151,8 @@ class DupeGuru(Broadcaster):
             "clean_empty_dirs": False,
             "ignore_hardlink_matches": False,
             "copymove_dest_type": DestType.RELATIVE,
-            "picture_cache_type": self.PICTURE_CACHE_TYPE,
+            "include_exists_check": True,
+            "rehash_ignore_mtime": False,
         }
         self.selected_dupes = []
         self.details_panel = DetailsPanel(self)
@@ -183,8 +182,7 @@ class DupeGuru(Broadcaster):
         self.view.create_results_window()
 
     def _get_picture_cache_path(self):
-        cache_type = self.options["picture_cache_type"]
-        cache_name = "cached_pictures.shelve" if cache_type == "shelve" else "cached_pictures.db"
+        cache_name = "cached_pictures.db"
         return op.join(self.appdata, cache_name)
 
     def _get_dupe_sort_key(self, dupe, get_group, key, delta):
@@ -555,7 +553,9 @@ class DupeGuru(Broadcaster):
                 # a workaround to make the damn thing work.
                 exepath, args = match.groups()
                 path, exename = op.split(exepath)
-                p = subprocess.Popen(exename + args, shell=True, cwd=path, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+                p = subprocess.Popen(
+                    exename + args, shell=True, cwd=path, stdout=subprocess.PIPE, stderr=subprocess.STDOUT
+                )
                 output = p.stdout.read()
                 logging.info("Custom command %s %s: %s", exename, args, output)
             else:
@@ -792,6 +792,7 @@ class DupeGuru(Broadcaster):
         Scans folders selected in :attr:`directories` and put the results in :attr:`results`
         """
         scanner = self.SCANNER_CLASS()
+        fs.filesdb.ignore_mtime = self.options["rehash_ignore_mtime"] is True
         if not self.directories.has_any_file():
             self.view.show_message(tr("The selected directories contain no scannable file."))
             return

core/fs.py

@@ -97,59 +97,68 @@ class FilesDB:
     schema_version = 1
     schema_version_description = "Changed from md5 to xxhash if available."
 
-    create_table_query = "CREATE TABLE IF NOT EXISTS files (path TEXT PRIMARY KEY, size INTEGER, mtime_ns INTEGER, entry_dt DATETIME, digest BLOB, digest_partial BLOB, digest_samples BLOB)"
+    create_table_query = """CREATE TABLE IF NOT EXISTS files (path TEXT PRIMARY KEY, size INTEGER, mtime_ns INTEGER,
+        entry_dt DATETIME, digest BLOB, digest_partial BLOB, digest_samples BLOB)"""
     drop_table_query = "DROP TABLE IF EXISTS files;"
     select_query = "SELECT {key} FROM files WHERE path=:path AND size=:size and mtime_ns=:mtime_ns"
+    select_query_ignore_mtime = "SELECT {key} FROM files WHERE path=:path AND size=:size"
     insert_query = """
-        INSERT INTO files (path, size, mtime_ns, entry_dt, {key}) VALUES (:path, :size, :mtime_ns, datetime('now'), :value)
+        INSERT INTO files (path, size, mtime_ns, entry_dt, {key})
+        VALUES (:path, :size, :mtime_ns, datetime('now'), :value)
         ON CONFLICT(path) DO UPDATE SET size=:size, mtime_ns=:mtime_ns, entry_dt=datetime('now'), {key}=:value;
     """
 
+    ignore_mtime = False
+
     def __init__(self):
         self.conn = None
-        self.cur = None
         self.lock = None
 
     def connect(self, path: Union[AnyStr, os.PathLike]) -> None:
         self.conn = sqlite3.connect(path, check_same_thread=False)
-        self.cur = self.conn.cursor()
         self.lock = Lock()
         self._check_upgrade()
 
     def _check_upgrade(self) -> None:
-        with self.lock:
-            has_schema = self.cur.execute(
+        with self.lock, self.conn as conn:
+            has_schema = conn.execute(
                 "SELECT NAME FROM sqlite_master WHERE type='table' AND name='schema_version'"
             ).fetchall()
             version = None
             if has_schema:
-                version = self.cur.execute("SELECT version FROM schema_version ORDER BY version DESC").fetchone()[0]
+                version = conn.execute("SELECT version FROM schema_version ORDER BY version DESC").fetchone()[0]
             else:
-                self.cur.execute("CREATE TABLE schema_version (version int PRIMARY KEY, description TEXT)")
+                conn.execute("CREATE TABLE schema_version (version int PRIMARY KEY, description TEXT)")
             if version != self.schema_version:
-                self.cur.execute(self.drop_table_query)
-                self.cur.execute(
+                conn.execute(self.drop_table_query)
+                conn.execute(
                     "INSERT OR REPLACE INTO schema_version VALUES (:version, :description)",
                     {"version": self.schema_version, "description": self.schema_version_description},
                 )
-            self.cur.execute(self.create_table_query)
-            self.conn.commit()
+            conn.execute(self.create_table_query)
 
     def clear(self) -> None:
-        with self.lock:
-            self.cur.execute(self.drop_table_query)
-            self.cur.execute(self.create_table_query)
+        with self.lock, self.conn as conn:
+            conn.execute(self.drop_table_query)
+            conn.execute(self.create_table_query)
 
     def get(self, path: Path, key: str) -> Union[bytes, None]:
         stat = path.stat()
         size = stat.st_size
         mtime_ns = stat.st_mtime_ns
         try:
-            with self.lock:
-                self.cur.execute(
-                    self.select_query.format(key=key), {"path": str(path), "size": size, "mtime_ns": mtime_ns}
-                )
-                result = self.cur.fetchone()
+            with self.conn as conn:
+                if self.ignore_mtime:
+                    cursor = conn.execute(
+                        self.select_query_ignore_mtime.format(key=key), {"path": str(path), "size": size}
+                    )
+                else:
+                    cursor = conn.execute(
+                        self.select_query.format(key=key),
+                        {"path": str(path), "size": size, "mtime_ns": mtime_ns},
+                    )
+                result = cursor.fetchone()
+                cursor.close()
 
             if result:
                 return result[0]
@@ -163,8 +172,8 @@ class FilesDB:
         size = stat.st_size
         mtime_ns = stat.st_mtime_ns
         try:
-            with self.lock:
-                self.cur.execute(
+            with self.lock, self.conn as conn:
+                conn.execute(
                     self.insert_query.format(key=key),
                     {"path": str(path), "size": size, "mtime_ns": mtime_ns, "value": value},
                 )
@@ -177,7 +186,6 @@ class FilesDB:
 
     def close(self) -> None:
         with self.lock:
-            self.cur.close()
             self.conn.close()
 
 
@@ -307,6 +315,14 @@ class File:
         """Returns whether this file wrapper class can handle ``path``."""
         return not path.is_symlink() and path.is_file()
 
+    def exists(self) -> bool:
+        """Safely check if the underlying file exists, treat error as non-existent"""
+        try:
+            return self.path.exists()
+        except OSError as ex:
+            logging.warning(f"Checking {self.path} raised: {ex}")
+            return False
+
     def rename(self, newname):
         if newname == self.name:
             return

core/pe/cache_shelve.py

@@ -1,141 +0,0 @@
-# Copyright 2016 Virgil Dupras
-#
-# This software is licensed under the "GPLv3" License as described in the "LICENSE" file,
-# which should be included with this package. The terms are also available at
-# http://www.gnu.org/licenses/gpl-3.0.html
-
-import os
-import os.path as op
-import shelve
-import tempfile
-from collections import namedtuple
-
-from core.pe.cache import bytes_to_colors, colors_to_bytes
-
-
-def wrap_path(path):
-    return f"path:{path}"
-
-
-def unwrap_path(key):
-    return key[5:]
-
-
-def wrap_id(path):
-    return f"id:{path}"
-
-
-def unwrap_id(key):
-    return int(key[3:])
-
-
-CacheRow = namedtuple("CacheRow", "id path blocks mtime")
-
-
-class ShelveCache:
-    """A class to cache picture blocks in a shelve backend."""
-
-    def __init__(self, db=None, readonly=False):
-        self.istmp = db is None
-        if self.istmp:
-            self.dtmp = tempfile.mkdtemp()
-            self.ftmp = db = op.join(self.dtmp, "tmpdb")
-        flag = "r" if readonly else "c"
-        self.shelve = shelve.open(db, flag)
-        self.maxid = self._compute_maxid()
-
-    def __contains__(self, key):
-        return wrap_path(key) in self.shelve
-
-    def __delitem__(self, key):
-        row = self.shelve[wrap_path(key)]
-        del self.shelve[wrap_path(key)]
-        del self.shelve[wrap_id(row.id)]
-
-    def __getitem__(self, key):
-        if isinstance(key, int):
-            skey = self.shelve[wrap_id(key)]
-        else:
-            skey = wrap_path(key)
-        return bytes_to_colors(self.shelve[skey].blocks)
-
-    def __iter__(self):
-        return (unwrap_path(k) for k in self.shelve if k.startswith("path:"))
-
-    def __len__(self):
-        return sum(1 for k in self.shelve if k.startswith("path:"))
-
-    def __setitem__(self, path_str, blocks):
-        blocks = colors_to_bytes(blocks)
-        if op.exists(path_str):
-            mtime = int(os.stat(path_str).st_mtime)
-        else:
-            mtime = 0
-        if path_str in self:
-            rowid = self.shelve[wrap_path(path_str)].id
-        else:
-            rowid = self._get_new_id()
-        row = CacheRow(rowid, path_str, blocks, mtime)
-        self.shelve[wrap_path(path_str)] = row
-        self.shelve[wrap_id(rowid)] = wrap_path(path_str)
-
-    def _compute_maxid(self):
-        return max((unwrap_id(k) for k in self.shelve if k.startswith("id:")), default=1)
-
-    def _get_new_id(self):
-        self.maxid += 1
-        return self.maxid
-
-    def clear(self):
-        self.shelve.clear()
-
-    def close(self):
-        if self.shelve is not None:
-            self.shelve.close()
-            if self.istmp:
-                os.remove(self.ftmp)
-                os.rmdir(self.dtmp)
-        self.shelve = None
-
-    def filter(self, func):
-        to_delete = [key for key in self if not func(key)]
-        for key in to_delete:
-            del self[key]
-
-    def get_id(self, path):
-        if path in self:
-            return self.shelve[wrap_path(path)].id
-        else:
-            raise ValueError(path)
-
-    def get_multiple(self, rowids):
-        for rowid in rowids:
-            try:
-                skey = self.shelve[wrap_id(rowid)]
-            except KeyError:
-                continue
-            yield (rowid, bytes_to_colors(self.shelve[skey].blocks))
-
-    def purge_outdated(self):
-        """Go through the cache and purge outdated records.
-
-        A record is outdated if the picture doesn't exist or if its mtime is greater than the one in
-        the db.
-        """
-        todelete = []
-        for path in self:
-            row = self.shelve[wrap_path(path)]
-            if row.mtime and op.exists(path):
-                picture_mtime = os.stat(path).st_mtime
-                if int(picture_mtime) <= row.mtime:
-                    # not outdated
-                    continue
-            todelete.append(path)
-        for path in todelete:
-            try:
-                del self[path]
-            except KeyError:
-                # I have no idea why a KeyError sometimes happen, but it does, as we can see in
-                # #402 and #439. I don't think it hurts to silently ignore the error, so that's
-                # what we do
-                pass

core/pe/matchblock.py

@@ -16,6 +16,7 @@ from hscommon.jobprogress import job
 
 from core.engine import Match
 from core.pe.block import avgdiff, DifferentBlockCountError, NoBlocksError
+from core.pe.cache_sqlite import SqliteCache
 
 # OPTIMIZATION NOTES:
 # The bottleneck of the matching phase is CPU, which is why we use multiprocessing. However, another
@@ -50,14 +51,7 @@ except Exception:
 
 
 def get_cache(cache_path, readonly=False):
-    if cache_path.endswith("shelve"):
-        from core.pe.cache_shelve import ShelveCache
-
-        return ShelveCache(cache_path, readonly=readonly)
-    else:
-        from core.pe.cache_sqlite import SqliteCache
-
-        return SqliteCache(cache_path, readonly=readonly)
+    return SqliteCache(cache_path, readonly=readonly)
 
 
 def prepare_pictures(pictures, cache_path, with_dimensions, j=job.nulljob):

core/pe/modules/block.c

@@ -245,4 +245,4 @@ PyObject *PyInit__block(void) {
   PyModule_AddObject(m, "DifferentBlockCountError", DifferentBlockCountError);
 
   return m;
-}
+}

core/pe/modules/block_osx.m

@@ -2,8 +2,8 @@
  * Created On: 2010-02-04
  * Copyright 2015 Hardcoded Software (http://www.hardcoded.net)
  *
- * This software is licensed under the "GPLv3" License as described in the "LICENSE" file, 
- * which should be included with this package. The terms are also available at 
+ * This software is licensed under the "GPLv3" License as described in the "LICENSE" file,
+ * which should be included with this package. The terms are also available at
  * http://www.gnu.org/licenses/gpl-3.0.html
 **/
 
@@ -22,7 +22,7 @@ pystring2cfstring(PyObject *pystring)
     UInt8 *s;
     CFIndex size;
     CFStringRef result;
-    
+
     if (PyUnicode_Check(pystring)) {
         encoded = PyUnicode_AsUTF8String(pystring);
         if (encoded == NULL) {
@@ -32,7 +32,7 @@ pystring2cfstring(PyObject *pystring)
         encoded = pystring;
         Py_INCREF(encoded);
     }
-    
+
     s = (UInt8*)PyBytes_AS_STRING(encoded);
     size = PyBytes_GET_SIZE(encoded);
     result = CFStringCreateWithBytes(NULL, s, size, kCFStringEncodingUTF8, FALSE);
@@ -50,20 +50,20 @@ static PyObject* block_osx_get_image_size(PyObject *self, PyObject *args)
     long width, height;
     PyObject *pwidth, *pheight;
     PyObject *result;
-    
+
     width = 0;
     height = 0;
     if (!PyArg_ParseTuple(args, "O", &path)) {
         return NULL;
     }
-    
+
     image_path = pystring2cfstring(path);
     if (image_path == NULL) {
         return PyErr_NoMemory();
     }
     image_url = CFURLCreateWithFileSystemPath(NULL, image_path, kCFURLPOSIXPathStyle, FALSE);
     CFRelease(image_path);
-    
+
     source = CGImageSourceCreateWithURL(image_url, NULL);
     CFRelease(image_url);
     if (source != NULL) {
@@ -75,7 +75,7 @@ static PyObject* block_osx_get_image_size(PyObject *self, PyObject *args)
         }
         CFRelease(source);
     }
-    
+
     pwidth = PyLong_FromLong(width);
     if (pwidth == NULL) {
         return NULL;
@@ -91,19 +91,19 @@ static PyObject* block_osx_get_image_size(PyObject *self, PyObject *args)
 }
 
 static CGContextRef
-MyCreateBitmapContext(int width, int height) 
+MyCreateBitmapContext(int width, int height)
 {
     CGContextRef context = NULL;
     CGColorSpaceRef colorSpace;
     void *bitmapData;
     int bitmapByteCount;
     int bitmapBytesPerRow;
-    
+
     bitmapBytesPerRow = (width * 4);
     bitmapByteCount = (bitmapBytesPerRow * height);
-    
+
     colorSpace = CGColorSpaceCreateWithName(kCGColorSpaceGenericRGB);
-    
+
     // calloc() must be used to allocate bitmapData here because the buffer has to be zeroed.
     // If it's not zeroes, when images with transparency are drawn in the context, this buffer
     // will stay with undefined pixels, which means that two pictures with the same pixels will
@@ -113,7 +113,7 @@ MyCreateBitmapContext(int width, int height)
         fprintf(stderr, "Memory not allocated!");
         return NULL;
     }
-    
+
     context = CGBitmapContextCreate(bitmapData, width, height, 8, bitmapBytesPerRow, colorSpace,
         (CGBitmapInfo)kCGImageAlphaNoneSkipLast);
     if (context== NULL) {
@@ -128,7 +128,7 @@ MyCreateBitmapContext(int width, int height)
 static PyObject* getblock(unsigned char *imageData, int imageWidth, int imageHeight, int boxX, int boxY, int boxW, int boxH)
 {
     int i,j, totalR, totalG, totalB;
-    
+
     totalR = totalG = totalB = 0;
     for(i=boxY; i<boxY+boxH; i++) {
         for(j=boxX; j<boxX+boxW; j++) {
@@ -142,7 +142,7 @@ static PyObject* getblock(unsigned char *imageData, int imageWidth, int imageHei
     totalR /= pixelCount;
     totalG /= pixelCount;
     totalB /= pixelCount;
-    
+
     return inttuple(3, totalR, totalG, totalB);
 }
 
@@ -155,27 +155,27 @@ static PyObject* block_osx_getblocks(PyObject *self, PyObject *args)
     CGImageRef image;
     size_t width, height, image_width, image_height;
     int block_count, block_width, block_height, orientation, i;
-    
+
     if (!PyArg_ParseTuple(args, "Oii", &path, &block_count, &orientation)) {
         return NULL;
     }
-    
+
     if (PySequence_Length(path) == 0) {
         PyErr_SetString(PyExc_ValueError, "empty path");
         return NULL;
     }
-    
+
     if ((orientation > 8) || (orientation < 0)) {
         orientation = 0; // simplifies checks later since we can only have values in 0-8
     }
-    
+
     image_path = pystring2cfstring(path);
     if (image_path == NULL) {
         return PyErr_NoMemory();
     }
     image_url = CFURLCreateWithFileSystemPath(NULL, image_path, kCFURLPOSIXPathStyle, FALSE);
     CFRelease(image_path);
-    
+
     source = CGImageSourceCreateWithURL(image_url, NULL);
     CFRelease(image_url);
     if (source == NULL) {
@@ -187,8 +187,8 @@ static PyObject* block_osx_getblocks(PyObject *self, PyObject *args)
         CFRelease(source);
         return PyErr_NoMemory();
     }
-    
-    
+
+
     width = image_width = CGImageGetWidth(image);
     height = image_height = CGImageGetHeight(image);
     if (orientation >= 5) {
@@ -196,9 +196,9 @@ static PyObject* block_osx_getblocks(PyObject *self, PyObject *args)
         width = image_height;
         height = image_width;
     }
-    
+
     CGContextRef context = MyCreateBitmapContext(width, height);
-    
+
     if (orientation == 2) {
         // Flip X
         CGContextTranslateCTM(context, width, 0);
@@ -207,7 +207,7 @@ static PyObject* block_osx_getblocks(PyObject *self, PyObject *args)
     else if (orientation == 3) {
         // Rot 180
         CGContextTranslateCTM(context, width, height);
-        CGContextRotateCTM(context, RADIANS(180)); 
+        CGContextRotateCTM(context, RADIANS(180));
     }
     else if (orientation == 4) {
         // Flip Y
@@ -242,21 +242,21 @@ static PyObject* block_osx_getblocks(PyObject *self, PyObject *args)
     CGContextDrawImage(context, myBoundingBox, image);
     unsigned char *bitmapData = CGBitmapContextGetData(context);
     CGContextRelease(context);
-    
+
     CGImageRelease(image);
     CFRelease(source);
     if (bitmapData == NULL) {
         return PyErr_NoMemory();
     }
-    
+
     block_width = max(width/block_count, 1);
     block_height = max(height/block_count, 1);
-    
+
     result = PyList_New(block_count * block_count);
     if (result == NULL) {
         return NULL;
     }
-    
+
     for(i=0; i<block_count; i++) {
         int j, top;
         top = min(i*block_height, height-block_height);
@@ -271,8 +271,8 @@ static PyObject* block_osx_getblocks(PyObject *self, PyObject *args)
             PyList_SET_ITEM(result, i*block_count+j, block);
         }
     }
-    
-    free(bitmapData); 
+
+    free(bitmapData);
     return result;
 }
 
@@ -302,4 +302,4 @@ PyInit__block_osx(void)
         return NULL;
     }
     return m;
-}
+}

core/pe/modules/cache.c

@@ -2,73 +2,68 @@
  * Created On: 2010-01-30
  * Copyright 2014 Hardcoded Software (http://www.hardcoded.net)
  *
- * This software is licensed under the "BSD" License as described in the "LICENSE" file, 
- * which should be included with this package. The terms are also available at 
- * http://www.hardcoded.net/licenses/bsd_license
+ * This software is licensed under the "BSD" License as described in the
+ * "LICENSE" file, which should be included with this package. The terms are
+ * also available at http://www.hardcoded.net/licenses/bsd_license
  */
 
 #include "common.h"
 
-static PyObject*
-cache_bytes_to_colors(PyObject *self, PyObject *args)
-{
-    char *y;
-    Py_ssize_t char_count, i, color_count;
-    PyObject *result;
-    unsigned long r, g, b;
-    Py_ssize_t ci;
-    PyObject *color_tuple;
+static PyObject *cache_bytes_to_colors(PyObject *self, PyObject *args) {
+  char *y;
+  Py_ssize_t char_count, i, color_count;
+  PyObject *result;
+  unsigned long r, g, b;
+  Py_ssize_t ci;
+  PyObject *color_tuple;
 
-    if (!PyArg_ParseTuple(args, "y#", &y, &char_count)) {
-        return NULL;
+  if (!PyArg_ParseTuple(args, "y#", &y, &char_count)) {
+    return NULL;
+  }
+
+  color_count = char_count / 3;
+  result = PyList_New(color_count);
+  if (result == NULL) {
+    return NULL;
+  }
+
+  for (i = 0; i < color_count; i++) {
+    ci = i * 3;
+    r = (unsigned char)y[ci];
+    g = (unsigned char)y[ci + 1];
+    b = (unsigned char)y[ci + 2];
+
+    color_tuple = inttuple(3, r, g, b);
+    if (color_tuple == NULL) {
+      Py_DECREF(result);
+      return NULL;
     }
-    
-    color_count = char_count / 3;
-    result = PyList_New(color_count);
-    if (result == NULL) {
-        return NULL;
-    }
-    
-    for (i=0; i<color_count; i++) {
-        ci = i * 3;
-        r = (unsigned char) y[ci];
-        g = (unsigned char) y[ci+1];
-        b = (unsigned char) y[ci+2];
-        
-        color_tuple = inttuple(3, r, g, b);
-        if (color_tuple == NULL) {
-            Py_DECREF(result);
-            return NULL;
-        }
-        PyList_SET_ITEM(result, i, color_tuple);
-    }
-    
-    return result;
+    PyList_SET_ITEM(result, i, color_tuple);
+  }
+
+  return result;
 }
 
 static PyMethodDef CacheMethods[] = {
-    {"bytes_to_colors",  cache_bytes_to_colors, METH_VARARGS, "Transform the bytes 's' into a list of 3 sized tuples."},
-    {NULL, NULL, 0, NULL}        /* Sentinel */
+    {"bytes_to_colors", cache_bytes_to_colors, METH_VARARGS,
+     "Transform the bytes 's' into a list of 3 sized tuples."},
+    {NULL, NULL, 0, NULL} /* Sentinel */
 };
 
-static struct PyModuleDef CacheDef = {
-    PyModuleDef_HEAD_INIT,
-    "_cache",
-    NULL,
-    -1,
-    CacheMethods,
-    NULL,
-    NULL,
-    NULL,
-    NULL
-};
+static struct PyModuleDef CacheDef = {PyModuleDef_HEAD_INIT,
+                                      "_cache",
+                                      NULL,
+                                      -1,
+                                      CacheMethods,
+                                      NULL,
+                                      NULL,
+                                      NULL,
+                                      NULL};
 
-PyObject *
-PyInit__cache(void)
-{
-    PyObject *m = PyModule_Create(&CacheDef);
-    if (m == NULL) {
-        return NULL;
-    }
-    return m;
-}
+PyObject *PyInit__cache(void) {
+  PyObject *m = PyModule_Create(&CacheDef);
+  if (m == NULL) {
+    return NULL;
+  }
+  return m;
+}

core/pe/modules/common.c

@@ -2,8 +2,8 @@
  * Created On: 2010-02-04
  * Copyright 2014 Hardcoded Software (http://www.hardcoded.net)
  *
- * This software is licensed under the "BSD" License as described in the "LICENSE" file, 
- * which should be included with this package. The terms are also available at 
+ * This software is licensed under the "BSD" License as described in the "LICENSE" file,
+ * which should be included with this package. The terms are also available at
  * http://www.hardcoded.net/licenses/bsd_license
  */
 
@@ -27,10 +27,10 @@ PyObject* inttuple(int n, ...)
     PyObject *pnumber;
     PyObject *result;
     va_list numbers;
-    
+
     va_start(numbers, n);
     result = PyTuple_New(n);
-    
+
     for (i=0; i<n; i++) {
         pnumber = PyLong_FromUnsignedLong(va_arg(numbers, long));
         if (pnumber == NULL) {
@@ -39,7 +39,7 @@ PyObject* inttuple(int n, ...)
         }
         PyTuple_SET_ITEM(result, i, pnumber);
     }
-    
+
     va_end(numbers);
     return result;
 }

core/pe/modules/common.h

@@ -2,8 +2,8 @@
  * Created On: 2010-02-04
  * Copyright 2015 Hardcoded Software (http://www.hardcoded.net)
  *
- * This software is licensed under the "GPLv3" License as described in the "LICENSE" file, 
- * which should be included with this package. The terms are also available at 
+ * This software is licensed under the "GPLv3" License as described in the "LICENSE" file,
+ * which should be included with this package. The terms are also available at
  * http://www.gnu.org/licenses/gpl-3.0.html
  */
 

core/pe/photo.py

@@ -29,7 +29,7 @@ class Photo(fs.File):
     __slots__ = fs.File.__slots__ + tuple(INITIAL_INFO.keys())
 
     # These extensions are supported on all platforms
-    HANDLED_EXTS = {"png", "jpg", "jpeg", "gif", "bmp", "tiff", "tif"}
+    HANDLED_EXTS = {"png", "jpg", "jpeg", "gif", "bmp", "tiff", "tif", "webp"}
 
     def _plat_get_dimensions(self):
         raise NotImplementedError()

core/scanner.py

@@ -171,7 +171,8 @@ class Scanner:
             matches = [m for m in matches if m.first.path not in toremove or m.second.path not in toremove]
         if not self.mix_file_kind:
             matches = [m for m in matches if get_file_ext(m.first.name) == get_file_ext(m.second.name)]
-        matches = [m for m in matches if m.first.path.exists() and m.second.path.exists()]
+        if self.include_exists_check:
+            matches = [m for m in matches if m.first.exists() and m.second.exists()]
         matches = [m for m in matches if not (m.first.is_ref and m.second.is_ref)]
         if ignore_list:
             matches = [m for m in matches if not ignore_list.are_ignored(str(m.first.path), str(m.second.path))]
@@ -212,3 +213,4 @@ class Scanner:
     large_size_threshold = 0
     big_file_size_threshold = 0
     word_weighting = False
+    include_exists_check = True

core/tests/cache_test.py

@@ -12,7 +12,6 @@ from hscommon.testutil import eq_
 try:
     from core.pe.cache import colors_to_bytes, bytes_to_colors
     from core.pe.cache_sqlite import SqliteCache
-    from core.pe.cache_shelve import ShelveCache
 except ImportError:
     skip("Can't import the cache module, probably hasn't been compiled.")
 
@@ -134,11 +133,6 @@ class TestCaseSqliteCache(BaseTestCaseCache):
         eq_(c["foo"], [(1, 2, 3)])
 
 
-class TestCaseShelveCache(BaseTestCaseCache):
-    def get_cache(self, dbname=None):
-        return ShelveCache(dbname)
-
-
 class TestCaseCacheSQLEscape:
     def get_cache(self):
         return SqliteCache()

core/tests/scanner_test.py

@@ -17,6 +17,7 @@ from core.scanner import Scanner, ScanType
 from core.me.scanner import ScannerME
 
 
+# TODO update this to be able to inherit from fs.File
 class NamedObject:
     def __init__(self, name="foobar", size=1, path=None):
         if path is None:
@@ -31,6 +32,9 @@ class NamedObject:
     def __repr__(self):
         return "<NamedObject {!r} {!r}>".format(self.name, self.path)
 
+    def exists(self):
+        return self.path.exists()
+
 
 no = NamedObject
 

help/changelog

@@ -57,7 +57,7 @@
 
 === 4.1.1 (2021-03-21)
 
-* Add Japanese 
+* Add Japanese
 * Update internationalization and translations to be up to date with current UI.
 * Minor translation and UI language updates
 * Fix language selection issues on Windows (#760)
@@ -419,7 +419,7 @@
 
 === 2.6.1 (2009-03-27)
 * **Fixed** an occasional crash caused by permission issues.
-* **Fixed** a bug where the "X discarded" notice would show a too large number of discarded 
+* **Fixed** a bug where the "X discarded" notice would show a too large number of discarded
       duplicates.
 
 === 2.6.0 (2008-09-10)
@@ -453,14 +453,14 @@
 * **Added** the "Remove empty folders" option.
 * **Fixed** results load/save issues.
 * **Fixed** occasional status bar inaccuracies when the results are filtered.
-           
+
 
 === 2.5.0 (2007-09-15)
 
 * **Added** post scan filtering.
 * **Fixed** issues with the rename feature under Windows
 * **Fixed** some user interface annoyances under Windows
-                
+
 
 === 2.4.8 (2007-04-14)
 
@@ -476,7 +476,7 @@
 
 * **Added** Re-orderable columns. In fact, I re-added the feature which was lost in the C# conversion in 2.4.0 (Windows).
 * **Changed** the behavior of the scanning engine when setting the hardness to 100. It will now only match files that have their words in the same order.
-* **Fixed** a bug with all the Delete/Move/Copy actions with certain kinds of files.            
+* **Fixed** a bug with all the Delete/Move/Copy actions with certain kinds of files.
 
 === 2.4.5 (2007-01-11)
 
@@ -514,7 +514,7 @@
 
 === 2.3.4 (2006-11-07)
 
-* **Improved** speed and memory usage of the scanning engine, again. Does it mean there was a lot of improvements to be made? Nah...            
+* **Improved** speed and memory usage of the scanning engine, again. Does it mean there was a lot of improvements to be made? Nah...
 
 === 2.3.3 (2006-11-02)
 
@@ -572,7 +572,7 @@
 === 2.2.3 (2006-06-15)
 
 * **Improved** duplicate scanning speed.
-* **Added** a warning that a file couldn't be renamed if a file with the same name already exists.            
+* **Added** a warning that a file couldn't be renamed if a file with the same name already exists.
 
 === 2.2.2 (2006-06-07)
 
@@ -616,9 +616,9 @@
 
 === 2.0.0 (2006-03-17)
 
-* Complete rewrite.         
+* Complete rewrite.
 * Now runs on Mac OS X.
 
 === 1.0.0 (2004-09-24)
 
-* Initial release.     
+* Initial release.

help/de/faq.rst

@@ -71,7 +71,7 @@ Häufig gestellte Fragen
         * Klicken Sie **Markieren --> Alle Markieren**.
 
 .. only:: edition_me
-    
+
     .. topic:: Ich möchte alle Stücke markieren, die mehr als 3 Sekunden von ihrer Referenz verschieden sind. Was kann ich tun?
 
         * Aktivieren Sie den :doc:`Nur Duplikate <results>` Modus.
@@ -83,7 +83,7 @@ Häufig gestellte Fragen
         * Klicken Sie auf **Entferne Ausgewählte von den Ergebnissen**.
 
     .. topic:: Ich möchte meine Stücke mit der höchsten Bitrate zur Referenz machen. Was kann ich tun?
-    
+
         * Aktivieren Sie den :doc:`Nur Duplikate <results>` Modus.
         * Aktivieren Sie den **Deltawerte** Modus.
         * Klicken Sie auf die "Bitrate" Spalte, um nach Bitrate zu sortieren.
@@ -92,9 +92,9 @@ Häufig gestellte Fragen
         * Klicken Sie auf **Mache Ausgewählte zur Referenz**.
 
     .. topic:: Ich möchte nicht das [live] und [remix] Versionen meiner Stücke als Duplikate erkannt werden. Was kann ich tun?
-    
+
         Ist Ihre Vergleichsschwelle niedrig genug, werden möglicherweise die live und remix Versionen in der Ergebnisliste landen. Das kann nicht verhindert werden, aber es gibt die Möglichkeit die Ergebnisse nach dem Scan zu entfernen, mittels dem Filter. Möchten Sie jedes Stück mit irgendetwas in eckigen Klammern [] im Dateinamen entfernen, so:
-    
+
         * **Windows**: Klicken Sie auf **Aktionen --> Filter anwenden**, geben "[*]" ein und klicken OK.
         * **Mac OS X**: Geben Sie "[*]" in das "Filter" Feld der Werkzeugleiste ein.
         * Klicken Sie auf **Markieren --> Alle Markieren**.

help/de/folders.rst

@@ -16,7 +16,7 @@ Jeder Ordner kann in einem von 3 Zuständen sein:
 * **Referenz:** Duplikate in diesem Ordner können **nicht** gelöscht werden. Dateien dieses Ordners können sich nur in der **Referenz** Position einer Duplikatgruppe befinden. Ist mehr als eine Datei des Referenzordners in derselben Duplikatgruppe, so wird nur Eine behalten. Die Anderen werden aus der Gruppe entfernt.
 * **Ausgeschlossen:** Dateien in diesem Verzeichnis sind nicht im Scan eingeschlossen.
 
-Der Standardzustand eines Ordners ist natürlich **Normal**. Sie können den **Referenz** Zustand für Ordner nutzen, in denen auf keinen Fall eine Datei gelöscht werden soll. 
+Der Standardzustand eines Ordners ist natürlich **Normal**. Sie können den **Referenz** Zustand für Ordner nutzen, in denen auf keinen Fall eine Datei gelöscht werden soll.
 
 Wenn sie einen Zustand für ein Verzeichnis setzen, erben alle Unterordner automatisch diesen Zustand, es sei denn Sie ändern den Zustand der Unterordner explizit.
 

help/de/index.rst

@@ -21,7 +21,7 @@ Inhalte:
 
 .. toctree::
     :maxdepth: 2
-    
+
     quick_start
     folders
     preferences

help/de/preferences.rst

@@ -4,9 +4,9 @@ Einstellungen
 .. only:: edition_se
 
     **Scan Typ:** Diese Option bestimmt nach welcher Eigenschaft die Dateien in einem Duplikate Scan verglichen werden. Wenn Sie **Dateiname** auswählen, wird dupeGuru jeden Dateinamen Wort für Wort vergleichen und, abhängig von den unteren Einstellungen, feststellen ob genügend Wörter übereinstimmen, um 2 Dateien als Duplikate zu betrachten. Wenn Sie **Inhalt** wählen, werden nur Dateien mit dem exakt gleichen Inhalt zusammenpassen.
-    
+
     Der **Ordner** Scan Typ ist etwas speziell. Wird er ausgewählt, scannt dupeGuru nach doppelten Ordnern anstelle von Dateien. Um festzustellen ob 2 Ordner identisch sind, werden alle Datein im Ordner gescannt und wenn die Inhalte aller Dateien der Ordner übereinstimmen, werden die Ordner als Duplikate erkannt.
-    
+
     **Filterempfindlichkeit:** Wenn Sie den **Dateiname** Scan Typ wählen, bestimmt diese Option wie ähnlich 2 Dateinamen für dupeGuru sein müssen, um Duplikate zu sein. Ist die Empfindlichkeit zum Beispiel 80, müssen 80% der Worte der 2 Dateinamen übereinstimmen. Um den Übereinstimmungsanteil herauszufinden, zählt dupeGuru zuerst die Gesamtzahl der Wörter **beider** Dateinamen, dann werden die gleichen Wörter gezählt (jedes Wort zählt als 2) und durch die Gesamtzahl der Wörter dividiert. Ist das Resultat größer oder gleich der Filterempfindlichkeit, haben wir ein Duplikat. Zum Beispiel, "a b c d" und "c d e" haben einen Übereinstimmungsanteil von 57 (4 gleiche Wörter, insgesamt 7 Wörter).
 
 .. only:: edition_me
@@ -33,7 +33,7 @@ Einstellungen
 .. only:: edition_pe
 
     **Scan Typ:** Diese option bestimmt, welcher Scan Typ bei Ihren Bildern angewendet wird. Der **Inhalte** Scan Typ vergleicht den Inhalt der Bilder auf eine ungenaue Art und Weise (so werden nicht nur exakte Duplikate gefunden, sondern auch Ähnliche). Der **EXIF Zeitstempel** Scan Typ schaut auf die EXIF Metadaten der Bilder (wenn vorhanden) und erkennt Bilder die den Selben haben. Er ist viel schneller als der Inhalte Scan. **Warnung:** Veränderte Bilder behalten oft den selben EXIF Zeitstempel, also achten Sie auf Falschpositive bei der Nutzung dieses Scans.
-    
+
     **Filterempfindlichkeit:** *Nur Inhalte Scan.* Je höher diese Einstellung, desto strenger ist der Filter (Mit anderen Worten, desto weniger Ergebnisse erhalten Sie). Die meisten Bilder der selben Qualität stimmen zu 100% überein, selbst wenn das Format anders ist (PNG und JPG zum Beispiel). Wie auch immer, wenn ein PNG mit einem JPG niederiger Qualität übereinstimmen soll, muss die Filterempfindlichkeit kleiner als 100 sein. Die Voreinstellung, 95, ist eine gute Wahl.
 
     **Bilder unterschiedlicher Abmessung gleich:** Wird diese Box gewählt, dürfen Bilder unterschiedlicher Abmessung in einer Duplikategruppe sein..
@@ -57,7 +57,7 @@ Auf jeden Fall behandelt dupeGuru Namenskonflikte indem es dem Ziel-Dateinamen e
 **Eigener Befehl:** Diese Einstellung bestimmt den Befehl der durch "Führe eigenen Befehl aus" ausgeführt wird. Sie können jede externe Anwendung durch diese Aktion aufrufen. Dies ist zum Beispiel hilfreich, wenn Sie eine gute diff-Anwendung installiert haben.
 
 Das Format des Befehls ist das Selbe wie in einer Befehlszeile, außer das 2 Platzhalter vorhanden sind: **%d** und **%r**. Diese Platzhalter werden durch den Pfad des markierten Duplikates (%d) und dem Pfad der Duplikatereferenz ersetzt (%r).
-  
+
 Wenn der Pfad Ihrer ausführbaren Datei Leerzeichen enthält, so schließen sie ihn bitte mit "" Zeichen ein. Sie sollten auch Platzhalter mit den Zitatzeichen einschließen, denn es ist möglich, das die Pfade der Duplikate und Referenzen ebenfalls Leerzeichen enthalten. Hier ist ein Beispiel eines eigenen Befehls::
-  
+
     "C:\Program Files\SuperDiffProg\SuperDiffProg.exe" "%d" "%r"

help/de/reprioritize.rst

@@ -22,4 +22,4 @@ criterion is used and so on and so on. For example, if your arguments are "Size
 "Filename (Doesn't end with a number)", the reference file that will be picked in a group will be
 the biggest file, and if two or more files have the same size, the one that has a filename that
 doesn't end with a number will be used. When all criteria result in ties, the order in which dupes
-previously were in the group will be used.
+previously were in the group will be used.

help/de/results.rst

@@ -98,4 +98,4 @@ Aktionen Menü
 * **Ausgewählte umbenennen:** Fragt nach einem neuen Namen und benennt die ausgewählte Datei um.
 
 .. todo:: Add Move and iPhoto/iTunes warning
-.. todo:: Add "Deletion Options" section.
+.. todo:: Add "Deletion Options" section.

help/en/developer/core/engine.rst

@@ -2,12 +2,12 @@ core.engine
 ===========