Update hash db code to upgrade schema

- Improves performance significantly
- Add xxhash to requirements.txt and sort requirements
- Rename md5 based members to digest
- Update all tests to use new member names and hashing methods

NOTE: May consider supporting multiple hashing algorithms in the future.

.github/workflows/default.yml

@@ -45,20 +45,20 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: [3.6, 3.7, 3.8, 3.9, '3.10']
+        python-version: [3.7, 3.8, 3.9, '3.10']
         exclude:
-          - os: macos-latest
-            python-version: 3.6
           - os: macos-latest
             python-version: 3.7
           - os: macos-latest
             python-version: 3.8
-          - os: windows-latest
-            python-version: 3.6
+          - os: macos-latest
+            python-version: 3.9
           - os: windows-latest
             python-version: 3.7
           - os: windows-latest
             python-version: 3.8
+          - os: windows-latest
+            python-version: 3.9
 
     steps:
     - uses: actions/checkout@v2

.sonarcloud.properties

@@ -1 +1 @@
-sonar.python.version=3.6, 3.7, 3.8, 3.9, 3.10
+sonar.python.version=3.7, 3.8, 3.9, 3.10

README.md

@@ -36,7 +36,7 @@ For windows instructions see the [Windows Instructions](Windows.md).
 For macos instructions (qt version) see the [macOS Instructions](macos.md).
 
 ### Prerequisites
-* [Python 3.6+][python]
+* [Python 3.7+][python]
 * PyQt5
 
 ### System Setup

Windows.md

@@ -2,7 +2,7 @@
 
 ### Prerequisites
 
-- [Python 3.6+][python]
+- [Python 3.7+][python]
 - [Visual Studio 2019][vs] or [Visual Studio Build Tools 2019][vsBuildTools] with the Windows 10 SDK
 - [nsis][nsis] (for installer creation)
 - [msys2][msys2] (for using makefile method)
@@ -16,7 +16,7 @@ After installing python it is recommended to update setuptools before compiling
 More details on setting up python for compiling packages on windows can be found on the [python wiki][pythonWindowsCompilers] Take note of the required vc++ versions.
 
 ### With build.py (preferred)
-To build with a different python version 3.6 vs 3.8 or 32 bit vs 64 bit specify that version instead of -3.8 to the `py` command below.  If you want to build additional versions while keeping all virtual environments setup use a different location for each virtual environment.
+To build with a different python version 3.7 vs 3.8 or 32 bit vs 64 bit specify that version instead of -3.8 to the `py` command below.  If you want to build additional versions while keeping all virtual environments setup use a different location for each virtual environment.
 
     $ cd <dupeGuru directory>
     $ py -3.8 -m venv .\env

build.py

@@ -150,8 +150,8 @@ def build_normal():
 
 
 def main():
-    if sys.version_info < (3, 6):
-        sys.exit("Python < 3.6 is unsupported.")
+    if sys.version_info < (3, 7):
+        sys.exit("Python < 3.7 is unsupported.")
     options = parse_args()
     if options.clean and Path("build").exists():
         shutil.rmtree("build")

core/engine.py

@@ -283,7 +283,7 @@ def getmatches_by_contents(files, bigsize=0, j=job.nulljob):
     """Returns a list of :class:`Match` within ``files`` if their contents is the same.
 
     :param bigsize: The size in bytes over which we consider files big enough to
-                    justify taking samples of md5. If 0, compute md5 as usual.
+                    justify taking samples of the file for hashing. If 0, compute digest as usual.
     :param j: A :ref:`job progress instance <jobs>`.
     """
     size2files = defaultdict(set)
@@ -300,15 +300,15 @@ def getmatches_by_contents(files, bigsize=0, j=job.nulljob):
             if first.is_ref and second.is_ref:
                 continue  # Don't spend time comparing two ref pics together.
             if first.size == 0 and second.size == 0:
-                # skip md5 for zero length files
+                # skip hashing for zero length files
                 result.append(Match(first, second, 100))
                 continue
-            if first.md5partial == second.md5partial:
+            if first.digest_partial == second.digest_partial:
                 if bigsize > 0 and first.size > bigsize:
-                    if first.md5samples == second.md5samples:
+                    if first.digest_samples == second.digest_samples:
                         result.append(Match(first, second, 100))
                 else:
-                    if first.md5 == second.md5:
+                    if first.digest == second.digest:
                         result.append(Match(first, second, 100))
         group_count += 1
         j.add_progress(desc=PROGRESS_MESSAGE % (len(result), group_count))

core/fs.py

@@ -11,12 +11,13 @@
 # resulting needless complexity and memory usage. It's been a while since I wanted to do that fork,
 # and I'm doing it now.
 
-import hashlib
+import os
+import xxhash
 from math import floor
 import logging
 import sqlite3
 from threading import Lock
-from typing import Any
+from typing import Any, AnyStr, Union
 
 from hscommon.path import Path
 from hscommon.util import nonone, get_file_ext
@@ -40,7 +41,7 @@ NOT_SET = object()
 # CPU.
 CHUNK_SIZE = 1024 * 1024  # 1 MiB
 
-# Minimum size below which partial hashes don't need to be computed
+# Minimum size below which partial hashing is not used
 MIN_FILE_SIZE = 3 * CHUNK_SIZE  # 3MiB, because we take 3 samples
 
 
@@ -83,9 +84,11 @@ class OperationError(FSError):
 
 
 class FilesDB:
+    schema_version = 1
+    schema_version_description = "Changed from md5 to xxhash"
 
-    create_table_query = "CREATE TABLE IF NOT EXISTS files (path TEXT PRIMARY KEY, size INTEGER, mtime_ns INTEGER, entry_dt DATETIME, md5 BLOB, md5partial BLOB)"
-    drop_table_query = "DROP TABLE files;"
+    create_table_query = "CREATE TABLE IF NOT EXISTS files (path TEXT PRIMARY KEY, size INTEGER, mtime_ns INTEGER, entry_dt DATETIME, digest BLOB, digest_partial BLOB, digest_samples BLOB)"
+    drop_table_query = "DROP TABLE IF EXISTS files;"
     select_query = "SELECT {key} FROM files WHERE path=:path AND size=:size and mtime_ns=:mtime_ns"
     insert_query = """
         INSERT INTO files (path, size, mtime_ns, entry_dt, {key}) VALUES (:path, :size, :mtime_ns, datetime('now'), :value)
@@ -97,24 +100,37 @@ class FilesDB:
         self.cur = None
         self.lock = None
 
-    def connect(self, path):
-        # type: (str, ) -> None
-
+    def connect(self, path: Union[AnyStr, os.PathLike]) -> None:
         self.conn = sqlite3.connect(path, check_same_thread=False)
         self.cur = self.conn.cursor()
-        self.cur.execute(self.create_table_query)
         self.lock = Lock()
+        self._check_upgrade()
 
-    def clear(self):
-        # type: () -> None
+    def _check_upgrade(self) -> None:
+        with self.lock:
+            has_schema = self.cur.execute(
+                "SELECT NAME FROM sqlite_master WHERE type='table' AND name='schema_version'"
+            ).fetchall()
+            version = None
+            if has_schema:
+                version = self.cur.execute("SELECT version FROM schema_version ORDER BY version DESC").fetchone()[0]
+            else:
+                self.cur.execute("CREATE TABLE schema_version (version int PRIMARY KEY, description TEXT)")
+            if version != self.schema_version:
+                self.cur.execute(self.drop_table_query)
+                self.cur.execute(
+                    "INSERT OR REPLACE INTO schema_version VALUES (:version, :description)",
+                    {"version": self.schema_version, "description": self.schema_version_description},
+                )
+            self.cur.execute(self.create_table_query)
+            self.conn.commit()
 
+    def clear(self) -> None:
         with self.lock:
             self.cur.execute(self.drop_table_query)
             self.cur.execute(self.create_table_query)
 
-    def get(self, path, key):
-        # type: (Path, str) -> bytes
-
+    def get(self, path: Path, key: str) -> Union[bytes, None]:
         stat = path.stat()
         size = stat.st_size
         mtime_ns = stat.st_mtime_ns
@@ -128,9 +144,7 @@ class FilesDB:
 
         return None
 
-    def put(self, path, key, value):
-        # type: (Path, str, Any) -> None
-
+    def put(self, path: Path, key: str, value: Any) -> None:
         stat = path.stat()
         size = stat.st_size
         mtime_ns = stat.st_mtime_ns
@@ -141,15 +155,11 @@ class FilesDB:
                 {"path": str(path), "size": size, "mtime_ns": mtime_ns, "value": value},
             )
 
-    def commit(self):
-        # type: () -> None
-
+    def commit(self) -> None:
         with self.lock:
             self.conn.commit()
 
-    def close(self):
-        # type: () -> None
-
+    def close(self) -> None:
         with self.lock:
             self.cur.close()
             self.conn.close()
@@ -161,7 +171,7 @@ filesdb = FilesDB()  # Singleton
 class File:
     """Represents a file and holds metadata to be used for scanning."""
 
-    INITIAL_INFO = {"size": 0, "mtime": 0, "md5": b"", "md5partial": b"", "md5samples": b""}
+    INITIAL_INFO = {"size": 0, "mtime": 0, "digest": b"", "digest_partial": b"", "digest_samples": b""}
     # Slots for File make us save quite a bit of memory. In a memory test I've made with a lot of
     # files, I saved 35% memory usage with "unread" files (no _read_info() call) and gains become
     # even greater when we take into account read attributes (70%!). Yeah, it's worth it.
@@ -187,32 +197,51 @@ class File:
                 result = self.INITIAL_INFO[attrname]
         return result
 
-    def _calc_md5(self):
+    def _calc_digest(self):
         # type: () -> bytes
 
         with self.path.open("rb") as fp:
-            md5 = hashlib.md5()
+            file_hash = xxhash.xxh128()
             # The goal here is to not run out of memory on really big files. However, the chunk
             # size has to be large enough so that the python loop isn't too costly in terms of
             # CPU.
             CHUNK_SIZE = 1024 * 1024  # 1 mb
             filedata = fp.read(CHUNK_SIZE)
             while filedata:
-                md5.update(filedata)
+                file_hash.update(filedata)
                 filedata = fp.read(CHUNK_SIZE)
-            return md5.digest()
+            return file_hash.digest()
 
-    def _calc_md5partial(self):
+    def _calc_digest_partial(self):
         # type: () -> bytes
 
-        # This offset is where we should start reading the file to get a partial md5
+        # This offset is where we should start reading the file to get a partial hash
         # For audio file, it should be where audio data starts
         offset, size = (0x4000, 0x4000)
 
         with self.path.open("rb") as fp:
             fp.seek(offset)
-            partialdata = fp.read(size)
-            return hashlib.md5(partialdata).digest()
+            partial_data = fp.read(size)
+            return xxhash.xxh128_digest(partial_data)
+
+    def _calc_digest_samples(self) -> bytes:
+        size = self.size
+        with self.path.open("rb") as fp:
+            # Chunk at 25% of the file
+            fp.seek(floor(size * 25 / 100), 0)
+            file_data = fp.read(CHUNK_SIZE)
+            file_hash = xxhash.xxh128(file_data)
+
+            # Chunk at 60% of the file
+            fp.seek(floor(size * 60 / 100), 0)
+            file_data = fp.read(CHUNK_SIZE)
+            file_hash.update(file_data)
+
+            # Last chunk of the file
+            fp.seek(-CHUNK_SIZE, 2)
+            file_data = fp.read(CHUNK_SIZE)
+            file_hash.update(file_data)
+            return file_hash.digest()
 
     def _read_info(self, field):
         # print(f"_read_info({field}) for {self}")
@@ -220,48 +249,35 @@ class File:
             stats = self.path.stat()
             self.size = nonone(stats.st_size, 0)
             self.mtime = nonone(stats.st_mtime, 0)
-        elif field == "md5partial":
+        elif field == "digest_partial":
             try:
-                self.md5partial = filesdb.get(self.path, "md5partial")
-                if self.md5partial is None:
-                    self.md5partial = self._calc_md5partial()
-                    filesdb.put(self.path, "md5partial", self.md5partial)
+                self.digest_partial = filesdb.get(self.path, "digest_partial")
+                if self.digest_partial is None:
+                    self.digest_partial = self._calc_digest_partial()
+                    filesdb.put(self.path, "digest_partial", self.digest_partial)
             except Exception as e:
-                logging.warning("Couldn't get md5partial for %s: %s", self.path, e)
-        elif field == "md5":
+                logging.warning("Couldn't get digest_partial for %s: %s", self.path, e)
+        elif field == "digest":
             try:
-                self.md5 = filesdb.get(self.path, "md5")
-                if self.md5 is None:
-                    self.md5 = self._calc_md5()
-                    filesdb.put(self.path, "md5", self.md5)
+                self.digest = filesdb.get(self.path, "digest")
+                if self.digest is None:
+                    self.digest = self._calc_digest()
+                    filesdb.put(self.path, "digest", self.digest)
             except Exception as e:
-                logging.warning("Couldn't get md5 for %s: %s", self.path, e)
-        elif field == "md5samples":
-            try:
-                with self.path.open("rb") as fp:
+                logging.warning("Couldn't get digest for %s: %s", self.path, e)
+        elif field == "digest_samples":
             size = self.size
             # Might as well hash such small files entirely.
             if size <= MIN_FILE_SIZE:
-                        setattr(self, field, self.md5)
+                setattr(self, field, self.digest)
                 return
-
-                    # Chunk at 25% of the file
-                    fp.seek(floor(size * 25 / 100), 0)
-                    filedata = fp.read(CHUNK_SIZE)
-                    md5 = hashlib.md5(filedata)
-
-                    # Chunk at 60% of the file
-                    fp.seek(floor(size * 60 / 100), 0)
-                    filedata = fp.read(CHUNK_SIZE)
-                    md5.update(filedata)
-
-                    # Last chunk of the file
-                    fp.seek(-CHUNK_SIZE, 2)
-                    filedata = fp.read(CHUNK_SIZE)
-                    md5.update(filedata)
-                    setattr(self, field, md5.digest())
+            try:
+                self.digest_samples = filesdb.get(self.path, "digest_samples")
+                if self.digest_samples is None:
+                    self.digest_samples = self._calc_digest_samples()
+                    filesdb.put(self.path, "digest_samples", self.digest_samples)
             except Exception as e:
-                logging.error(f"Error computing md5samples: {e}")
+                logging.warning(f"Couldn't get digest_samples for {self.path}: {e}")
 
     def _read_all_info(self, attrnames=None):
         """Cache all possible info.
@@ -314,7 +330,7 @@ class File:
 class Folder(File):
     """A wrapper around a folder path.
 
-    It has the size/md5 info of a File, but its value is the sum of its subitems.
+    It has the size/digest info of a File, but its value is the sum of its subitems.
     """
 
     __slots__ = File.__slots__ + ("_subfolders",)
@@ -335,19 +351,18 @@ class Folder(File):
             self.size = size
             stats = self.path.stat()
             self.mtime = nonone(stats.st_mtime, 0)
-        elif field in {"md5", "md5partial", "md5samples"}:
+        elif field in {"digest", "digest_partial", "digest_samples"}:
             # What's sensitive here is that we must make sure that subfiles'
-            # md5 are always added up in the same order, but we also want a
-            # different md5 if a file gets moved in a different subdirectory.
+            # digest are always added up in the same order, but we also want a
+            # different digest if a file gets moved in a different subdirectory.
 
-            def get_dir_md5_concat():
+            def get_dir_digest_concat():
                 items = self._all_items()
                 items.sort(key=lambda f: f.path)
-                md5s = [getattr(f, field) for f in items]
-                return b"".join(md5s)
+                digests = [getattr(f, field) for f in items]
+                return b"".join(digests)
 
-            md5 = hashlib.md5(get_dir_md5_concat())
-            digest = md5.digest()
+            digest = xxhash.xxh128_digest(get_dir_digest_concat())
             setattr(self, field, digest)
 
     @property

core/me/fs.py

@@ -97,11 +97,6 @@ class MusicFile(fs.File):
             "dupe_count": format_dupe_count(dupe_count),
         }
 
-    def _get_md5partial_offset_and_size(self):
-        # No longer calculating the offset and audio size, just whole file
-        size = self.path.stat().st_size
-        return (0, size)
-
     def _read_info(self, field):
         fs.File._read_info(self, field)
         if field in TAG_FIELDS:

core/pe/matchblock.py

@@ -238,7 +238,7 @@ def getmatches(pictures, cache_path, threshold, match_scaled=False, j=job.nulljo
     for ref_id, other_id, percentage in myiter:
         ref = id2picture[ref_id]
         other = id2picture[other_id]
-        if percentage == 100 and ref.md5 != other.md5:
+        if percentage == 100 and ref.digest != other.digest:
             percentage = 99
         if percentage >= threshold:
             ref.dimensions  # pre-read dimensions for display in results

core/tests/base.py

@@ -86,9 +86,9 @@ class NamedObject:
             folder = "basepath"
         self._folder = Path(folder)
         self.size = size
-        self.md5partial = name
-        self.md5 = name
-        self.md5samples = name
+        self.digest_partial = name
+        self.digest = name
+        self.digest_samples = name
         if with_words:
             self.words = getwords(name)
         self.is_ref = False

core/tests/engine_test.py

@@ -530,7 +530,7 @@ class TestCaseGetMatches:
 
 
 class TestCaseGetMatchesByContents:
-    def test_big_file_partial_hashes(self):
+    def test_big_file_partial_hashing(self):
         smallsize = 1
         bigsize = 100 * 1024 * 1024  # 100MB
         f = [
@@ -539,17 +539,17 @@ class TestCaseGetMatchesByContents:
             no("smallfoo", size=smallsize),
             no("smallbar", size=smallsize),
         ]
-        f[0].md5 = f[0].md5partial = f[0].md5samples = "foobar"
-        f[1].md5 = f[1].md5partial = f[1].md5samples = "foobar"
-        f[2].md5 = f[2].md5partial = "bleh"
-        f[3].md5 = f[3].md5partial = "bleh"
+        f[0].digest = f[0].digest_partial = f[0].digest_samples = "foobar"
+        f[1].digest = f[1].digest_partial = f[1].digest_samples = "foobar"
+        f[2].digest = f[2].digest_partial = "bleh"
+        f[3].digest = f[3].digest_partial = "bleh"
         r = getmatches_by_contents(f, bigsize=bigsize)
         eq_(len(r), 2)
-        # User disabled optimization for big files, compute hashes as usual
+        # User disabled optimization for big files, compute digests as usual
         r = getmatches_by_contents(f, bigsize=0)
         eq_(len(r), 2)
-        # Other file is now slightly different, md5partial is still the same
-        f[1].md5 = f[1].md5samples = "foobardiff"
+        # Other file is now slightly different, digest_partial is still the same
+        f[1].digest = f[1].digest_samples = "foobardiff"
         r = getmatches_by_contents(f, bigsize=bigsize)
         # Successfully filter it out
         eq_(len(r), 1)

core/tests/fs_test.py

@@ -6,7 +6,7 @@
 # which should be included with this package. The terms are also available at
 # http://www.gnu.org/licenses/gpl-3.0.html
 
-import hashlib
+import xxhash
 from os import urandom
 
 from hscommon.path import Path
@@ -52,54 +52,54 @@ def test_size_aggregates_subfiles(tmpdir):
     eq_(b.size, 12)
 
 
-def test_md5_aggregate_subfiles_sorted(tmpdir):
-    # dir.allfiles can return child in any order. Thus, bundle.md5 must aggregate
-    # all files' md5 it contains, but it must make sure that it does so in the
+def test_digest_aggregate_subfiles_sorted(tmpdir):
+    # dir.allfiles can return child in any order. Thus, bundle.digest must aggregate
+    # all files' digests it contains, but it must make sure that it does so in the
     # same order everytime.
     p = create_fake_fs_with_random_data(Path(str(tmpdir)))
     b = fs.Folder(p)
-    md51 = fs.File(p["dir1"]["file1.test"]).md5
-    md52 = fs.File(p["dir2"]["file2.test"]).md5
-    md53 = fs.File(p["dir3"]["file3.test"]).md5
-    md54 = fs.File(p["file1.test"]).md5
-    md55 = fs.File(p["file2.test"]).md5
-    md56 = fs.File(p["file3.test"]).md5
-    # The expected md5 is the md5 of md5s for folders and the direct md5 for files
-    folder_md51 = hashlib.md5(md51).digest()
-    folder_md52 = hashlib.md5(md52).digest()
-    folder_md53 = hashlib.md5(md53).digest()
-    md5 = hashlib.md5(folder_md51 + folder_md52 + folder_md53 + md54 + md55 + md56)
-    eq_(b.md5, md5.digest())
+    digest1 = fs.File(p["dir1"]["file1.test"]).digest
+    digest2 = fs.File(p["dir2"]["file2.test"]).digest
+    digest3 = fs.File(p["dir3"]["file3.test"]).digest
+    digest4 = fs.File(p["file1.test"]).digest
+    digest5 = fs.File(p["file2.test"]).digest
+    digest6 = fs.File(p["file3.test"]).digest
+    # The expected digest is the hash of digests for folders and the direct digest for files
+    folder_digest1 = xxhash.xxh128_digest(digest1)
+    folder_digest2 = xxhash.xxh128_digest(digest2)
+    folder_digest3 = xxhash.xxh128_digest(digest3)
+    digest = xxhash.xxh128_digest(folder_digest1 + folder_digest2 + folder_digest3 + digest4 + digest5 + digest6)
+    eq_(b.digest, digest)
 
 
-def test_partial_md5_aggregate_subfile_sorted(tmpdir):
+def test_partial_digest_aggregate_subfile_sorted(tmpdir):
     p = create_fake_fs_with_random_data(Path(str(tmpdir)))
     b = fs.Folder(p)
-    md51 = fs.File(p["dir1"]["file1.test"]).md5partial
-    md52 = fs.File(p["dir2"]["file2.test"]).md5partial
-    md53 = fs.File(p["dir3"]["file3.test"]).md5partial
-    md54 = fs.File(p["file1.test"]).md5partial
-    md55 = fs.File(p["file2.test"]).md5partial
-    md56 = fs.File(p["file3.test"]).md5partial
-    # The expected md5 is the md5 of md5s for folders and the direct md5 for files
-    folder_md51 = hashlib.md5(md51).digest()
-    folder_md52 = hashlib.md5(md52).digest()
-    folder_md53 = hashlib.md5(md53).digest()
-    md5 = hashlib.md5(folder_md51 + folder_md52 + folder_md53 + md54 + md55 + md56)
-    eq_(b.md5partial, md5.digest())
+    digest1 = fs.File(p["dir1"]["file1.test"]).digest_partial
+    digest2 = fs.File(p["dir2"]["file2.test"]).digest_partial
+    digest3 = fs.File(p["dir3"]["file3.test"]).digest_partial
+    digest4 = fs.File(p["file1.test"]).digest_partial
+    digest5 = fs.File(p["file2.test"]).digest_partial
+    digest6 = fs.File(p["file3.test"]).digest_partial
+    # The expected digest is the hash of digests for folders and the direct digest for files
+    folder_digest1 = xxhash.xxh128_digest(digest1)
+    folder_digest2 = xxhash.xxh128_digest(digest2)
+    folder_digest3 = xxhash.xxh128_digest(digest3)
+    digest = xxhash.xxh128_digest(folder_digest1 + folder_digest2 + folder_digest3 + digest4 + digest5 + digest6)
+    eq_(b.digest_partial, digest)
 
-    md51 = fs.File(p["dir1"]["file1.test"]).md5samples
-    md52 = fs.File(p["dir2"]["file2.test"]).md5samples
-    md53 = fs.File(p["dir3"]["file3.test"]).md5samples
-    md54 = fs.File(p["file1.test"]).md5samples
-    md55 = fs.File(p["file2.test"]).md5samples
-    md56 = fs.File(p["file3.test"]).md5samples
-    # The expected md5 is the md5 of md5s for folders and the direct md5 for files
-    folder_md51 = hashlib.md5(md51).digest()
-    folder_md52 = hashlib.md5(md52).digest()
-    folder_md53 = hashlib.md5(md53).digest()
-    md5 = hashlib.md5(folder_md51 + folder_md52 + folder_md53 + md54 + md55 + md56)
-    eq_(b.md5samples, md5.digest())
+    digest1 = fs.File(p["dir1"]["file1.test"]).digest_samples
+    digest2 = fs.File(p["dir2"]["file2.test"]).digest_samples
+    digest3 = fs.File(p["dir3"]["file3.test"]).digest_samples
+    digest4 = fs.File(p["file1.test"]).digest_samples
+    digest5 = fs.File(p["file2.test"]).digest_samples
+    digest6 = fs.File(p["file3.test"]).digest_samples
+    # The expected digest is the digest of digests for folders and the direct digest for files
+    folder_digest1 = xxhash.xxh128_digest(digest1)
+    folder_digest2 = xxhash.xxh128_digest(digest2)
+    folder_digest3 = xxhash.xxh128_digest(digest3)
+    digest = xxhash.xxh128_digest(folder_digest1 + folder_digest2 + folder_digest3 + digest4 + digest5 + digest6)
+    eq_(b.digest_samples, digest)
 
 
 def test_has_file_attrs(tmpdir):

core/tests/scanner_test.py

@@ -123,19 +123,19 @@ def test_content_scan(fake_fileexists):
     s = Scanner()
     s.scan_type = ScanType.CONTENTS
     f = [no("foo"), no("bar"), no("bleh")]
-    f[0].md5 = f[0].md5partial = f[0].md5samples = "foobar"
-    f[1].md5 = f[1].md5partial = f[1].md5samples = "foobar"
-    f[2].md5 = f[2].md5partial = f[1].md5samples = "bleh"
+    f[0].digest = f[0].digest_partial = f[0].digest_samples = "foobar"
+    f[1].digest = f[1].digest_partial = f[1].digest_samples = "foobar"
+    f[2].digest = f[2].digest_partial = f[1].digest_samples = "bleh"
     r = s.get_dupe_groups(f)
     eq_(len(r), 1)
     eq_(len(r[0]), 2)
-    eq_(s.discarded_file_count, 0)  # don't count the different md5 as discarded!
+    eq_(s.discarded_file_count, 0)  # don't count the different digest as discarded!
 
 
 def test_content_scan_compare_sizes_first(fake_fileexists):
     class MyFile(no):
         @property
-        def md5(self):
+        def digest(self):
             raise AssertionError()
 
     s = Scanner()
@@ -161,14 +161,14 @@ def test_ignore_file_size(fake_fileexists):
         no("largeignore1", large_size + 1),
         no("largeignore2", large_size + 1),
     ]
-    f[0].md5 = f[0].md5partial = f[0].md5samples = "smallignore"
-    f[1].md5 = f[1].md5partial = f[1].md5samples = "smallignore"
-    f[2].md5 = f[2].md5partial = f[2].md5samples = "small"
-    f[3].md5 = f[3].md5partial = f[3].md5samples = "small"
-    f[4].md5 = f[4].md5partial = f[4].md5samples = "large"
-    f[5].md5 = f[5].md5partial = f[5].md5samples = "large"
-    f[6].md5 = f[6].md5partial = f[6].md5samples = "largeignore"
-    f[7].md5 = f[7].md5partial = f[7].md5samples = "largeignore"
+    f[0].digest = f[0].digest_partial = f[0].digest_samples = "smallignore"
+    f[1].digest = f[1].digest_partial = f[1].digest_samples = "smallignore"
+    f[2].digest = f[2].digest_partial = f[2].digest_samples = "small"
+    f[3].digest = f[3].digest_partial = f[3].digest_samples = "small"
+    f[4].digest = f[4].digest_partial = f[4].digest_samples = "large"
+    f[5].digest = f[5].digest_partial = f[5].digest_samples = "large"
+    f[6].digest = f[6].digest_partial = f[6].digest_samples = "largeignore"
+    f[7].digest = f[7].digest_partial = f[7].digest_samples = "largeignore"
 
     r = s.get_dupe_groups(f)
     # No ignores
@@ -197,21 +197,21 @@ def test_big_file_partial_hashes(fake_fileexists):
     s.big_file_size_threshold = bigsize
 
     f = [no("bigfoo", bigsize), no("bigbar", bigsize), no("smallfoo", smallsize), no("smallbar", smallsize)]
-    f[0].md5 = f[0].md5partial = f[0].md5samples = "foobar"
-    f[1].md5 = f[1].md5partial = f[1].md5samples = "foobar"
-    f[2].md5 = f[2].md5partial = "bleh"
-    f[3].md5 = f[3].md5partial = "bleh"
+    f[0].digest = f[0].digest_partial = f[0].digest_samples = "foobar"
+    f[1].digest = f[1].digest_partial = f[1].digest_samples = "foobar"
+    f[2].digest = f[2].digest_partial = "bleh"
+    f[3].digest = f[3].digest_partial = "bleh"
     r = s.get_dupe_groups(f)
     eq_(len(r), 2)
 
-    # md5partial is still the same, but the file is actually different
-    f[1].md5 = f[1].md5samples = "difffoobar"
-    # here we compare the full md5s, as the user disabled the optimization
+    # digest_partial is still the same, but the file is actually different
+    f[1].digest = f[1].digest_samples = "difffoobar"
+    # here we compare the full digests, as the user disabled the optimization
     s.big_file_size_threshold = 0
     r = s.get_dupe_groups(f)
     eq_(len(r), 1)
 
-    # here we should compare the md5samples, and see they are different
+    # here we should compare the digest_samples, and see they are different
     s.big_file_size_threshold = bigsize
     r = s.get_dupe_groups(f)
     eq_(len(r), 1)
@@ -221,9 +221,9 @@ def test_min_match_perc_doesnt_matter_for_content_scan(fake_fileexists):
     s = Scanner()
     s.scan_type = ScanType.CONTENTS
     f = [no("foo"), no("bar"), no("bleh")]
-    f[0].md5 = f[0].md5partial = f[0].md5samples = "foobar"
-    f[1].md5 = f[1].md5partial = f[1].md5samples = "foobar"
-    f[2].md5 = f[2].md5partial = f[2].md5samples = "bleh"
+    f[0].digest = f[0].digest_partial = f[0].digest_samples = "foobar"
+    f[1].digest = f[1].digest_partial = f[1].digest_samples = "foobar"
+    f[2].digest = f[2].digest_partial = f[2].digest_samples = "bleh"
     s.min_match_percentage = 101
     r = s.get_dupe_groups(f)
     eq_(len(r), 1)
@@ -234,12 +234,16 @@ def test_min_match_perc_doesnt_matter_for_content_scan(fake_fileexists):
     eq_(len(r[0]), 2)
 
 
-def test_content_scan_doesnt_put_md5_in_words_at_the_end(fake_fileexists):
+def test_content_scan_doesnt_put_digest_in_words_at_the_end(fake_fileexists):
     s = Scanner()
     s.scan_type = ScanType.CONTENTS
     f = [no("foo"), no("bar")]
-    f[0].md5 = f[0].md5partial = f[0].md5samples = "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
-    f[1].md5 = f[1].md5partial = f[1].md5samples = "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+    f[0].digest = f[0].digest_partial = f[
+        0
+    ].digest_samples = "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
+    f[1].digest = f[1].digest_partial = f[
+        1
+    ].digest_samples = "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"
     r = s.get_dupe_groups(f)
     # FIXME looks like we are missing something here?
     r[0]
@@ -587,21 +591,21 @@ def test_folder_scan_exclude_subfolder_matches(fake_fileexists):
     s = Scanner()
     s.scan_type = ScanType.FOLDERS
     topf1 = no("top folder 1", size=42)
-    topf1.md5 = topf1.md5partial = topf1.md5samples = b"some_md5_1"
+    topf1.digest = topf1.digest_partial = topf1.digest_samples = b"some_digest__1"
     topf1.path = Path("/topf1")
     topf2 = no("top folder 2", size=42)
-    topf2.md5 = topf2.md5partial = topf2.md5samples = b"some_md5_1"
+    topf2.digest = topf2.digest_partial = topf2.digest_samples = b"some_digest__1"
     topf2.path = Path("/topf2")
     subf1 = no("sub folder 1", size=41)
-    subf1.md5 = subf1.md5partial = subf1.md5samples = b"some_md5_2"
+    subf1.digest = subf1.digest_partial = subf1.digest_samples = b"some_digest__2"
     subf1.path = Path("/topf1/sub")
     subf2 = no("sub folder 2", size=41)
-    subf2.md5 = subf2.md5partial = subf2.md5samples = b"some_md5_2"
+    subf2.digest = subf2.digest_partial = subf2.digest_samples = b"some_digest__2"
     subf2.path = Path("/topf2/sub")
     eq_(len(s.get_dupe_groups([topf1, topf2, subf1, subf2])), 1)  # only top folders
     # however, if another folder matches a subfolder, keep in in the matches
     otherf = no("other folder", size=41)
-    otherf.md5 = otherf.md5partial = otherf.md5samples = b"some_md5_2"
+    otherf.digest = otherf.digest_partial = otherf.digest_samples = b"some_digest__2"
     otherf.path = Path("/otherfolder")
     eq_(len(s.get_dupe_groups([topf1, topf2, subf1, subf2, otherf])), 2)
 
@@ -624,9 +628,9 @@ def test_dont_count_ref_files_as_discarded(fake_fileexists):
     o1 = no("foo", path="p1")
     o2 = no("foo", path="p2")
     o3 = no("foo", path="p3")
-    o1.md5 = o1.md5partial = o1.md5samples = "foobar"
-    o2.md5 = o2.md5partial = o2.md5samples = "foobar"
-    o3.md5 = o3.md5partial = o3.md5samples = "foobar"
+    o1.digest = o1.digest_partial = o1.digest_samples = "foobar"
+    o2.digest = o2.digest_partial = o2.digest_samples = "foobar"
+    o3.digest = o3.digest_partial = o3.digest_samples = "foobar"
     o1.is_ref = True
     o2.is_ref = True
     eq_(len(s.get_dupe_groups([o1, o2, o3])), 1)

hscommon/trans.py

@@ -150,7 +150,9 @@ def install_gettext_trans_under_qt(base_folder, lang=None):
     if not lang:
         lang = str(QLocale.system().name())[:2]
     localename = get_locale_name(lang)
-    if localename is not None:
+    if localename is None:
+        lang = "en"
+        localename = get_locale_name(lang)
     try:
         locale.setlocale(locale.LC_ALL, localename)
     except locale.Error:

macos.md

@@ -2,7 +2,7 @@
 
 ### Prerequisites
 
-- [Python 3.6+][python]
+- [Python 3.7+][python]
 - [Xcode 12.3][xcode] or just Xcode command line tools (older versions can be used if not interested in arm macs)
 - [Homebrew][homebrew]
 - [qt5](https://www.qt.io/)
@@ -12,7 +12,7 @@
 2. Install [Homebrew][homebrew], if not on the path after install (arm based Macs) create `~/.zshrc` 
 with `export PATH="/opt/homebrew/bin:$PATH"`. Will need to reload terminal or source the file to take 
 effect.
-3. Install qt5 with `brew`. If you are using a version of macos without system python 3.6+ then you will 
+3. Install qt5 with `brew`. If you are using a version of macos without system python 3.7+ then you will 
 also need to install that via brew or with pyenv.
 
         $ brew install qt5

pkg/debian/control

@@ -10,7 +10,7 @@ Vcs-Git: https://github.com/arsenetar/dupeguru.git
 
 Package: {pkgname}
 Architecture: {arch}
-Depends: ${shlibs:Depends}, python3 (>=3.6), python3-pyqt5, python3-mutagen
+Depends: ${shlibs:Depends}, python3 (>=3.7), python3-pyqt5, python3-mutagen
 Provides: dupeguru-se, dupeguru-me, dupeguru-pe
 Replaces: dupeguru-se, dupeguru-me, dupeguru-pe
 Conflicts: dupeguru-se, dupeguru-me, dupeguru-pe

qt/pe/image_viewer.py

@@ -1041,26 +1041,44 @@ class ScrollAreaImageViewer(QScrollArea):
         """After scaling, no mouse position, default to center."""
         # scrollBar.setMaximum(scrollBar.maximum() - scrollBar.minimum() + scrollBar.pageStep())
         self._horizontalScrollBar.setValue(
-            int(factor * self._horizontalScrollBar.value() + ((factor - 1) * self._horizontalScrollBar.pageStep() / 2))
+            int(
+                factor * self._horizontalScrollBar.value()
+                + ((factor - 1) * self._horizontalScrollBar.pageStep() / 2)
+            )
         )
         self._verticalScrollBar.setValue(
-            int(factor * self._verticalScrollBar.value() + ((factor - 1) * self._verticalScrollBar.pageStep() / 2))
+            int(
+                factor * self._verticalScrollBar.value()
+                + ((factor - 1) * self._verticalScrollBar.pageStep() / 2)
+            )
         )
 
     def adjustScrollBarsScaled(self, delta):
         """After scaling with the mouse, update relative to mouse position."""
-        self._horizontalScrollBar.setValue(self._horizontalScrollBar.value() + delta.x())
-        self._verticalScrollBar.setValue(self._verticalScrollBar.value() + delta.y())
+        self._horizontalScrollBar.setValue(
+            int(self._horizontalScrollBar.value() + delta.x())
+        )
+        self._verticalScrollBar.setValue(
+            int(self._verticalScrollBar.value() + delta.y())
+        )
 
     def adjustScrollBarsAuto(self):
         """After panning, update accordingly."""
-        self.horizontalScrollBar().setValue(self.horizontalScrollBar().value() - self._mousePanningDelta.x())
-        self.verticalScrollBar().setValue(self.verticalScrollBar().value() - self._mousePanningDelta.y())
+        self.horizontalScrollBar().setValue(
+            int(self.horizontalScrollBar().value() - self._mousePanningDelta.x())
+        )
+        self.verticalScrollBar().setValue(
+            int(self.verticalScrollBar().value() - self._mousePanningDelta.y())
+        )
 
     def adjustScrollBarCentered(self):
         """Just center in the middle."""
-        self._horizontalScrollBar.setValue(int(self._horizontalScrollBar.maximum() / 2))
-        self._verticalScrollBar.setValue(int(self._verticalScrollBar.maximum() / 2))
+        self._horizontalScrollBar.setValue(
+            int(self._horizontalScrollBar.maximum() / 2)
+        )
+        self._verticalScrollBar.setValue(
+            int(self._verticalScrollBar.maximum() / 2)
+        )
 
     def resetCenter(self):
         """Resets origin"""

requirements.txt

@@ -1,7 +1,7 @@
-Send2Trash>=1.3.0
-sphinx>=3.0.0
-polib>=1.1.0
-mutagen>=1.44.0
 distro>=1.5.0
+mutagen>=1.44.0
 PyQt5 >=5.14.1,<6.0; sys_platform != 'linux'
 pywin32>=228; sys_platform == 'win32'
+Send2Trash>=1.3.0
+sphinx>=3.0.0
+xxhash>=3.0.0,<4.0.0

setup.cfg

@@ -18,16 +18,16 @@ classifiers =
     Operating System :: MacOS :: MacOS X
     Operating System :: Microsoft :: Windows
     Operating System :: POSIX
-    Programming Language :: Python :: 3.6
     Programming Language :: Python :: 3.7
     Programming Language :: Python :: 3.8
     Programming Language :: Python :: 3.9
+    Programming Language :: Python :: 3.10
     Programming Language :: Python :: 3 :: Only
     Topic :: Desktop Environment :: File Managers
 
 [options]
 packages = find:
-python_requires = >=3.6
+python_requires = >=3.7
 install_requires = 
     Send2Trash>=1.3.0
     polib>=1.1.0