More testing with pyproject.toml

Still have some build migration to do and other cleanup.

.github/workflows/default.yml

@@ -52,4 +52,14 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           name: modules ${{ matrix.python-version }}
-          path: ${{ github.workspace }}/**/*.so
+          path: build/**/*.so
+  merge-artifacts:
+    needs: [test]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Merge Artifacts
+        uses: actions/upload-artifact/merge@v4
+        with:
+          name: modules
+          pattern: modules*
+          delete-merged: true

.vscode/launch.json

@@ -6,7 +6,7 @@
     "configurations": [
         {
             "name": "DupuGuru",
-            "type": "python",
+            "type": "debugpy",
             "request": "launch",
             "program": "run.py",
             "console": "integratedTerminal",

.vscode/settings.json

@@ -12,5 +12,6 @@
     "[python]": {
         "editor.formatOnSave": true,
         "editor.defaultFormatter": "ms-python.black-formatter"
-    }
+    },
+    "python.testing.pytestEnabled": true
 }

core/pe/cache_sqlite.py

@@ -158,7 +158,7 @@ class SqliteCache:
         ids = ",".join(map(str, rowids))
         sql = (
             "select rowid, blocks, blocks2, blocks3, blocks4, blocks5, blocks6, blocks7, blocks8 "
-            f"from pictures where rowid in {ids}"
+            f"from pictures where rowid in ({ids})"
         )
         cur = self.con.execute(sql)
         return (

core/pe/matchblock.py

@@ -54,7 +54,7 @@ def get_cache(cache_path, readonly=False):
     return SqliteCache(cache_path, readonly=readonly)
 
 
-def prepare_pictures(pictures, cache_path, with_dimensions, j=job.nulljob):
+def prepare_pictures(pictures, cache_path, with_dimensions, match_rotated, j=job.nulljob):
     # The MemoryError handlers in there use logging without first caring about whether or not
     # there is enough memory left to carry on the operation because it is assumed that the
     # MemoryError happens when trying to read an image file, which is freed from memory by the
@@ -76,8 +76,14 @@ def prepare_pictures(pictures, cache_path, with_dimensions, j=job.nulljob):
             if with_dimensions:
                 picture.dimensions  # pre-read dimensions
             try:
-                if picture.unicode_path not in cache:
+                if picture.unicode_path not in cache or (
-                    blocks = [picture.get_blocks(BLOCK_COUNT_PER_SIDE, orientation) for orientation in range(1, 9)]
+                    match_rotated and any(block == [] for block in cache[picture.unicode_path])
+                ):
+                    if match_rotated:
+                        blocks = [picture.get_blocks(BLOCK_COUNT_PER_SIDE, orientation) for orientation in range(1, 9)]
+                    else:
+                        blocks = [[]] * 8
+                        blocks[max(picture.get_orientation() - 1, 0)] = picture.get_blocks(BLOCK_COUNT_PER_SIDE)
                     cache[picture.unicode_path] = blocks
                 prepared.append(picture)
             except (OSError, ValueError) as e:
@@ -187,7 +193,7 @@ def getmatches(pictures, cache_path, threshold, match_scaled=False, match_rotate
         j.set_progress(comparison_count, progress_msg)
 
     j = j.start_subjob([3, 7])
-    pictures = prepare_pictures(pictures, cache_path, with_dimensions=not match_scaled, j=j)
+    pictures = prepare_pictures(pictures, cache_path, not match_scaled, match_rotated, j=j)
     j = j.start_subjob([9, 1], tr("Preparing for matching"))
     cache = get_cache(cache_path)
     id2picture = {}

core/pe/photo.py

@@ -37,7 +37,7 @@ class Photo(fs.File):
     def _plat_get_blocks(self, block_count_per_side, orientation):
         raise NotImplementedError()
 
-    def _get_orientation(self):
+    def get_orientation(self):
         if not hasattr(self, "_cached_orientation"):
             try:
                 with self.path.open("rb") as fp:
@@ -95,13 +95,13 @@ class Photo(fs.File):
         fs.File._read_info(self, field)
         if field == "dimensions":
             self.dimensions = self._plat_get_dimensions()
-            if self._get_orientation() in {5, 6, 7, 8}:
+            if self.get_orientation() in {5, 6, 7, 8}:
                 self.dimensions = (self.dimensions[1], self.dimensions[0])
         elif field == "exif_timestamp":
             self.exif_timestamp = self._get_exif_timestamp()
 
     def get_blocks(self, block_count_per_side, orientation: int = None):
         if orientation is None:
-            return self._plat_get_blocks(block_count_per_side, self._get_orientation())
+            return self._plat_get_blocks(block_count_per_side, self.get_orientation())
         else:
             return self._plat_get_blocks(block_count_per_side, orientation)

core/prioritize.py

@@ -96,6 +96,8 @@ class FilenameCategory(CriterionCategory):
     DOESNT_END_WITH_NUMBER = 1
     LONGEST = 2
     SHORTEST = 3
+    LONGEST_PATH = 4
+    SHORTEST_PATH = 5
 
     def format_criterion_value(self, value):
         return {
@@ -103,6 +105,8 @@ class FilenameCategory(CriterionCategory):
             self.DOESNT_END_WITH_NUMBER: tr("Doesn't end with number"),
             self.LONGEST: tr("Longest"),
             self.SHORTEST: tr("Shortest"),
+            self.LONGEST_PATH: tr("Longest Path"),
+            self.SHORTEST_PATH: tr("Shortest Path"),
         }[value]
 
     def extract_value(self, dupe):
@@ -116,6 +120,10 @@ class FilenameCategory(CriterionCategory):
                 return 0 if ends_with_digit else 1
             else:
                 return 1 if ends_with_digit else 0
+        elif crit_value == self.LONGEST_PATH:
+            return len(str(dupe.folder_path)) * -1
+        elif crit_value == self.SHORTEST_PATH:
+            return len(str(dupe.folder_path))
         else:
             value = len(value)
             if crit_value == self.LONGEST:
@@ -130,6 +138,8 @@ class FilenameCategory(CriterionCategory):
                 self.DOESNT_END_WITH_NUMBER,
                 self.LONGEST,
                 self.SHORTEST,
+                self.LONGEST_PATH,
+                self.SHORTEST_PATH,
             ]
         ]
 

core/tests/cache_test.py

@@ -59,13 +59,13 @@ class BaseTestCaseCache:
 
     def test_set_then_retrieve_blocks(self):
         c = self.get_cache()
-        b = [(0, 0, 0), (1, 2, 3)]
+        b = [[(0, 0, 0), (1, 2, 3)]] * 8
         c["foo"] = b
         eq_(b, c["foo"])
 
     def test_delitem(self):
         c = self.get_cache()
-        c["foo"] = ""
+        c["foo"] = [[]] * 8
         del c["foo"]
         assert "foo" not in c
         with raises(KeyError):
@@ -74,16 +74,16 @@ class BaseTestCaseCache:
     def test_persistance(self, tmpdir):
         DBNAME = tmpdir.join("hstest.db")
         c = self.get_cache(str(DBNAME))
-        c["foo"] = [(1, 2, 3)]
+        c["foo"] = [[(1, 2, 3)]] * 8
         del c
         c = self.get_cache(str(DBNAME))
-        eq_([(1, 2, 3)], c["foo"])
+        eq_([[(1, 2, 3)]] * 8, c["foo"])
 
     def test_filter(self):
         c = self.get_cache()
-        c["foo"] = ""
+        c["foo"] = [[]] * 8
-        c["bar"] = ""
+        c["bar"] = [[]] * 8
-        c["baz"] = ""
+        c["baz"] = [[]] * 8
         c.filter(lambda p: p != "bar")  # only 'bar' is removed
         eq_(2, len(c))
         assert "foo" in c
@@ -92,9 +92,9 @@ class BaseTestCaseCache:
 
     def test_clear(self):
         c = self.get_cache()
-        c["foo"] = ""
+        c["foo"] = [[]] * 8
-        c["bar"] = ""
+        c["bar"] = [[]] * 8
-        c["baz"] = ""
+        c["baz"] = [[]] * 8
         c.clear()
         eq_(0, len(c))
         assert "foo" not in c
@@ -104,7 +104,7 @@ class BaseTestCaseCache:
     def test_by_id(self):
         # it's possible to use the cache by referring to the files by their row_id
         c = self.get_cache()
-        b = [(0, 0, 0), (1, 2, 3)]
+        b = [[(0, 0, 0), (1, 2, 3)]] * 8
         c["foo"] = b
         foo_id = c.get_id("foo")
         eq_(c[foo_id], b)
@@ -127,10 +127,10 @@ class TestCaseSqliteCache(BaseTestCaseCache):
         fp.write("invalid sqlite content")
         fp.close()
         c = self.get_cache(dbname)  # should not raise a DatabaseError
-        c["foo"] = [(1, 2, 3)]
+        c["foo"] = [[(1, 2, 3)]] * 8
         del c
         c = self.get_cache(dbname)
-        eq_(c["foo"], [(1, 2, 3)])
+        eq_(c["foo"], [[(1, 2, 3)]] * 8)
 
 
 class TestCaseCacheSQLEscape:
@@ -152,7 +152,7 @@ class TestCaseCacheSQLEscape:
 
     def test_delitem(self):
         c = self.get_cache()
-        c["foo'bar"] = []
+        c["foo'bar"] = [[]] * 8
         try:
             del c["foo'bar"]
         except KeyError:

pyproject.toml

@@ -1,9 +1,86 @@
 [build-system]
-requires = ["setuptools"]
+requires = ["setuptools >= 75.3.1"]
 build-backend = "setuptools.build_meta"
+
+[project]
+name = "dupeGuru"
+description = "dupeGuru is a tool to find duplicate files on your computer."
+authors = [
+    {name = "Andrew Senetar", email = "arsenetar@voltaicideas.net"}
+]
+readme = "README.md"
+license = "GPL-3.0-or-later"
+license-files = ["LICENSE"]
+keywords = ["deduplication"]
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Intended Audience :: End Users/Desktop",
+    "Operating System :: MacOS :: MacOS X",
+    "Operating System :: Microsoft :: Windows",
+    "Operating System :: POSIX",
+    "Programming Language :: Python :: 3 :: Only",
+    "Programming Language :: Python :: 3.7",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Desktop Environment :: File Managers",
+]
+requires-python = ">=3.7, <3.13"
+
+dynamic = ["version"]
+
+dependencies = [
+    "distro>=1.8.0,<2.0.0",
+    "mutagen>=1.46.0,<2.0.0",
+    "polib>=1.1.0,<2.0.0",
+    "PyQt5 >=5.15.0,<6.0; sys_platform != 'linux'",
+    "pywin32>=304; sys_platform == 'win32'",
+    "semantic-version>=2.0.0,<3.0.0",
+    "Send2Trash>=1.8.2",
+    "xxhash>=3.0.0,<4.0.0",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=7,<8",
+    "flake8",
+    "black",
+]
+build = [
+    "dupeGuru[dev]",
+    "sphinx>=5.3.0,<8.0.0",
+    "pyinstaller>=5.6,<6.0; sys_platform != 'linux'"
+]
+
+[project.urls]
+Homepage = "https://dupeguru.voltaicideas.net/"
+Documentation = "https://dupeguru.voltaicideas.net/help/en/"
+Repository = "https://github.com/arsenetar/dupeguru.git"
+Issues = "https://github.com/arsenetar/dupeguru/issues"
+Releases = "https://github.com/arsenetar/dupeguru/releases"
+
+[project.gui-scripts]
+dupeguru = "dupeguru.__main__:main"
+
 [tool.black]
 line-length = 120
+
 [tool.isort]
 # make it compatible with black
 profile = "black"
 skip_gitignore = true
+
+[tool.setuptools.packages.find]
+include = ["core", "hscommon", "qt"]
+
+[tool.setuptools.dynamic]
+version = {attr = "core.__version__"}
+
+[tool.setuptools]
+ext-modules = [
+    {name = "core.pe._block", sources = ["core/pe/modules/block.c", "core/pe/modules/common.c"], include-dirs = ["core/pe/modules"]},
+    {name = "core.pe._cache", sources = ["core/pe/modules/cache.c", "core/pe/modules/common.c"], include-dirs = ["core/pe/modules"]},
+    {name = "qt.pe._block_qt", sources = ["qt/pe/modules/block.c"]},
+]

requirements-extra.txt

@@ -1,4 +0,0 @@
-pytest>=7,<8
-flake8
-black
-pyinstaller>=5.6,<6.0; sys_platform != 'linux'

requirements.txt

@@ -1,9 +0,0 @@
-distro>=1.8.0,<2.0.0
-mutagen>=1.46.0,<2.0.0
-polib>=1.1.0,<2.0.0
-PyQt5 >=5.15.0,<6.0; sys_platform != 'linux'
-pywin32>=304; sys_platform == 'win32'
-semantic-version>=2.0.0,<3.0.0
-Send2Trash>=1.8.2,<2.0.0
-sphinx>=5.3.0,<8.0.0
-xxhash>=3.0.0,<4.0.0

setup.cfg

@@ -1,48 +0,0 @@
-[metadata]
-name = dupeGuru
-version = attr: core.__version__
-url = https://github.com/arsenetar/dupeguru
-project_urls =
-    Bug Reports = https://github.com/arsenetar/dupeguru/issues
-author = Andrew Senetar
-author_email = arsenetar@voltaicideas.net
-license = GPLv3
-license_files = license
-description = dupeGuru is a tool to find duplicate files on your computer.
-long_description = file:README.md
-long_description_content_type = text/markdown
-classifiers =
-    Development Status :: 5 - Production/Stable
-    Intended Audience :: End Users/Desktop
-    License :: OSI Approved :: GNU General Public License v3 (GPLv3)
-    Operating System :: MacOS :: MacOS X
-    Operating System :: Microsoft :: Windows
-    Operating System :: POSIX
-    Programming Language :: Python :: 3.7
-    Programming Language :: Python :: 3.8
-    Programming Language :: Python :: 3.9
-    Programming Language :: Python :: 3.10
-    Programming Language :: Python :: 3 :: Only
-    Topic :: Desktop Environment :: File Managers
-
-[options]
-packages = find:
-python_requires = >=3.7
-install_requires =
-    Send2Trash>=1.8.2,<2.0.0
-    mutagen>=1.46.0,<2.0.0
-    distro>=1.8.0,<2.0.0
-    PyQt5 >=5.15.0,<6.0; sys_platform != 'linux'
-    pywin32>=228; sys_platform == 'win32'
-    semantic-version>=2.0.0,<3.0.0
-    xxhash>=3.0.0,<4.0.0
-setup_requires =
-    sphinx>=3.0.0
-    polib>=1.1.0
-tests_require =
-    pytest >=6,<7
-include_package_data = true
-
-[options.entry_points]
-console_scripts =
-    dupeguru = run.py

setup.py

@@ -1,26 +0,0 @@
-from setuptools import setup, Extension
-from pathlib import Path
-
-exts = [
-    Extension(
-        "core.pe._block",
-        [
-            str(Path("core", "pe", "modules", "block.c")),
-            str(Path("core", "pe", "modules", "common.c")),
-        ],
-        include_dirs=[str(Path("core", "pe", "modules"))],
-    ),
-    Extension(
-        "core.pe._cache",
-        [
-            str(Path("core", "pe", "modules", "cache.c")),
-            str(Path("core", "pe", "modules", "common.c")),
-        ],
-        include_dirs=[str(Path("core", "pe", "modules"))],
-    ),
-    Extension("qt.pe._block_qt", [str(Path("qt", "pe", "modules", "block.c"))]),
-]
-
-headers = [str(Path("core", "pe", "modules", "common.h"))]
-
-setup(ext_modules=exts, headers=headers)