Compare commits

...

8 Commits

Author SHA1 Message Date
Andrew Senetar 1e651a1603
Merge pull request #1089 from arsenetar/as/pre-commit
feat: Add pre-commit, include python 3.11 in tests
2023-01-09 23:18:13 -06:00
Andrew Senetar 78f4145910
chore: Remove unused qtlib.pot file 2023-01-09 23:02:19 -06:00
Andrew Senetar 46d1afb566
chore: Apply whitespace fixes from hooks
- Remove trailing whitespace
- Correct single newline at end of files (skip for json)
- Update to formatting in a few places due to black
2023-01-09 22:58:08 -06:00
Andrew Senetar a5e31f15f0
Merge pull request #1088 from arsenetar/as/remove-shelve
feat: Remove shelve picture cache
2023-01-09 22:48:37 -06:00
Andrew Senetar 0cf6c9a1a2
ci: Update to include python 3.11 & pre-commit 2023-01-09 22:44:10 -06:00
Andrew Senetar 6db2fa2be6
fix: Correct flake8 config
- Add exclude pattern for flake8 when running with pre-commit as it does
  not fully honor the exclude paths.
- Cleanup exclude paths for flake8 in tox.ini
- Re-enable line length check and correct three affected files
2023-01-09 22:35:12 -06:00
Andrew Senetar 2dd2a801cc
feat: Add pre-commit and commitlint 2023-01-09 21:53:22 -06:00
Andrew Senetar 83f5e80427
feat: Remove shelve picture cache
- Remove shelve picture cache as it has had a fair number of historical
  issues.  Original issue for which it was added should be long
  resolved.  Additionally this allows additional consolidation of the
  various cache code and potentially dbs in the future.
- Remove all related preferences and related code for changing cache
  backend between sqlite and shelve.
2023-01-06 00:35:23 -06:00
92 changed files with 299 additions and 449 deletions

View File

@ -9,43 +9,22 @@ on:
branches: [master]
jobs:
lint:
pre-commit:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Set up Python 3.10
uses: actions/setup-python@v2
- uses: actions/checkout@v3
- name: Set up Python 3.11
uses: actions/setup-python@v4
with:
python-version: "3.10"
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -r requirements.txt -r requirements-extra.txt
- name: Lint with flake8
run: |
flake8 .
format:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Set up Python 3.10
uses: actions/setup-python@v2
with:
python-version: "3.10"
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -r requirements.txt -r requirements-extra.txt
- name: Check format with black
run: |
black .
python-version: "3.11"
- uses: pre-commit/action@v3.0.0
test:
needs: [lint, format]
needs: [pre-commit]
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [ubuntu-latest, macos-latest, windows-latest]
python-version: [3.7, 3.8, 3.9, "3.10"]
python-version: [3.7, 3.8, 3.9, "3.10", "3.11"]
exclude:
- os: macos-latest
python-version: 3.7
@ -53,17 +32,21 @@ jobs:
python-version: 3.8
- os: macos-latest
python-version: 3.9
- os: macos-latest
python-version: "3.10"
- os: windows-latest
python-version: 3.7
- os: windows-latest
python-version: 3.8
- os: windows-latest
python-version: 3.9
- os: windows-latest
python-version: "3.10"
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies

24
.pre-commit-config.yaml Normal file
View File

@ -0,0 +1,24 @@
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v2.3.0
hooks:
- id: check-yaml
- id: check-toml
- id: end-of-file-fixer
exclude: ".*.json"
- id: trailing-whitespace
- repo: https://github.com/psf/black
rev: 22.10.0
hooks:
- id: black
- repo: https://github.com/PyCQA/flake8
rev: 6.0.0
hooks:
- id: flake8
exclude: ^(.tox|env|build|dist|help|qt/dg_rc.py|pkg).*
- repo: https://github.com/alessandrojcm/commitlint-pre-commit-hook
rev: v9.3.0
hooks:
- id: commitlint
stages: [commit-msg]
additional_dependencies: ["@commitlint/config-conventional"]

View File

@ -1 +1 @@
sonar.python.version=3.7, 3.8, 3.9, 3.10
sonar.python.version=3.7, 3.8, 3.9, 3.10, 3.11

View File

@ -18,4 +18,3 @@ file_filter = locale/<lang>/LC_MESSAGES/ui.po
source_file = locale/ui.pot
source_lang = en
type = PO

View File

@ -619,4 +619,3 @@ Program, unless a warranty or assumption of liability accompanies a
copy of the Program in return for a fee.
END OF TERMS AND CONDITIONS

17
commitlint.config.js Normal file
View File

@ -0,0 +1,17 @@
const Configuration = {
/*
* Resolve and load @commitlint/config-conventional from node_modules.
* Referenced packages must be installed
*/
extends: ['@commitlint/config-conventional'],
/*
* Any rules defined here will override rules from @commitlint/config-conventional
*/
rules: {
'header-max-length': [2, 'always', 72],
'subject-case': [2, 'always', 'sentence-case'],
'scope-enum': [2, 'always'],
},
};
module.exports = Configuration;

View File

@ -126,8 +126,6 @@ class DupeGuru(Broadcaster):
NAME = PROMPT_NAME = "dupeGuru"
PICTURE_CACHE_TYPE = "sqlite" # set to 'shelve' for a ShelveCache
def __init__(self, view, portable=False):
if view.get_default(DEBUG_MODE_PREFERENCE):
logging.getLogger().setLevel(logging.DEBUG)
@ -153,7 +151,6 @@ class DupeGuru(Broadcaster):
"clean_empty_dirs": False,
"ignore_hardlink_matches": False,
"copymove_dest_type": DestType.RELATIVE,
"picture_cache_type": self.PICTURE_CACHE_TYPE,
"include_exists_check": True,
"rehash_ignore_mtime": False,
}
@ -185,8 +182,7 @@ class DupeGuru(Broadcaster):
self.view.create_results_window()
def _get_picture_cache_path(self):
cache_type = self.options["picture_cache_type"]
cache_name = "cached_pictures.shelve" if cache_type == "shelve" else "cached_pictures.db"
cache_name = "cached_pictures.db"
return op.join(self.appdata, cache_name)
def _get_dupe_sort_key(self, dupe, get_group, key, delta):

View File

@ -97,12 +97,14 @@ class FilesDB:
schema_version = 1
schema_version_description = "Changed from md5 to xxhash if available."
create_table_query = "CREATE TABLE IF NOT EXISTS files (path TEXT PRIMARY KEY, size INTEGER, mtime_ns INTEGER, entry_dt DATETIME, digest BLOB, digest_partial BLOB, digest_samples BLOB)"
create_table_query = """CREATE TABLE IF NOT EXISTS files (path TEXT PRIMARY KEY, size INTEGER, mtime_ns INTEGER,
entry_dt DATETIME, digest BLOB, digest_partial BLOB, digest_samples BLOB)"""
drop_table_query = "DROP TABLE IF EXISTS files;"
select_query = "SELECT {key} FROM files WHERE path=:path AND size=:size and mtime_ns=:mtime_ns"
select_query_ignore_mtime = "SELECT {key} FROM files WHERE path=:path AND size=:size"
insert_query = """
INSERT INTO files (path, size, mtime_ns, entry_dt, {key}) VALUES (:path, :size, :mtime_ns, datetime('now'), :value)
INSERT INTO files (path, size, mtime_ns, entry_dt, {key})
VALUES (:path, :size, :mtime_ns, datetime('now'), :value)
ON CONFLICT(path) DO UPDATE SET size=:size, mtime_ns=:mtime_ns, entry_dt=datetime('now'), {key}=:value;
"""
@ -153,7 +155,8 @@ class FilesDB:
self.cur.execute(self.select_query_ignore_mtime.format(key=key), {"path": str(path), "size": size})
else:
self.cur.execute(
self.select_query.format(key=key), {"path": str(path), "size": size, "mtime_ns": mtime_ns}
self.select_query.format(key=key),
{"path": str(path), "size": size, "mtime_ns": mtime_ns},
)
result = self.cur.fetchone()

View File

@ -1,141 +0,0 @@
# Copyright 2016 Virgil Dupras
#
# This software is licensed under the "GPLv3" License as described in the "LICENSE" file,
# which should be included with this package. The terms are also available at
# http://www.gnu.org/licenses/gpl-3.0.html
import os
import os.path as op
import shelve
import tempfile
from collections import namedtuple
from core.pe.cache import string_to_colors, colors_to_string
def wrap_path(path):
return f"path:{path}"
def unwrap_path(key):
return key[5:]
def wrap_id(path):
return f"id:{path}"
def unwrap_id(key):
return int(key[3:])
CacheRow = namedtuple("CacheRow", "id path blocks mtime")
class ShelveCache:
"""A class to cache picture blocks in a shelve backend."""
def __init__(self, db=None, readonly=False):
self.istmp = db is None
if self.istmp:
self.dtmp = tempfile.mkdtemp()
self.ftmp = db = op.join(self.dtmp, "tmpdb")
flag = "r" if readonly else "c"
self.shelve = shelve.open(db, flag)
self.maxid = self._compute_maxid()
def __contains__(self, key):
return wrap_path(key) in self.shelve
def __delitem__(self, key):
row = self.shelve[wrap_path(key)]
del self.shelve[wrap_path(key)]
del self.shelve[wrap_id(row.id)]
def __getitem__(self, key):
if isinstance(key, int):
skey = self.shelve[wrap_id(key)]
else:
skey = wrap_path(key)
return string_to_colors(self.shelve[skey].blocks)
def __iter__(self):
return (unwrap_path(k) for k in self.shelve if k.startswith("path:"))
def __len__(self):
return sum(1 for k in self.shelve if k.startswith("path:"))
def __setitem__(self, path_str, blocks):
blocks = colors_to_string(blocks)
if op.exists(path_str):
mtime = int(os.stat(path_str).st_mtime)
else:
mtime = 0
if path_str in self:
rowid = self.shelve[wrap_path(path_str)].id
else:
rowid = self._get_new_id()
row = CacheRow(rowid, path_str, blocks, mtime)
self.shelve[wrap_path(path_str)] = row
self.shelve[wrap_id(rowid)] = wrap_path(path_str)
def _compute_maxid(self):
return max((unwrap_id(k) for k in self.shelve if k.startswith("id:")), default=1)
def _get_new_id(self):
self.maxid += 1
return self.maxid
def clear(self):
self.shelve.clear()
def close(self):
if self.shelve is not None:
self.shelve.close()
if self.istmp:
os.remove(self.ftmp)
os.rmdir(self.dtmp)
self.shelve = None
def filter(self, func):
to_delete = [key for key in self if not func(key)]
for key in to_delete:
del self[key]
def get_id(self, path):
if path in self:
return self.shelve[wrap_path(path)].id
else:
raise ValueError(path)
def get_multiple(self, rowids):
for rowid in rowids:
try:
skey = self.shelve[wrap_id(rowid)]
except KeyError:
continue
yield (rowid, string_to_colors(self.shelve[skey].blocks))
def purge_outdated(self):
"""Go through the cache and purge outdated records.
A record is outdated if the picture doesn't exist or if its mtime is greater than the one in
the db.
"""
todelete = []
for path in self:
row = self.shelve[wrap_path(path)]
if row.mtime and op.exists(path):
picture_mtime = os.stat(path).st_mtime
if int(picture_mtime) <= row.mtime:
# not outdated
continue
todelete.append(path)
for path in todelete:
try:
del self[path]
except KeyError:
# I have no idea why a KeyError sometimes happen, but it does, as we can see in
# #402 and #439. I don't think it hurts to silently ignore the error, so that's
# what we do
pass

View File

@ -16,6 +16,7 @@ from hscommon.jobprogress import job
from core.engine import Match
from core.pe.block import avgdiff, DifferentBlockCountError, NoBlocksError
from core.pe.cache_sqlite import SqliteCache
# OPTIMIZATION NOTES:
# The bottleneck of the matching phase is CPU, which is why we use multiprocessing. However, another
@ -50,14 +51,7 @@ except Exception:
def get_cache(cache_path, readonly=False):
if cache_path.endswith("shelve"):
from core.pe.cache_shelve import ShelveCache
return ShelveCache(cache_path, readonly=readonly)
else:
from core.pe.cache_sqlite import SqliteCache
return SqliteCache(cache_path, readonly=readonly)
return SqliteCache(cache_path, readonly=readonly)
def prepare_pictures(pictures, cache_path, with_dimensions, j=job.nulljob):

View File

@ -12,7 +12,6 @@ from hscommon.testutil import eq_
try:
from core.pe.cache import colors_to_string, string_to_colors
from core.pe.cache_sqlite import SqliteCache
from core.pe.cache_shelve import ShelveCache
except ImportError:
skip("Can't import the cache module, probably hasn't been compiled.")
@ -133,11 +132,6 @@ class TestCaseSqliteCache(BaseTestCaseCache):
eq_(c["foo"], [(1, 2, 3)])
class TestCaseShelveCache(BaseTestCaseCache):
def get_cache(self, dbname=None):
return ShelveCache(dbname)
class TestCaseCacheSQLEscape:
def get_cache(self):
return SqliteCache()

View File

@ -15,4 +15,3 @@ hscommon.gui.progress_window
.. autoclass:: ProgressWindowView
:members:
:private-members:

View File

@ -15,4 +15,3 @@ hscommon.gui.tree
.. autoclass:: Node
:members:
:private-members:

View File

@ -13,4 +13,3 @@ hscommon
util
jobprogress/*
gui/*

View File

@ -14,4 +14,3 @@ hscommon.jobprogress.job
.. autoclass:: NullJob
:members:

View File

@ -9,4 +9,3 @@ hscommon.jobprogress.performer
.. autoclass:: ThreadedJobPerformer
:members:

View File

@ -178,4 +178,3 @@ Preferences are stored elsewhere:
.. _Github: https://github.com/arsenetar/dupeguru
.. _open an issue: https://github.com/arsenetar/dupeguru/wiki/issue-labels

View File

@ -12,4 +12,3 @@
* Եթե համոզված եք, որ կրկնօրինակը արդյունքներում կա, ապա սեղմեք **Խմբագրել-->Նշել բոլորը**, և ապա **Գործողություններ-->Ուղարկել Նշվածը Աղբարկղ**:
Սա միայն բազային ստուգում է: Կան բազմաթիվ կարգավորումներ, որոնք հնարավորություն են տալիս նշելու տարբեր արդյունքներ և մի քանի եղանակներ արդյունքների փոփոխման: Մանրամասների համար կարդացեք Օգնության ֆայլը:

View File

@ -23,4 +23,3 @@ dupeGuru-ը փորձում է որոշել, թե որ կրկնօրինակներ
մեծագույն ֆայլը և եթե երկու կամ ավելի ֆայլեր ունեն նույն չափը, ապա մեկը ունի ֆայլի անուն, որը
չի ավարտվում թվով, կօգտագործվի: Երբ փաստարկի արդյունքը կապված է, կարգը, որի սխալները
նախկինում էին, խումբը պետք է օգտագործվի:

View File

@ -114,4 +114,3 @@
Якщо все це не так, `контакт УГ підтримки <http://www.hardcoded.net/support>`_, ми зрозуміти це.
.. todo:: This FAQ qestion is outdated, see english version.

View File

@ -41,7 +41,8 @@ def trget(domain: str) -> Callable[[str], str]:
def set_tr(
new_tr: Callable[[str, Union[str, None]], str], new_trget: Union[Callable[[str], Callable[[str], str]], None] = None
new_tr: Callable[[str, Union[str, None]], str],
new_trget: Union[Callable[[str], Callable[[str], str]], None] = None,
) -> None:
global _trfunc, _trget
_trfunc = new_tr

View File

@ -114,4 +114,3 @@ msgstr ""
#: core\prioritize.py:158
msgid "Size"
msgstr ""

View File

@ -243,4 +243,3 @@ msgstr ""
#: core\se\scanner.py:18
msgid "Folders"
msgstr ""

View File

@ -1,6 +0,0 @@
msgid ""
msgstr ""
"Content-Type: text/plain; charset=utf-8\n"
"Content-Transfer-Encoding: utf-8\n"

View File

@ -1114,3 +1114,10 @@ msgstr ""
#: qt\progress_window.py:65
msgid "Are you sure you want to cancel? All progress will be lost."
msgstr ""
#: qt\exclude_list_dialog.py:161
msgid ""
"These (case sensitive) python regular expressions will filter out files during scans.<br>Directores will also have their <strong>default state</strong> set to Excluded in the Directories tab if their name happens to match one of the selected regular expressions.<br>For each file collected, two tests are performed to determine whether or not to completely ignore it:<br><li>1. Regular expressions with no path separator in them will be compared to the file name only.</li>\n"
"<li>2. Regular expressions with at least one path separator in them will be compared to the full path to the file.</li><br>Example: if you want to filter out .PNG files from the \"My Pictures\" directory only:<br><code>.*My\\sPictures\\\\.*\\.png</code><br><br>You can test the regular expression with the \"test string\" button after pasting a fake path in the test field:<br><code>C:\\\\User\\My Pictures\\test.png</code><br><br>\n"
"Matching regular expressions will be highlighted.<br>If there is at least one highlight, the path or filename tested will be ignored during scans.<br><br>Directories and files starting with a period '.' are filtered out by default.<br><br>"
msgstr ""

View File

@ -348,4 +348,3 @@ dupeguru (2.9.2-1) unstable; urgency=low
* Fixed selection glitches, especially while renaming. (#93)
-- Virgil Dupras <hsoft@hardcoded.net> Wed, 10 Feb 2010 00:00:00 +0000

View File

@ -192,7 +192,6 @@ class DupeGuru(QObject):
scanned_tags.add("year")
self.model.options["scanned_tags"] = scanned_tags
self.model.options["match_scaled"] = self.prefs.match_scaled
self.model.options["picture_cache_type"] = self.prefs.picture_cache_type
self.model.options["include_exists_check"] = self.prefs.include_exists_check
self.model.options["rehash_ignore_mtime"] = self.prefs.rehash_ignore_mtime

View File

@ -165,8 +165,8 @@ Directores will also have their <strong>default state</strong> set to Excluded \
in the Directories tab if their name happens to match one of the selected regular expressions.<br>\
For each file collected, two tests are performed to determine whether or not to completely ignore it:<br>\
<li>1. Regular expressions with no path separator in them will be compared to the file name only.</li>
<li>2. Regular expressions with at least one path separator in them will be compared to the full path to the file.</li><br>
Example: if you want to filter out .PNG files from the "My Pictures" directory only:<br>\
<li>2. Regular expressions with at least one path separator in them will be compared to the full path to the file.</li>\
<br>Example: if you want to filter out .PNG files from the "My Pictures" directory only:<br>\
<code>.*My\\sPictures\\\\.*\\.png</code><br><br>\
You can test the regular expression with the "test string" button after pasting a fake path in the test field:<br>\
<code>C:\\\\User\\My Pictures\\test.png</code><br><br>

View File

@ -31,7 +31,10 @@ class File(PhotoBase):
image = image.convertToFormat(QImage.Format_RGB888)
if type(orientation) != int:
logging.warning(
"Orientation for file '%s' was a %s '%s', not an int.", str(self.path), type(orientation), orientation
"Orientation for file '%s' was a %s '%s', not an int.",
str(self.path),
type(orientation),
orientation,
)
try:
orientation = int(orientation)

View File

@ -4,11 +4,9 @@
# which should be included with this package. The terms are also available at
# http://www.gnu.org/licenses/gpl-3.0.html
from PyQt5.QtWidgets import QFormLayout
from PyQt5.QtCore import Qt
from hscommon.trans import trget
from hscommon.plat import ISLINUX
from qt.radio_box import RadioBox
from core.scanner import ScanType
from core.app import AppMode
@ -35,11 +33,6 @@ class PreferencesDialog(PreferencesDialogBase):
)
self.widgetsVLayout.addWidget(self.ignoreHardlinkMatches)
self.cacheTypeRadio = RadioBox(self, items=["Sqlite", "Shelve"], spread=False)
cache_form = QFormLayout()
cache_form.setLabelAlignment(Qt.AlignLeft)
cache_form.addRow(tr("Picture cache mode:"), self.cacheTypeRadio)
self.widgetsVLayout.addLayout(cache_form)
self._setupBottomPart()
def _setupDisplayPage(self):
@ -64,7 +57,6 @@ show scrollbars to span the view around"
def _load(self, prefs, setchecked, section):
setchecked(self.matchScaledBox, prefs.match_scaled)
self.cacheTypeRadio.selected_index = 1 if prefs.picture_cache_type == "shelve" else 0
# Update UI state based on selected scan type
scan_type = prefs.get_scan_type(AppMode.PICTURE)
@ -75,6 +67,5 @@ show scrollbars to span the view around"
def _save(self, prefs, ischecked):
prefs.match_scaled = ischecked(self.matchScaledBox)
prefs.picture_cache_type = "shelve" if self.cacheTypeRadio.selected_index == 1 else "sqlite"
prefs.details_dialog_override_theme_icons = ischecked(self.details_dialog_override_theme_icons)
prefs.details_dialog_viewers_show_scrollbars = ischecked(self.details_dialog_viewers_show_scrollbars)

View File

@ -225,7 +225,6 @@ class Preferences(PreferencesBase):
self.scan_tag_genre = get("ScanTagGenre", self.scan_tag_genre)
self.scan_tag_year = get("ScanTagYear", self.scan_tag_year)
self.match_scaled = get("MatchScaled", self.match_scaled)
self.picture_cache_type = get("PictureCacheType", self.picture_cache_type)
def reset(self):
self.filter_hardness = 95
@ -278,7 +277,6 @@ class Preferences(PreferencesBase):
self.scan_tag_genre = False
self.scan_tag_year = False
self.match_scaled = False
self.picture_cache_type = "sqlite"
def _save_values(self, settings):
set_ = self.set_value
@ -332,7 +330,6 @@ class Preferences(PreferencesBase):
set_("ScanTagGenre", self.scan_tag_genre)
set_("ScanTagYear", self.scan_tag_year)
set_("MatchScaled", self.match_scaled)
set_("PictureCacheType", self.picture_cache_type)
# scan_type is special because we save it immediately when we set it.
def get_scan_type(self, app_mode):

View File

@ -146,7 +146,8 @@ On MacOS, the tab bar will fill up the window's width instead."
)
self.use_native_dialogs.setToolTip(
tr(
"For actions such as file/folder selection use the OS native dialogs.\nSome native dialogs have limited functionality."
"For actions such as file/folder selection use the OS native dialogs.\n\
Some native dialogs have limited functionality."
)
)
layout.addWidget(self.use_native_dialogs)
@ -217,7 +218,8 @@ use the modifier key to drag the floating window around"
def _setup_advanced_page(self):
tab_label = QLabel(
tr(
"These options are for advanced users or for very specific situations, most users should not have to modify these."
"These options are for advanced users or for very specific situations, \
most users should not have to modify these."
),
wordWrap=True,
)

View File

@ -16,7 +16,7 @@ deps =
-r{toxinidir}/requirements-extra.txt
[flake8]
exclude = .tox,env,build,cocoalib,cocoa,help,./qt/dg_rc.py,cocoa/run_template.py,./pkg
exclude = .tox,env*,build,help,qt/dg_rc.py,pkg
max-line-length = 120
select = C,E,F,W,B,B950
extend-ignore = E203, E501, W503
extend-ignore = E203,W503