1
0
mirror of https://github.com/arsenetar/dupeguru.git synced 2025-05-10 18:59:49 +00:00

Compare commits

..

No commits in common. "1e651a1603873ea6b46ff04cd9471e41aa1a4ca3" and "091cae0cc69f00fb80f667e1e9a48a9bfe4d94fa" have entirely different histories.

92 changed files with 449 additions and 299 deletions

View File

@ -9,22 +9,43 @@ on:
branches: [master]
jobs:
pre-commit:
lint:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Set up Python 3.11
uses: actions/setup-python@v4
- uses: actions/checkout@v2
- name: Set up Python 3.10
uses: actions/setup-python@v2
with:
python-version: "3.11"
- uses: pre-commit/action@v3.0.0
python-version: "3.10"
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -r requirements.txt -r requirements-extra.txt
- name: Lint with flake8
run: |
flake8 .
format:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Set up Python 3.10
uses: actions/setup-python@v2
with:
python-version: "3.10"
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -r requirements.txt -r requirements-extra.txt
- name: Check format with black
run: |
black .
test:
needs: [pre-commit]
needs: [lint, format]
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [ubuntu-latest, macos-latest, windows-latest]
python-version: [3.7, 3.8, 3.9, "3.10", "3.11"]
python-version: [3.7, 3.8, 3.9, "3.10"]
exclude:
- os: macos-latest
python-version: 3.7
@ -32,21 +53,17 @@ jobs:
python-version: 3.8
- os: macos-latest
python-version: 3.9
- os: macos-latest
python-version: "3.10"
- os: windows-latest
python-version: 3.7
- os: windows-latest
python-version: 3.8
- os: windows-latest
python-version: 3.9
- os: windows-latest
python-version: "3.10"
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v2
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies

View File

@ -1,24 +0,0 @@
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v2.3.0
hooks:
- id: check-yaml
- id: check-toml
- id: end-of-file-fixer
exclude: ".*.json"
- id: trailing-whitespace
- repo: https://github.com/psf/black
rev: 22.10.0
hooks:
- id: black
- repo: https://github.com/PyCQA/flake8
rev: 6.0.0
hooks:
- id: flake8
exclude: ^(.tox|env|build|dist|help|qt/dg_rc.py|pkg).*
- repo: https://github.com/alessandrojcm/commitlint-pre-commit-hook
rev: v9.3.0
hooks:
- id: commitlint
stages: [commit-msg]
additional_dependencies: ["@commitlint/config-conventional"]

View File

@ -1 +1 @@
sonar.python.version=3.7, 3.8, 3.9, 3.10, 3.11
sonar.python.version=3.7, 3.8, 3.9, 3.10

View File

@ -18,3 +18,4 @@ file_filter = locale/<lang>/LC_MESSAGES/ui.po
source_file = locale/ui.pot
source_lang = en
type = PO

View File

@ -619,3 +619,4 @@ Program, unless a warranty or assumption of liability accompanies a
copy of the Program in return for a fee.
END OF TERMS AND CONDITIONS

View File

@ -1,17 +0,0 @@
const Configuration = {
/*
* Resolve and load @commitlint/config-conventional from node_modules.
* Referenced packages must be installed
*/
extends: ['@commitlint/config-conventional'],
/*
* Any rules defined here will override rules from @commitlint/config-conventional
*/
rules: {
'header-max-length': [2, 'always', 72],
'subject-case': [2, 'always', 'sentence-case'],
'scope-enum': [2, 'always'],
},
};
module.exports = Configuration;

View File

@ -126,6 +126,8 @@ class DupeGuru(Broadcaster):
NAME = PROMPT_NAME = "dupeGuru"
PICTURE_CACHE_TYPE = "sqlite" # set to 'shelve' for a ShelveCache
def __init__(self, view, portable=False):
if view.get_default(DEBUG_MODE_PREFERENCE):
logging.getLogger().setLevel(logging.DEBUG)
@ -151,6 +153,7 @@ class DupeGuru(Broadcaster):
"clean_empty_dirs": False,
"ignore_hardlink_matches": False,
"copymove_dest_type": DestType.RELATIVE,
"picture_cache_type": self.PICTURE_CACHE_TYPE,
"include_exists_check": True,
"rehash_ignore_mtime": False,
}
@ -182,7 +185,8 @@ class DupeGuru(Broadcaster):
self.view.create_results_window()
def _get_picture_cache_path(self):
cache_name = "cached_pictures.db"
cache_type = self.options["picture_cache_type"]
cache_name = "cached_pictures.shelve" if cache_type == "shelve" else "cached_pictures.db"
return op.join(self.appdata, cache_name)
def _get_dupe_sort_key(self, dupe, get_group, key, delta):

View File

@ -97,14 +97,12 @@ class FilesDB:
schema_version = 1
schema_version_description = "Changed from md5 to xxhash if available."
create_table_query = """CREATE TABLE IF NOT EXISTS files (path TEXT PRIMARY KEY, size INTEGER, mtime_ns INTEGER,
entry_dt DATETIME, digest BLOB, digest_partial BLOB, digest_samples BLOB)"""
create_table_query = "CREATE TABLE IF NOT EXISTS files (path TEXT PRIMARY KEY, size INTEGER, mtime_ns INTEGER, entry_dt DATETIME, digest BLOB, digest_partial BLOB, digest_samples BLOB)"
drop_table_query = "DROP TABLE IF EXISTS files;"
select_query = "SELECT {key} FROM files WHERE path=:path AND size=:size and mtime_ns=:mtime_ns"
select_query_ignore_mtime = "SELECT {key} FROM files WHERE path=:path AND size=:size"
insert_query = """
INSERT INTO files (path, size, mtime_ns, entry_dt, {key})
VALUES (:path, :size, :mtime_ns, datetime('now'), :value)
INSERT INTO files (path, size, mtime_ns, entry_dt, {key}) VALUES (:path, :size, :mtime_ns, datetime('now'), :value)
ON CONFLICT(path) DO UPDATE SET size=:size, mtime_ns=:mtime_ns, entry_dt=datetime('now'), {key}=:value;
"""
@ -155,8 +153,7 @@ class FilesDB:
self.cur.execute(self.select_query_ignore_mtime.format(key=key), {"path": str(path), "size": size})
else:
self.cur.execute(
self.select_query.format(key=key),
{"path": str(path), "size": size, "mtime_ns": mtime_ns},
self.select_query.format(key=key), {"path": str(path), "size": size, "mtime_ns": mtime_ns}
)
result = self.cur.fetchone()

141
core/pe/cache_shelve.py Normal file
View File

@ -0,0 +1,141 @@
# Copyright 2016 Virgil Dupras
#
# This software is licensed under the "GPLv3" License as described in the "LICENSE" file,
# which should be included with this package. The terms are also available at
# http://www.gnu.org/licenses/gpl-3.0.html
import os
import os.path as op
import shelve
import tempfile
from collections import namedtuple
from core.pe.cache import string_to_colors, colors_to_string
def wrap_path(path):
return f"path:{path}"
def unwrap_path(key):
return key[5:]
def wrap_id(path):
return f"id:{path}"
def unwrap_id(key):
return int(key[3:])
CacheRow = namedtuple("CacheRow", "id path blocks mtime")
class ShelveCache:
"""A class to cache picture blocks in a shelve backend."""
def __init__(self, db=None, readonly=False):
self.istmp = db is None
if self.istmp:
self.dtmp = tempfile.mkdtemp()
self.ftmp = db = op.join(self.dtmp, "tmpdb")
flag = "r" if readonly else "c"
self.shelve = shelve.open(db, flag)
self.maxid = self._compute_maxid()
def __contains__(self, key):
return wrap_path(key) in self.shelve
def __delitem__(self, key):
row = self.shelve[wrap_path(key)]
del self.shelve[wrap_path(key)]
del self.shelve[wrap_id(row.id)]
def __getitem__(self, key):
if isinstance(key, int):
skey = self.shelve[wrap_id(key)]
else:
skey = wrap_path(key)
return string_to_colors(self.shelve[skey].blocks)
def __iter__(self):
return (unwrap_path(k) for k in self.shelve if k.startswith("path:"))
def __len__(self):
return sum(1 for k in self.shelve if k.startswith("path:"))
def __setitem__(self, path_str, blocks):
blocks = colors_to_string(blocks)
if op.exists(path_str):
mtime = int(os.stat(path_str).st_mtime)
else:
mtime = 0
if path_str in self:
rowid = self.shelve[wrap_path(path_str)].id
else:
rowid = self._get_new_id()
row = CacheRow(rowid, path_str, blocks, mtime)
self.shelve[wrap_path(path_str)] = row
self.shelve[wrap_id(rowid)] = wrap_path(path_str)
def _compute_maxid(self):
return max((unwrap_id(k) for k in self.shelve if k.startswith("id:")), default=1)
def _get_new_id(self):
self.maxid += 1
return self.maxid
def clear(self):
self.shelve.clear()
def close(self):
if self.shelve is not None:
self.shelve.close()
if self.istmp:
os.remove(self.ftmp)
os.rmdir(self.dtmp)
self.shelve = None
def filter(self, func):
to_delete = [key for key in self if not func(key)]
for key in to_delete:
del self[key]
def get_id(self, path):
if path in self:
return self.shelve[wrap_path(path)].id
else:
raise ValueError(path)
def get_multiple(self, rowids):
for rowid in rowids:
try:
skey = self.shelve[wrap_id(rowid)]
except KeyError:
continue
yield (rowid, string_to_colors(self.shelve[skey].blocks))
def purge_outdated(self):
"""Go through the cache and purge outdated records.
A record is outdated if the picture doesn't exist or if its mtime is greater than the one in
the db.
"""
todelete = []
for path in self:
row = self.shelve[wrap_path(path)]
if row.mtime and op.exists(path):
picture_mtime = os.stat(path).st_mtime
if int(picture_mtime) <= row.mtime:
# not outdated
continue
todelete.append(path)
for path in todelete:
try:
del self[path]
except KeyError:
# I have no idea why a KeyError sometimes happen, but it does, as we can see in
# #402 and #439. I don't think it hurts to silently ignore the error, so that's
# what we do
pass

View File

@ -16,7 +16,6 @@ from hscommon.jobprogress import job
from core.engine import Match
from core.pe.block import avgdiff, DifferentBlockCountError, NoBlocksError
from core.pe.cache_sqlite import SqliteCache
# OPTIMIZATION NOTES:
# The bottleneck of the matching phase is CPU, which is why we use multiprocessing. However, another
@ -51,6 +50,13 @@ except Exception:
def get_cache(cache_path, readonly=False):
if cache_path.endswith("shelve"):
from core.pe.cache_shelve import ShelveCache
return ShelveCache(cache_path, readonly=readonly)
else:
from core.pe.cache_sqlite import SqliteCache
return SqliteCache(cache_path, readonly=readonly)

View File

@ -12,6 +12,7 @@ from hscommon.testutil import eq_
try:
from core.pe.cache import colors_to_string, string_to_colors
from core.pe.cache_sqlite import SqliteCache
from core.pe.cache_shelve import ShelveCache
except ImportError:
skip("Can't import the cache module, probably hasn't been compiled.")
@ -132,6 +133,11 @@ class TestCaseSqliteCache(BaseTestCaseCache):
eq_(c["foo"], [(1, 2, 3)])
class TestCaseShelveCache(BaseTestCaseCache):
def get_cache(self, dbname=None):
return ShelveCache(dbname)
class TestCaseCacheSQLEscape:
def get_cache(self):
return SqliteCache()

View File

@ -15,3 +15,4 @@ hscommon.gui.progress_window
.. autoclass:: ProgressWindowView
:members:
:private-members:

View File

@ -15,3 +15,4 @@ hscommon.gui.tree
.. autoclass:: Node
:members:
:private-members:

View File

@ -13,3 +13,4 @@ hscommon
util
jobprogress/*
gui/*

View File

@ -14,3 +14,4 @@ hscommon.jobprogress.job
.. autoclass:: NullJob
:members:

View File

@ -9,3 +9,4 @@ hscommon.jobprogress.performer
.. autoclass:: ThreadedJobPerformer
:members:

View File

@ -178,3 +178,4 @@ Preferences are stored elsewhere:
.. _Github: https://github.com/arsenetar/dupeguru
.. _open an issue: https://github.com/arsenetar/dupeguru/wiki/issue-labels

View File

@ -12,3 +12,4 @@
* Եթե համոզված եք, որ կրկնօրինակը արդյունքներում կա, ապա սեղմեք **Խմբագրել-->Նշել բոլորը**, և ապա **Գործողություններ-->Ուղարկել Նշվածը Աղբարկղ**:
Սա միայն բազային ստուգում է: Կան բազմաթիվ կարգավորումներ, որոնք հնարավորություն են տալիս նշելու տարբեր արդյունքներ և մի քանի եղանակներ արդյունքների փոփոխման: Մանրամասների համար կարդացեք Օգնության ֆայլը:

View File

@ -23,3 +23,4 @@ dupeGuru-ը փորձում է որոշել, թե որ կրկնօրինակներ
մեծագույն ֆայլը և եթե երկու կամ ավելի ֆայլեր ունեն նույն չափը, ապա մեկը ունի ֆայլի անուն, որը
չի ավարտվում թվով, կօգտագործվի: Երբ փաստարկի արդյունքը կապված է, կարգը, որի սխալները
նախկինում էին, խումբը պետք է օգտագործվի:

View File

@ -114,3 +114,4 @@
Якщо все це не так, `контакт УГ підтримки <http://www.hardcoded.net/support>`_, ми зрозуміти це.
.. todo:: This FAQ qestion is outdated, see english version.

View File

@ -41,8 +41,7 @@ def trget(domain: str) -> Callable[[str], str]:
def set_tr(
new_tr: Callable[[str, Union[str, None]], str],
new_trget: Union[Callable[[str], Callable[[str], str]], None] = None,
new_tr: Callable[[str, Union[str, None]], str], new_trget: Union[Callable[[str], Callable[[str], str]], None] = None
) -> None:
global _trfunc, _trget
_trfunc = new_tr

View File

@ -114,3 +114,4 @@ msgstr ""
#: core\prioritize.py:158
msgid "Size"
msgstr ""

View File

@ -243,3 +243,4 @@ msgstr ""
#: core\se\scanner.py:18
msgid "Folders"
msgstr ""

6
locale/qtlib.pot Normal file
View File

@ -0,0 +1,6 @@
msgid ""
msgstr ""
"Content-Type: text/plain; charset=utf-8\n"
"Content-Transfer-Encoding: utf-8\n"

View File

@ -1114,10 +1114,3 @@ msgstr ""
#: qt\progress_window.py:65
msgid "Are you sure you want to cancel? All progress will be lost."
msgstr ""
#: qt\exclude_list_dialog.py:161
msgid ""
"These (case sensitive) python regular expressions will filter out files during scans.<br>Directores will also have their <strong>default state</strong> set to Excluded in the Directories tab if their name happens to match one of the selected regular expressions.<br>For each file collected, two tests are performed to determine whether or not to completely ignore it:<br><li>1. Regular expressions with no path separator in them will be compared to the file name only.</li>\n"
"<li>2. Regular expressions with at least one path separator in them will be compared to the full path to the file.</li><br>Example: if you want to filter out .PNG files from the \"My Pictures\" directory only:<br><code>.*My\\sPictures\\\\.*\\.png</code><br><br>You can test the regular expression with the \"test string\" button after pasting a fake path in the test field:<br><code>C:\\\\User\\My Pictures\\test.png</code><br><br>\n"
"Matching regular expressions will be highlighted.<br>If there is at least one highlight, the path or filename tested will be ignored during scans.<br><br>Directories and files starting with a period '.' are filtered out by default.<br><br>"
msgstr ""

View File

@ -348,3 +348,4 @@ dupeguru (2.9.2-1) unstable; urgency=low
* Fixed selection glitches, especially while renaming. (#93)
-- Virgil Dupras <hsoft@hardcoded.net> Wed, 10 Feb 2010 00:00:00 +0000

View File

@ -192,6 +192,7 @@ class DupeGuru(QObject):
scanned_tags.add("year")
self.model.options["scanned_tags"] = scanned_tags
self.model.options["match_scaled"] = self.prefs.match_scaled
self.model.options["picture_cache_type"] = self.prefs.picture_cache_type
self.model.options["include_exists_check"] = self.prefs.include_exists_check
self.model.options["rehash_ignore_mtime"] = self.prefs.rehash_ignore_mtime

View File

@ -165,8 +165,8 @@ Directores will also have their <strong>default state</strong> set to Excluded \
in the Directories tab if their name happens to match one of the selected regular expressions.<br>\
For each file collected, two tests are performed to determine whether or not to completely ignore it:<br>\
<li>1. Regular expressions with no path separator in them will be compared to the file name only.</li>
<li>2. Regular expressions with at least one path separator in them will be compared to the full path to the file.</li>\
<br>Example: if you want to filter out .PNG files from the "My Pictures" directory only:<br>\
<li>2. Regular expressions with at least one path separator in them will be compared to the full path to the file.</li><br>
Example: if you want to filter out .PNG files from the "My Pictures" directory only:<br>\
<code>.*My\\sPictures\\\\.*\\.png</code><br><br>\
You can test the regular expression with the "test string" button after pasting a fake path in the test field:<br>\
<code>C:\\\\User\\My Pictures\\test.png</code><br><br>

View File

@ -31,10 +31,7 @@ class File(PhotoBase):
image = image.convertToFormat(QImage.Format_RGB888)
if type(orientation) != int:
logging.warning(
"Orientation for file '%s' was a %s '%s', not an int.",
str(self.path),
type(orientation),
orientation,
"Orientation for file '%s' was a %s '%s', not an int.", str(self.path), type(orientation), orientation
)
try:
orientation = int(orientation)

View File

@ -4,9 +4,11 @@
# which should be included with this package. The terms are also available at
# http://www.gnu.org/licenses/gpl-3.0.html
from PyQt5.QtWidgets import QFormLayout
from PyQt5.QtCore import Qt
from hscommon.trans import trget
from hscommon.plat import ISLINUX
from qt.radio_box import RadioBox
from core.scanner import ScanType
from core.app import AppMode
@ -33,6 +35,11 @@ class PreferencesDialog(PreferencesDialogBase):
)
self.widgetsVLayout.addWidget(self.ignoreHardlinkMatches)
self.cacheTypeRadio = RadioBox(self, items=["Sqlite", "Shelve"], spread=False)
cache_form = QFormLayout()
cache_form.setLabelAlignment(Qt.AlignLeft)
cache_form.addRow(tr("Picture cache mode:"), self.cacheTypeRadio)
self.widgetsVLayout.addLayout(cache_form)
self._setupBottomPart()
def _setupDisplayPage(self):
@ -57,6 +64,7 @@ show scrollbars to span the view around"
def _load(self, prefs, setchecked, section):
setchecked(self.matchScaledBox, prefs.match_scaled)
self.cacheTypeRadio.selected_index = 1 if prefs.picture_cache_type == "shelve" else 0
# Update UI state based on selected scan type
scan_type = prefs.get_scan_type(AppMode.PICTURE)
@ -67,5 +75,6 @@ show scrollbars to span the view around"
def _save(self, prefs, ischecked):
prefs.match_scaled = ischecked(self.matchScaledBox)
prefs.picture_cache_type = "shelve" if self.cacheTypeRadio.selected_index == 1 else "sqlite"
prefs.details_dialog_override_theme_icons = ischecked(self.details_dialog_override_theme_icons)
prefs.details_dialog_viewers_show_scrollbars = ischecked(self.details_dialog_viewers_show_scrollbars)

View File

@ -225,6 +225,7 @@ class Preferences(PreferencesBase):
self.scan_tag_genre = get("ScanTagGenre", self.scan_tag_genre)
self.scan_tag_year = get("ScanTagYear", self.scan_tag_year)
self.match_scaled = get("MatchScaled", self.match_scaled)
self.picture_cache_type = get("PictureCacheType", self.picture_cache_type)
def reset(self):
self.filter_hardness = 95
@ -277,6 +278,7 @@ class Preferences(PreferencesBase):
self.scan_tag_genre = False
self.scan_tag_year = False
self.match_scaled = False
self.picture_cache_type = "sqlite"
def _save_values(self, settings):
set_ = self.set_value
@ -330,6 +332,7 @@ class Preferences(PreferencesBase):
set_("ScanTagGenre", self.scan_tag_genre)
set_("ScanTagYear", self.scan_tag_year)
set_("MatchScaled", self.match_scaled)
set_("PictureCacheType", self.picture_cache_type)
# scan_type is special because we save it immediately when we set it.
def get_scan_type(self, app_mode):

View File

@ -146,8 +146,7 @@ On MacOS, the tab bar will fill up the window's width instead."
)
self.use_native_dialogs.setToolTip(
tr(
"For actions such as file/folder selection use the OS native dialogs.\n\
Some native dialogs have limited functionality."
"For actions such as file/folder selection use the OS native dialogs.\nSome native dialogs have limited functionality."
)
)
layout.addWidget(self.use_native_dialogs)
@ -218,8 +217,7 @@ use the modifier key to drag the floating window around"
def _setup_advanced_page(self):
tab_label = QLabel(
tr(
"These options are for advanced users or for very specific situations, \
most users should not have to modify these."
"These options are for advanced users or for very specific situations, most users should not have to modify these."
),
wordWrap=True,
)

View File

@ -16,7 +16,7 @@ deps =
-r{toxinidir}/requirements-extra.txt
[flake8]
exclude = .tox,env*,build,help,qt/dg_rc.py,pkg
exclude = .tox,env,build,cocoalib,cocoa,help,./qt/dg_rc.py,cocoa/run_template.py,./pkg
max-line-length = 120
select = C,E,F,W,B,B950
extend-ignore = E203,W503
extend-ignore = E203, E501, W503