feat: Add additional scan time options
- Add option to include file existence check at end of scan, speeds up end of scan operation time considerably, however if user has removed or moved files since starting a scan there could be later errors when interacting with results. Defaults to existing behavior of including the check, until it can be verified later dialogs and actions handle non-existent items better. - Add option to ignore differences in mtime when checking hash cache. Option is present in advanced tab of preferences. Closes #1022. - Regenerate pot files for translations
This commit is contained in:
parent
1db93fd142
commit
e30a135451
GPG Key ID:
C63300DCE48AB2F1
|
|
@ -154,6 +154,8 @@ class DupeGuru(Broadcaster):
|
|||
"ignore_hardlink_matches": False,
|
||||
"copymove_dest_type": DestType.RELATIVE,
|
||||
"picture_cache_type": self.PICTURE_CACHE_TYPE,
|
||||
"include_exists_check": True,
|
||||
"rehash_ignore_mtime": False,
|
||||
}
|
||||
self.selected_dupes = []
|
||||
self.details_panel = DetailsPanel(self)
|
||||
|
|
@ -555,7 +557,9 @@ class DupeGuru(Broadcaster):
|
|||
# a workaround to make the damn thing work.
|
||||
exepath, args = match.groups()
|
||||
path, exename = op.split(exepath)
|
||||
p = subprocess.Popen(exename + args, shell=True, cwd=path, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
||||
p = subprocess.Popen(
|
||||
exename + args, shell=True, cwd=path, stdout=subprocess.PIPE, stderr=subprocess.STDOUT
|
||||
)
|
||||
output = p.stdout.read()
|
||||
logging.info("Custom command %s %s: %s", exename, args, output)
|
||||
else:
|
||||
|
|
@ -792,6 +796,7 @@ class DupeGuru(Broadcaster):
|
|||
Scans folders selected in :attr:`directories` and put the results in :attr:`results`
|
||||
"""
|
||||
scanner = self.SCANNER_CLASS()
|
||||
fs.filesdb.ignore_mtime = self.options["rehash_ignore_mtime"] is True
|
||||
if not self.directories.has_any_file():
|
||||
self.view.show_message(tr("The selected directories contain no scannable file."))
|
||||
return
|
||||
|
|
|
|||
12
core/fs.py
12
core/fs.py
|
|
@ -100,11 +100,14 @@ class FilesDB:
|
|||
create_table_query = "CREATE TABLE IF NOT EXISTS files (path TEXT PRIMARY KEY, size INTEGER, mtime_ns INTEGER, entry_dt DATETIME, digest BLOB, digest_partial BLOB, digest_samples BLOB)"
|
||||
drop_table_query = "DROP TABLE IF EXISTS files;"
|
||||
select_query = "SELECT {key} FROM files WHERE path=:path AND size=:size and mtime_ns=:mtime_ns"
|
||||
select_query_ignore_mtime = "SELECT {key} FROM files WHERE path=:path AND size=:size"
|
||||
insert_query = """
|
||||
INSERT INTO files (path, size, mtime_ns, entry_dt, {key}) VALUES (:path, :size, :mtime_ns, datetime('now'), :value)
|
||||
ON CONFLICT(path) DO UPDATE SET size=:size, mtime_ns=:mtime_ns, entry_dt=datetime('now'), {key}=:value;
|
||||
"""
|
||||
|
||||
ignore_mtime = False
|
||||
|
||||
def __init__(self):
|
||||
self.conn = None
|
||||
self.cur = None
|
||||
|
|
@ -146,9 +149,12 @@ class FilesDB:
|
|||
mtime_ns = stat.st_mtime_ns
|
||||
try:
|
||||
with self.lock:
|
||||
self.cur.execute(
|
||||
self.select_query.format(key=key), {"path": str(path), "size": size, "mtime_ns": mtime_ns}
|
||||
)
|
||||
if self.ignore_mtime:
|
||||
self.cur.execute(self.select_query_ignore_mtime.format(key=key), {"path": str(path), "size": size})
|
||||
else:
|
||||
self.cur.execute(
|
||||
self.select_query.format(key=key), {"path": str(path), "size": size, "mtime_ns": mtime_ns}
|
||||
)
|
||||
result = self.cur.fetchone()
|
||||
|
||||
if result:
|
||||
|
|
|
|||
|
|
@ -171,7 +171,8 @@ class Scanner:
|
|||
matches = [m for m in matches if m.first.path not in toremove or m.second.path not in toremove]
|
||||
if not self.mix_file_kind:
|
||||
matches = [m for m in matches if get_file_ext(m.first.name) == get_file_ext(m.second.name)]
|
||||
matches = [m for m in matches if m.first.path.exists() and m.second.path.exists()]
|
||||
if self.include_exists_check:
|
||||
matches = [m for m in matches if m.first.path.exists() and m.second.path.exists()]
|
||||
matches = [m for m in matches if not (m.first.is_ref and m.second.is_ref)]
|
||||
if ignore_list:
|
||||
matches = [m for m in matches if not ignore_list.are_ignored(str(m.first.path), str(m.second.path))]
|
||||
|
|
@ -212,3 +213,4 @@ class Scanner:
|
|||
large_size_threshold = 0
|
||||
big_file_size_threshold = 0
|
||||
word_weighting = False
|
||||
include_exists_check = True
|
||||
|
|
|
|||
|
|
@ -36,83 +36,83 @@ msgstr ""
|
|||
msgid "Sending to Trash"
|
||||
msgstr ""
|
||||
|
||||
#: core\app.py:291
|
||||
#: core\app.py:293
|
||||
msgid "A previous action is still hanging in there. You can't start a new one yet. Wait a few seconds, then try again."
|
||||
msgstr ""
|
||||
|
||||
#: core\app.py:302
|
||||
#: core\app.py:304
|
||||
msgid "No duplicates found."
|
||||
msgstr ""
|
||||
|
||||
#: core\app.py:317
|
||||
#: core\app.py:319
|
||||
msgid "All marked files were copied successfully."
|
||||
msgstr ""
|
||||
|
||||
#: core\app.py:319
|
||||
#: core\app.py:321
|
||||
msgid "All marked files were moved successfully."
|
||||
msgstr ""
|
||||
|
||||
#: core\app.py:321
|
||||
#: core\app.py:323
|
||||
msgid "All marked files were deleted successfully."
|
||||
msgstr ""
|
||||
|
||||
#: core\app.py:323
|
||||
#: core\app.py:325
|
||||
msgid "All marked files were successfully sent to Trash."
|
||||
msgstr ""
|
||||
|
||||
#: core\app.py:328
|
||||
#: core\app.py:330
|
||||
msgid "Could not load file: {}"
|
||||
msgstr ""
|
||||
|
||||
#: core\app.py:384
|
||||
#: core\app.py:386
|
||||
msgid "'{}' already is in the list."
|
||||
msgstr ""
|
||||
|
||||
#: core\app.py:386
|
||||
#: core\app.py:388
|
||||
msgid "'{}' does not exist."
|
||||
msgstr ""
|
||||
|
||||
#: core\app.py:394
|
||||
#: core\app.py:396
|
||||
msgid "All selected %d matches are going to be ignored in all subsequent scans. Continue?"
|
||||
msgstr ""
|
||||
|
||||
#: core\app.py:471
|
||||
#: core\app.py:473
|
||||
msgid "Select a directory to copy marked files to"
|
||||
msgstr ""
|
||||
|
||||
#: core\app.py:473
|
||||
#: core\app.py:475
|
||||
msgid "Select a directory to move marked files to"
|
||||
msgstr ""
|
||||
|
||||
#: core\app.py:512
|
||||
#: core\app.py:514
|
||||
msgid "Select a destination for your exported CSV"
|
||||
msgstr ""
|
||||
|
||||
#: core\app.py:518 core\app.py:773 core\app.py:783
|
||||
#: core\app.py:520 core\app.py:781 core\app.py:791
|
||||
msgid "Couldn't write to file: {}"
|
||||
msgstr ""
|
||||
|
||||
#: core\app.py:541
|
||||
#: core\app.py:543
|
||||
msgid "You have no custom command set up. Set it up in your preferences."
|
||||
msgstr ""
|
||||
|
||||
#: core\app.py:697 core\app.py:709
|
||||
#: core\app.py:705 core\app.py:717
|
||||
msgid "You are about to remove %d files from results. Continue?"
|
||||
msgstr ""
|
||||
|
||||
#: core\app.py:745
|
||||
#: core\app.py:753
|
||||
msgid "{} duplicate groups were changed by the re-prioritization."
|
||||
msgstr ""
|
||||
|
||||
#: core\app.py:792
|
||||
#: core\app.py:801
|
||||
msgid "The selected directories contain no scannable file."
|
||||
msgstr ""
|
||||
|
||||
#: core\app.py:808
|
||||
#: core\app.py:817
|
||||
msgid "Collecting files to scan"
|
||||
msgstr ""
|
||||
|
||||
#: core\app.py:858
|
||||
#: core\app.py:867
|
||||
msgid "%s (%d discarded)"
|
||||
msgstr ""
|
||||
|
||||
|
|
|
|||
|
|
@ -1092,3 +1092,17 @@ msgstr ""
|
|||
#: qt\search_edit.py:78
|
||||
msgid "Search..."
|
||||
msgstr ""
|
||||
|
||||
#: qt\preferences_dialog.py:219
|
||||
msgid ""
|
||||
"These options are for advanced users or for very specific situations, most "
|
||||
"users should not have to modify these."
|
||||
msgstr ""
|
||||
|
||||
#: qt\preferences_dialog.py:225
|
||||
msgid "Include existence check after scan completion"
|
||||
msgstr ""
|
||||
|
||||
#: qt\preferences_dialog.py:227
|
||||
msgid "Ignore difference in mtime when loading cached digests"
|
||||
msgstr ""
|
||||
|
|
|
|||
|
|
@ -193,6 +193,8 @@ class DupeGuru(QObject):
|
|||
self.model.options["scanned_tags"] = scanned_tags
|
||||
self.model.options["match_scaled"] = self.prefs.match_scaled
|
||||
self.model.options["picture_cache_type"] = self.prefs.picture_cache_type
|
||||
self.model.options["include_exists_check"] = self.prefs.include_exists_check
|
||||
self.model.options["rehash_ignore_mtime"] = self.prefs.rehash_ignore_mtime
|
||||
|
||||
if self.details_dialog:
|
||||
self.details_dialog.update_options()
|
||||
|
|
|
|||
|
|
@ -161,6 +161,8 @@ class Preferences(PreferencesBase):
|
|||
self.ignore_hardlink_matches = get("IgnoreHardlinkMatches", self.ignore_hardlink_matches)
|
||||
self.use_regexp = get("UseRegexp", self.use_regexp)
|
||||
self.remove_empty_folders = get("RemoveEmptyFolders", self.remove_empty_folders)
|
||||
self.rehash_ignore_mtime = get("RehashIgnoreMTime", self.rehash_ignore_mtime)
|
||||
self.include_exists_check = get("IncludeExistsCheck", self.include_exists_check)
|
||||
self.debug_mode = get("DebugMode", self.debug_mode)
|
||||
self.profile_scan = get("ProfileScan", self.profile_scan)
|
||||
self.destination_type = get("DestinationType", self.destination_type)
|
||||
|
|
@ -231,6 +233,8 @@ class Preferences(PreferencesBase):
|
|||
self.use_regexp = False
|
||||
self.ignore_hardlink_matches = False
|
||||
self.remove_empty_folders = False
|
||||
self.rehash_ignore_mtime = False
|
||||
self.include_exists_check = True
|
||||
self.debug_mode = False
|
||||
self.profile_scan = False
|
||||
self.destination_type = 1
|
||||
|
|
@ -283,6 +287,8 @@ class Preferences(PreferencesBase):
|
|||
set_("IgnoreHardlinkMatches", self.ignore_hardlink_matches)
|
||||
set_("UseRegexp", self.use_regexp)
|
||||
set_("RemoveEmptyFolders", self.remove_empty_folders)
|
||||
set_("RehashIgnoreMTime", self.rehash_ignore_mtime)
|
||||
set_("IncludeExistsCheck", self.include_exists_check)
|
||||
set_("DebugMode", self.debug_mode)
|
||||
set_("ProfileScan", self.profile_scan)
|
||||
set_("DestinationType", self.destination_type)
|
||||
|
|
|
|||
|
|
@ -47,8 +47,9 @@ class Sections(Flag):
|
|||
|
||||
GENERAL = auto()
|
||||
DISPLAY = auto()
|
||||
ADVANCED = auto()
|
||||
DEBUG = auto()
|
||||
ALL = GENERAL | DISPLAY | DEBUG
|
||||
ALL = GENERAL | DISPLAY | ADVANCED | DEBUG
|
||||
|
||||
|
||||
class PreferencesDialogBase(QDialog):
|
||||
|
|
@ -213,6 +214,19 @@ use the modifier key to drag the floating window around"
|
|||
details_groupbox.setLayout(self.details_groupbox_layout)
|
||||
self.displayVLayout.addWidget(details_groupbox)
|
||||
|
||||
def _setup_advanced_page(self):
|
||||
tab_label = QLabel(
|
||||
tr(
|
||||
"These options are for advanced users or for very specific situations, most users should not have to modify these."
|
||||
),
|
||||
wordWrap=True,
|
||||
)
|
||||
self.advanced_vlayout.addWidget(tab_label)
|
||||
self._setupAddCheckbox("include_exists_check_box", tr("Include existence check after scan completion"))
|
||||
self.advanced_vlayout.addWidget(self.include_exists_check_box)
|
||||
self._setupAddCheckbox("rehash_ignore_mtime_box", tr("Ignore difference in mtime when loading cached digests"))
|
||||
self.advanced_vlayout.addWidget(self.rehash_ignore_mtime_box)
|
||||
|
||||
def _setupDebugPage(self):
|
||||
self._setupAddCheckbox("debugModeBox", tr("Debug mode (restart required)"))
|
||||
self._setupAddCheckbox("profile_scan_box", tr("Profile scan operation"))
|
||||
|
|
@ -244,16 +258,20 @@ use the modifier key to drag the floating window around"
|
|||
self.tabwidget = QTabWidget()
|
||||
self.page_general = QWidget()
|
||||
self.page_display = QWidget()
|
||||
self.page_advanced = QWidget()
|
||||
self.page_debug = QWidget()
|
||||
self.widgetsVLayout = QVBoxLayout()
|
||||
self.page_general.setLayout(self.widgetsVLayout)
|
||||
self.displayVLayout = QVBoxLayout()
|
||||
self.displayVLayout.setSpacing(5) # arbitrary value, might conflict with style
|
||||
self.page_display.setLayout(self.displayVLayout)
|
||||
self.advanced_vlayout = QVBoxLayout()
|
||||
self.page_advanced.setLayout(self.advanced_vlayout)
|
||||
self.debugVLayout = QVBoxLayout()
|
||||
self.page_debug.setLayout(self.debugVLayout)
|
||||
self._setupPreferenceWidgets()
|
||||
self._setupDisplayPage()
|
||||
self._setup_advanced_page()
|
||||
self._setupDebugPage()
|
||||
# self.mainVLayout.addLayout(self.widgetsVLayout)
|
||||
self.buttonBox = QDialogButtonBox(self)
|
||||
|
|
@ -265,9 +283,11 @@ use the modifier key to drag the floating window around"
|
|||
self.layout().setSizeConstraint(QLayout.SetFixedSize)
|
||||
self.tabwidget.addTab(self.page_general, tr("General"))
|
||||
self.tabwidget.addTab(self.page_display, tr("Display"))
|
||||
self.tabwidget.addTab(self.page_advanced, tr("Advanced"))
|
||||
self.tabwidget.addTab(self.page_debug, tr("Debug"))
|
||||
self.displayVLayout.addStretch(0)
|
||||
self.widgetsVLayout.addStretch(0)
|
||||
self.advanced_vlayout.addStretch(0)
|
||||
self.debugVLayout.addStretch(0)
|
||||
|
||||
def _load(self, prefs, setchecked, section):
|
||||
|
|
@ -318,6 +338,9 @@ use the modifier key to drag the floating window around"
|
|||
except KeyError:
|
||||
selected_lang = self.supportedLanguages["en"]
|
||||
self.languageComboBox.setCurrentText(selected_lang)
|
||||
if section & Sections.ADVANCED:
|
||||
setchecked(self.rehash_ignore_mtime_box, prefs.rehash_ignore_mtime)
|
||||
setchecked(self.include_exists_check_box, prefs.include_exists_check)
|
||||
if section & Sections.DEBUG:
|
||||
setchecked(self.debugModeBox, prefs.debug_mode)
|
||||
setchecked(self.profile_scan_box, prefs.profile_scan)
|
||||
|
|
@ -334,6 +357,8 @@ use the modifier key to drag the floating window around"
|
|||
prefs.use_regexp = ischecked(self.useRegexpBox)
|
||||
prefs.remove_empty_folders = ischecked(self.removeEmptyFoldersBox)
|
||||
prefs.ignore_hardlink_matches = ischecked(self.ignoreHardlinkMatches)
|
||||
prefs.rehash_ignore_mtime = ischecked(self.rehash_ignore_mtime_box)
|
||||
prefs.include_exists_check = ischecked(self.include_exists_check_box)
|
||||
prefs.debug_mode = ischecked(self.debugModeBox)
|
||||
prefs.profile_scan = ischecked(self.profile_scan_box)
|
||||
prefs.reference_bold_font = ischecked(self.reference_bold_font)
|
||||
|
|
|
|||
Loading…
Reference in New Issue