v4.0.3

I forgot to remove a sparkle reference in the build script.

.gitmodules

@@ -7,6 +7,3 @@
 [submodule "cocoalib"]
 	path = cocoalib
 	url = https://github.com/hsoft/cocoalib.git
-[submodule "cocoa/Sparkle"]
-	path = cocoa/Sparkle
-	url = https://github.com/sparkle-project/Sparkle.git

README.md

@@ -60,7 +60,6 @@ This folder contains the source for dupeGuru. Its documentation is in `help`, bu
 There are also other sub-folder that comes from external repositories and are part of this repo as
 git submodules:
 
-* Sparkle: An auto-update library for the OS X version.
 * hscommon: A collection of helpers used across HS applications.
 * cocoalib: A collection of helpers used across Cocoa UI codebases of HS applications.
 * qtlib: A collection of helpers used across Qt UI codebases of HS applications.

build.py

@@ -106,12 +106,6 @@ def build_xibless(dest='cocoa/autogen'):
         )
 
 def build_cocoa(dev):
-    sparkle_framework_path = op.join('cocoa', 'Sparkle', 'build', 'Release', 'Sparkle.framework')
-    if not op.exists(sparkle_framework_path):
-        print("Building Sparkle")
-        os.chdir(op.join('cocoa', 'Sparkle'))
-        print_and_do('make build')
-        os.chdir(op.join('..', '..'))
     print("Creating OS X app structure")
     app = cocoa_app()
     app_version = get_module_version('core')
@@ -160,7 +154,7 @@ def build_cocoa(dev):
     image_path = 'cocoa/dupeguru.icns'
     resources = [image_path, 'cocoa/dsa_pub.pem', 'build/dg_cocoa.py', 'build/help']
     app.copy_resources(*resources, use_symlinks=dev)
-    app.copy_frameworks('build/Python', sparkle_framework_path)
+    app.copy_frameworks('build/Python')
     print("Creating the run.py file")
     tmpl = open('cocoa/run_template.py', 'rt').read()
     run_contents = tmpl.replace('{{app_path}}', app.dest)

cocoa/AppDelegate.h

@@ -7,7 +7,6 @@ http://www.gnu.org/licenses/gpl-3.0.html
 */
 
 #import <Cocoa/Cocoa.h>
-#import <Sparkle/SUUpdater.h>
 #import "PyDupeGuru.h"
 #import "ResultWindow.h"
 #import "ResultTable.h"
@@ -24,7 +23,6 @@ http://www.gnu.org/licenses/gpl-3.0.html
 {
     NSMenu *recentResultsMenu;
     NSMenu *columnsMenu;
-    SUUpdater *updater;
     
     PyDupeGuru *model;
     ResultWindow *_resultWindow;
@@ -41,7 +39,6 @@ http://www.gnu.org/licenses/gpl-3.0.html
 
 @property (readwrite, retain) NSMenu *recentResultsMenu;
 @property (readwrite, retain) NSMenu *columnsMenu;
-@property (readwrite, retain) SUUpdater *updater;
 
 /* Virtual */
 + (NSDictionary *)defaultPreferences;

cocoa/AppDelegate.m

@@ -22,7 +22,6 @@ http://www.gnu.org/licenses/gpl-3.0.html
 
 @synthesize recentResultsMenu;
 @synthesize columnsMenu;
-@synthesize updater;
 
 + (NSDictionary *)defaultPreferences
 {
@@ -70,7 +69,6 @@ http://www.gnu.org/licenses/gpl-3.0.html
     self = [super init];
     model = [[PyDupeGuru alloc] init];
     [model bindCallback:createCallback(@"DupeGuruView", self)];
-    [self setUpdater:[SUUpdater sharedUpdater]];
     NSMutableIndexSet *contentsIndexes = [NSMutableIndexSet indexSet];
     [contentsIndexes addIndex:1];
     [contentsIndexes addIndex:2];
@@ -92,12 +90,6 @@ http://www.gnu.org/licenses/gpl-3.0.html
     // We can only finalize initialization once the main menu has been created, which cannot happen
     // before AppDelegate is created.
     NSUserDefaults *ud = [NSUserDefaults standardUserDefaults];
-    /* Because the pref pane is lazily loaded, we have to manually do the update check if the
-       preference is set.
-    */
-    if ([ud boolForKey:@"SUEnableAutomaticChecks"]) {
-        [[SUUpdater sharedUpdater] checkForUpdatesInBackground];
-    }
     _recentResults = [[HSRecentFiles alloc] initWithName:@"recentResults" menu:recentResultsMenu];
     [_recentResults setDelegate:self];
     _directoryPanel = [[DirectoryPanel alloc] initWithParentApp:self];

cocoa/inter/app.py

@@ -1,7 +1,6 @@
 import logging
 
 from objp.util import pyref, dontwrap
-from hscommon.path import Path, pathify
 from cocoa import install_exception_hook, install_cocoa_logger, patch_threaded_job_performer
 from cocoa.inter import PyBaseApp, BaseAppView
 
@@ -11,15 +10,17 @@ from .directories import Directories, Bundle
 from .photo import Photo
 
 class DupeGuru(DupeGuruBase):
+    PICTURE_CACHE_TYPE = 'shelve'
+
     def __init__(self, view):
         DupeGuruBase.__init__(self, view)
         self.directories = Directories()
-    
+
     def selected_dupe_path(self):
         if not self.selected_dupes:
             return None
         return self.selected_dupes[0].path
-    
+
     def selected_dupe_ref_path(self):
         if not self.selected_dupes:
             return None
@@ -27,7 +28,7 @@ class DupeGuru(DupeGuruBase):
         if ref is self.selected_dupes[0]: # we don't want the same pic to be displayed on both sides
             return None
         return ref.path
-    
+
     def _get_fileclasses(self):
         result = DupeGuruBase._get_fileclasses(self)
         if self.app_mode == AppMode.Standard:
@@ -51,133 +52,133 @@ class PyDupeGuru(PyBaseApp):
         install_cocoa_logger()
         patch_threaded_job_performer()
         self.model = DupeGuru(self)
-    
+
     #---Sub-proxies
     def detailsPanel(self) -> pyref:
         return self.model.details_panel
-    
+
     def directoryTree(self) -> pyref:
         return self.model.directory_tree
-    
+
     def problemDialog(self) -> pyref:
         return self.model.problem_dialog
-    
+
     def statsLabel(self) -> pyref:
         return self.model.stats_label
-    
+
     def resultTable(self) -> pyref:
         return self.model.result_table
-    
+
     def ignoreListDialog(self) -> pyref:
         return self.model.ignore_list_dialog
-    
+
     def progressWindow(self) -> pyref:
         return self.model.progress_window
-    
+
     def deletionOptions(self) -> pyref:
         return self.model.deletion_options
-    
+
     #---Directories
     def addDirectory_(self, directory: str):
         self.model.add_directory(directory)
-    
+
     #---Results
     def doScan(self):
         self.model.start_scanning()
-    
+
     def exportToXHTML(self):
         self.model.export_to_xhtml()
-    
+
     def exportToCSV(self):
         self.model.export_to_csv()
-    
+
     def loadSession(self):
         self.model.load()
-    
+
     def loadResultsFrom_(self, filename: str):
         self.model.load_from(filename)
-    
+
     def markAll(self):
         self.model.mark_all()
-    
+
     def markNone(self):
         self.model.mark_none()
-    
+
     def markInvert(self):
         self.model.mark_invert()
-    
+
     def purgeIgnoreList(self):
         self.model.purge_ignore_list()
-    
+
     def toggleSelectedMark(self):
         self.model.toggle_selected_mark_state()
-    
+
     def saveSession(self):
         self.model.save()
-    
+
     def saveResultsAs_(self, filename: str):
         self.model.save_as(filename)
-    
+
     #---Actions
     def addSelectedToIgnoreList(self):
         self.model.add_selected_to_ignore_list()
-    
+
     def deleteMarked(self):
         self.model.delete_marked()
-    
+
     def applyFilter_(self, filter: str):
         self.model.apply_filter(filter)
-    
+
     def makeSelectedReference(self):
         self.model.make_selected_reference()
-    
+
     def copyMarked(self):
         self.model.copy_or_move_marked(copy=True)
-    
+
     def moveMarked(self):
         self.model.copy_or_move_marked(copy=False)
-    
+
     def openSelected(self):
         self.model.open_selected()
-    
+
     def removeMarked(self):
         self.model.remove_marked()
-    
+
     def removeSelected(self):
         self.model.remove_selected()
-    
+
     def revealSelected(self):
         self.model.reveal_selected()
 
     def invokeCustomCommand(self):
         self.model.invoke_custom_command()
-    
+
     def showIgnoreList(self):
         self.model.ignore_list_dialog.show()
-    
+
     def clearPictureCache(self):
         self.model.clear_picture_cache()
-    
+
     #---Information
     def getScanOptions(self) -> list:
         return [o.label for o in self.model.SCANNER_CLASS.get_scan_options()]
-    
+
     def resultsAreModified(self) -> bool:
         return self.model.results.is_modified
-    
+
     def getSelectedDupePath(self) -> str:
         return str(self.model.selected_dupe_path())
-    
+
     def getSelectedDupeRefPath(self) -> str:
         return str(self.model.selected_dupe_ref_path())
-    
+
     #---Properties
     def getAppMode(self) -> int:
         return self.model.app_mode
 
     def setAppMode_(self, app_mode: int):
         self.model.app_mode = app_mode
-    
+
     def setScanType_(self, scan_type_index: int):
         scan_options = self.model.SCANNER_CLASS.get_scan_options()
         try:
@@ -188,13 +189,13 @@ class PyDupeGuru(PyBaseApp):
 
     def setMinMatchPercentage_(self, percentage: int):
         self.model.options['min_match_percentage'] = int(percentage)
-    
+
     def setWordWeighting_(self, words_are_weighted: bool):
         self.model.options['word_weighting'] = words_are_weighted
-    
+
     def setMatchSimilarWords_(self, match_similar_words: bool):
         self.model.options['match_similar_words'] = match_similar_words
-    
+
     def setSizeThreshold_(self, size_threshold: int):
         self.model.options['size_threshold'] = size_threshold
 
@@ -208,44 +209,44 @@ class PyDupeGuru(PyBaseApp):
 
     def setMatchScaled_(self, match_scaled: bool):
         self.model.options['match_scaled'] = match_scaled
-    
+
     def setMixFileKind_(self, mix_file_kind: bool):
         self.model.options['mix_file_kind'] = mix_file_kind
-    
+
     def setEscapeFilterRegexp_(self, escape_filter_regexp: bool):
         self.model.options['escape_filter_regexp'] = escape_filter_regexp
-    
+
     def setRemoveEmptyFolders_(self, remove_empty_folders: bool):
         self.model.options['clean_empty_dirs'] = remove_empty_folders
-    
+
     def setIgnoreHardlinkMatches_(self, ignore_hardlink_matches: bool):
         self.model.options['ignore_hardlink_matches'] = ignore_hardlink_matches
-    
+
     def setCopyMoveDestType_(self, copymove_dest_type: int):
         self.model.options['copymove_dest_type'] = copymove_dest_type
-    
+
     #--- model --> view
     @dontwrap
     def ask_yes_no(self, prompt):
         return self.callback.askYesNoWithPrompt_(prompt)
-    
+
     @dontwrap
     def create_results_window(self):
         self.callback.createResultsWindow()
-    
+
     @dontwrap
     def show_results_window(self):
         self.callback.showResultsWindow()
-    
+
     @dontwrap
     def show_problem_dialog(self):
         self.callback.showProblemDialog()
-    
+
     @dontwrap
     def select_dest_folder(self, prompt):
         return self.callback.selectDestFolderWithPrompt_(prompt)
-    
+
     @dontwrap
     def select_dest_file(self, prompt, extension):
         return self.callback.selectDestFileWithPrompt_extension_(prompt, extension)
-    
+

cocoa/ui/main_menu.py

@@ -12,7 +12,6 @@ windowMenu = result.addMenu("Window")
 helpMenu = result.addMenu("Help")
 
 appMenu.addItem("About dupeGuru", Action(owner, 'showAboutBox'))
-appMenu.addItem("Check for update...", Action(owner.updater, 'checkForUpdates:'))
 appMenu.addSeparator()
 appMenu.addItem("Preferences...", Action(owner, 'showPreferencesPanel'), 'cmd+,')
 appMenu.addSeparator()

cocoa/wscript

@@ -31,8 +31,6 @@ def configure(conf):
     conf.env.FRAMEWORK_COCOA = 'Cocoa'
     conf.env.ARCH_COCOA = ['x86_64']
     conf.env.MACOSX_DEPLOYMENT_TARGET = '10.8'
-    conf.env.CFLAGS = ['-F'+op.abspath('Sparkle/build/Release')]
-    conf.env.LINKFLAGS = ['-F'+op.abspath('Sparkle/build/Release')]
 
 def build(ctx):
     # What do we compile?
@@ -62,7 +60,7 @@ def build(ctx):
         # Because our python lib's install name is "@rpath/Python", we need to set the executable's
         # rpath. Fortunately, WAF supports it and we just need to supply the "rpath" argument.
         rpath         = '@executable_path/../Frameworks',
-        framework     = ['Sparkle', 'Quartz'],
+        framework     = ['Quartz'],
     )
 
 from waflib import TaskGen

core/__init__.py

@@ -1,3 +1,3 @@
-__version__ = '4.0.2'
+__version__ = '4.0.3'
 __appname__ = 'dupeGuru'
 

core/app.py

@@ -116,6 +116,8 @@ class DupeGuru(Broadcaster):
 
     NAME = PROMPT_NAME = "dupeGuru"
 
+    PICTURE_CACHE_TYPE = 'sqlite' # set to 'shelve' for a ShelveCache
+
     def __init__(self, view):
         if view.get_default(DEBUG_MODE_PREFERENCE):
             logging.getLogger().setLevel(logging.DEBUG)
@@ -138,7 +140,7 @@ class DupeGuru(Broadcaster):
             'clean_empty_dirs': False,
             'ignore_hardlink_matches': False,
             'copymove_dest_type': DestType.Relative,
-            'cache_path': op.join(self.appdata, 'cached_pictures.db'),
+            'picture_cache_type': self.PICTURE_CACHE_TYPE
         }
         self.selected_dupes = []
         self.details_panel = DetailsPanel(self)
@@ -166,6 +168,11 @@ class DupeGuru(Broadcaster):
         self.result_table.connect()
         self.view.create_results_window()
 
+    def _get_picture_cache_path(self):
+        cache_type = self.options['picture_cache_type']
+        cache_name = 'cached_pictures.shelve' if cache_type == 'shelve' else 'cached_pictures.db'
+        return op.join(self.appdata, cache_name)
+
     def _get_dupe_sort_key(self, dupe, get_group, key, delta):
         if self.app_mode in (AppMode.Music, AppMode.Picture):
             if key == 'folder_path':
@@ -405,9 +412,10 @@ class DupeGuru(Broadcaster):
                 path = path.parent()
 
     def clear_picture_cache(self):
-        cache = pe.cache.Cache(self.options['cache_path'])
+        try:
-        cache.clear()
+            os.remove(self._get_picture_cache_path())
-        cache.close()
+        except FileNotFoundError:
+            pass # we don't care
 
     def copy_or_move(self, dupe, copy: bool, destination: str, dest_type: DestType):
         source_path = dupe.path
@@ -754,6 +762,8 @@ class DupeGuru(Broadcaster):
         for k, v in self.options.items():
             if hasattr(scanner, k):
                 setattr(scanner, k, v)
+        if self.app_mode == AppMode.Picture:
+            scanner.cache_path = self._get_picture_cache_path()
         self.results.groups = []
         self._recreate_result_table()
         self._results_changed()

core/pe/cache.py

@@ -1,17 +1,10 @@
-# Created By: Virgil Dupras
+# Copyright 2016 Virgil Dupras
-# Created On: 2006/09/14
-# Copyright 2015 Hardcoded Software (http://www.hardcoded.net)
 #
 # This software is licensed under the "GPLv3" License as described in the "LICENSE" file,
 # which should be included with this package. The terms are also available at
 # http://www.gnu.org/licenses/gpl-3.0.html
 
-import os
+from ._cache import string_to_colors # noqa
-import os.path as op
-import logging
-import sqlite3 as sqlite
-
-from ._cache import string_to_colors
 
 def colors_to_string(colors):
     """Transform the 3 sized tuples 'colors' into a hex string.
@@ -19,7 +12,7 @@ def colors_to_string(colors):
     [(0,100,255)] --> 0064ff
     [(1,2,3),(4,5,6)] --> 010203040506
     """
-    return ''.join(['%02x%02x%02x' % (r, g, b) for r, g, b in colors])
+    return ''.join('%02x%02x%02x' % (r, g, b) for r, g, b in colors)
 
 # This function is an important bottleneck of dupeGuru PE. It has been converted to C.
 # def string_to_colors(s):
@@ -31,132 +24,3 @@ def colors_to_string(colors):
 #         result.append((number >> 16, (number >> 8) & 0xff, number & 0xff))
 #     return result
 
-class Cache:
-    """A class to cache picture blocks.
-    """
-    def __init__(self, db=':memory:'):
-        self.dbname = db
-        self.con = None
-        self._create_con()
-
-    def __contains__(self, key):
-        sql = "select count(*) from pictures where path = ?"
-        result = self.con.execute(sql, [key]).fetchall()
-        return result[0][0] > 0
-
-    def __delitem__(self, key):
-        if key not in self:
-            raise KeyError(key)
-        sql = "delete from pictures where path = ?"
-        self.con.execute(sql, [key])
-
-    # Optimized
-    def __getitem__(self, key):
-        if isinstance(key, int):
-            sql = "select blocks from pictures where rowid = ?"
-        else:
-            sql = "select blocks from pictures where path = ?"
-        result = self.con.execute(sql, [key]).fetchone()
-        if result:
-            result = string_to_colors(result[0])
-            return result
-        else:
-            raise KeyError(key)
-
-    def __iter__(self):
-        sql = "select path from pictures"
-        result = self.con.execute(sql)
-        return (row[0] for row in result)
-
-    def __len__(self):
-        sql = "select count(*) from pictures"
-        result = self.con.execute(sql).fetchall()
-        return result[0][0]
-
-    def __setitem__(self, path_str, blocks):
-        blocks = colors_to_string(blocks)
-        if op.exists(path_str):
-            mtime = int(os.stat(path_str).st_mtime)
-        else:
-            mtime = 0
-        if path_str in self:
-            sql = "update pictures set blocks = ?, mtime = ? where path = ?"
-        else:
-            sql = "insert into pictures(blocks,mtime,path) values(?,?,?)"
-        try:
-            self.con.execute(sql, [blocks, mtime, path_str])
-        except sqlite.OperationalError:
-            logging.warning('Picture cache could not set value for key %r', path_str)
-        except sqlite.DatabaseError as e:
-            logging.warning('DatabaseError while setting value for key %r: %s', path_str, str(e))
-
-    def _create_con(self, second_try=False):
-        def create_tables():
-            logging.debug("Creating picture cache tables.")
-            self.con.execute("drop table if exists pictures")
-            self.con.execute("drop index if exists idx_path")
-            self.con.execute("create table pictures(path TEXT, mtime INTEGER, blocks TEXT)")
-            self.con.execute("create index idx_path on pictures (path)")
-
-        self.con = sqlite.connect(self.dbname, isolation_level=None)
-        try:
-            self.con.execute("select path, mtime, blocks from pictures where 1=2")
-        except sqlite.OperationalError: # new db
-            create_tables()
-        except sqlite.DatabaseError as e: # corrupted db
-            if second_try:
-                raise # Something really strange is happening
-            logging.warning('Could not create picture cache because of an error: %s', str(e))
-            self.con.close()
-            os.remove(self.dbname)
-            self._create_con(second_try=True)
-
-    def clear(self):
-        self.close()
-        if self.dbname != ':memory:':
-            os.remove(self.dbname)
-        self._create_con()
-
-    def close(self):
-        if self.con is not None:
-            self.con.close()
-        self.con = None
-
-    def filter(self, func):
-        to_delete = [key for key in self if not func(key)]
-        for key in to_delete:
-            del self[key]
-
-    def get_id(self, path):
-        sql = "select rowid from pictures where path = ?"
-        result = self.con.execute(sql, [path]).fetchone()
-        if result:
-            return result[0]
-        else:
-            raise ValueError(path)
-
-    def get_multiple(self, rowids):
-        sql = "select rowid, blocks from pictures where rowid in (%s)" % ','.join(map(str, rowids))
-        cur = self.con.execute(sql)
-        return ((rowid, string_to_colors(blocks)) for rowid, blocks in cur)
-
-    def purge_outdated(self):
-        """Go through the cache and purge outdated records.
-
-        A record is outdated if the picture doesn't exist or if its mtime is greater than the one in
-        the db.
-        """
-        todelete = []
-        sql = "select rowid, path, mtime from pictures"
-        cur = self.con.execute(sql)
-        for rowid, path_str, mtime in cur:
-            if mtime and op.exists(path_str):
-                picture_mtime = os.stat(path_str).st_mtime
-                if int(picture_mtime) <= mtime:
-                    # not outdated
-                    continue
-            todelete.append(rowid)
-        if todelete:
-            sql = "delete from pictures where rowid in (%s)" % ','.join(map(str, todelete))
-            self.con.execute(sql)
-

core/pe/cache_shelve.py

@@ -0,0 +1,131 @@
+# Copyright 2016 Virgil Dupras
+#
+# This software is licensed under the "GPLv3" License as described in the "LICENSE" file,
+# which should be included with this package. The terms are also available at
+# http://www.gnu.org/licenses/gpl-3.0.html
+
+import os
+import os.path as op
+import shelve
+import tempfile
+from collections import namedtuple
+
+from .cache import string_to_colors, colors_to_string
+
+def wrap_path(path):
+    return 'path:{}'.format(path)
+
+def unwrap_path(key):
+    return key[5:]
+
+def wrap_id(path):
+    return 'id:{}'.format(path)
+
+def unwrap_id(key):
+    return int(key[3:])
+
+CacheRow = namedtuple('CacheRow', 'id path blocks mtime')
+
+class ShelveCache:
+    """A class to cache picture blocks in a shelve backend.
+    """
+    def __init__(self, db=None, readonly=False):
+        self.istmp = db is None
+        if self.istmp:
+            self.dtmp = tempfile.mkdtemp()
+            self.ftmp = db = op.join(self.dtmp, 'tmpdb')
+        flag = 'r' if readonly else 'c'
+        self.shelve = shelve.open(db, flag)
+        self.maxid = self._compute_maxid()
+
+    def __contains__(self, key):
+        return wrap_path(key) in self.shelve
+
+    def __delitem__(self, key):
+        row = self.shelve[wrap_path(key)]
+        del self.shelve[wrap_path(key)]
+        del self.shelve[wrap_id(row.id)]
+
+    def __getitem__(self, key):
+        if isinstance(key, int):
+            skey = self.shelve[wrap_id(key)]
+        else:
+            skey = wrap_path(key)
+        return string_to_colors(self.shelve[skey].blocks)
+
+    def __iter__(self):
+        return (unwrap_path(k) for k in self.shelve if k.startswith('path:'))
+
+    def __len__(self):
+        return sum(1 for k in self.shelve if k.startswith('path:'))
+
+    def __setitem__(self, path_str, blocks):
+        blocks = colors_to_string(blocks)
+        if op.exists(path_str):
+            mtime = int(os.stat(path_str).st_mtime)
+        else:
+            mtime = 0
+        if path_str in self:
+            rowid = self.shelve[wrap_path(path_str)].id
+        else:
+            rowid = self._get_new_id()
+        row = CacheRow(rowid, path_str, blocks, mtime)
+        self.shelve[wrap_path(path_str)] = row
+        self.shelve[wrap_id(rowid)] = wrap_path(path_str)
+
+    def _compute_maxid(self):
+        return max((unwrap_id(k) for k in self.shelve if k.startswith('id:')), default=1)
+
+    def _get_new_id(self):
+        self.maxid += 1
+        return self.maxid
+
+    def clear(self):
+        self.shelve.clear()
+
+    def close(self):
+        if self.shelve is not None:
+            self.shelve.close()
+            if self.istmp:
+                os.remove(self.ftmp)
+                os.rmdir(self.dtmp)
+        self.shelve = None
+
+    def filter(self, func):
+        to_delete = [key for key in self if not func(key)]
+        for key in to_delete:
+            del self[key]
+
+    def get_id(self, path):
+        if path in self:
+            return self.shelve[wrap_path(path)].id
+        else:
+            raise ValueError(path)
+
+    def get_multiple(self, rowids):
+        for rowid in rowids:
+            try:
+                skey = self.shelve[wrap_id(rowid)]
+            except KeyError:
+                continue
+            yield (rowid, string_to_colors(self.shelve[skey].blocks))
+
+    def purge_outdated(self):
+        """Go through the cache and purge outdated records.
+
+        A record is outdated if the picture doesn't exist or if its mtime is greater than the one in
+        the db.
+        """
+        todelete = []
+        for path in self:
+            row = self.shelve[wrap_path(path)]
+            if row.mtime and op.exists(path):
+                picture_mtime = os.stat(path).st_mtime
+                if int(picture_mtime) <= row.mtime:
+                    # not outdated
+                    continue
+            todelete.append(path)
+        for path in todelete:
+            del self[path]
+
+

core/pe/cache_sqlite.py

@@ -0,0 +1,143 @@
+# Copyright 2016 Virgil Dupras
+#
+# This software is licensed under the "GPLv3" License as described in the "LICENSE" file,
+# which should be included with this package. The terms are also available at
+# http://www.gnu.org/licenses/gpl-3.0.html
+
+import os
+import os.path as op
+import logging
+import sqlite3 as sqlite
+
+from .cache import string_to_colors, colors_to_string
+
+class SqliteCache:
+    """A class to cache picture blocks in a sqlite backend.
+    """
+    def __init__(self, db=':memory:', readonly=False):
+        # readonly is not used in the sqlite version of the cache
+        self.dbname = db
+        self.con = None
+        self._create_con()
+
+    def __contains__(self, key):
+        sql = "select count(*) from pictures where path = ?"
+        result = self.con.execute(sql, [key]).fetchall()
+        return result[0][0] > 0
+
+    def __delitem__(self, key):
+        if key not in self:
+            raise KeyError(key)
+        sql = "delete from pictures where path = ?"
+        self.con.execute(sql, [key])
+
+    # Optimized
+    def __getitem__(self, key):
+        if isinstance(key, int):
+            sql = "select blocks from pictures where rowid = ?"
+        else:
+            sql = "select blocks from pictures where path = ?"
+        result = self.con.execute(sql, [key]).fetchone()
+        if result:
+            result = string_to_colors(result[0])
+            return result
+        else:
+            raise KeyError(key)
+
+    def __iter__(self):
+        sql = "select path from pictures"
+        result = self.con.execute(sql)
+        return (row[0] for row in result)
+
+    def __len__(self):
+        sql = "select count(*) from pictures"
+        result = self.con.execute(sql).fetchall()
+        return result[0][0]
+
+    def __setitem__(self, path_str, blocks):
+        blocks = colors_to_string(blocks)
+        if op.exists(path_str):
+            mtime = int(os.stat(path_str).st_mtime)
+        else:
+            mtime = 0
+        if path_str in self:
+            sql = "update pictures set blocks = ?, mtime = ? where path = ?"
+        else:
+            sql = "insert into pictures(blocks,mtime,path) values(?,?,?)"
+        try:
+            self.con.execute(sql, [blocks, mtime, path_str])
+        except sqlite.OperationalError:
+            logging.warning('Picture cache could not set value for key %r', path_str)
+        except sqlite.DatabaseError as e:
+            logging.warning('DatabaseError while setting value for key %r: %s', path_str, str(e))
+
+    def _create_con(self, second_try=False):
+        def create_tables():
+            logging.debug("Creating picture cache tables.")
+            self.con.execute("drop table if exists pictures")
+            self.con.execute("drop index if exists idx_path")
+            self.con.execute("create table pictures(path TEXT, mtime INTEGER, blocks TEXT)")
+            self.con.execute("create index idx_path on pictures (path)")
+
+        self.con = sqlite.connect(self.dbname, isolation_level=None)
+        try:
+            self.con.execute("select path, mtime, blocks from pictures where 1=2")
+        except sqlite.OperationalError: # new db
+            create_tables()
+        except sqlite.DatabaseError as e: # corrupted db
+            if second_try:
+                raise # Something really strange is happening
+            logging.warning('Could not create picture cache because of an error: %s', str(e))
+            self.con.close()
+            os.remove(self.dbname)
+            self._create_con(second_try=True)
+
+    def clear(self):
+        self.close()
+        if self.dbname != ':memory:':
+            os.remove(self.dbname)
+        self._create_con()
+
+    def close(self):
+        if self.con is not None:
+            self.con.close()
+        self.con = None
+
+    def filter(self, func):
+        to_delete = [key for key in self if not func(key)]
+        for key in to_delete:
+            del self[key]
+
+    def get_id(self, path):
+        sql = "select rowid from pictures where path = ?"
+        result = self.con.execute(sql, [path]).fetchone()
+        if result:
+            return result[0]
+        else:
+            raise ValueError(path)
+
+    def get_multiple(self, rowids):
+        sql = "select rowid, blocks from pictures where rowid in (%s)" % ','.join(map(str, rowids))
+        cur = self.con.execute(sql)
+        return ((rowid, string_to_colors(blocks)) for rowid, blocks in cur)
+
+    def purge_outdated(self):
+        """Go through the cache and purge outdated records.
+
+        A record is outdated if the picture doesn't exist or if its mtime is greater than the one in
+        the db.
+        """
+        todelete = []
+        sql = "select rowid, path, mtime from pictures"
+        cur = self.con.execute(sql)
+        for rowid, path_str, mtime in cur:
+            if mtime and op.exists(path_str):
+                picture_mtime = os.stat(path_str).st_mtime
+                if int(picture_mtime) <= mtime:
+                    # not outdated
+                    continue
+            todelete.append(rowid)
+        if todelete:
+            sql = "delete from pictures where rowid in (%s)" % ','.join(map(str, todelete))
+            self.con.execute(sql)
+

core/pe/matchblock.py

@@ -16,7 +16,6 @@ from hscommon.jobprogress import job
 
 from core.engine import Match
 from .block import avgdiff, DifferentBlockCountError, NoBlocksError
-from .cache import Cache
 
 # OPTIMIZATION NOTES:
 # The bottleneck of the matching phase is CPU, which is why we use multiprocessing. However, another
@@ -49,12 +48,20 @@ except Exception:
     logging.warning("Had problems to determine cpu count on launch.")
     RESULTS_QUEUE_LIMIT = 8
 
+def get_cache(cache_path, readonly=False):
+    if cache_path.endswith('shelve'):
+        from .cache_shelve import ShelveCache
+        return ShelveCache(cache_path, readonly=readonly)
+    else:
+        from .cache_sqlite import SqliteCache
+        return SqliteCache(cache_path, readonly=readonly)
+
 def prepare_pictures(pictures, cache_path, with_dimensions, j=job.nulljob):
     # The MemoryError handlers in there use logging without first caring about whether or not
     # there is enough memory left to carry on the operation because it is assumed that the
     # MemoryError happens when trying to read an image file, which is freed from memory by the
     # time that MemoryError is raised.
-    cache = Cache(cache_path)
+    cache = get_cache(cache_path)
     cache.purge_outdated()
     prepared = [] # only pictures for which there was no error getting blocks
     try:
@@ -109,7 +116,7 @@ def async_compare(ref_ids, other_ids, dbname, threshold, picinfo):
     # The list of ids in ref_ids have to be compared to the list of ids in other_ids. other_ids
     # can be None. In this case, ref_ids has to be compared with itself
     # picinfo is a dictionary {pic_id: (dimensions, is_ref)}
-    cache = Cache(dbname)
+    cache = get_cache(dbname, readonly=True)
     limit = 100 - threshold
     ref_pairs = list(cache.get_multiple(ref_ids))
     if other_ids is not None:
@@ -159,7 +166,7 @@ def getmatches(pictures, cache_path, threshold, match_scaled=False, j=job.nulljo
     j = j.start_subjob([3, 7])
     pictures = prepare_pictures(pictures, cache_path, with_dimensions=not match_scaled, j=j)
     j = j.start_subjob([9, 1], tr("Preparing for matching"))
-    cache = Cache(cache_path)
+    cache = get_cache(cache_path)
     id2picture = {}
     for picture in pictures:
         try:

core/tests/cache_test.py

@@ -1,4 +1,4 @@
-# Copyright 2016 Hardcoded Software (http://www.hardcoded.net)
+# Copyright 2016 Virgil Dupras
 #
 # This software is licensed under the "GPLv3" License as described in the "LICENSE" file,
 # which should be included with this package. The terms are also available at
@@ -10,7 +10,9 @@ from pytest import raises, skip
 from hscommon.testutil import eq_
 
 try:
-    from ..pe.cache import Cache, colors_to_string, string_to_colors
+    from ..pe.cache import colors_to_string, string_to_colors
+    from ..pe.cache_sqlite import SqliteCache
+    from ..pe.cache_shelve import ShelveCache
 except ImportError:
     skip("Can't import the cache module, probably hasn't been compiled.")
 
@@ -44,21 +46,24 @@ class TestCasestring_to_colors:
         eq_([], string_to_colors('102'))
 
 
-class TestCaseCache:
+class BaseTestCaseCache:
+    def get_cache(self, dbname=None):
+        raise NotImplementedError()
+
     def test_empty(self):
-        c = Cache()
+        c = self.get_cache()
         eq_(0, len(c))
         with raises(KeyError):
             c['foo']
 
     def test_set_then_retrieve_blocks(self):
-        c = Cache()
+        c = self.get_cache()
         b = [(0, 0, 0), (1, 2, 3)]
         c['foo'] = b
         eq_(b, c['foo'])
 
     def test_delitem(self):
-        c = Cache()
+        c = self.get_cache()
         c['foo'] = ''
         del c['foo']
         assert 'foo' not in c
@@ -67,14 +72,14 @@ class TestCaseCache:
 
     def test_persistance(self, tmpdir):
         DBNAME = tmpdir.join('hstest.db')
-        c = Cache(str(DBNAME))
+        c = self.get_cache(str(DBNAME))
         c['foo'] = [(1, 2, 3)]
         del c
-        c = Cache(str(DBNAME))
+        c = self.get_cache(str(DBNAME))
         eq_([(1, 2, 3)], c['foo'])
 
     def test_filter(self):
-        c = Cache()
+        c = self.get_cache()
         c['foo'] = ''
         c['bar'] = ''
         c['baz'] = ''
@@ -85,7 +90,7 @@ class TestCaseCache:
         assert 'bar' not in c
 
     def test_clear(self):
-        c = Cache()
+        c = self.get_cache()
         c['foo'] = ''
         c['bar'] = ''
         c['baz'] = ''
@@ -95,6 +100,22 @@ class TestCaseCache:
         assert 'baz' not in c
         assert 'bar' not in c
 
+    def test_by_id(self):
+        # it's possible to use the cache by referring to the files by their row_id
+        c = self.get_cache()
+        b = [(0, 0, 0), (1, 2, 3)]
+        c['foo'] = b
+        foo_id = c.get_id('foo')
+        eq_(c[foo_id], b)
+
+
+class TestCaseSqliteCache(BaseTestCaseCache):
+    def get_cache(self, dbname=None):
+        if dbname:
+            return SqliteCache(dbname)
+        else:
+            return SqliteCache()
+
     def test_corrupted_db(self, tmpdir, monkeypatch):
         # If we don't do this monkeypatching, we get a weird exception about trying to flush a
         # closed file. I've tried setting logging level and stuff, but nothing worked. So, there we
@@ -104,37 +125,37 @@ class TestCaseCache:
         fp = open(dbname, 'w')
         fp.write('invalid sqlite content')
         fp.close()
-        c = Cache(dbname) # should not raise a DatabaseError
+        c = self.get_cache(dbname) # should not raise a DatabaseError
         c['foo'] = [(1, 2, 3)]
         del c
-        c = Cache(dbname)
+        c = self.get_cache(dbname)
         eq_(c['foo'], [(1, 2, 3)])
 
-    def test_by_id(self):
+
-        # it's possible to use the cache by referring to the files by their row_id
+class TestCaseShelveCache(BaseTestCaseCache):
-        c = Cache()
+    def get_cache(self, dbname=None):
-        b = [(0, 0, 0), (1, 2, 3)]
+        return ShelveCache(dbname)
-        c['foo'] = b
-        foo_id = c.get_id('foo')
-        eq_(c[foo_id], b)
 
 
 class TestCaseCacheSQLEscape:
+    def get_cache(self):
+        return SqliteCache()
+
     def test_contains(self):
-        c = Cache()
+        c = self.get_cache()
         assert "foo'bar" not in c
 
     def test_getitem(self):
-        c = Cache()
+        c = self.get_cache()
         with raises(KeyError):
             c["foo'bar"]
 
     def test_setitem(self):
-        c = Cache()
+        c = self.get_cache()
         c["foo'bar"] = []
 
     def test_delitem(self):
-        c = Cache()
+        c = self.get_cache()
         c["foo'bar"] = []
         try:
             del c["foo'bar"]

help/changelog

@@ -1,6 +1,13 @@
+=== 4.0.3 (2016-11-24)
+
+* Add new picture cache backend: shelve
+* Make shelve picture cache backend the active one on MacOS to fix #394 more
+  elegantly. [cocoa]
+* Remove Sparkle (auto-updates) due to technical limitations. [cocoa]
+
 === 4.0.2 (2016-10-09)
 
-* Fix systematic crash in Picture Mode under MacOs Sierra. (#394)
+* Fix systematic crash in Picture Mode under MacOS Sierra. (#394)
 * No change for Linux. Just keeping version in sync.
 
 === 4.0.1 (2016-08-24)

qt/app.py

@@ -115,6 +115,7 @@ class DupeGuru(QObject):
             scanned_tags.add('year')
         self.model.options['scanned_tags'] = scanned_tags
         self.model.options['match_scaled'] = self.prefs.match_scaled
+        self.model.options['picture_cache_type'] = self.prefs.picture_cache_type
 
     #--- Private
     def _get_details_dialog_class(self):

qt/pe/preferences_dialog.py

@@ -4,7 +4,9 @@
 # which should be included with this package. The terms are also available at
 # http://www.gnu.org/licenses/gpl-3.0.html
 
+from PyQt5.QtWidgets import QLabel
 from hscommon.trans import trget
+from qtlib.radio_box import RadioBox
 from core.scanner import ScanType
 from core.app import AppMode
 
@@ -28,10 +30,14 @@ class PreferencesDialog(PreferencesDialogBase):
         self.widgetsVLayout.addWidget(self.ignoreHardlinkMatches)
         self._setupAddCheckbox('debugModeBox', tr("Debug mode (restart required)"))
         self.widgetsVLayout.addWidget(self.debugModeBox)
+        self.widgetsVLayout.addWidget(QLabel(tr("Picture cache mode:")))
+        self.cacheTypeRadio = RadioBox(self, items=["Sqlite", "Shelve"], spread=False)
+        self.widgetsVLayout.addWidget(self.cacheTypeRadio)
         self._setupBottomPart()
 
     def _load(self, prefs, setchecked):
         setchecked(self.matchScaledBox, prefs.match_scaled)
+        self.cacheTypeRadio.selected_index = 1 if prefs.picture_cache_type == 'shelve' else 0
 
         # Update UI state based on selected scan type
         scan_type = prefs.get_scan_type(AppMode.Picture)
@@ -40,4 +46,5 @@ class PreferencesDialog(PreferencesDialogBase):
 
     def _save(self, prefs, ischecked):
         prefs.match_scaled = ischecked(self.matchScaledBox)
+        prefs.picture_cache_type = 'shelve' if self.cacheTypeRadio.selected_index == 1 else 'sqlite'
 

qt/preferences.py

@@ -44,6 +44,7 @@ class Preferences(PreferencesBase):
         self.scan_tag_genre = get('ScanTagGenre', self.scan_tag_genre)
         self.scan_tag_year = get('ScanTagYear', self.scan_tag_year)
         self.match_scaled = get('MatchScaled', self.match_scaled)
+        self.picture_cache_type = get('PictureCacheType', self.picture_cache_type)
 
     def reset(self):
         self.filter_hardness = 95
@@ -74,6 +75,7 @@ class Preferences(PreferencesBase):
         self.scan_tag_genre = False
         self.scan_tag_year = False
         self.match_scaled = False
+        self.picture_cache_type = 'sqlite'
 
     def _save_values(self, settings):
         set_ = self.set_value
@@ -105,6 +107,7 @@ class Preferences(PreferencesBase):
         set_('ScanTagGenre', self.scan_tag_genre)
         set_('ScanTagYear', self.scan_tag_year)
         set_('MatchScaled', self.match_scaled)
+        set_('PictureCacheType', self.picture_cache_type)
 
     # scan_type is special because we save it immediately when we set it.
     def get_scan_type(self, app_mode):

tox.ini

@@ -14,5 +14,5 @@ deps =
 [flake8]
 exclude = .tox,env,build,hscommon,qtlib,cocoalib,cocoa,help,./qt/dg_rc.py,qt/run_template.py,cocoa/run_template.py,./run.py,./pkg
 max-line-length = 120
-ignore = W391,W293,E302,E261,E226,E227,W291,E262,E303,E265,E731
+ignore = W391,W293,E302,E261,E226,E227,W291,E262,E303,E265,E731,E305