mirror of
https://github.com/arsenetar/dupeguru.git
synced 2026-01-22 14:41:39 +00:00
Changed the build system (that commit is *huge*)
--HG-- rename : base/cocoa/AppDelegate.h => cocoa/base/AppDelegate.h rename : base/cocoa/AppDelegate.m => cocoa/base/AppDelegate.m rename : base/cocoa/Consts.h => cocoa/base/Consts.h rename : base/cocoa/DetailsPanel.h => cocoa/base/DetailsPanel.h rename : base/cocoa/DetailsPanel.m => cocoa/base/DetailsPanel.m rename : base/cocoa/DirectoryPanel.h => cocoa/base/DirectoryPanel.h rename : base/cocoa/DirectoryPanel.m => cocoa/base/DirectoryPanel.m rename : base/cocoa/PyDupeGuru.h => cocoa/base/PyDupeGuru.h rename : base/cocoa/ResultWindow.h => cocoa/base/ResultWindow.h rename : base/cocoa/ResultWindow.m => cocoa/base/ResultWindow.m rename : base/cocoa/dsa_pub.pem => cocoa/base/dsa_pub.pem rename : base/cocoa/xib/DetailsPanel.xib => cocoa/base/xib/DetailsPanel.xib rename : base/cocoa/xib/DirectoryPanel.xib => cocoa/base/xib/DirectoryPanel.xib rename : base/cocoa/xib/MainMenu.xib => cocoa/base/xib/MainMenu.xib rename : me/cocoa/AppDelegate.h => cocoa/me/AppDelegate.h rename : me/cocoa/AppDelegate.m => cocoa/me/AppDelegate.m rename : me/cocoa/Consts.h => cocoa/me/Consts.h rename : me/cocoa/DetailsPanel.h => cocoa/me/DetailsPanel.h rename : me/cocoa/DetailsPanel.m => cocoa/me/DetailsPanel.m rename : me/cocoa/DirectoryPanel.h => cocoa/me/DirectoryPanel.h rename : me/cocoa/DirectoryPanel.m => cocoa/me/DirectoryPanel.m rename : me/cocoa/Info.plist => cocoa/me/Info.plist rename : me/cocoa/PyDupeGuru.h => cocoa/me/PyDupeGuru.h rename : me/cocoa/ResultWindow.h => cocoa/me/ResultWindow.h rename : me/cocoa/ResultWindow.m => cocoa/me/ResultWindow.m rename : me/cocoa/dupeguru.icns => cocoa/me/dupeguru.icns rename : me/cocoa/dupeguru.xcodeproj/project.pbxproj => cocoa/me/dupeguru.xcodeproj/project.pbxproj rename : me/cocoa/gen.py => cocoa/me/gen.py rename : me/cocoa/main.m => cocoa/me/main.m rename : me/cocoa/py/dg_cocoa.py => cocoa/me/py/dg_cocoa.py rename : me/cocoa/py/setup.py => cocoa/me/py/setup.py rename : me/cocoa/xib/Preferences.xib => cocoa/me/xib/Preferences.xib rename : pe/cocoa/AppDelegate.h => cocoa/pe/AppDelegate.h rename : pe/cocoa/AppDelegate.m => cocoa/pe/AppDelegate.m rename : pe/cocoa/Consts.h => cocoa/pe/Consts.h rename : pe/cocoa/DetailsPanel.h => cocoa/pe/DetailsPanel.h rename : pe/cocoa/DetailsPanel.m => cocoa/pe/DetailsPanel.m rename : pe/cocoa/DirectoryPanel.h => cocoa/pe/DirectoryPanel.h rename : pe/cocoa/DirectoryPanel.m => cocoa/pe/DirectoryPanel.m rename : pe/cocoa/Info.plist => cocoa/pe/Info.plist rename : pe/cocoa/PictureBlocks.h => cocoa/pe/PictureBlocks.h rename : pe/cocoa/PictureBlocks.m => cocoa/pe/PictureBlocks.m rename : pe/cocoa/PyDupeGuru.h => cocoa/pe/PyDupeGuru.h rename : pe/cocoa/ResultWindow.h => cocoa/pe/ResultWindow.h rename : pe/cocoa/ResultWindow.m => cocoa/pe/ResultWindow.m rename : pe/cocoa/dupeguru.icns => cocoa/pe/dupeguru.icns rename : pe/cocoa/dupeguru.xcodeproj/project.pbxproj => cocoa/pe/dupeguru.xcodeproj/project.pbxproj rename : pe/cocoa/gen.py => cocoa/pe/gen.py rename : pe/cocoa/main.m => cocoa/pe/main.m rename : pe/cocoa/py/dg_cocoa.py => cocoa/pe/py/dg_cocoa.py rename : pe/cocoa/py/setup.py => cocoa/pe/py/setup.py rename : pe/cocoa/xib/DetailsPanel.xib => cocoa/pe/xib/DetailsPanel.xib rename : pe/cocoa/xib/Preferences.xib => cocoa/pe/xib/Preferences.xib rename : se/cocoa/AppDelegate.h => cocoa/se/AppDelegate.h rename : se/cocoa/AppDelegate.m => cocoa/se/AppDelegate.m rename : se/cocoa/Consts.h => cocoa/se/Consts.h rename : se/cocoa/DetailsPanel.h => cocoa/se/DetailsPanel.h rename : se/cocoa/DetailsPanel.m => cocoa/se/DetailsPanel.m rename : se/cocoa/DirectoryPanel.h => cocoa/se/DirectoryPanel.h rename : se/cocoa/DirectoryPanel.m => cocoa/se/DirectoryPanel.m rename : se/cocoa/Info.plist => cocoa/se/Info.plist rename : se/cocoa/PyDupeGuru.h => cocoa/se/PyDupeGuru.h rename : se/cocoa/ResultWindow.h => cocoa/se/ResultWindow.h rename : se/cocoa/ResultWindow.m => cocoa/se/ResultWindow.m rename : se/cocoa/dupeguru.icns => cocoa/se/dupeguru.icns rename : se/cocoa/dupeguru.xcodeproj/project.pbxproj => cocoa/se/dupeguru.xcodeproj/project.pbxproj rename : se/cocoa/gen.py => cocoa/se/gen.py rename : se/cocoa/main.m => cocoa/se/main.m rename : se/cocoa/py/dg_cocoa.py => cocoa/se/py/dg_cocoa.py rename : se/cocoa/py/setup.py => cocoa/se/py/setup.py rename : se/cocoa/xib/Preferences.xib => cocoa/se/xib/Preferences.xib rename : base/core/LICENSE => core/LICENSE rename : base/core/__init__.py => core/__init__.py rename : base/core/app.py => core/app.py rename : base/core/app_cocoa.py => core/app_cocoa.py rename : base/core/data.py => core/data.py rename : base/core/directories.py => core/directories.py rename : base/core/engine.py => core/engine.py rename : base/core/export.py => core/export.py rename : base/core/fs.py => core/fs.py rename : base/core/ignore.py => core/ignore.py rename : base/core/results.py => core/results.py rename : base/core/scanner.py => core/scanner.py rename : base/core/tests/__init__.py => core/tests/__init__.py rename : base/core/tests/app_cocoa_test.py => core/tests/app_cocoa_test.py rename : base/core/tests/app_test.py => core/tests/app_test.py rename : base/core/tests/data.py => core/tests/data.py rename : base/core/tests/directories_test.py => core/tests/directories_test.py rename : base/core/tests/engine_test.py => core/tests/engine_test.py rename : base/core/tests/ignore_test.py => core/tests/ignore_test.py rename : base/core/tests/results_test.py => core/tests/results_test.py rename : base/core/tests/scanner_test.py => core/tests/scanner_test.py rename : me/core/__init__.py => core_me/__init__.py rename : me/core/app_cocoa.py => core_me/app_cocoa.py rename : me/core/data.py => core_me/data.py rename : me/core/fs.py => core_me/fs.py rename : me/core/scanner.py => core_me/scanner.py rename : me/core/tests/__init__.py => core_me/tests/__init__.py rename : me/core/tests/scanner_test.py => core_me/tests/scanner_test.py rename : pe/core/LICENSE => core_pe/LICENSE rename : pe/core/__init__.py => core_pe/__init__.py rename : pe/core/app_cocoa.py => core_pe/app_cocoa.py rename : pe/core/block.py => core_pe/block.py rename : pe/core/cache.py => core_pe/cache.py rename : pe/core/data.py => core_pe/data.py rename : pe/core/gen.py => core_pe/gen.py rename : pe/core/matchbase.py => core_pe/matchbase.py rename : pe/core/modules/block/block.pyx => core_pe/modules/block/block.pyx rename : pe/core/modules/block/setup.py => core_pe/modules/block/setup.py rename : pe/core/modules/cache/cache.pyx => core_pe/modules/cache/cache.pyx rename : pe/core/modules/cache/setup.py => core_pe/modules/cache/setup.py rename : pe/core/scanner.py => core_pe/scanner.py rename : pe/core/tests/__init__.py => core_pe/tests/__init__.py rename : pe/core/tests/block_test.py => core_pe/tests/block_test.py rename : pe/core/tests/cache_test.py => core_pe/tests/cache_test.py rename : se/core/LICENSE => core_se/LICENSE rename : se/core/__init__.py => core_se/__init__.py rename : se/core/app_cocoa.py => core_se/app_cocoa.py rename : se/core/data.py => core_se/data.py rename : se/core/fs.py => core_se/fs.py rename : se/core/tests/__init__.py => core_se/tests/__init__.py rename : se/core/tests/fs_test.py => core_se/tests/fs_test.py rename : me/help/LICENSE => help_me/LICENSE rename : me/help/__init__.py => help_me/__init__.py rename : me/help/changelog.yaml => help_me/changelog.yaml rename : me/help/gen.py => help_me/gen.py rename : me/help/skeleton/hardcoded.css => help_me/skeleton/hardcoded.css rename : me/help/skeleton/images/hs_title.png => help_me/skeleton/images/hs_title.png rename : me/help/templates/base_dg.mako => help_me/templates/base_dg.mako rename : me/help/templates/credits.mako => help_me/templates/credits.mako rename : me/help/templates/directories.mako => help_me/templates/directories.mako rename : me/help/templates/faq.mako => help_me/templates/faq.mako rename : me/help/templates/intro.mako => help_me/templates/intro.mako rename : me/help/templates/power_marker.mako => help_me/templates/power_marker.mako rename : me/help/templates/preferences.mako => help_me/templates/preferences.mako rename : me/help/templates/quick_start.mako => help_me/templates/quick_start.mako rename : me/help/templates/results.mako => help_me/templates/results.mako rename : me/help/templates/versions.mako => help_me/templates/versions.mako rename : pe/help/LICENSE => help_pe/LICENSE rename : pe/help/__init__.py => help_pe/__init__.py rename : pe/help/changelog.yaml => help_pe/changelog.yaml rename : pe/help/gen.py => help_pe/gen.py rename : pe/help/skeleton/hardcoded.css => help_pe/skeleton/hardcoded.css rename : pe/help/skeleton/images/hs_title.png => help_pe/skeleton/images/hs_title.png rename : pe/help/templates/base_dg.mako => help_pe/templates/base_dg.mako rename : pe/help/templates/credits.mako => help_pe/templates/credits.mako rename : pe/help/templates/directories.mako => help_pe/templates/directories.mako rename : pe/help/templates/faq.mako => help_pe/templates/faq.mako rename : pe/help/templates/intro.mako => help_pe/templates/intro.mako rename : pe/help/templates/power_marker.mako => help_pe/templates/power_marker.mako rename : pe/help/templates/preferences.mako => help_pe/templates/preferences.mako rename : pe/help/templates/quick_start.mako => help_pe/templates/quick_start.mako rename : pe/help/templates/results.mako => help_pe/templates/results.mako rename : pe/help/templates/versions.mako => help_pe/templates/versions.mako rename : se/help/LICENSE => help_se/LICENSE rename : se/help/changelog.yaml => help_se/changelog.yaml rename : se/help/gen.py => help_se/gen.py rename : se/help/skeleton/hardcoded.css => help_se/skeleton/hardcoded.css rename : se/help/skeleton/images/hs_title.png => help_se/skeleton/images/hs_title.png rename : se/help/templates/base_dg.mako => help_se/templates/base_dg.mako rename : se/help/templates/credits.mako => help_se/templates/credits.mako rename : se/help/templates/directories.mako => help_se/templates/directories.mako rename : se/help/templates/faq.mako => help_se/templates/faq.mako rename : se/help/templates/intro.mako => help_se/templates/intro.mako rename : se/help/templates/power_marker.mako => help_se/templates/power_marker.mako rename : se/help/templates/preferences.mako => help_se/templates/preferences.mako rename : se/help/templates/quick_start.mako => help_se/templates/quick_start.mako rename : se/help/templates/results.mako => help_se/templates/results.mako rename : se/help/templates/versions.mako => help_se/templates/versions.mako rename : base/qt/WARNING => qt/WARNING rename : base/qt/__init__.py => qt/base/__init__.py rename : base/qt/app.py => qt/base/app.py rename : base/qt/details_table.py => qt/base/details_table.py rename : base/qt/dg.qrc => qt/base/dg.qrc rename : base/qt/directories_dialog.py => qt/base/directories_dialog.py rename : base/qt/directories_dialog.ui => qt/base/directories_dialog.ui rename : base/qt/directories_model.py => qt/base/directories_model.py rename : base/qt/main_window.py => qt/base/main_window.py rename : base/qt/main_window.ui => qt/base/main_window.ui rename : base/qt/platform.py => qt/base/platform.py rename : base/qt/platform_osx.py => qt/base/platform_osx.py rename : base/qt/platform_win.py => qt/base/platform_win.py rename : base/qt/preferences.py => qt/base/preferences.py rename : base/qt/results_model.py => qt/base/results_model.py rename : me/qt/app.py => qt/me/app.py rename : me/qt/build.py => qt/me/build.py rename : me/qt/details_dialog.py => qt/me/details_dialog.py rename : me/qt/details_dialog.ui => qt/me/details_dialog.ui rename : me/qt/dgme.spec => qt/me/dgme.spec rename : me/qt/gen.py => qt/me/gen.py rename : me/qt/installer.aip => qt/me/installer.aip rename : me/qt/preferences.py => qt/me/preferences.py rename : me/qt/preferences_dialog.py => qt/me/preferences_dialog.py rename : me/qt/preferences_dialog.ui => qt/me/preferences_dialog.ui rename : me/qt/profile.py => qt/me/profile.py rename : me/qt/start.py => qt/me/start.py rename : me/qt/verinfo => qt/me/verinfo rename : pe/qt/app.py => qt/pe/app.py rename : pe/qt/block.py => qt/pe/block.py rename : pe/qt/build.py => qt/pe/build.py rename : pe/qt/details_dialog.py => qt/pe/details_dialog.py rename : pe/qt/details_dialog.ui => qt/pe/details_dialog.ui rename : pe/qt/dgpe.spec => qt/pe/dgpe.spec rename : pe/qt/gen.py => qt/pe/gen.py rename : pe/qt/installer.aip => qt/pe/installer.aip rename : pe/qt/main_window.py => qt/pe/main_window.py rename : pe/qt/modules/block/block.pyx => qt/pe/modules/block/block.pyx rename : pe/qt/modules/block/setup.py => qt/pe/modules/block/setup.py rename : pe/qt/preferences.py => qt/pe/preferences.py rename : pe/qt/preferences_dialog.py => qt/pe/preferences_dialog.py rename : pe/qt/preferences_dialog.ui => qt/pe/preferences_dialog.ui rename : pe/qt/profile.py => qt/pe/profile.py rename : pe/qt/start.py => qt/pe/start.py rename : pe/qt/verinfo => qt/pe/verinfo rename : se/qt/app.py => qt/se/app.py rename : se/qt/build.py => qt/se/build.py rename : se/qt/details_dialog.py => qt/se/details_dialog.py rename : se/qt/details_dialog.ui => qt/se/details_dialog.ui rename : se/qt/dgse.spec => qt/se/dgse.spec rename : se/qt/gen.py => qt/se/gen.py rename : se/qt/installer.aip => qt/se/installer.aip rename : se/qt/preferences.py => qt/se/preferences.py rename : se/qt/preferences_dialog.py => qt/se/preferences_dialog.py rename : se/qt/preferences_dialog.ui => qt/se/preferences_dialog.ui rename : se/qt/profile.py => qt/se/profile.py rename : se/qt/start.py => qt/se/start.py rename : se/qt/verinfo => qt/se/verinfo extra : convert_revision : svn%3Ac306627e-7827-47d3-bdf0-9a457c9553a1/trunk%40285
This commit is contained in:
hsoft
11
core_pe/LICENSE
Normal file
11
core_pe/LICENSE
Normal file
@@ -0,0 +1,11 @@
|
||||
Copyright 2009 Hardcoded Software Inc. (http://www.hardcoded.net)
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
|
||||
* Neither the name of Hardcoded Software Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
|
||||
* If the source code has been published less than two years ago, any redistribution, in whole or in part, must retain full licensing functionality, without any attempt to change, obscure or in other ways circumvent its intent.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
0
core_pe/__init__.py
Normal file
0
core_pe/__init__.py
Normal file
211
core_pe/app_cocoa.py
Normal file
211
core_pe/app_cocoa.py
Normal file
@@ -0,0 +1,211 @@
|
||||
# Created By: Virgil Dupras
|
||||
# Created On: 2006/11/13
|
||||
# $Id$
|
||||
# Copyright 2009 Hardcoded Software (http://www.hardcoded.net)
|
||||
#
|
||||
# This software is licensed under the "HS" License as described in the "LICENSE" file,
|
||||
# which should be included with this package. The terms are also available at
|
||||
# http://www.hardcoded.net/licenses/hs_license
|
||||
|
||||
import os.path as op
|
||||
import logging
|
||||
import plistlib
|
||||
import re
|
||||
|
||||
from Foundation import *
|
||||
from AppKit import *
|
||||
from appscript import app, k, CommandError
|
||||
|
||||
from hsutil import io
|
||||
from hsutil.str import get_file_ext
|
||||
from hsutil.path import Path
|
||||
from hsutil.cocoa import as_fetch
|
||||
|
||||
from core import fs
|
||||
from core import app_cocoa, directories
|
||||
from . import data
|
||||
from .cache import string_to_colors, Cache
|
||||
from .scanner import ScannerPE
|
||||
|
||||
mainBundle = NSBundle.mainBundle()
|
||||
PictureBlocks = mainBundle.classNamed_('PictureBlocks')
|
||||
assert PictureBlocks is not None
|
||||
|
||||
class Photo(fs.File):
|
||||
INITIAL_INFO = fs.File.INITIAL_INFO.copy()
|
||||
INITIAL_INFO.update({
|
||||
'dimensions': (0,0),
|
||||
})
|
||||
HANDLED_EXTS = set(['png', 'jpg', 'jpeg', 'gif', 'psd', 'bmp', 'tiff', 'tif', 'nef', 'cr2'])
|
||||
|
||||
@classmethod
|
||||
def can_handle(cls, path):
|
||||
return fs.File.can_handle(path) and get_file_ext(path[-1]) in cls.HANDLED_EXTS
|
||||
|
||||
def _read_info(self, field):
|
||||
fs.File._read_info(self, field)
|
||||
if field == 'dimensions':
|
||||
size = PictureBlocks.getImageSize_(unicode(self.path))
|
||||
self.dimensions = (size.width, size.height)
|
||||
|
||||
def get_blocks(self, block_count_per_side):
|
||||
try:
|
||||
blocks = PictureBlocks.getBlocksFromImagePath_blockCount_(unicode(self.path), block_count_per_side)
|
||||
except Exception as e:
|
||||
raise IOError('The reading of "%s" failed with "%s"' % (unicode(self.path), unicode(e)))
|
||||
if not blocks:
|
||||
raise IOError('The picture %s could not be read' % unicode(self.path))
|
||||
return string_to_colors(blocks)
|
||||
|
||||
|
||||
class IPhoto(Photo):
|
||||
@property
|
||||
def display_path(self):
|
||||
return Path(('iPhoto Library', self.name))
|
||||
|
||||
def get_iphoto_database_path():
|
||||
ud = NSUserDefaults.standardUserDefaults()
|
||||
prefs = ud.persistentDomainForName_('com.apple.iApps')
|
||||
if prefs is None:
|
||||
raise directories.InvalidPathError()
|
||||
if 'iPhotoRecentDatabases' not in prefs:
|
||||
raise directories.InvalidPathError()
|
||||
plisturl = NSURL.URLWithString_(prefs['iPhotoRecentDatabases'][0])
|
||||
return Path(plisturl.path())
|
||||
|
||||
def get_iphoto_pictures(plistpath):
|
||||
if not io.exists(plistpath):
|
||||
raise InvalidPath(self)
|
||||
s = io.open(plistpath).read()
|
||||
# There was a case where a guy had 0x10 chars in his plist, causing expat errors on loading
|
||||
s = s.replace('\x10', '')
|
||||
# It seems that iPhoto sometimes doesn't properly escape & chars. The regexp below is to find
|
||||
# any & char that is not a &-based entity (&, ", etc.). based on TextMate's XML
|
||||
# bundle's regexp
|
||||
s, count = re.subn(r'&(?![a-zA-Z0-9_-]+|#[0-9]+|#x[0-9a-fA-F]+;)', '', s)
|
||||
if count:
|
||||
logging.warning("%d invalid XML entities replacement made", count)
|
||||
plist = plistlib.readPlistFromString(s)
|
||||
result = []
|
||||
for photo_data in plist['Master Image List'].values():
|
||||
if photo_data['MediaType'] != 'Image':
|
||||
continue
|
||||
photo_path = Path(photo_data['ImagePath'])
|
||||
photo = IPhoto(photo_path)
|
||||
result.append(photo)
|
||||
return result
|
||||
|
||||
class Directories(directories.Directories):
|
||||
def __init__(self):
|
||||
directories.Directories.__init__(self, fileclasses=[Photo])
|
||||
try:
|
||||
self.iphoto_libpath = get_iphoto_database_path()
|
||||
self.set_state(self.iphoto_libpath[:-1], directories.STATE_EXCLUDED)
|
||||
except directories.InvalidPathError:
|
||||
self.iphoto_libpath = None
|
||||
|
||||
def _get_files(self, from_path):
|
||||
if from_path == Path('iPhoto Library'):
|
||||
if self.iphoto_libpath is None:
|
||||
return []
|
||||
is_ref = self.get_state(from_path) == directories.STATE_REFERENCE
|
||||
photos = get_iphoto_pictures(self.iphoto_libpath)
|
||||
for photo in photos:
|
||||
photo.is_ref = is_ref
|
||||
return photos
|
||||
else:
|
||||
return directories.Directories._get_files(self, from_path)
|
||||
|
||||
@staticmethod
|
||||
def get_subfolders(path):
|
||||
if path == Path('iPhoto Library'):
|
||||
return []
|
||||
else:
|
||||
return directories.Directories.get_subfolders(path)
|
||||
|
||||
def add_path(self, path):
|
||||
if path == Path('iPhoto Library'):
|
||||
if path in self:
|
||||
raise AlreadyThereError()
|
||||
self._dirs.append(path)
|
||||
else:
|
||||
directories.Directories.add_path(self, path)
|
||||
|
||||
|
||||
class DupeGuruPE(app_cocoa.DupeGuru):
|
||||
def __init__(self):
|
||||
app_cocoa.DupeGuru.__init__(self, data, 'dupeGuru Picture Edition', appid=5)
|
||||
self.scanner = ScannerPE()
|
||||
self.directories = Directories()
|
||||
p = op.join(self.appdata, 'cached_pictures.db')
|
||||
self.scanner.cached_blocks = Cache(p)
|
||||
|
||||
def _do_delete(self, j):
|
||||
def op(dupe):
|
||||
j.add_progress()
|
||||
return self._do_delete_dupe(dupe)
|
||||
|
||||
marked = [dupe for dupe in self.results.dupes if self.results.is_marked(dupe)]
|
||||
self.path2iphoto = {}
|
||||
if any(isinstance(dupe, IPhoto) for dupe in marked):
|
||||
j = j.start_subjob([6, 4], "Probing iPhoto. Don\'t touch it during the operation!")
|
||||
try:
|
||||
a = app('iPhoto')
|
||||
a.activate(timeout=0)
|
||||
a.select(a.photo_library_album(timeout=0), timeout=0)
|
||||
photos = as_fetch(a.photo_library_album().photos, k.item)
|
||||
for photo in j.iter_with_progress(photos):
|
||||
try:
|
||||
self.path2iphoto[unicode(photo.image_path(timeout=0))] = photo
|
||||
except CommandError:
|
||||
pass
|
||||
except (CommandError, RuntimeError):
|
||||
pass
|
||||
j.start_job(self.results.mark_count, "Sending dupes to the Trash")
|
||||
self.last_op_error_count = self.results.perform_on_marked(op, True)
|
||||
del self.path2iphoto
|
||||
|
||||
def _do_delete_dupe(self, dupe):
|
||||
if isinstance(dupe, IPhoto):
|
||||
if unicode(dupe.path) in self.path2iphoto:
|
||||
photo = self.path2iphoto[unicode(dupe.path)]
|
||||
try:
|
||||
a = app('iPhoto')
|
||||
a.remove(photo, timeout=0)
|
||||
return True
|
||||
except (CommandError, RuntimeError):
|
||||
return False
|
||||
else:
|
||||
logging.warning(u"Could not find photo %s in iPhoto Library", unicode(dupe.path))
|
||||
return False
|
||||
else:
|
||||
return app_cocoa.DupeGuru._do_delete_dupe(self, dupe)
|
||||
|
||||
def _do_load(self, j):
|
||||
self.directories.load_from_file(op.join(self.appdata, 'last_directories.xml'))
|
||||
self.results.load_from_xml(op.join(self.appdata, 'last_results.xml'), self._get_file, j)
|
||||
|
||||
def _get_file(self, str_path):
|
||||
p = Path(str_path)
|
||||
if (self.directories.iphoto_libpath is not None) and (p in self.directories.iphoto_libpath[:-1]):
|
||||
return IPhoto(p)
|
||||
return app_cocoa.DupeGuru._get_file(self, str_path)
|
||||
|
||||
def copy_or_move(self, dupe, copy, destination, dest_type):
|
||||
if isinstance(dupe, IPhoto):
|
||||
copy = True
|
||||
return app_cocoa.DupeGuru.copy_or_move(self, dupe, copy, destination, dest_type)
|
||||
|
||||
def selected_dupe_path(self):
|
||||
if not self.selected_dupes:
|
||||
return None
|
||||
return self.selected_dupes[0].path
|
||||
|
||||
def selected_dupe_ref_path(self):
|
||||
if not self.selected_dupes:
|
||||
return None
|
||||
ref = self.results.get_group_of_duplicate(self.selected_dupes[0]).ref
|
||||
if ref is self.selected_dupes[0]: # we don't want the same pic to be displayed on both sides
|
||||
return None
|
||||
return ref.path
|
||||
|
||||
123
core_pe/block.py
Normal file
123
core_pe/block.py
Normal file
@@ -0,0 +1,123 @@
|
||||
# Created By: Virgil Dupras
|
||||
# Created On: 2006/09/01
|
||||
# $Id$
|
||||
# Copyright 2009 Hardcoded Software (http://www.hardcoded.net)
|
||||
#
|
||||
# This software is licensed under the "HS" License as described in the "LICENSE" file,
|
||||
# which should be included with this package. The terms are also available at
|
||||
# http://www.hardcoded.net/licenses/hs_license
|
||||
|
||||
from _block import NoBlocksError, DifferentBlockCountError, avgdiff, getblocks2
|
||||
|
||||
# Converted to Cython
|
||||
# def getblock(image):
|
||||
# """Returns a 3 sized tuple containing the mean color of 'image'.
|
||||
#
|
||||
# image: a PIL image or crop.
|
||||
# """
|
||||
# if image.size[0]:
|
||||
# pixel_count = image.size[0] * image.size[1]
|
||||
# red = green = blue = 0
|
||||
# for r,g,b in image.getdata():
|
||||
# red += r
|
||||
# green += g
|
||||
# blue += b
|
||||
# return (red // pixel_count, green // pixel_count, blue // pixel_count)
|
||||
# else:
|
||||
# return (0,0,0)
|
||||
|
||||
# This is not used anymore
|
||||
# def getblocks(image,blocksize):
|
||||
# """Returns a list of blocks (3 sized tuples).
|
||||
#
|
||||
# image: A PIL image to base the blocks on.
|
||||
# blocksize: The size of the blocks to be create. This is a single integer, defining
|
||||
# both width and height (blocks are square).
|
||||
# """
|
||||
# if min(image.size) < blocksize:
|
||||
# return ()
|
||||
# result = []
|
||||
# for i in xrange(image.size[1] // blocksize):
|
||||
# for j in xrange(image.size[0] // blocksize):
|
||||
# box = (blocksize * j, blocksize * i, blocksize * (j + 1), blocksize * (i + 1))
|
||||
# crop = image.crop(box)
|
||||
# result.append(getblock(crop))
|
||||
# return result
|
||||
|
||||
# Converted to Cython
|
||||
# def getblocks2(image,block_count_per_side):
|
||||
# """Returns a list of blocks (3 sized tuples).
|
||||
#
|
||||
# image: A PIL image to base the blocks on.
|
||||
# block_count_per_side: This integer determine the number of blocks the function will return.
|
||||
# If it is 10, for example, 100 blocks will be returns (10 width, 10 height). The blocks will not
|
||||
# necessarely cover square areas. The area covered by each block will be proportional to the image
|
||||
# itself.
|
||||
# """
|
||||
# if not image.size[0]:
|
||||
# return []
|
||||
# width,height = image.size
|
||||
# block_width = max(width // block_count_per_side,1)
|
||||
# block_height = max(height // block_count_per_side,1)
|
||||
# result = []
|
||||
# for ih in range(block_count_per_side):
|
||||
# top = min(ih * block_height, height - block_height)
|
||||
# bottom = top + block_height
|
||||
# for iw in range(block_count_per_side):
|
||||
# left = min(iw * block_width, width - block_width)
|
||||
# right = left + block_width
|
||||
# box = (left,top,right,bottom)
|
||||
# crop = image.crop(box)
|
||||
# result.append(getblock(crop))
|
||||
# return result
|
||||
|
||||
# Converted to Cython
|
||||
# def diff(first, second):
|
||||
# """Returns the difference between the first block and the second.
|
||||
#
|
||||
# It returns an absolute sum of the 3 differences (RGB).
|
||||
# """
|
||||
# r1, g1, b1 = first
|
||||
# r2, g2, b2 = second
|
||||
# return abs(r1 - r2) + abs(g1 - g2) + abs(b1 - b2)
|
||||
|
||||
# Converted to Cython
|
||||
# def avgdiff(first, second, limit=768, min_iterations=1):
|
||||
# """Returns the average diff between first blocks and seconds.
|
||||
#
|
||||
# If the result surpasses limit, limit + 1 is returned, except if less than min_iterations
|
||||
# iterations have been made in the blocks.
|
||||
# """
|
||||
# if len(first) != len(second):
|
||||
# raise DifferentBlockCountError
|
||||
# if not first:
|
||||
# raise NoBlocksError
|
||||
# count = len(first)
|
||||
# sum = 0
|
||||
# zipped = izip(xrange(1, count + 1), first, second)
|
||||
# for i, first, second in zipped:
|
||||
# sum += diff(first, second)
|
||||
# if sum > limit * i and i >= min_iterations:
|
||||
# return limit + 1
|
||||
# result = sum // count
|
||||
# if (not result) and sum:
|
||||
# result = 1
|
||||
# return result
|
||||
|
||||
# This is not used anymore
|
||||
# def maxdiff(first,second,limit=768):
|
||||
# """Returns the max diff between first blocks and seconds.
|
||||
#
|
||||
# If the result surpasses limit, the first max being over limit is returned.
|
||||
# """
|
||||
# if len(first) != len(second):
|
||||
# raise DifferentBlockCountError
|
||||
# if not first:
|
||||
# raise NoBlocksError
|
||||
# result = 0
|
||||
# zipped = zip(first,second)
|
||||
# for first,second in zipped:
|
||||
# result = max(result,diff(first,second))
|
||||
# if result > limit:
|
||||
# return result
|
||||
# return result
|
||||
133
core_pe/cache.py
Normal file
133
core_pe/cache.py
Normal file
@@ -0,0 +1,133 @@
|
||||
# Created By: Virgil Dupras
|
||||
# Created On: 2006/09/14
|
||||
# $Id$
|
||||
# Copyright 2009 Hardcoded Software (http://www.hardcoded.net)
|
||||
#
|
||||
# This software is licensed under the "HS" License as described in the "LICENSE" file,
|
||||
# which should be included with this package. The terms are also available at
|
||||
# http://www.hardcoded.net/licenses/hs_license
|
||||
|
||||
import os
|
||||
import logging
|
||||
import sqlite3 as sqlite
|
||||
|
||||
import hsutil.sqlite
|
||||
|
||||
from _cache import string_to_colors
|
||||
|
||||
def colors_to_string(colors):
|
||||
"""Transform the 3 sized tuples 'colors' into a hex string.
|
||||
|
||||
[(0,100,255)] --> 0064ff
|
||||
[(1,2,3),(4,5,6)] --> 010203040506
|
||||
"""
|
||||
return ''.join(['%02x%02x%02x' % (r,g,b) for r,g,b in colors])
|
||||
|
||||
# This function is an important bottleneck of dupeGuru PE. It has been converted to Cython.
|
||||
# def string_to_colors(s):
|
||||
# """Transform the string 's' in a list of 3 sized tuples.
|
||||
# """
|
||||
# result = []
|
||||
# for i in xrange(0, len(s), 6):
|
||||
# number = int(s[i:i+6], 16)
|
||||
# result.append((number >> 16, (number >> 8) & 0xff, number & 0xff))
|
||||
# return result
|
||||
|
||||
class Cache(object):
|
||||
"""A class to cache picture blocks.
|
||||
"""
|
||||
def __init__(self, db=':memory:', threaded=True):
|
||||
def create_tables():
|
||||
sql = "create table pictures(path TEXT, blocks TEXT)"
|
||||
self.con.execute(sql);
|
||||
sql = "create index idx_path on pictures (path)"
|
||||
self.con.execute(sql)
|
||||
|
||||
self.dbname = db
|
||||
if threaded:
|
||||
self.con = hsutil.sqlite.ThreadedConn(db, True)
|
||||
else:
|
||||
self.con = sqlite.connect(db, isolation_level=None)
|
||||
try:
|
||||
self.con.execute("select * from pictures where 1=2")
|
||||
except sqlite.OperationalError: # new db
|
||||
create_tables()
|
||||
except sqlite.DatabaseError, e: # corrupted db
|
||||
logging.warning('Could not create picture cache because of an error: %s', str(e))
|
||||
self.con.close()
|
||||
os.remove(db)
|
||||
if threaded:
|
||||
self.con = hsutil.sqlite.ThreadedConn(db, True)
|
||||
else:
|
||||
self.con = sqlite.connect(db, isolation_level=None)
|
||||
create_tables()
|
||||
|
||||
def __contains__(self, key):
|
||||
sql = "select count(*) from pictures where path = ?"
|
||||
result = self.con.execute(sql, [key]).fetchall()
|
||||
return result[0][0] > 0
|
||||
|
||||
def __delitem__(self, key):
|
||||
if key not in self:
|
||||
raise KeyError(key)
|
||||
sql = "delete from pictures where path = ?"
|
||||
self.con.execute(sql, [key])
|
||||
|
||||
# Optimized
|
||||
def __getitem__(self, key):
|
||||
if isinstance(key, int):
|
||||
sql = "select blocks from pictures where rowid = ?"
|
||||
else:
|
||||
sql = "select blocks from pictures where path = ?"
|
||||
result = self.con.execute(sql, [key]).fetchone()
|
||||
if result:
|
||||
result = string_to_colors(result[0])
|
||||
return result
|
||||
else:
|
||||
raise KeyError(key)
|
||||
|
||||
def __iter__(self):
|
||||
sql = "select path from pictures"
|
||||
result = self.con.execute(sql)
|
||||
return (row[0] for row in result)
|
||||
|
||||
def __len__(self):
|
||||
sql = "select count(*) from pictures"
|
||||
result = self.con.execute(sql).fetchall()
|
||||
return result[0][0]
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
value = colors_to_string(value)
|
||||
if key in self:
|
||||
sql = "update pictures set blocks = ? where path = ?"
|
||||
else:
|
||||
sql = "insert into pictures(blocks,path) values(?,?)"
|
||||
try:
|
||||
self.con.execute(sql, [value, key])
|
||||
except sqlite.OperationalError:
|
||||
logging.warning('Picture cache could not set %r for key %r', value, key)
|
||||
except sqlite.DatabaseError, e:
|
||||
logging.warning('DatabaseError while setting %r for key %r: %s', value, key, str(e))
|
||||
|
||||
def clear(self):
|
||||
sql = "delete from pictures"
|
||||
self.con.execute(sql)
|
||||
|
||||
def filter(self, func):
|
||||
to_delete = [key for key in self if not func(key)]
|
||||
for key in to_delete:
|
||||
del self[key]
|
||||
|
||||
def get_id(self, path):
|
||||
sql = "select rowid from pictures where path = ?"
|
||||
result = self.con.execute(sql, [path]).fetchone()
|
||||
if result:
|
||||
return result[0]
|
||||
else:
|
||||
raise ValueError(path)
|
||||
|
||||
def get_multiple(self, rowids):
|
||||
sql = "select rowid, blocks from pictures where rowid in (%s)" % ','.join(map(str, rowids))
|
||||
cur = self.con.execute(sql)
|
||||
return ((rowid, string_to_colors(blocks)) for rowid, blocks in cur)
|
||||
|
||||
78
core_pe/data.py
Normal file
78
core_pe/data.py
Normal file
@@ -0,0 +1,78 @@
|
||||
# Created By: Virgil Dupras
|
||||
# Created On: 2006/03/15
|
||||
# $Id$
|
||||
# Copyright 2009 Hardcoded Software (http://www.hardcoded.net)
|
||||
#
|
||||
# This software is licensed under the "HS" License as described in the "LICENSE" file,
|
||||
# which should be included with this package. The terms are also available at
|
||||
# http://www.hardcoded.net/licenses/hs_license
|
||||
|
||||
from hsutil.str import format_size
|
||||
from core.data import format_path, format_timestamp, format_perc, format_dupe_count, cmp_value
|
||||
|
||||
def format_dimensions(dimensions):
|
||||
return '%d x %d' % (dimensions[0], dimensions[1])
|
||||
|
||||
COLUMNS = [
|
||||
{'attr':'name','display':'Filename'},
|
||||
{'attr':'path','display':'Directory'},
|
||||
{'attr':'size','display':'Size (KB)'},
|
||||
{'attr':'extension','display':'Kind'},
|
||||
{'attr':'dimensions','display':'Dimensions'},
|
||||
{'attr':'ctime','display':'Creation'},
|
||||
{'attr':'mtime','display':'Modification'},
|
||||
{'attr':'percentage','display':'Match %'},
|
||||
{'attr':'dupe_count','display':'Dupe Count'},
|
||||
]
|
||||
|
||||
METADATA_TO_READ = ['size', 'ctime', 'mtime', 'dimensions']
|
||||
|
||||
def GetDisplayInfo(dupe,group,delta=False):
|
||||
if (dupe is None) or (group is None):
|
||||
return ['---'] * len(COLUMNS)
|
||||
size = dupe.size
|
||||
ctime = dupe.ctime
|
||||
mtime = dupe.mtime
|
||||
m = group.get_match_of(dupe)
|
||||
if m:
|
||||
percentage = m.percentage
|
||||
dupe_count = 0
|
||||
if delta:
|
||||
r = group.ref
|
||||
size -= r.size
|
||||
ctime -= r.ctime
|
||||
mtime -= r.mtime
|
||||
else:
|
||||
percentage = group.percentage
|
||||
dupe_count = len(group.dupes)
|
||||
dupe_path = getattr(dupe, 'display_path', dupe.path)
|
||||
return [
|
||||
dupe.name,
|
||||
format_path(dupe_path),
|
||||
format_size(size, 0, 1, False),
|
||||
dupe.extension,
|
||||
format_dimensions(dupe.dimensions),
|
||||
format_timestamp(ctime, delta and m),
|
||||
format_timestamp(mtime, delta and m),
|
||||
format_perc(percentage),
|
||||
format_dupe_count(dupe_count)
|
||||
]
|
||||
|
||||
def GetDupeSortKey(dupe, get_group, key, delta):
|
||||
if key == 7:
|
||||
m = get_group().get_match_of(dupe)
|
||||
return m.percentage
|
||||
if key == 8:
|
||||
return 0
|
||||
r = cmp_value(getattr(dupe, COLUMNS[key]['attr']))
|
||||
if delta and (key in (2, 5, 6)):
|
||||
r -= cmp_value(getattr(get_group().ref, COLUMNS[key]['attr']))
|
||||
return r
|
||||
|
||||
def GetGroupSortKey(group, key):
|
||||
if key == 7:
|
||||
return group.percentage
|
||||
if key == 8:
|
||||
return len(group)
|
||||
return cmp_value(getattr(group.ref, COLUMNS[key]['attr']))
|
||||
|
||||
32
core_pe/gen.py
Normal file
32
core_pe/gen.py
Normal file
@@ -0,0 +1,32 @@
|
||||
#!/usr/bin/env python
|
||||
# Created By: Virgil Dupras
|
||||
# Created On: 2009-05-26
|
||||
# $Id$
|
||||
# Copyright 2009 Hardcoded Software (http://www.hardcoded.net)
|
||||
#
|
||||
# This software is licensed under the "HS" License as described in the "LICENSE" file,
|
||||
# which should be included with this package. The terms are also available at
|
||||
# http://www.hardcoded.net/licenses/hs_license
|
||||
|
||||
import os
|
||||
import os.path as op
|
||||
|
||||
def move(src, dst):
|
||||
if not op.exists(src):
|
||||
return
|
||||
if op.exists(dst):
|
||||
os.remove(dst)
|
||||
print 'Moving %s --> %s' % (src, dst)
|
||||
os.rename(src, dst)
|
||||
|
||||
# The CC=gcc-4.0 thing is because, in Snow Leopard, gcc-4.2 can't compile these units.
|
||||
os.environ['CC'] = 'gcc-4.0'
|
||||
os.chdir(op.join('modules', 'block'))
|
||||
os.system('python setup.py build_ext --inplace')
|
||||
os.chdir(op.join('..', 'cache'))
|
||||
os.system('python setup.py build_ext --inplace')
|
||||
os.chdir(op.join('..', '..'))
|
||||
move(op.join('modules', 'block', '_block.so'), '_block.so')
|
||||
move(op.join('modules', 'block', '_block.pyd'), '_block.pyd')
|
||||
move(op.join('modules', 'cache', '_cache.so'), '_cache.so')
|
||||
move(op.join('modules', 'cache', '_cache.pyd'), '_cache.pyd')
|
||||
127
core_pe/matchbase.py
Normal file
127
core_pe/matchbase.py
Normal file
@@ -0,0 +1,127 @@
|
||||
# Created By: Virgil Dupras
|
||||
# Created On: 2007/02/25
|
||||
# $Id$
|
||||
# Copyright 2009 Hardcoded Software (http://www.hardcoded.net)
|
||||
#
|
||||
# This software is licensed under the "HS" License as described in the "LICENSE" file,
|
||||
# which should be included with this package. The terms are also available at
|
||||
# http://www.hardcoded.net/licenses/hs_license
|
||||
|
||||
import logging
|
||||
import multiprocessing
|
||||
from Queue import Empty
|
||||
from collections import defaultdict
|
||||
|
||||
from hsutil import job
|
||||
from hsutil.misc import dedupe
|
||||
|
||||
from core.engine import Match
|
||||
from .block import avgdiff, DifferentBlockCountError, NoBlocksError
|
||||
from .cache import Cache
|
||||
|
||||
MIN_ITERATIONS = 3
|
||||
BLOCK_COUNT_PER_SIDE = 15
|
||||
|
||||
# Enough so that we're sure that the main thread will not wait after a result.get() call
|
||||
# cpucount*2 should be enough to be sure that the spawned process will not wait after the results
|
||||
# collection made by the main process.
|
||||
RESULTS_QUEUE_LIMIT = multiprocessing.cpu_count() * 2
|
||||
|
||||
def prepare_pictures(pictures, cached_blocks, j=job.nulljob):
|
||||
# The MemoryError handlers in there use logging without first caring about whether or not
|
||||
# there is enough memory left to carry on the operation because it is assumed that the
|
||||
# MemoryError happens when trying to read an image file, which is freed from memory by the
|
||||
# time that MemoryError is raised.
|
||||
prepared = [] # only pictures for which there was no error getting blocks
|
||||
try:
|
||||
for picture in j.iter_with_progress(pictures, 'Analyzed %d/%d pictures'):
|
||||
picture.dimensions
|
||||
picture.unicode_path = unicode(picture.path)
|
||||
try:
|
||||
if picture.unicode_path not in cached_blocks:
|
||||
blocks = picture.get_blocks(BLOCK_COUNT_PER_SIDE)
|
||||
cached_blocks[picture.unicode_path] = blocks
|
||||
prepared.append(picture)
|
||||
except IOError as e:
|
||||
logging.warning(unicode(e))
|
||||
except MemoryError:
|
||||
logging.warning(u'Ran out of memory while reading %s of size %d' % (picture.unicode_path, picture.size))
|
||||
if picture.size < 10 * 1024 * 1024: # We're really running out of memory
|
||||
raise
|
||||
except MemoryError:
|
||||
logging.warning('Ran out of memory while preparing pictures')
|
||||
return prepared
|
||||
|
||||
def get_match(first, second, percentage):
|
||||
if percentage < 0:
|
||||
percentage = 0
|
||||
return Match(first, second, percentage)
|
||||
|
||||
def async_compare(ref_id, other_ids, dbname, threshold):
|
||||
cache = Cache(dbname, threaded=False)
|
||||
limit = 100 - threshold
|
||||
ref_blocks = cache[ref_id]
|
||||
pairs = cache.get_multiple(other_ids)
|
||||
results = []
|
||||
for other_id, other_blocks in pairs:
|
||||
try:
|
||||
diff = avgdiff(ref_blocks, other_blocks, limit, MIN_ITERATIONS)
|
||||
percentage = 100 - diff
|
||||
except (DifferentBlockCountError, NoBlocksError):
|
||||
percentage = 0
|
||||
if percentage >= threshold:
|
||||
results.append((ref_id, other_id, percentage))
|
||||
cache.con.close()
|
||||
return results
|
||||
|
||||
def getmatches(pictures, cached_blocks, threshold=75, match_scaled=False, j=job.nulljob):
|
||||
def empty_out_queue(queue, into):
|
||||
try:
|
||||
while True:
|
||||
into.append(queue.get(block=False))
|
||||
except Empty:
|
||||
pass
|
||||
|
||||
j = j.start_subjob([3, 7])
|
||||
pictures = prepare_pictures(pictures, cached_blocks, j)
|
||||
j = j.start_subjob([9, 1], 'Preparing for matching')
|
||||
cache = cached_blocks
|
||||
id2picture = {}
|
||||
dimensions2pictures = defaultdict(set)
|
||||
for picture in pictures:
|
||||
try:
|
||||
picture.cache_id = cache.get_id(picture.unicode_path)
|
||||
id2picture[picture.cache_id] = picture
|
||||
if not match_scaled:
|
||||
dimensions2pictures[picture.dimensions].add(picture)
|
||||
except ValueError:
|
||||
pass
|
||||
pictures = [p for p in pictures if hasattr(p, 'cache_id')]
|
||||
pool = multiprocessing.Pool()
|
||||
async_results = []
|
||||
matches = []
|
||||
pictures_copy = set(pictures)
|
||||
for ref in j.iter_with_progress(pictures, 'Matched %d/%d pictures'):
|
||||
others = pictures_copy if match_scaled else dimensions2pictures[ref.dimensions]
|
||||
others.remove(ref)
|
||||
if others:
|
||||
cache_ids = [f.cache_id for f in others]
|
||||
args = (ref.cache_id, cache_ids, cached_blocks.dbname, threshold)
|
||||
async_results.append(pool.apply_async(async_compare, args))
|
||||
if len(async_results) > RESULTS_QUEUE_LIMIT:
|
||||
result = async_results.pop(0)
|
||||
matches.extend(result.get())
|
||||
for result in async_results: # process the rest of the results
|
||||
matches.extend(result.get())
|
||||
|
||||
result = []
|
||||
for ref_id, other_id, percentage in j.iter_with_progress(matches, 'Verified %d/%d matches', every=10):
|
||||
ref = id2picture[ref_id]
|
||||
other = id2picture[other_id]
|
||||
if percentage == 100 and ref.md5 != other.md5:
|
||||
percentage = 99
|
||||
if percentage >= threshold:
|
||||
result.append(get_match(ref, other, percentage))
|
||||
return result
|
||||
|
||||
multiprocessing.freeze_support()
|
||||
97
core_pe/modules/block/block.pyx
Normal file
97
core_pe/modules/block/block.pyx
Normal file
@@ -0,0 +1,97 @@
|
||||
# Created By: Virgil Dupras
|
||||
# Created On: 2009-04-23
|
||||
# $Id$
|
||||
# Copyright 2009 Hardcoded Software (http://www.hardcoded.net)
|
||||
#
|
||||
# This software is licensed under the "HS" License as described in the "LICENSE" file,
|
||||
# which should be included with this package. The terms are also available at
|
||||
# http://www.hardcoded.net/licenses/hs_license
|
||||
|
||||
cdef extern from "stdlib.h":
|
||||
int abs(int n) # required so that abs() is applied on ints, not python objects
|
||||
|
||||
class NoBlocksError(Exception):
|
||||
"""avgdiff/maxdiff has been called with empty lists"""
|
||||
|
||||
class DifferentBlockCountError(Exception):
|
||||
"""avgdiff/maxdiff has been called with 2 block lists of different size."""
|
||||
|
||||
|
||||
cdef object getblock(object image):
|
||||
"""Returns a 3 sized tuple containing the mean color of 'image'.
|
||||
|
||||
image: a PIL image or crop.
|
||||
"""
|
||||
cdef int pixel_count, red, green, blue, r, g, b
|
||||
if image.size[0]:
|
||||
pixel_count = image.size[0] * image.size[1]
|
||||
red = green = blue = 0
|
||||
for r, g, b in image.getdata():
|
||||
red += r
|
||||
green += g
|
||||
blue += b
|
||||
return (red // pixel_count, green // pixel_count, blue // pixel_count)
|
||||
else:
|
||||
return (0, 0, 0)
|
||||
|
||||
def getblocks2(image, int block_count_per_side):
|
||||
"""Returns a list of blocks (3 sized tuples).
|
||||
|
||||
image: A PIL image to base the blocks on.
|
||||
block_count_per_side: This integer determine the number of blocks the function will return.
|
||||
If it is 10, for example, 100 blocks will be returns (10 width, 10 height). The blocks will not
|
||||
necessarely cover square areas. The area covered by each block will be proportional to the image
|
||||
itself.
|
||||
"""
|
||||
if not image.size[0]:
|
||||
return []
|
||||
cdef int width, height, block_width, block_height, ih, iw, top, bottom, left, right
|
||||
width, height = image.size
|
||||
block_width = max(width // block_count_per_side, 1)
|
||||
block_height = max(height // block_count_per_side, 1)
|
||||
result = []
|
||||
for ih in range(block_count_per_side):
|
||||
top = min(ih * block_height, height - block_height)
|
||||
bottom = top + block_height
|
||||
for iw in range(block_count_per_side):
|
||||
left = min(iw * block_width, width - block_width)
|
||||
right = left + block_width
|
||||
box = (left, top, right, bottom)
|
||||
crop = image.crop(box)
|
||||
result.append(getblock(crop))
|
||||
return result
|
||||
|
||||
cdef int diff(first, second):
|
||||
"""Returns the difference between the first block and the second.
|
||||
|
||||
It returns an absolute sum of the 3 differences (RGB).
|
||||
"""
|
||||
cdef int r1, g1, b1, r2, g2, b2
|
||||
r1, g1, b1 = first
|
||||
r2, g2, b2 = second
|
||||
return abs(r1 - r2) + abs(g1 - g2) + abs(b1 - b2)
|
||||
|
||||
def avgdiff(first, second, int limit, int min_iterations):
|
||||
"""Returns the average diff between first blocks and seconds.
|
||||
|
||||
If the result surpasses limit, limit + 1 is returned, except if less than min_iterations
|
||||
iterations have been made in the blocks.
|
||||
"""
|
||||
cdef int count, sum, i, iteration_count
|
||||
count = len(first)
|
||||
if count != len(second):
|
||||
raise DifferentBlockCountError()
|
||||
if not count:
|
||||
raise NoBlocksError()
|
||||
sum = 0
|
||||
for i in range(count):
|
||||
iteration_count = i + 1
|
||||
item1 = first[i]
|
||||
item2 = second[i]
|
||||
sum += diff(item1, item2)
|
||||
if sum > limit * iteration_count and iteration_count >= min_iterations:
|
||||
return limit + 1
|
||||
result = sum // count
|
||||
if (not result) and sum:
|
||||
result = 1
|
||||
return result
|
||||
17
core_pe/modules/block/setup.py
Normal file
17
core_pe/modules/block/setup.py
Normal file
@@ -0,0 +1,17 @@
|
||||
# Created By: Virgil Dupras
|
||||
# Created On: 2009-04-23
|
||||
# $Id$
|
||||
# Copyright 2009 Hardcoded Software (http://www.hardcoded.net)
|
||||
#
|
||||
# This software is licensed under the "HS" License as described in the "LICENSE" file,
|
||||
# which should be included with this package. The terms are also available at
|
||||
# http://www.hardcoded.net/licenses/hs_license
|
||||
|
||||
from distutils.core import setup
|
||||
from distutils.extension import Extension
|
||||
from Cython.Distutils import build_ext
|
||||
|
||||
setup(
|
||||
cmdclass = {'build_ext': build_ext},
|
||||
ext_modules = [Extension("_block", ["block.pyx"])]
|
||||
)
|
||||
37
core_pe/modules/cache/cache.pyx
vendored
Normal file
37
core_pe/modules/cache/cache.pyx
vendored
Normal file
@@ -0,0 +1,37 @@
|
||||
# Created By: Virgil Dupras
|
||||
# Created On: 2009-04-23
|
||||
# $Id$
|
||||
# Copyright 2009 Hardcoded Software (http://www.hardcoded.net)
|
||||
#
|
||||
# This software is licensed under the "HS" License as described in the "LICENSE" file,
|
||||
# which should be included with this package. The terms are also available at
|
||||
# http://www.hardcoded.net/licenses/hs_license
|
||||
|
||||
# ok, this is hacky and stuff, but I don't know C well enough to play with char buffers, copy
|
||||
# them around and stuff
|
||||
cdef int xchar_to_int(char c):
|
||||
if 48 <= c <= 57: # 0-9
|
||||
return c - 48
|
||||
elif 65 <= c <= 70: # A-F
|
||||
return c - 55
|
||||
elif 97 <= c <= 102: # a-f
|
||||
return c - 87
|
||||
|
||||
def string_to_colors(s):
|
||||
"""Transform the string 's' in a list of 3 sized tuples.
|
||||
"""
|
||||
result = []
|
||||
cdef int i, char_count, r, g, b
|
||||
cdef char* cs
|
||||
char_count = len(s)
|
||||
char_count = (char_count // 6) * 6
|
||||
cs = s
|
||||
for i in range(0, char_count, 6):
|
||||
r = xchar_to_int(cs[i]) << 4
|
||||
r += xchar_to_int(cs[i+1])
|
||||
g = xchar_to_int(cs[i+2]) << 4
|
||||
g += xchar_to_int(cs[i+3])
|
||||
b = xchar_to_int(cs[i+4]) << 4
|
||||
b += xchar_to_int(cs[i+5])
|
||||
result.append((r, g, b))
|
||||
return result
|
||||
17
core_pe/modules/cache/setup.py
vendored
Normal file
17
core_pe/modules/cache/setup.py
vendored
Normal file
@@ -0,0 +1,17 @@
|
||||
# Created By: Virgil Dupras
|
||||
# Created On: 2009-04-23
|
||||
# $Id$
|
||||
# Copyright 2009 Hardcoded Software (http://www.hardcoded.net)
|
||||
#
|
||||
# This software is licensed under the "HS" License as described in the "LICENSE" file,
|
||||
# which should be included with this package. The terms are also available at
|
||||
# http://www.hardcoded.net/licenses/hs_license
|
||||
|
||||
from distutils.core import setup
|
||||
from distutils.extension import Extension
|
||||
from Cython.Distutils import build_ext
|
||||
|
||||
setup(
|
||||
cmdclass = {'build_ext': build_ext},
|
||||
ext_modules = [Extension("_cache", ["cache.pyx"])]
|
||||
)
|
||||
22
core_pe/scanner.py
Normal file
22
core_pe/scanner.py
Normal file
@@ -0,0 +1,22 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# Created By: Virgil Dupras
|
||||
# Created On: 2009-10-18
|
||||
# $Id$
|
||||
# Copyright 2009 Hardcoded Software (http://www.hardcoded.net)
|
||||
#
|
||||
# This software is licensed under the "HS" License as described in the "LICENSE" file,
|
||||
# which should be included with this package. The terms are also available at
|
||||
# http://www.hardcoded.net/licenses/hs_license
|
||||
|
||||
from core.scanner import Scanner
|
||||
|
||||
from . import matchbase
|
||||
|
||||
class ScannerPE(Scanner):
|
||||
cached_blocks = None
|
||||
match_scaled = False
|
||||
threshold = 75
|
||||
|
||||
def _getmatches(self, files, j):
|
||||
return matchbase.getmatches(files, self.cached_blocks, self.threshold, self.match_scaled, j)
|
||||
|
||||
0
core_pe/tests/__init__.py
Normal file
0
core_pe/tests/__init__.py
Normal file
309
core_pe/tests/block_test.py
Normal file
309
core_pe/tests/block_test.py
Normal file
@@ -0,0 +1,309 @@
|
||||
# Created By: Virgil Dupras
|
||||
# Created On: 2006/09/01
|
||||
# $Id$
|
||||
# Copyright 2009 Hardcoded Software (http://www.hardcoded.net)
|
||||
#
|
||||
# This software is licensed under the "HS" License as described in the "LICENSE" file,
|
||||
# which should be included with this package. The terms are also available at
|
||||
# http://www.hardcoded.net/licenses/hs_license
|
||||
# The commented out tests are tests for function that have been converted to pure C for speed
|
||||
|
||||
import unittest
|
||||
|
||||
from ..block import *
|
||||
|
||||
def my_avgdiff(first, second, limit=768, min_iter=3): # this is so I don't have to re-write every call
|
||||
return avgdiff(first, second, limit, min_iter)
|
||||
|
||||
BLACK = (0,0,0)
|
||||
RED = (0xff,0,0)
|
||||
GREEN = (0,0xff,0)
|
||||
BLUE = (0,0,0xff)
|
||||
|
||||
class FakeImage(object):
|
||||
def __init__(self, size, data):
|
||||
self.size = size
|
||||
self.data = data
|
||||
|
||||
def getdata(self):
|
||||
return self.data
|
||||
|
||||
def crop(self, box):
|
||||
pixels = []
|
||||
for i in range(box[1], box[3]):
|
||||
for j in range(box[0], box[2]):
|
||||
pixel = self.data[i * self.size[0] + j]
|
||||
pixels.append(pixel)
|
||||
return FakeImage((box[2] - box[0], box[3] - box[1]), pixels)
|
||||
|
||||
def empty():
|
||||
return FakeImage((0,0), [])
|
||||
|
||||
def single_pixel(): #one red pixel
|
||||
return FakeImage((1, 1), [(0xff,0,0)])
|
||||
|
||||
def four_pixels():
|
||||
pixels = [RED,(0,0x80,0xff),(0x80,0,0),(0,0x40,0x80)]
|
||||
return FakeImage((2, 2), pixels)
|
||||
|
||||
class TCgetblock(unittest.TestCase):
|
||||
def test_single_pixel(self):
|
||||
im = single_pixel()
|
||||
[b] = getblocks2(im, 1)
|
||||
self.assertEqual(RED,b)
|
||||
|
||||
def test_no_pixel(self):
|
||||
im = empty()
|
||||
self.assertEqual([], getblocks2(im, 1))
|
||||
|
||||
def test_four_pixels(self):
|
||||
im = four_pixels()
|
||||
[b] = getblocks2(im, 1)
|
||||
meanred = (0xff + 0x80) // 4
|
||||
meangreen = (0x80 + 0x40) // 4
|
||||
meanblue = (0xff + 0x80) // 4
|
||||
self.assertEqual((meanred,meangreen,meanblue),b)
|
||||
|
||||
|
||||
# class TCdiff(unittest.TestCase):
|
||||
# def test_diff(self):
|
||||
# b1 = (10, 20, 30)
|
||||
# b2 = (1, 2, 3)
|
||||
# self.assertEqual(9 + 18 + 27,diff(b1,b2))
|
||||
#
|
||||
# def test_diff_negative(self):
|
||||
# b1 = (10, 20, 30)
|
||||
# b2 = (1, 2, 3)
|
||||
# self.assertEqual(9 + 18 + 27,diff(b2,b1))
|
||||
#
|
||||
# def test_diff_mixed_positive_and_negative(self):
|
||||
# b1 = (1, 5, 10)
|
||||
# b2 = (10, 1, 15)
|
||||
# self.assertEqual(9 + 4 + 5,diff(b1,b2))
|
||||
#
|
||||
|
||||
# class TCgetblocks(unittest.TestCase):
|
||||
# def test_empty_image(self):
|
||||
# im = empty()
|
||||
# blocks = getblocks(im,1)
|
||||
# self.assertEqual(0,len(blocks))
|
||||
#
|
||||
# def test_one_block_image(self):
|
||||
# im = four_pixels()
|
||||
# blocks = getblocks2(im, 1)
|
||||
# self.assertEqual(1,len(blocks))
|
||||
# block = blocks[0]
|
||||
# meanred = (0xff + 0x80) // 4
|
||||
# meangreen = (0x80 + 0x40) // 4
|
||||
# meanblue = (0xff + 0x80) // 4
|
||||
# self.assertEqual((meanred,meangreen,meanblue),block)
|
||||
#
|
||||
# def test_not_enough_height_to_fit_a_block(self):
|
||||
# im = FakeImage((2,1), [BLACK, BLACK])
|
||||
# blocks = getblocks(im,2)
|
||||
# self.assertEqual(0,len(blocks))
|
||||
#
|
||||
# def xtest_dont_include_leftovers(self):
|
||||
# # this test is disabled because getblocks is not used and getblock in cdeffed
|
||||
# pixels = [
|
||||
# RED,(0,0x80,0xff),BLACK,
|
||||
# (0x80,0,0),(0,0x40,0x80),BLACK,
|
||||
# BLACK,BLACK,BLACK
|
||||
# ]
|
||||
# im = FakeImage((3,3), pixels)
|
||||
# blocks = getblocks(im,2)
|
||||
# block = blocks[0]
|
||||
# #Because the block is smaller than the image, only blocksize must be considered.
|
||||
# meanred = (0xff + 0x80) // 4
|
||||
# meangreen = (0x80 + 0x40) // 4
|
||||
# meanblue = (0xff + 0x80) // 4
|
||||
# self.assertEqual((meanred,meangreen,meanblue),block)
|
||||
#
|
||||
# def xtest_two_blocks(self):
|
||||
# # this test is disabled because getblocks is not used and getblock in cdeffed
|
||||
# pixels = [BLACK for i in xrange(4 * 2)]
|
||||
# pixels[0] = RED
|
||||
# pixels[1] = (0,0x80,0xff)
|
||||
# pixels[4] = (0x80,0,0)
|
||||
# pixels[5] = (0,0x40,0x80)
|
||||
# im = FakeImage((4, 2), pixels)
|
||||
# blocks = getblocks(im,2)
|
||||
# self.assertEqual(2,len(blocks))
|
||||
# block = blocks[0]
|
||||
# #Because the block is smaller than the image, only blocksize must be considered.
|
||||
# meanred = (0xff + 0x80) // 4
|
||||
# meangreen = (0x80 + 0x40) // 4
|
||||
# meanblue = (0xff + 0x80) // 4
|
||||
# self.assertEqual((meanred,meangreen,meanblue),block)
|
||||
# self.assertEqual(BLACK,blocks[1])
|
||||
#
|
||||
# def test_four_blocks(self):
|
||||
# pixels = [BLACK for i in xrange(4 * 4)]
|
||||
# pixels[0] = RED
|
||||
# pixels[1] = (0,0x80,0xff)
|
||||
# pixels[4] = (0x80,0,0)
|
||||
# pixels[5] = (0,0x40,0x80)
|
||||
# im = FakeImage((4, 4), pixels)
|
||||
# blocks = getblocks2(im, 2)
|
||||
# self.assertEqual(4,len(blocks))
|
||||
# block = blocks[0]
|
||||
# #Because the block is smaller than the image, only blocksize must be considered.
|
||||
# meanred = (0xff + 0x80) // 4
|
||||
# meangreen = (0x80 + 0x40) // 4
|
||||
# meanblue = (0xff + 0x80) // 4
|
||||
# self.assertEqual((meanred,meangreen,meanblue),block)
|
||||
# self.assertEqual(BLACK,blocks[1])
|
||||
# self.assertEqual(BLACK,blocks[2])
|
||||
# self.assertEqual(BLACK,blocks[3])
|
||||
#
|
||||
|
||||
class TCgetblocks2(unittest.TestCase):
|
||||
def test_empty_image(self):
|
||||
im = empty()
|
||||
blocks = getblocks2(im,1)
|
||||
self.assertEqual(0,len(blocks))
|
||||
|
||||
def test_one_block_image(self):
|
||||
im = four_pixels()
|
||||
blocks = getblocks2(im,1)
|
||||
self.assertEqual(1,len(blocks))
|
||||
block = blocks[0]
|
||||
meanred = (0xff + 0x80) // 4
|
||||
meangreen = (0x80 + 0x40) // 4
|
||||
meanblue = (0xff + 0x80) // 4
|
||||
self.assertEqual((meanred,meangreen,meanblue),block)
|
||||
|
||||
def test_four_blocks_all_black(self):
|
||||
im = FakeImage((2, 2), [BLACK, BLACK, BLACK, BLACK])
|
||||
blocks = getblocks2(im,2)
|
||||
self.assertEqual(4,len(blocks))
|
||||
for block in blocks:
|
||||
self.assertEqual(BLACK,block)
|
||||
|
||||
def test_two_pixels_image_horizontal(self):
|
||||
pixels = [RED,BLUE]
|
||||
im = FakeImage((2, 1), pixels)
|
||||
blocks = getblocks2(im,2)
|
||||
self.assertEqual(4,len(blocks))
|
||||
self.assertEqual(RED,blocks[0])
|
||||
self.assertEqual(BLUE,blocks[1])
|
||||
self.assertEqual(RED,blocks[2])
|
||||
self.assertEqual(BLUE,blocks[3])
|
||||
|
||||
def test_two_pixels_image_vertical(self):
|
||||
pixels = [RED,BLUE]
|
||||
im = FakeImage((1, 2), pixels)
|
||||
blocks = getblocks2(im,2)
|
||||
self.assertEqual(4,len(blocks))
|
||||
self.assertEqual(RED,blocks[0])
|
||||
self.assertEqual(RED,blocks[1])
|
||||
self.assertEqual(BLUE,blocks[2])
|
||||
self.assertEqual(BLUE,blocks[3])
|
||||
|
||||
|
||||
class TCavgdiff(unittest.TestCase):
|
||||
def test_empty(self):
|
||||
self.assertRaises(NoBlocksError, my_avgdiff, [], [])
|
||||
|
||||
def test_two_blocks(self):
|
||||
im = empty()
|
||||
b1 = (5,10,15)
|
||||
b2 = (255,250,245)
|
||||
b3 = (0,0,0)
|
||||
b4 = (255,0,255)
|
||||
blocks1 = [b1,b2]
|
||||
blocks2 = [b3,b4]
|
||||
expected1 = 5 + 10 + 15
|
||||
expected2 = 0 + 250 + 10
|
||||
expected = (expected1 + expected2) // 2
|
||||
self.assertEqual(expected, my_avgdiff(blocks1, blocks2))
|
||||
|
||||
def test_blocks_not_the_same_size(self):
|
||||
b = (0,0,0)
|
||||
self.assertRaises(DifferentBlockCountError,my_avgdiff,[b,b],[b])
|
||||
|
||||
def test_first_arg_is_empty_but_not_second(self):
|
||||
#Don't return 0 (as when the 2 lists are empty), raise!
|
||||
b = (0,0,0)
|
||||
self.assertRaises(DifferentBlockCountError,my_avgdiff,[],[b])
|
||||
|
||||
def test_limit(self):
|
||||
ref = (0,0,0)
|
||||
b1 = (10,10,10) #avg 30
|
||||
b2 = (20,20,20) #avg 45
|
||||
b3 = (30,30,30) #avg 60
|
||||
blocks1 = [ref,ref,ref]
|
||||
blocks2 = [b1,b2,b3]
|
||||
self.assertEqual(45,my_avgdiff(blocks1,blocks2,44))
|
||||
|
||||
def test_min_iterations(self):
|
||||
ref = (0,0,0)
|
||||
b1 = (10,10,10) #avg 30
|
||||
b2 = (20,20,20) #avg 45
|
||||
b3 = (10,10,10) #avg 40
|
||||
blocks1 = [ref,ref,ref]
|
||||
blocks2 = [b1,b2,b3]
|
||||
self.assertEqual(40,my_avgdiff(blocks1,blocks2,45 - 1,3))
|
||||
|
||||
# Bah, I don't know why this test fails, but I don't think it matters very much
|
||||
# def test_just_over_the_limit(self):
|
||||
# #A score just over the limit might return exactly the limit due to truncating. We should
|
||||
# #ceil() the result in this case.
|
||||
# ref = (0,0,0)
|
||||
# b1 = (10,0,0)
|
||||
# b2 = (11,0,0)
|
||||
# blocks1 = [ref,ref]
|
||||
# blocks2 = [b1,b2]
|
||||
# self.assertEqual(11,my_avgdiff(blocks1,blocks2,10))
|
||||
#
|
||||
def test_return_at_least_1_at_the_slightest_difference(self):
|
||||
ref = (0,0,0)
|
||||
b1 = (1,0,0)
|
||||
blocks1 = [ref for i in xrange(250)]
|
||||
blocks2 = [ref for i in xrange(250)]
|
||||
blocks2[0] = b1
|
||||
self.assertEqual(1,my_avgdiff(blocks1,blocks2))
|
||||
|
||||
def test_return_0_if_there_is_no_difference(self):
|
||||
ref = (0,0,0)
|
||||
blocks1 = [ref,ref]
|
||||
blocks2 = [ref,ref]
|
||||
self.assertEqual(0,my_avgdiff(blocks1,blocks2))
|
||||
|
||||
|
||||
# class TCmaxdiff(unittest.TestCase):
|
||||
# def test_empty(self):
|
||||
# self.assertRaises(NoBlocksError,maxdiff,[],[])
|
||||
#
|
||||
# def test_two_blocks(self):
|
||||
# b1 = (5,10,15)
|
||||
# b2 = (255,250,245)
|
||||
# b3 = (0,0,0)
|
||||
# b4 = (255,0,255)
|
||||
# blocks1 = [b1,b2]
|
||||
# blocks2 = [b3,b4]
|
||||
# expected1 = 5 + 10 + 15
|
||||
# expected2 = 0 + 250 + 10
|
||||
# expected = max(expected1,expected2)
|
||||
# self.assertEqual(expected,maxdiff(blocks1,blocks2))
|
||||
#
|
||||
# def test_blocks_not_the_same_size(self):
|
||||
# b = (0,0,0)
|
||||
# self.assertRaises(DifferentBlockCountError,maxdiff,[b,b],[b])
|
||||
#
|
||||
# def test_first_arg_is_empty_but_not_second(self):
|
||||
# #Don't return 0 (as when the 2 lists are empty), raise!
|
||||
# b = (0,0,0)
|
||||
# self.assertRaises(DifferentBlockCountError,maxdiff,[],[b])
|
||||
#
|
||||
# def test_limit(self):
|
||||
# b1 = (5,10,15)
|
||||
# b2 = (255,250,245)
|
||||
# b3 = (0,0,0)
|
||||
# b4 = (255,0,255)
|
||||
# blocks1 = [b1,b2]
|
||||
# blocks2 = [b3,b4]
|
||||
# expected1 = 5 + 10 + 15
|
||||
# expected2 = 0 + 250 + 10
|
||||
# self.assertEqual(expected1,maxdiff(blocks1,blocks2,expected1 - 1))
|
||||
#
|
||||
155
core_pe/tests/cache_test.py
Normal file
155
core_pe/tests/cache_test.py
Normal file
@@ -0,0 +1,155 @@
|
||||
# Created By: Virgil Dupras
|
||||
# Created On: 2006/09/14
|
||||
# $Id$
|
||||
# Copyright 2009 Hardcoded Software (http://www.hardcoded.net)
|
||||
#
|
||||
# This software is licensed under the "HS" License as described in the "LICENSE" file,
|
||||
# which should be included with this package. The terms are also available at
|
||||
# http://www.hardcoded.net/licenses/hs_license
|
||||
|
||||
from StringIO import StringIO
|
||||
import os.path as op
|
||||
import os
|
||||
import threading
|
||||
|
||||
from hsutil.testcase import TestCase
|
||||
|
||||
from ..cache import *
|
||||
|
||||
class TCcolors_to_string(TestCase):
|
||||
def test_no_color(self):
|
||||
self.assertEqual('',colors_to_string([]))
|
||||
|
||||
def test_single_color(self):
|
||||
self.assertEqual('000000',colors_to_string([(0,0,0)]))
|
||||
self.assertEqual('010101',colors_to_string([(1,1,1)]))
|
||||
self.assertEqual('0a141e',colors_to_string([(10,20,30)]))
|
||||
|
||||
def test_two_colors(self):
|
||||
self.assertEqual('000102030405',colors_to_string([(0,1,2),(3,4,5)]))
|
||||
|
||||
|
||||
class TCstring_to_colors(TestCase):
|
||||
def test_empty(self):
|
||||
self.assertEqual([],string_to_colors(''))
|
||||
|
||||
def test_single_color(self):
|
||||
self.assertEqual([(0,0,0)],string_to_colors('000000'))
|
||||
self.assertEqual([(2,3,4)],string_to_colors('020304'))
|
||||
self.assertEqual([(10,20,30)],string_to_colors('0a141e'))
|
||||
|
||||
def test_two_colors(self):
|
||||
self.assertEqual([(10,20,30),(40,50,60)],string_to_colors('0a141e28323c'))
|
||||
|
||||
def test_incomplete_color(self):
|
||||
# don't return anything if it's not a complete color
|
||||
self.assertEqual([],string_to_colors('102'))
|
||||
|
||||
|
||||
class TCCache(TestCase):
|
||||
def test_empty(self):
|
||||
c = Cache()
|
||||
self.assertEqual(0,len(c))
|
||||
self.assertRaises(KeyError,c.__getitem__,'foo')
|
||||
|
||||
def test_set_then_retrieve_blocks(self):
|
||||
c = Cache()
|
||||
b = [(0,0,0),(1,2,3)]
|
||||
c['foo'] = b
|
||||
self.assertEqual(b,c['foo'])
|
||||
|
||||
def test_delitem(self):
|
||||
c = Cache()
|
||||
c['foo'] = ''
|
||||
del c['foo']
|
||||
self.assert_('foo' not in c)
|
||||
self.assertRaises(KeyError,c.__delitem__,'foo')
|
||||
|
||||
def test_persistance(self):
|
||||
DBNAME = op.join(self.tmpdir(), 'hstest.db')
|
||||
c = Cache(DBNAME)
|
||||
c['foo'] = [(1,2,3)]
|
||||
del c
|
||||
c = Cache(DBNAME)
|
||||
self.assertEqual([(1,2,3)],c['foo'])
|
||||
del c
|
||||
os.remove(DBNAME)
|
||||
|
||||
def test_filter(self):
|
||||
c = Cache()
|
||||
c['foo'] = ''
|
||||
c['bar'] = ''
|
||||
c['baz'] = ''
|
||||
c.filter(lambda p:p != 'bar') #only 'bar' is removed
|
||||
self.assertEqual(2,len(c))
|
||||
self.assert_('foo' in c)
|
||||
self.assert_('baz' in c)
|
||||
self.assert_('bar' not in c)
|
||||
|
||||
def test_clear(self):
|
||||
c = Cache()
|
||||
c['foo'] = ''
|
||||
c['bar'] = ''
|
||||
c['baz'] = ''
|
||||
c.clear()
|
||||
self.assertEqual(0,len(c))
|
||||
self.assert_('foo' not in c)
|
||||
self.assert_('baz' not in c)
|
||||
self.assert_('bar' not in c)
|
||||
|
||||
def test_corrupted_db(self):
|
||||
dbname = op.join(self.tmpdir(), 'foo.db')
|
||||
fp = open(dbname, 'w')
|
||||
fp.write('invalid sqlite content')
|
||||
fp.close()
|
||||
c = Cache(dbname) # should not raise a DatabaseError
|
||||
c['foo'] = [(1, 2, 3)]
|
||||
del c
|
||||
c = Cache(dbname)
|
||||
self.assertEqual(c['foo'], [(1, 2, 3)])
|
||||
|
||||
def test_by_id(self):
|
||||
# it's possible to use the cache by referring to the files by their row_id
|
||||
c = Cache()
|
||||
b = [(0,0,0),(1,2,3)]
|
||||
c['foo'] = b
|
||||
foo_id = c.get_id('foo')
|
||||
self.assertEqual(c[foo_id], b)
|
||||
|
||||
|
||||
class TCCacheSQLEscape(TestCase):
|
||||
def test_contains(self):
|
||||
c = Cache()
|
||||
self.assert_("foo'bar" not in c)
|
||||
|
||||
def test_getitem(self):
|
||||
c = Cache()
|
||||
self.assertRaises(KeyError, c.__getitem__, "foo'bar")
|
||||
|
||||
def test_setitem(self):
|
||||
c = Cache()
|
||||
c["foo'bar"] = []
|
||||
|
||||
def test_delitem(self):
|
||||
c = Cache()
|
||||
c["foo'bar"] = []
|
||||
try:
|
||||
del c["foo'bar"]
|
||||
except KeyError:
|
||||
self.fail()
|
||||
|
||||
|
||||
class TCCacheThreaded(TestCase):
|
||||
def test_access_cache(self):
|
||||
def thread_run():
|
||||
try:
|
||||
c['foo'] = [(1,2,3)]
|
||||
except sqlite.ProgrammingError:
|
||||
self.fail()
|
||||
|
||||
c = Cache()
|
||||
t = threading.Thread(target=thread_run)
|
||||
t.start()
|
||||
t.join()
|
||||
self.assertEqual([(1,2,3)], c['foo'])
|
||||
|
||||
Reference in New Issue
Block a user