arsenetar/dupeguru
1
0
Fork 0
mirror of https://github.com/arsenetar/dupeguru.git synced 2025-03-10 05:34:36 +00:00
Code Releases Wiki Activity

[#70 state:fixed] Wrapped get_groups() into a try..except MemoryError

Browse Source 
--HG--
extra : convert_revision : svn%3Ac306627e-7827-47d3-bdf0-9a457c9553a1/trunk%40185
This commit is contained in:
hsoft 2009-10-14 14:42:18 +00:00
parent 46a27d5988
commit 7f34389b8e
1 changed files with 27 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
base/py/engine.py
View File

@ -334,6 +334,7 @@ def get_groups(matches, j=job.nulljob):
matches.sort(key=lambda match: -match.percentage) matches.sort(key=lambda match: -match.percentage)
dupe2group = {} dupe2group = {}
groups = [] groups = []
try:
for match in j.iter_with_progress(matches, 'Grouped %d/%d matches', JOB_REFRESH_RATE): for match in j.iter_with_progress(matches, 'Grouped %d/%d matches', JOB_REFRESH_RATE):
first, second, _ = match first, second, _ = match
first_group = dupe2group.get(first) first_group = dupe2group.get(first)
@ -357,6 +358,11 @@ def get_groups(matches, j=job.nulljob):
dupe2group[first] = target_group dupe2group[first] = target_group
dupe2group[second] = target_group dupe2group[second] = target_group
target_group.add_match(match) target_group.add_match(match)
except MemoryError:
del dupe2group
del matches
# should free enough memory to continue
logging.warning('Memory Overflow. Groups: {0}'.format(len(groups)))
# Now that we have a group, we have to discard groups' matches and see if there're any "orphan" # Now that we have a group, we have to discard groups' matches and see if there're any "orphan"
# matches, that is, matches that were candidate in a group but that none of their 2 files were # matches, that is, matches that were candidate in a group but that none of their 2 files were
# accepted in the group. With these orphan groups, it's safe to build additional groups # accepted in the group. With these orphan groups, it's safe to build additional groups