arsenetar
/
dupeguru
mirror of https://github.com/arsenetar/dupeguru.git
1
0
Fork 0
Code Releases Wiki Activity

[#70 state:fixed] Wrapped get_groups() into a try..except MemoryError 

--HG--
extra : convert_revision : svn%3Ac306627e-7827-47d3-bdf0-9a457c9553a1/trunk%40185
This commit is contained in:
hsoft 2009-10-14 14:42:18 +00:00
parent 46a27d5988
commit 7f34389b8e
1 changed files with 27 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
base/py/engine.py
View File

@ -334,29 +334,35 @@ def get_groups(matches, j=job.nulljob):
matches.sort(key=lambda match: -match.percentage) matches.sort(key=lambda match: -match.percentage)
dupe2group = {} dupe2group = {}
groups = [] groups = []
for match in j.iter_with_progress(matches, 'Grouped %d/%d matches', JOB_REFRESH_RATE): try:
first, second, _ = match for match in j.iter_with_progress(matches, 'Grouped %d/%d matches', JOB_REFRESH_RATE):
first_group = dupe2group.get(first) first, second, _ = match
second_group = dupe2group.get(second) first_group = dupe2group.get(first)
if first_group: second_group = dupe2group.get(second)
if second_group: if first_group:
if first_group is second_group: if second_group:
target_group = first_group if first_group is second_group:
target_group = first_group
else:
continue
else: else:
continue target_group = first_group
dupe2group[second] = target_group
else: else:
target_group = first_group if second_group:
dupe2group[second] = target_group target_group = second_group
else: dupe2group[first] = target_group
if second_group: else:
target_group = second_group target_group = Group()
dupe2group[first] = target_group groups.append(target_group)
else: dupe2group[first] = target_group
target_group = Group() dupe2group[second] = target_group
groups.append(target_group) target_group.add_match(match)
dupe2group[first] = target_group except MemoryError:
dupe2group[second] = target_group del dupe2group
target_group.add_match(match) del matches
# should free enough memory to continue
logging.warning('Memory Overflow. Groups: {0}'.format(len(groups)))
# Now that we have a group, we have to discard groups' matches and see if there're any "orphan" # Now that we have a group, we have to discard groups' matches and see if there're any "orphan"
# matches, that is, matches that were candidate in a group but that none of their 2 files were # matches, that is, matches that were candidate in a group but that none of their 2 files were
# accepted in the group. With these orphan groups, it's safe to build additional groups # accepted in the group. With these orphan groups, it's safe to build additional groups