arsenetar/dupeguru
1
0
Fork 0
mirror of https://github.com/arsenetar/dupeguru.git synced 2026-01-22 14:41:39 +00:00
Code Releases Wiki Activity

Apply pyupgrade changes

Browse Source
This commit is contained in:
Andrew Senetar
2022-04-27 20:53:12 -05:00
parent e0061d7bc1
commit 63dd4d4561
36 changed files with 89 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
core/engine.py
View File

@@ -166,7 +166,7 @@ def reduce_common_words(word_dict, threshold):
The exception to this removal are the objects where all the words of the object are common.
Because if we remove them, we will miss some duplicates!
"""
uncommon_words = set(word for word, objects in word_dict.items() if len(objects) < threshold)
uncommon_words = {word for word, objects in word_dict.items() if len(objects) < threshold}
for word, objects in list(word_dict.items()):
if len(objects) < threshold:
continue
@@ -409,7 +409,7 @@ class Group:
You can call this after the duplicate scanning process to free a bit of memory.
"""
discarded = set(m for m in self.matches if not all(obj in self.unordered for obj in [m.first, m.second]))
discarded = {m for m in self.matches if not all(obj in self.unordered for obj in [m.first, m.second])}
self.matches -= discarded
self.candidates = defaultdict(set)
return discarded
@@ -456,7 +456,7 @@ class Group:
self._matches_for_ref = None
if (len(self) > 1) and any(not getattr(item, "is_ref", False) for item in self):
if discard_matches:
self.matches = set(m for m in self.matches if item not in m)
self.matches = {m for m in self.matches if item not in m}
else:
self._clear()
except ValueError:
@@ -529,7 +529,7 @@ def get_groups(matches):
del dupe2group
del matches
# should free enough memory to continue
logging.warning("Memory Overflow. Groups: {0}".format(len(groups)))
logging.warning(f"Memory Overflow. Groups: {len(groups)}")
# Now that we have a group, we have to discard groups' matches and see if there're any "orphan"
# matches, that is, matches that were candidate in a group but that none of their 2 files were
# accepted in the group. With these orphan groups, it's safe to build additional groups