diff --git a/core/engine.py b/core/engine.py
index fdf943a9..039f030e 100644
--- a/core/engine.py
+++ b/core/engine.py
@@ -216,6 +216,8 @@ def getmatches_by_contents(files, sizeattr='size', partial=False, j=job.nulljob)
     j.start_job(len(possible_matches), '0 matches found')
     for group in possible_matches:
         for first, second in itertools.combinations(group, 2):
+            if first.is_ref and second.is_ref:
+                continue # Don't spend time comparing two ref pics together.
             if first.md5partial == second.md5partial:
                 if partial or first.md5 == second.md5:
                     result.append(Match(first, second, 100))
diff --git a/core_pe/matchbase.py b/core_pe/matchbase.py
index a3cee041..678455e8 100644
--- a/core_pe/matchbase.py
+++ b/core_pe/matchbase.py
@@ -103,6 +103,9 @@ def getmatches(pictures, cached_blocks, threshold=75, match_scaled=False, j=job.
     for ref in j.iter_with_progress(pictures, 'Matched %d/%d pictures'):
         others = pictures_copy if match_scaled else dimensions2pictures[ref.dimensions]
         others.remove(ref)
+        if ref.is_ref:
+            # Don't spend time comparing two ref pics together.
+            others = [pic for pic in others if not pic.is_ref]
         if others:
             cache_ids = [f.cache_id for f in others]
             args = (ref.cache_id, cache_ids, cached_blocks.dbname, threshold)