diff --git a/core/app.py b/core/app.py
index 6eb1168f..61100c38 100644
--- a/core/app.py
+++ b/core/app.py
@@ -442,14 +442,16 @@ class DupeGuru(RegistrableApplication, Broadcaster):
         for dupe in dupes:
             g = self.results.get_group_of_duplicate(dupe)
             if g not in changed_groups:
-                self.results.make_ref(dupe)
-                changed_groups.add(g)
+                if self.results.make_ref(dupe):
+                    changed_groups.add(g)
         # It's not always obvious to users what this action does, so to make it a bit clearer,
         # we change our selection to the ref of all changed groups. However, we also want to keep
         # the files that were ref before and weren't changed by the action. In effect, what this
         # does is that we keep our old selection, but remove all non-ref dupes from it.
-        self.selected_dupes = [d for d in self.selected_dupes
-            if self.results.get_group_of_duplicate(d).ref is d]
+        # If no group was changed, however, we don't touch the selection.
+        if changed_groups:
+            self.selected_dupes = [d for d in self.selected_dupes
+                if self.results.get_group_of_duplicate(d).ref is d]
         self.notify('results_changed')
     
     def mark_all(self):
diff --git a/core/engine.py b/core/engine.py
index 6273a391..ae2ca956 100644
--- a/core/engine.py
+++ b/core/engine.py
@@ -325,14 +325,15 @@ class Group:
     
     def switch_ref(self, with_dupe):
         if self.ref.is_ref:
-            return
+            return False
         try:
             self.ordered.remove(with_dupe)
             self.ordered.insert(0, with_dupe)
             self._percentage = None
             self._matches_for_ref = None
+            return True
         except ValueError:
-            pass
+            return False
     
     dupes = property(lambda self: self[1:])
     
diff --git a/core/results.py b/core/results.py
index fd2229d4..153d3d64 100644
--- a/core/results.py
+++ b/core/results.py
@@ -244,8 +244,9 @@ class Results(Markable):
     def make_ref(self, dupe):
         g = self.get_group_of_duplicate(dupe)
         r = g.ref
+        if not g.switch_ref(dupe):
+            return False
         self._remove_mark_flag(dupe)
-        g.switch_ref(dupe);
         if not r.is_ref:
             self.__total_count += 1
             self.__total_size += r.size
@@ -254,6 +255,7 @@ class Results(Markable):
             self.__total_size -= dupe.size
         self.__dupes = None
         self.is_modified = True
+        return True
     
     def perform_on_marked(self, func, remove_from_results):
         # Performs `func` on all marked dupes. If an EnvironmentError is raised during the call,