diff --git a/core/engine.py b/core/engine.py
index 898a5082..479d357d 100644
--- a/core/engine.py
+++ b/core/engine.py
@@ -290,7 +290,8 @@ class Group:
     
     def prioritize(self, key_func, tie_breaker=None):
         # tie_breaker(ref, dupe) --> True if dupe should be ref
-        self.ordered.sort(key=key_func)
+        master_key_func = lambda x: (-x.is_ref, key_func(x))
+        self.ordered.sort(key=master_key_func)
         if tie_breaker is None:
             return
         ref = self.ref
@@ -318,6 +319,8 @@ class Group:
             pass
     
     def switch_ref(self, with_dupe):
+        if self.ref.is_ref:
+            return
         try:
             self.ordered.remove(with_dupe)
             self.ordered.insert(0, with_dupe)
diff --git a/core/scanner.py b/core/scanner.py
index 2beadb17..ba12d34a 100644
--- a/core/scanner.py
+++ b/core/scanner.py
@@ -80,7 +80,7 @@ class Scanner:
     
     @staticmethod
     def _key_func(dupe):
-        return (not dupe.is_ref, -dupe.size)
+        return -dupe.size
     
     @staticmethod
     def _tie_breaker(ref, dupe):
diff --git a/core/tests/app_test.py b/core/tests/app_test.py
index ffc416ef..32f65128 100644
--- a/core/tests/app_test.py
+++ b/core/tests/app_test.py
@@ -416,6 +416,8 @@ class TestCaseDupeGuru_renameSelected:
         fp = open(str(p + 'foo bar 3'),mode='w')
         fp.close()
         files = fs.get_files(p)
+        for f in files:
+            f.is_ref = False
         matches = engine.getmatches(files)
         groups = engine.get_groups(matches)
         g = groups[0]
diff --git a/core/tests/engine_test.py b/core/tests/engine_test.py
index acfaa62e..7bd42100 100644
--- a/core/tests/engine_test.py
+++ b/core/tests/engine_test.py
@@ -599,6 +599,16 @@ class TestCaseGroup:
         g.switch_ref(NamedObject('',True))
         assert o2 is g.ref
     
+    def test_switch_ref_from_ref_dir(self):
+        # When the ref dupe is from a ref dir, switch_ref() does nothing
+        o1 = no(with_words=True)
+        o2 = no(with_words=True)
+        o1.is_ref = True
+        g = Group()
+        g.add_match(get_match(o1, o2))
+        g.switch_ref(o2)
+        assert o1 is g.ref
+    
     def test_get_match_of(self):
         g = Group()
         for m in get_match_triangle():
@@ -684,6 +694,15 @@ class TestCaseGroup:
         g.prioritize(key_func, tie_breaker)
         assert g.ref is o2
     
+    def test_prioritize_with_ref_dupe(self):
+        # when the ref dupe of a group is from a ref dir, make it stay on top.
+        g = get_test_group()
+        o1, o2, o3 = g
+        o1.is_ref = True
+        o2.size = 2
+        g.prioritize(lambda x: -x.size)
+        assert g.ref is o1
+    
     def test_list_like(self):
         g = Group()
         o1,o2 = (NamedObject("foo",True),NamedObject("bar",True))
diff --git a/core/tests/results_test.py b/core/tests/results_test.py
index 4ab6dee5..e9b49c77 100644
--- a/core/tests/results_test.py
+++ b/core/tests/results_test.py
@@ -794,14 +794,4 @@ class TestCaseResultsRefFile:
     def test_stat_line(self):
         expected = '0 / 2 (0.00 B / 2.00 B) duplicates marked.'
         eq_(expected, self.results.stat_line)
-    
-    def test_make_ref(self):
-        d = self.results.groups[0].dupes[1] #non-ref
-        r = self.results.groups[0].ref
-        self.results.make_ref(d)
-        expected = '0 / 1 (0.00 B / 1.00 B) duplicates marked.'
-        eq_(expected, self.results.stat_line)
-        self.results.make_ref(r)
-        expected = '0 / 2 (0.00 B / 2.00 B) duplicates marked.'
-        eq_(expected, self.results.stat_line)
-    
+    
\ No newline at end of file
diff --git a/core_me/scanner.py b/core_me/scanner.py
index 7b371d64..a35c75a3 100644
--- a/core_me/scanner.py
+++ b/core_me/scanner.py
@@ -11,5 +11,5 @@ from core.scanner import Scanner as ScannerBase
 class ScannerME(ScannerBase):
     @staticmethod
     def _key_func(dupe):
-        return (not dupe.is_ref, -dupe.bitrate, -dupe.size)
+        return (-dupe.bitrate, -dupe.size)