Merge pull request #879 from glubsy/fix_unicode

Fix stripping (japanese) unicode characters

core/engine.py

@@ -26,8 +26,19 @@ def getwords(s):
     # We decompose the string so that ascii letters with accents can be part of the word.
     s = normalize("NFD", s)
     s = multi_replace(s, "-_&+():;\\[]{}.,<>/?~!@#$*", " ").lower()
+    # logging.debug(f"DEBUG chars for: {s}\n"
+    #               f"{[c for c in s if ord(c) != 32]}\n"
+    #               f"{[ord(c) for c in s if ord(c) != 32]}")
+    # HACK We shouldn't ignore non-ascii characters altogether. Any Unicode char
+    # above common european characters that cannot be "sanitized" (ie. stripped
+    # of their accents, etc.) are preserved as is. The arbitrary limit is
+    # obtained from this one: ord("\u037e") GREEK QUESTION MARK
     s = "".join(
-        c for c in s if c in string.ascii_letters + string.digits + string.whitespace
+        c for c in s
+        if (ord(c) <= 894
+            and c in string.ascii_letters + string.digits + string.whitespace
+            )
+        or ord(c) > 894
     )
     return [_f for _f in s.split(" ") if _f]  # remove empty elements
 

core/tests/engine_test.py

@@ -69,6 +69,10 @@ class TestCasegetwords:
         eq_(["a", "b", "c", "d"], getwords("a b c d"))
         eq_(["a", "b", "c", "d"], getwords(" a  b  c d "))
 
+    def test_unicode(self):
+        eq_(["e", "c", "0", "a", "o", "u", "e", "u"], getwords("é ç 0 à ö û è ¤ ù"))
+        eq_(["02", "君のこころは輝いてるかい？", "国木田花丸", "solo", "ver"], getwords("02 君のこころは輝いてるかい？ 国木田花丸 Solo Ver"))
+
     def test_splitter_chars(self):
         eq_(
             [chr(i) for i in range(ord("a"), ord("z") + 1)],
@@ -85,7 +89,7 @@ class TestCasegetwords:
         eq_(["foo", "bar"], getwords("FOO BAR"))
 
     def test_decompose_unicode(self):
-        eq_(getwords("foo\xe9bar"), ["fooebar"])
+        eq_(["fooebar"], getwords("foo\xe9bar"))
 
 
 class TestCasegetfields:

macos.md

@@ -11,7 +11,7 @@
 1. Install Xcode if desired
 2. Install [Homebrew][homebrew], if not on the path after install (arm based Macs) create `~/.zshrc` 
 with `export PATH="/opt/homebrew/bin:$PATH"`. Will need to reload terminal or source the file to take 
-affect.
+effect.
 3. Install qt5 with `brew`. If you are using a version of macos without system python 3.6+ then you will 
 also need to install that via brew or with pyenv.
 

qt/app.py

@@ -271,6 +271,9 @@ class DupeGuru(QObject):
         self.willSavePrefs.emit()
         self.prefs.save()
         self.model.save()
+        # Workaround for #857, hide() or close().
+        if self.details_dialog is not None:
+            self.details_dialog.close()
         QApplication.quit()
 
     # --- Signals

qt/details_dialog.py

@@ -51,7 +51,7 @@ class DetailsDialog(QDockWidget):
             if not self.titleBarWidget():  # default title bar
                 self.setTitleBarWidget(QWidget())  # disables title bar
                 # Windows (and MacOS?) users cannot move a floating window which
-                # has not native decoration so we force it to dock for now
+                # has no native decoration so we force it to dock for now
                 if not ISLINUX:
                     self.setFloating(False)
         elif self.titleBarWidget() is not None:  # title bar is disabled

qt/pe/details_dialog.py

@@ -19,7 +19,6 @@ tr = trget("ui")
 class DetailsDialog(DetailsDialogBase):
     def __init__(self, parent, app):
         self.vController = None
-        self.app = app
         super().__init__(parent, app)
 
     def _setupUi(self):

qt/preferences.py

@@ -102,7 +102,7 @@ class Preferences(PreferencesBase):
         self.details_dialog_override_theme_icons = False if not ISLINUX else True
         self.details_dialog_viewers_show_scrollbars = True
         self.result_table_ref_foreground_color = QColor(Qt.blue)
-        self.result_table_ref_background_color = QColor(Qt.darkGray)
+        self.result_table_ref_background_color = QColor(Qt.lightGray)
         self.result_table_delta_foreground_color = QColor(255, 142, 40)  # orange
         self.resultWindowIsMaximized = False
         self.resultWindowRect = None