Fix text normalization bug (#16)

py/helper.py

@@ -8,6 +8,8 @@ from unicodedata import normalize
 import numpy as np
 import onnxruntime as ort
 
+import re
+
 
 class UnicodeProcessor:
     def __init__(self, unicode_indexer_path: str):
@@ -15,8 +17,96 @@ class UnicodeProcessor:
             self.indexer = json.load(f)
 
     def _preprocess_text(self, text: str) -> str:
-        # TODO: add more preprocessing
+        # TODO: Need advanced normalizer for better performance
         text = normalize("NFKD", text)
+
+        # FIXME: this should be fixed for non-English languages
+
+        # Remove emojis (wide Unicode range)
+        emoji_pattern = re.compile(
+            "[\U0001f600-\U0001f64f"  # emoticons
+            "\U0001f300-\U0001f5ff"  # symbols & pictographs
+            "\U0001f680-\U0001f6ff"  # transport & map symbols
+            "\U0001f700-\U0001f77f"
+            "\U0001f780-\U0001f7ff"
+            "\U0001f800-\U0001f8ff"
+            "\U0001f900-\U0001f9ff"
+            "\U0001fa00-\U0001fa6f"
+            "\U0001fa70-\U0001faff"
+            "\u2600-\u26ff"
+            "\u2700-\u27bf"
+            "\U0001f1e6-\U0001f1ff]+",
+            flags=re.UNICODE,
+        )
+        text = emoji_pattern.sub("", text)
+
+        # Replace various dashes and symbols
+        replacements = {
+            "–": "-",
+            "‑": "-",
+            "—": "-",
+            "¯": " ",
+            "_": " ",
+            "“": '"',
+            "”": '"',
+            "‘": "'",
+            "’": "'",
+            "´": "'",
+            "`": "'",
+            "[": " ",
+            "]": " ",
+            "|": " ",
+            "/": " ",
+            "#": " ",
+            "→": " ",
+            "←": " ",
+        }
+        for k, v in replacements.items():
+            text = text.replace(k, v)
+
+        # Remove combining diacritics # FIXME: this should be fixed for non-English languages
+        text = re.sub(
+            r"[\u0302\u0303\u0304\u0305\u0306\u0307\u0308\u030A\u030B\u030C\u0327\u0328\u0329\u032A\u032B\u032C\u032D\u032E\u032F]",
+            "",
+            text,
+        )
+
+        # Remove special symbols
+        text = re.sub(r"[♥☆♡©\\]", "", text)
+
+        # Replace known expressions
+        expr_replacements = {
+            "@": " at ",
+            "e.g.,": "for example, ",
+            "i.e.,": "that is, ",
+        }
+        for k, v in expr_replacements.items():
+            text = text.replace(k, v)
+
+        # Fix spacing around punctuation
+        text = re.sub(r" ,", ",", text)
+        text = re.sub(r" \.", ".", text)
+        text = re.sub(r" !", "!", text)
+        text = re.sub(r" \?", "?", text)
+        text = re.sub(r" ;", ";", text)
+        text = re.sub(r" :", ":", text)
+        text = re.sub(r" '", "'", text)
+
+        # Remove duplicate quotes
+        while '""' in text:
+            text = text.replace('""', '"')
+        while "''" in text:
+            text = text.replace("''", "'")
+        while "``" in text:
+            text = text.replace("``", "`")
+
+        # Remove extra spaces
+        text = re.sub(r"\s+", " ", text).strip()
+
+        # If text doesn't end with punctuation, quotes, or closing brackets, add a period
+        if not re.search(r"[.!?;:,'\"')\]}…。」』】〉》›»]$", text):
+            text += "."
+
         return text
 
     def _get_text_mask(self, text_ids_lengths: np.ndarray) -> np.ndarray: