初始化项目，由ModelHub XC社区提供模型

Model: Yusiko/khazri-2-mini
Source: Original Platform

khazri_tokenizer_runtime.py

@@ -0,0 +1,41 @@
+
+import re
+
+def normalize_text(text: str) -> str:
+    if not isinstance(text, str):
+        return ""
+    text = text.replace("\r\n", "\n").replace("\r", "\n")
+    text = re.sub(r"[ \t]+$", "", text, flags=re.MULTILINE)
+    text = re.sub(r"\n{5,}", "\n\n\n\n", text)
+    return text.strip()
+
+def encode_structural_whitespace(text: str) -> str:
+    text = normalize_text(text)
+    if not text:
+        return ""
+    out_lines = []
+    for line in text.split("\n"):
+        line = line.replace("\t", " <|tab|> ")
+        m = re.match(r"^( +)", line)
+        if m:
+            n = len(m.group(1))
+            rest = line[n:]
+            tags = []
+            while n >= 4:
+                tags.append("<|indent_4|>")
+                n -= 4
+            while n >= 2:
+                tags.append("<|indent_2|>")
+                n -= 2
+            if n == 1:
+                rest = " " + rest
+            line = (" ".join(tags) + (" " if tags and rest else "") + rest)
+        out_lines.append(line)
+    return " <|nl|> ".join(out_lines).strip()
+
+def decode_structural_whitespace(text: str) -> str:
+    text = text.replace(" <|nl|> ", "\n").replace("<|nl|>", "\n")
+    text = text.replace(" <|tab|> ", "\t").replace("<|tab|>", "\t")
+    text = text.replace("<|indent_4|> ", "    ").replace("<|indent_4|>", "    ")
+    text = text.replace("<|indent_2|> ", "  ").replace("<|indent_2|>", "  ")
+    return text