初始化项目，由ModelHub XC社区提供模型

Model: NeTSlab/gpt2_parfind_en_zh_equal
Source: Original Platform

boundary_discovery.py

@@ -0,0 +1,49 @@
+import re
+from typing import Iterable, List
+
+
+DEFAULT_ANCHOR_RE = re.compile(r"[.!?;:,…·。！？；：，、（）()\[\]{}\"'«»\n\r\t]+")
+
+
+def anchor_sequences(
+    text: str,
+    space_marker: str = "_",
+    min_sequence_length: int = 2,
+) -> List[str]:
+    """Return true-anchored sequences with internal spaces preserved as markers.
+
+    This mirrors the MorPiece boundary-discovery preparation step closely:
+    split only on strong punctuation/newline anchors, keep spaces inside a span
+    as a soft cue rather than a delimiter, and drop very short fragments.
+    """
+    pieces = []
+    for seg in DEFAULT_ANCHOR_RE.split(text):
+        if not seg:
+            continue
+        filtered = "".join(ch for ch in seg if ch.isalpha() or ch in (" ", "'", "-"))
+        filtered = re.sub(r"\s+", " ", filtered).strip()
+        if len(filtered) < min_sequence_length:
+            continue
+        pieces.append(filtered.replace(" ", space_marker))
+    return pieces
+
+
+def collect_boundary_units(
+    lines: Iterable[str],
+    space_marker: str = "_",
+    min_sequence_length: int = 2,
+    shorter_first: bool = True,
+) -> List[str]:
+    """Collect anchored training units from a raw-text iterator."""
+    units = []
+    for line in lines:
+        units.extend(
+            anchor_sequences(
+                line,
+                space_marker=space_marker,
+                min_sequence_length=min_sequence_length,
+            )
+        )
+    if shorter_first:
+        units.sort(key=len)
+    return units