50 lines
1.5 KiB
Python
50 lines
1.5 KiB
Python
import re
|
|
from typing import Iterable, List
|
|
|
|
|
|
DEFAULT_ANCHOR_RE = re.compile(r"[.!?;:,…·。！？；：，、（）()\[\]{}\"'«»\n\r\t]+")
|
|
|
|
|
|
def anchor_sequences(
|
|
text: str,
|
|
space_marker: str = "_",
|
|
min_sequence_length: int = 2,
|
|
) -> List[str]:
|
|
"""Return true-anchored sequences with internal spaces preserved as markers.
|
|
|
|
This mirrors the MorPiece boundary-discovery preparation step closely:
|
|
split only on strong punctuation/newline anchors, keep spaces inside a span
|
|
as a soft cue rather than a delimiter, and drop very short fragments.
|
|
"""
|
|
pieces = []
|
|
for seg in DEFAULT_ANCHOR_RE.split(text):
|
|
if not seg:
|
|
continue
|
|
filtered = "".join(ch for ch in seg if ch.isalpha() or ch in (" ", "'", "-"))
|
|
filtered = re.sub(r"\s+", " ", filtered).strip()
|
|
if len(filtered) < min_sequence_length:
|
|
continue
|
|
pieces.append(filtered.replace(" ", space_marker))
|
|
return pieces
|
|
|
|
|
|
def collect_boundary_units(
|
|
lines: Iterable[str],
|
|
space_marker: str = "_",
|
|
min_sequence_length: int = 2,
|
|
shorter_first: bool = True,
|
|
) -> List[str]:
|
|
"""Collect anchored training units from a raw-text iterator."""
|
|
units = []
|
|
for line in lines:
|
|
units.extend(
|
|
anchor_sequences(
|
|
line,
|
|
space_marker=space_marker,
|
|
min_sequence_length=min_sequence_length,
|
|
)
|
|
)
|
|
if shorter_first:
|
|
units.sort(key=len)
|
|
return units
|