import re
from typing import Iterable, List


DEFAULT_ANCHOR_RE = re.compile(r"[.!?;:,…·。！？；：，、（）()\[\]{}\"'«»\n\r\t]+")


def anchor_sequences(
    text: str,
    space_marker: str = "_",
    min_sequence_length: int = 2,
) -> List[str]:
    """Return true-anchored sequences with internal spaces preserved as markers.

    This mirrors the MorPiece boundary-discovery preparation step closely:
    split only on strong punctuation/newline anchors, keep spaces inside a span
    as a soft cue rather than a delimiter, and drop very short fragments.
    """
    pieces = []
    for seg in DEFAULT_ANCHOR_RE.split(text):
        if not seg:
            continue
        filtered = "".join(ch for ch in seg if ch.isalpha() or ch in (" ", "'", "-"))
        filtered = re.sub(r"\s+", " ", filtered).strip()
        if len(filtered) < min_sequence_length:
            continue
        pieces.append(filtered.replace(" ", space_marker))
    return pieces


def collect_boundary_units(
    lines: Iterable[str],
    space_marker: str = "_",
    min_sequence_length: int = 2,
    shorter_first: bool = True,
) -> List[str]:
    """Collect anchored training units from a raw-text iterator."""
    units = []
    for line in lines:
        units.extend(
            anchor_sequences(
                line,
                space_marker=space_marker,
                min_sequence_length=min_sequence_length,
            )
        )
    if shorter_first:
        units.sort(key=len)
    return units