229 lines
9.0 KiB
Python
229 lines
9.0 KiB
Python
from __future__ import annotations
|
|
|
|
from collections.abc import Mapping, Sequence
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
|
|
import regex as re
|
|
|
|
from tools.docs_codegen.errors import DocsCodegenError, make_docs_codegen_error
|
|
from tools.docs_codegen.utils import trim_blank_edges
|
|
|
|
MODEL_CODE_DEFAULTS_PATH = Path("docs/source/tutorials/models")
|
|
MODEL_CODE_REQUIRED_OPTION_NAMES = ("block_name", "converter_tag", "test_case_path")
|
|
MODEL_CODE_OPTION_NAMES = (*MODEL_CODE_REQUIRED_OPTION_NAMES, "case_index", "host_index")
|
|
MODEL_CODE_OPEN_RE = re.compile(r"^\s*```{model-code}\s*$")
|
|
MODEL_CODE_CLOSE_RE = re.compile(r"^\s*```\s*$")
|
|
MODEL_CODE_OPTION_RE = re.compile(r"^\s*:([A-Za-z0-9_-]+):\s*(.*?)\s*$")
|
|
BLOCK_NAME_RE = re.compile(r"^[A-Za-z0-9._-]+$")
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ModelCodeBlock:
|
|
"""One ``model-code`` block discovered in a documentation page."""
|
|
|
|
doc_path: Path
|
|
block_name: str
|
|
converter_tag: str
|
|
test_case_path: str
|
|
extra_options: tuple[tuple[str, str], ...] = ()
|
|
directive_line: int | None = None
|
|
raw_block_lines: tuple[str, ...] = ()
|
|
|
|
@property
|
|
def key(self) -> tuple[str, str]:
|
|
"""Identity used to detect duplicate blocks: ``(doc_path, block_name)``."""
|
|
return (self.doc_path.as_posix(), self.block_name)
|
|
|
|
def get_option(self, name: str) -> str | None:
|
|
"""Return the value of an extra (non-required) directive option, or ``None``."""
|
|
for key, value in self.extra_options:
|
|
if key == name:
|
|
return value
|
|
return None
|
|
|
|
|
|
class BlockScanner:
|
|
"""Scan markdown files for ``model-code`` directives."""
|
|
|
|
def __init__(
|
|
self,
|
|
*,
|
|
documents_root: str | Path = MODEL_CODE_DEFAULTS_PATH,
|
|
repo_root: str | Path | None = None,
|
|
) -> None:
|
|
self.documents_root = Path(documents_root)
|
|
self.repo_root = Path(repo_root) if repo_root is not None else None
|
|
|
|
# -- Public API ----------------------------------------------------------
|
|
|
|
def scan_default_blocks(self) -> list[ModelCodeBlock]:
|
|
"""Scan every markdown file under ``documents_root`` for model-code blocks."""
|
|
base = self._base
|
|
models_dir = base / self.documents_root
|
|
if not models_dir.exists():
|
|
raise make_docs_codegen_error(
|
|
"tutorials models directory does not exist",
|
|
doc_path=self.documents_root,
|
|
)
|
|
|
|
blocks: list[ModelCodeBlock] = []
|
|
for absolute_doc_path in sorted(models_dir.rglob("*.md")):
|
|
blocks.extend(self.scan_document_blocks(absolute_doc_path.relative_to(base)))
|
|
return blocks
|
|
|
|
def scan_document_blocks(self, doc_path: str | Path) -> list[ModelCodeBlock]:
|
|
"""Parse all model-code directive fences in a single markdown document."""
|
|
repo_relative_doc_path = self._normalize_document_path(doc_path)
|
|
absolute_doc_path = self._base / repo_relative_doc_path
|
|
if not absolute_doc_path.exists():
|
|
raise make_docs_codegen_error("document file does not exist", doc_path=repo_relative_doc_path)
|
|
|
|
lines = absolute_doc_path.read_text(encoding="utf-8").splitlines()
|
|
blocks: list[ModelCodeBlock] = []
|
|
line_index = 0
|
|
|
|
while line_index < len(lines):
|
|
if not MODEL_CODE_OPEN_RE.match(lines[line_index]):
|
|
line_index += 1
|
|
continue
|
|
|
|
directive_line = line_index + 1
|
|
line_index += 1
|
|
options: dict[str, str] = {}
|
|
body_lines: list[str] = []
|
|
in_body = False
|
|
|
|
while line_index < len(lines):
|
|
line = lines[line_index]
|
|
if MODEL_CODE_CLOSE_RE.match(line):
|
|
break
|
|
|
|
option_match = MODEL_CODE_OPTION_RE.match(line)
|
|
if not in_body and option_match:
|
|
options[option_match.group(1)] = option_match.group(2).strip()
|
|
else:
|
|
in_body = True
|
|
body_lines.append(line)
|
|
line_index += 1
|
|
|
|
if line_index >= len(lines) or not MODEL_CODE_CLOSE_RE.match(lines[line_index]):
|
|
raise make_docs_codegen_error(
|
|
"unclosed model-code directive fence",
|
|
doc_path=repo_relative_doc_path,
|
|
line=directive_line,
|
|
)
|
|
|
|
blocks.append(
|
|
self.build_block(
|
|
options,
|
|
doc_path=repo_relative_doc_path,
|
|
directive_line=directive_line,
|
|
body_lines=body_lines,
|
|
)
|
|
)
|
|
line_index += 1
|
|
|
|
self._validate_unique_block_names(blocks)
|
|
return blocks
|
|
|
|
def select_document_blocks(self, doc_path: str | Path, block_name: str | None = None) -> list[ModelCodeBlock]:
|
|
"""Scan a document and optionally keep only the block named ``block_name``."""
|
|
blocks = self.scan_document_blocks(doc_path)
|
|
if block_name is None:
|
|
return blocks
|
|
|
|
selected_blocks = [block for block in blocks if block.block_name == block_name]
|
|
if not selected_blocks:
|
|
raise make_docs_codegen_error(
|
|
f"block_name '{block_name}' not found in document",
|
|
doc_path=self._normalize_document_path(doc_path),
|
|
)
|
|
return selected_blocks
|
|
|
|
def build_block(
|
|
self,
|
|
options: Mapping[str, str],
|
|
*,
|
|
doc_path: str | Path,
|
|
directive_line: int | None = None,
|
|
body_lines: Sequence[str] = (),
|
|
) -> ModelCodeBlock:
|
|
"""Validate directive options and assemble a ``ModelCodeBlock``."""
|
|
repo_relative_doc_path = self._normalize_document_path(doc_path)
|
|
|
|
def fail(message: str) -> DocsCodegenError:
|
|
return make_docs_codegen_error(
|
|
message,
|
|
doc_path=repo_relative_doc_path,
|
|
line=directive_line,
|
|
test_case_path=options.get("test_case_path"),
|
|
block_name=options.get("block_name"),
|
|
converter_tag=options.get("converter_tag"),
|
|
)
|
|
|
|
missing = [name for name in MODEL_CODE_REQUIRED_OPTION_NAMES if name not in options]
|
|
if missing:
|
|
raise fail(f"model-code block missing required metadata: {', '.join(missing)}")
|
|
|
|
extra = sorted(set(options).difference(MODEL_CODE_OPTION_NAMES))
|
|
if extra:
|
|
raise fail(f"model-code block contains unsupported metadata: {', '.join(extra)}")
|
|
|
|
normalized_options = {name: options[name].strip() for name in MODEL_CODE_OPTION_NAMES if name in options}
|
|
empty_values = [name for name, value in normalized_options.items() if not value]
|
|
if empty_values:
|
|
raise fail(f"model-code block contains empty metadata: {', '.join(empty_values)}")
|
|
|
|
if not BLOCK_NAME_RE.fullmatch(normalized_options["block_name"]):
|
|
raise fail("block_name may only contain letters, numbers, dots, underscores, and dashes")
|
|
|
|
return ModelCodeBlock(
|
|
doc_path=repo_relative_doc_path,
|
|
block_name=normalized_options["block_name"],
|
|
converter_tag=normalized_options["converter_tag"],
|
|
test_case_path=normalized_options["test_case_path"],
|
|
extra_options=tuple(
|
|
(name, normalized_options[name])
|
|
for name in MODEL_CODE_OPTION_NAMES
|
|
if name not in MODEL_CODE_REQUIRED_OPTION_NAMES and name in normalized_options
|
|
),
|
|
directive_line=directive_line,
|
|
raw_block_lines=tuple(trim_blank_edges(body_lines)),
|
|
)
|
|
|
|
# -- Internal helpers ----------------------------------------------------
|
|
|
|
@staticmethod
|
|
def _normalize_document_path(doc_path: str | Path) -> Path:
|
|
"""Validate that a document path is repository-relative and contained."""
|
|
candidate = Path(doc_path)
|
|
if candidate.is_absolute():
|
|
raise make_docs_codegen_error("document path must be repository-relative", doc_path=candidate)
|
|
if ".." in candidate.parts:
|
|
raise make_docs_codegen_error("document path must stay within the repository", doc_path=candidate)
|
|
return candidate
|
|
|
|
@staticmethod
|
|
def _validate_unique_block_names(blocks: Sequence[ModelCodeBlock]) -> None:
|
|
"""Reject documents that declare the same block_name twice."""
|
|
seen: dict[tuple[str, str], ModelCodeBlock] = {}
|
|
for block in blocks:
|
|
previous = seen.get(block.key)
|
|
if previous is None:
|
|
seen[block.key] = block
|
|
continue
|
|
|
|
raise make_docs_codegen_error(
|
|
"duplicated block_name "
|
|
f"'{block.block_name}' in document; previous declaration is on line {previous.directive_line}",
|
|
block=block,
|
|
)
|
|
|
|
# -- Path helpers --------------------------------------------------------
|
|
|
|
@property
|
|
def _base(self) -> Path:
|
|
"""Directory that repo-relative paths resolve against for filesystem I/O."""
|
|
return self.repo_root if self.repo_root is not None else Path.cwd()
|