初始化项目，由ModelHub XC社区提供模型

Model: ZelligeAI/tessera-compressor
Source: Original Platform

scripts/compress.py

@@ -0,0 +1,142 @@
+#!/usr/bin/env python3
+"""
+compress.py — Compress English reasoning text into the telegraphic CJK register
+using tessera-compressor behind any OpenAI-compatible endpoint (vLLM, llama.cpp
+server, etc.). No API keys or HF token required; the endpoint is yours.
+
+This is the same harness the compressor was accepted under: segment the block,
+group sentences into step-sized passages, classify each passage, compress it
+against the chain built so far, then run the deterministic fidelity gate. A
+passage that fails the gate falls back to a rules-only compression, so a bad
+model output costs savings, never content.
+
+Serve the model first, e.g.:
+    vllm serve ZelligeAI/tessera-compressor --port 8001
+or with the GGUF:
+    llama-server -m gguf/compressor-v31-q8_0.gguf --port 8001   # from the repo root
+
+Then:
+    # one block from a text file
+    python compress.py --in think.txt --endpoint http://localhost:8001/v1
+
+    # a JSONL corpus: {"id": ..., "text": ...} per line
+    python compress.py --in blocks.jsonl --out compressed.jsonl \
+        --endpoint http://localhost:8001/v1
+
+Token counting: the fidelity gate compares token counts under a target
+tokenizer. For results matching the accepted harness, point --tokenizer at the
+model you are producing training data FOR (default: the compressor's own
+tokenizer, which is close but not identical to the Qwen3.5 target used in the
+acceptance run).
+"""
+import argparse
+import json
+import sys
+
+from openai import OpenAI
+from tokenizers import Tokenizer
+
+from segmenting import segment, group_steps, classify_passage, facts, gate
+from tokenmax import _apply_subs
+
+PASSAGE_SYSTEM = (
+    "你是推理压缩器。Re-notate the NEXT PASSAGE of a reasoning chain into telegraphic "
+    "CJK/symbol notation. Every NEW logical step, fact, number and identifier must "
+    "survive — unless already stated in the chain. Never restate chain content. "
+    "[passage=load]: step-lossless telegraphic. [passage=narr]: minimal stubs "
+    "(试X→否). Output only the re-notated continuation."
+)
+
+MAX_NEW_TOKENS = 512
+
+
+def compress_block(text, client, model, ntok):
+    """Compress one reasoning block. Returns (compressed_text, stats)."""
+    segs = group_steps(segment(text))
+    chain, seen = [], set()
+    stats = {"segments": len(segs), "model_ok": 0, "fallback": 0,
+             "narr_skipped": 0, "code": 0, "calls": 0}
+
+    for kind, s in segs:
+        if kind == "code":
+            chain.append(s)
+            seen |= facts(s)
+            stats["code"] += 1
+            continue
+        cls = classify_passage(s, seen, ntok)
+        novel = facts(s) - seen
+        rules_s, _ = _apply_subs(s)
+        if not rules_s.strip():
+            continue
+        tail = "\n".join(chain)[-500:] or "(start)"
+        stats["calls"] += 1
+        r = client.chat.completions.create(
+            model=model, temperature=0.0, max_tokens=MAX_NEW_TOKENS,
+            messages=[
+                {"role": "system", "content": PASSAGE_SYSTEM},
+                {"role": "user", "content": f"[passage={cls}]\n链:\n{tail}\n\n段:\n{s[:2000]}"},
+            ],
+            extra_body={"repetition_penalty": 1.15},
+        )
+        out = (r.choices[0].message.content or "").strip()
+
+        if out == "∅" and cls == "narr" and not novel:
+            stats["narr_skipped"] += 1
+            seen |= facts(s)
+            continue
+        if gate(s, rules_s, out, ntok, novel=novel) is None:
+            chain.append(out)
+            stats["model_ok"] += 1
+        else:
+            chain.append(rules_s)
+            stats["fallback"] += 1
+        seen |= facts(s)
+
+    return "\n".join(chain), stats
+
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__,
+                                 formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--in", dest="inp", required=True,
+                    help=".txt (one block) or .jsonl ({'id','text'} per line)")
+    ap.add_argument("--out", default=None, help="output JSONL (default: stdout)")
+    ap.add_argument("--endpoint", default="http://localhost:8001/v1")
+    ap.add_argument("--model", default="ZelligeAI/tessera-compressor",
+                    help="served model name at the endpoint")
+    ap.add_argument("--tokenizer", default="ZelligeAI/tessera-compressor",
+                    help="HF repo id or local tokenizer.json for gate token counts")
+    args = ap.parse_args()
+
+    if args.tokenizer.endswith(".json"):
+        tok = Tokenizer.from_file(args.tokenizer)
+    else:
+        tok = Tokenizer.from_pretrained(args.tokenizer)
+
+    def ntok(s):
+        return len(tok.encode(s).ids) if s else 0
+
+    client = OpenAI(base_url=args.endpoint, api_key="none")
+
+    if args.inp.endswith(".jsonl"):
+        rows = [json.loads(l) for l in open(args.inp) if l.strip()]
+    else:
+        rows = [{"id": args.inp, "text": open(args.inp).read()}]
+
+    sink = open(args.out, "w") if args.out else sys.stdout
+    for row in rows:
+        compressed, stats = compress_block(row["text"], client, args.model, ntok)
+        rec = {"id": row.get("id"), "compressed": compressed,
+               "src_tokens": ntok(row["text"]), "out_tokens": ntok(compressed),
+               "harness": stats}
+        sink.write(json.dumps(rec, ensure_ascii=False) + "\n")
+        sink.flush()
+        print(f"[{row.get('id')}] {rec['src_tokens']} -> {rec['out_tokens']} tokens "
+              f"(model_ok={stats['model_ok']} fallback={stats['fallback']})",
+              file=sys.stderr)
+    if args.out:
+        sink.close()
+
+
+if __name__ == "__main__":
+    main()

scripts/requirements.txt

@@ -0,0 +1,3 @@
+openai>=1.0
+tokenizers>=0.15
+transformers>=4.40  # tokenmax.py standalone CLI only

scripts/segmenting.py

@@ -0,0 +1,121 @@
+"""
+segmenting.py — Passage segmentation, classification, and fidelity gates for the
+tessera-compressor harness.
+
+Extracted from the harness the compressor was accepted under (same functions the
+teacher mint used). Pure text processing: no network, no credentials.
+
+Flow: segment -> group_steps -> classify_passage per passage -> model call ->
+gate -> rules fallback on failure. A failed passage costs a few dozen tokens of
+savings, never content.
+"""
+import re
+
+CJK = re.compile(r'[一-鿿㐀-䶿]')
+NUM = re.compile(r'\d+(?:\.\d+)?')
+IDENT = re.compile(r'`[^`\n]+`|\b[A-Za-z]+(?:_[A-Za-z0-9]+)+\b|\b[a-z]+[A-Z][A-Za-z0-9]*\b')
+FENCE = re.compile(r'```.*?```', re.DOTALL)
+SENT_SPLIT = re.compile(r'(?<=[.!?;])\s+')
+_LIST_MARKER = re.compile(r'(?:^|[\n\s(])(\d{1,2})[.)]\s')
+_OPS = set('+-*/=<>≤≥≠∈∀∃¬→⇒%^{}[]')
+
+
+def segment(text):
+    """Split a reasoning block into ordered segments; code fences are atomic and marked."""
+    segs = []  # (kind, text) kind ∈ {'code','prose'}
+    pos = 0
+    for m in FENCE.finditer(text):
+        before = text[pos:m.start()]
+        segs.extend(('prose', s) for s in _split_prose(before))
+        segs.append(('code', m.group(0)))
+        pos = m.end()
+    segs.extend(('prose', s) for s in _split_prose(text[pos:]))
+    return [(k, s) for k, s in segs if s.strip()]
+
+
+def _split_prose(text):
+    out = []
+    for line in text.split('\n'):
+        line = line.strip()
+        if not line:
+            continue
+        out.extend(s.strip() for s in SENT_SPLIT.split(line) if s.strip())
+    return out
+
+
+def group_steps(segs, max_words=160, max_sents=10):
+    """Merge consecutive prose sentences into step-sized passages; code stays atomic."""
+    out, buf, words = [], [], 0
+
+    def flush():
+        nonlocal buf, words
+        if buf:
+            out.append(('prose', ' '.join(buf)))
+            buf, words = [], 0
+
+    for kind, s in segs:
+        if kind == 'code':
+            flush()
+            out.append((kind, s))
+            continue
+        buf.append(s)
+        words += len(s.split())
+        if words >= max_words or len(buf) >= max_sents:
+            flush()
+    flush()
+    return out
+
+
+def facts(s):
+    """Numbers + identifiers that must survive compression.
+    List-enumeration markers ("1. Load...") are structure, not facts."""
+    nums = set(NUM.findall(s)) - set(_LIST_MARKER.findall(s))
+    idents = set(i.strip('`') for i in IDENT.findall(s))
+    return nums | idents
+
+
+def facts_preserved(src, out):
+    """Substring presence — regex \\b breaks against adjacent CJK chars.
+    Returns the list of MISSING facts (empty list = all preserved)."""
+    out_n = out.replace(',', '')
+    return [f for f in facts(src) if f.replace(',', '') not in out_n]
+
+
+def classify_passage(seg, seen_facts, ntok):
+    """'load' = fact-dense or novel-fact-bearing (step-faithful treatment);
+       'narr' = search/narrative (stub treatment).
+    ntok is a callable: text -> token count under your target tokenizer."""
+    f = facts(seg)
+    novel = f - seen_facts
+    toks = max(ntok(seg), 1)
+    dens = (len(NUM.findall(seg)) + len(IDENT.findall(seg))
+            + sum(seg.count(o) for o in _OPS)) / toks
+    if novel and (dens >= 0.08 or len(novel) >= 3):
+        return 'load'
+    if dens >= 0.15:
+        return 'load'
+    return 'narr'
+
+
+def gate(src_seg, rules_seg, out, ntok, novel=None):
+    """Deterministic per-passage fidelity gate.
+    Returns None if the model output is admissible, else a short fail-reason
+    string; on failure the caller uses rules_seg instead.
+
+    novel: the passage's facts that are NOT already in the accumulated chain.
+    The prompt tells the model never to restate chain content, so only novel
+    facts are required to survive (matching the acceptance harness). Pass None
+    to require every fact of the passage (stricter, for chainless use)."""
+    if not out or not out.strip():
+        return "empty"
+    if '```' in out:
+        return "fence"
+    if len(out) > 2 * len(src_seg) + 40:           # explanation/blow-up guard
+        return "blowup"
+    required = facts(src_seg) if novel is None else novel
+    out_n = out.replace(',', '')
+    if any(f.replace(',', '') not in out_n for f in required):
+        return "facts"
+    if ntok(out) > ntok(rules_seg):                # must not exceed the rules-only version
+        return "tokens"
+    return None

scripts/tokenmax.py

@@ -0,0 +1,431 @@
+"""
+tokenmax.py — Deterministic token-maxing post-processor for compressed think blocks.
+
+Applies ONLY substitutions that are verified to save tokens on the Qwen 248K tokenizer
+(OmniCoder-9B / Qwen3.5). Every substitution was tested in-context (not isolation) to
+confirm real token savings without boundary interference.
+
+Design:
+  - LLM does semantic compression (what to keep vs drop)
+  - This code enforces consistent notation deterministically
+  - GUARD: only returns the processed version if ntok(result) < ntok(original)
+  - Idempotent: safe to run multiple times
+
+Usage:
+    from caveman.compress.tokenmax import tokenmax, tokenmax_with_stats
+    compressed = tokenmax(think_text, tokenizer)
+    compressed, stats = tokenmax_with_stats(think_text, tokenizer)
+
+Verified: 2026-05-31 on Qwen 248K vocab. 27/28 substitutions save in-context.
+Zero false positives. One zero-effect (贪心 for "greedy" — boundary-dependent).
+"""
+
+import re
+from typing import Optional
+
+# ── Phase 1: Filler drops ──────────────────────────────────────────────
+# Phrases that carry zero information in compressed reasoning.
+# Only patterns that are NEVER load-bearing in a think block.
+_FILLER_PATTERNS = [
+    # Metacognition (the model narrating its own process)
+    r"\bI need to\b",
+    r"\bwe need to\b",
+    r"\bI will\b",
+    r"\bI'll\b",
+    r"\blet me\b",
+    r"\blet's\b",
+    r"\bI want to\b",
+    r"\bI should\b",
+    r"\bwe should\b",
+    # Hedging
+    r"\bprobably\b",
+    r"\bbasically\b",
+    r"\bessentially\b",
+    r"\bit seems like\b",
+    # Filler transitions
+    r"\bin order to\b",
+    r"\bfirst of all\b",
+    r"\bin other words\b",
+    r"\bon the other hand\b",
+    r"\bmore specifically\b",
+    r"\bto be more precise\b",
+    r"\band so on\b",
+    # Conversational padding (require word boundary at end to avoid "Greatest", "Perfectly")
+    r"\bGreat\b[,!.]?\s*",
+    r"\bPerfect\b[,!.]?\s*",
+    # Obvious statements
+    r"\bAs (?:we|you) can see\b",
+    r"\bAs mentioned (?:above|earlier|before)\b",
+]
+
+# ── Phase 2: Phrase → cheapest token ───────────────────────────────────
+# Ordered LONGEST FIRST to prevent partial matches.
+# Each entry: (regex_pattern, replacement, category)
+# Categories: 'cjk', 'symbol', 'abbrev' — for stats tracking.
+_SUBSTITUTIONS = [
+    # ── COMPOUND PATTERNS FIRST (must fire before their components) ──
+
+    # Verbose comparison phrases (+5t savings)
+    (r'\bis\s+greater\s+than\s+or\s+equal\s+to\b', '≥', 'symbol'),   # +5t
+    (r'\bis\s+less\s+than\s+or\s+equal\s+to\b', '≤', 'symbol'),      # +5t
+
+    # Verbose discourse phrases (+3t savings)
+    (r'\bwe\s+can\s+see\s+that\b', '可知', 'cjk'),                    # +3t
+    (r'\bat\s+the\s+same\s+time\b', '同时', 'cjk'),                   # +3t
+    (r'\bthat\s+is\s+to\s+say\b', '即', 'cjk'),                      # +3t
+
+    # Multi-word phrases (+2t savings)
+    (r'\bin\s+this\s+case\b', '此时', 'cjk'),                         # +2t
+    (r'\bthe\s+number\s+of\b', '个数', 'cjk'),                        # +2t
+    (r'\bis\s+equal\s+to\b', '等于', 'cjk'),                          # +2t
+
+    # Complexity boilerplate (biggest per-occurrence savings)
+    (r'[Oo]\(n\)\s*time[,;]?\s*[Oo]\(n\)\s*space\.?', 'O(n|n).', 'abbrev'),
+    (r'[Oo]\(n\)\s*time[,;]?\s*[Oo]\(1\)\s*space\.?', 'O(n|1).', 'abbrev'),
+    (r'[Oo]\(n\s*log\s*n\)\s*time[,;]?\s*[Oo]\(n\)\s*space', 'O(n㏒n|n)', 'abbrev'),
+    (r'[Oo]\(n\s*log\s*n\)\s*time[,;]?\s*[Oo]\(1\)\s*space', 'O(n㏒n|1)', 'abbrev'),
+    (r'[Tt]ime\s*complexity[:\s]+', 'T=', 'abbrev'),
+    (r'[Ss]pace\s*complexity[:\s]+', 'S=', 'abbrev'),
+
+    # Multi-word compounds (BEFORE their single-word components)
+    (r'\bassume without loss of generality\b', '设 不妨', 'cjk'),  # before "assume"
+    (r'\bproof by contradiction\b', '反证', 'cjk'),                # before "proof", "contradiction"
+    (r'\bnecessary and sufficient\b', '充要', 'cjk'),              # before "sufficient"
+    (r'\bnot equal(?:\s+to)?\b', '≠', 'symbol'),                  # before "is not", "does not"
+    (r'\bif and only if\b', 'iff', 'abbrev'),                     # before "for all"
+    (r'\bmuch greater than\b', '≫', 'symbol'),                    # before "greater than"
+    (r'\bkeep track(?:\s+of)?\b', '记录', 'cjk'),                 # before article strip
+    (r'\bin ascending order\b', 'asc', 'abbrev'),
+    (r'\bin descending order\b', 'desc', 'abbrev'),
+    (r'\bmaximum value\b', '最大值', 'cjk'),
+    (r'\bminimum value\b', '最小值', 'cjk'),
+    (r'\breturn value\b', '返回値', 'cjk'),
+    (r'\brather than\b', '而非', 'cjk'),
+    (r'\baccording to\b', '按照', 'cjk'),
+
+    # DS compounds (before components)
+    (r'\bdoubly linked list\b', 'DLL', 'abbrev'),      # BEFORE "linked list"
+    (r'\bbinary indexed tree\b', 'BIT', 'abbrev'),     # BEFORE "binary"
+    (r'\bminimum spanning tree\b', 'MST', 'abbrev'),
+    (r'\bdepth[- ]first search\b', 'DFS', 'abbrev'),
+    (r'\bbreadth[- ]first search\b', 'BFS', 'abbrev'),
+    (r'\bdynamic programming\b', 'DP', 'abbrev'),
+    (r'\bdivide and conquer\b', '分治', 'cjk'),
+    (r'\bmonot(?:onic|one)\s*stack\b', '单调栈', 'cjk'),
+    (r'\btime limit exceeded\b', '超时', 'cjk'),
+    (r'\bout of bounds\b', '越界', 'cjk'),
+    (r'\bremove duplicates?\b', '去重', 'cjk'),
+    (r'\benumerate all\b', '穷举', 'cjk'),
+    (r'\bbinary search\b', '二分', 'cjk'),
+    (r'\bsliding window\b', 'sw', 'abbrev'),
+    (r'\bunion[- ]find\b', 'UF', 'abbrev'),
+    (r'\btopological sort\b', '拓扑序', 'cjk'),
+    (r'\bshortest path\b', 'sp', 'abbrev'),
+    (r'\blinked list\b', 'LL', 'abbrev'),
+    (r'\bpriority queue\b', 'heap', 'abbrev'),
+    (r'\bprefix sum\b', 'ps', 'abbrev'),
+    (r'\bbrute force\b', '暴力', 'cjk'),
+    (r'\bno solution\b', '无解', 'cjk'),
+    (r'\bedge cases?\b', '边界', 'cjk'),
+    (r'\bbase case\b', 'bc', 'abbrev'),
+    (r'\bworst case\b', 'wc', 'abbrev'),
+
+    # ── SINGLE-WORD SUBSTITUTIONS (safe after compounds consumed) ──
+
+    # +4t savings
+    (r'\bobviously\b', '显然', 'cjk'),
+    # +3t savings
+    (r'\bredundant\b', '冗余', 'cjk'),
+    (r'\bsatisf(?:y|ies|ied)\b', '满足', 'cjk'),            # +2t, 333x in data
+    (r'\bunsorted\b', '无序', 'cjk'),
+    (r'\bdue to\b', '由于', 'cjk'),
+    (r'\bhence\b', '故', 'cjk'),
+    (r'\bnamely\b', '即', 'cjk'),
+    (r'\bassume\b', '设', 'cjk'),
+    (r'\bsuppose\b', '设', 'cjk'),
+    (r'\bderive\b', '推导', 'cjk'),
+    # +2t savings
+    (r'\bmonotone\b', '单调', 'cjk'),
+    (r'\bconvergent\b', '收敛', 'cjk'),
+    (r'\bdivergent\b', '发散', 'cjk'),
+    (r'\bcommutative\b', '交换', 'cjk'),
+    (r'\bdeterministic\b', '确定', 'cjk'),
+    (r'\bprobabilistic\b', '概率', 'cjk'),
+    (r'\bprove\b', '证明', 'cjk'),
+    (r'\bproof\b', '证明', 'cjk'),
+    (r'\bflip\b', '翻转', 'cjk'),
+    (r'\bsorted\b(?!\s*[=(\[])', '有序', 'cjk'),  # not before = ( [ (assignment/call)
+    # +1t savings
+    (r'\bbacktrack(?:ing)?\b', '回溯', 'cjk'),
+    (r'\btravers(?:e|al|ing)\b', '遍历', 'cjk'),
+    (r'\brecursi(?:on|ve|vely)\b', '递归', 'cjk'),
+    (r'\bcontradiction\b', '矛盾', 'cjk'),
+    (r'\bsufficient\b', '充分', 'cjk'),
+    (r'\bequivalent\b', '等价', 'cjk'),
+    (r'\bsymmetric\b', '对称', 'cjk'),
+    (r'\binvariant\b', '不变', 'cjk'),
+    (r'\bexponential\b', '指数', 'cjk'),
+    (r'\bpermutation\b', '排列', 'cjk'),
+    (r'\badjacent\b', '相邻', 'cjk'),
+    (r'\boptimal\b', '最优', 'cjk'),
+    (r'\bfeasible\b', '可行', 'cjk'),
+    (r'\binduction\b', '归纳', 'cjk'),
+    (r'\bmaintain\b', '维护', 'cjk'),
+    (r'\bswap\b(?!\s*[=(\[])', '交换', 'cjk'),  # not before = ( [ (assignment/call)
+    (r'\bcumulative\b', '累积', 'cjk'),
+    (r'\bquotient\b', '商', 'cjk'),
+    (r'\bmemoiz(?:ation|e)\b', 'memo', 'abbrev'),
+    # +2t savings (mined from v19 data)
+    (r'\bmathematical\b', '数学', 'cjk'),
+    (r'\bcorresponding(?:ly)?\b', '对应', 'cjk'),
+    (r'\brequirement\b', '需求', 'cjk'),
+    # +1t savings (mined from v19 data)
+    (r'\bcomplexity\b', '复杂度', 'cjk'),
+    (r'\bsimilarly\b', '同理', 'cjk'),
+    (r'\bsubstitut(?:e|ion)\b', '代入', 'cjk'),
+    (r'\bincreasing(?:ly)?\b', '递增', 'cjk'),
+    (r'\bdecreasing(?:ly)?\b', '递减', 'cjk'),
+    (r'\brespectively\b', '分别', 'cjk'),
+    (r'\bnecessarily\b', '必然', 'cjk'),
+    (r'\btransformation\b', '变换', 'cjk'),
+    (r'\bprerequisite\b', '前提', 'cjk'),
+    (r'\bconsequently\b', '从而', 'cjk'),
+    (r'\boverlapping\b', '重叠', 'cjk'),
+    (r'\bcontribut(?:e|ion)\b', '贡献', 'cjk'),
+    (r'\bindependent(?:ly)?\b', '独立', 'cjk'),
+    (r'\bimpossible\b', '不可能', 'cjk'),
+    (r'\biterat(?:e|ion|ing)\b', '迭代', 'cjk'),
+    (r'\benumerat(?:e|ion|ing)\b', '枚举', 'cjk'),
+
+    # ── LOGIC SYMBOLS ──
+    (r'\btherefore\b', '⇒', 'symbol'),
+    (r'\bthus\b', '⇒', 'symbol'),
+    (r'\bsuch that\b', 'st', 'abbrev'),
+    (r'\bthere exists?\b', '∃', 'symbol'),
+    (r'\bfor each\b', '∀', 'symbol'),
+    (r'\bfor every\b', '∀', 'symbol'),
+    (r'\bfor all\b', '∀', 'symbol'),
+    (r'\bdoes not\b', '¬', 'symbol'),
+    (r"\bdoesn't\b", '¬', 'symbol'),
+    (r'\bis not\b(?!\s+(?:None|null|undefined|empty|zero|0))', '非', 'cjk'),  # protect "is not None" etc
+    (r'\bat least\b', '≥', 'symbol'),
+    (r'\bat most\b', '≤', 'symbol'),
+    (r'\bgreater than\b', '>', 'symbol'),
+    (r'\bless than\b', '<', 'symbol'),
+
+    # ── ARTICLE STRIPPING (last — lowest priority) ──
+    (r'\bthe\b\s+(?!(?:same|only|first|last|next|other)\b)', '', 'filler'),  # protect "the same", "the only" etc
+    (r'\ba\b\s+(?=[bcdfghjklmnpqrstvwxyz])', '', 'filler'),
+    (r'\ban\b\s+', '', 'filler'),
+]
+
+# ── Compile once ───────────────────────────────────────────────────────
+_FILLER_COMPILED = [(re.compile(p, re.IGNORECASE), '') for p in _FILLER_PATTERNS]
+_SUBS_COMPILED = [(re.compile(p, re.IGNORECASE), r, cat) for p, r, cat in _SUBSTITUTIONS]
+
+
+def _ntok(text: str, tokenizer) -> int:
+    """Token count using the provided tokenizer."""
+    return len(tokenizer.encode(text, add_special_tokens=False))
+
+
+def _protect_code_fences(text: str) -> tuple[str, list]:
+    """Extract code-fenced blocks, replace with placeholders.
+    Returns (text_with_placeholders, list_of_extracted_blocks)."""
+    blocks = []
+    def _replace(m):
+        blocks.append(m.group(0))
+        return f'\x00CODEFENCE{len(blocks)-1}\x00'
+    # Match ```...``` and inline `...` (non-greedy)
+    protected = re.sub(r'```.*?```|`[^`\n]+`', _replace, text, flags=re.DOTALL)
+    return protected, blocks
+
+
+def _restore_code_fences(text: str, blocks: list) -> str:
+    """Restore code-fenced blocks from placeholders."""
+    for i, block in enumerate(blocks):
+        text = text.replace(f'\x00CODEFENCE{i}\x00', block)
+    return text
+
+
+def _apply_subs(text: str) -> tuple[str, dict]:
+    """Apply all substitutions, return (result, stats).
+    Code fences (``` and inline `) are protected from substitution."""
+    stats = {'filler_drops': 0, 'cjk': 0, 'symbol': 0, 'abbrev': 0, 'total_subs': 0}
+
+    # Phase 0: protect code fences from substitution
+    text, code_blocks = _protect_code_fences(text)
+
+    # Phase 1: filler drops
+    for pat, repl in _FILLER_COMPILED:
+        text, n = pat.subn(repl, text)
+        if n:
+            stats['filler_drops'] += n
+            stats['total_subs'] += n
+
+    # Phase 2: substitutions
+    for pat, repl, cat in _SUBS_COMPILED:
+        text, n = pat.subn(repl, text)
+        if n:
+            stats[cat] = stats.get(cat, 0) + n
+            stats['total_subs'] += n
+
+    # Phase 3: restore code fences
+    text = _restore_code_fences(text, code_blocks)
+
+    # Phase 4: whitespace normalization
+    text = re.sub(r'[ \t]+', ' ', text)
+    text = re.sub(r'\n{3,}', '\n\n', text)
+    text = re.sub(r' *\n *', '\n', text)
+    text = text.strip()
+
+    return text, stats
+
+
+def tokenmax(text: str, tokenizer, force_cjk: bool = False) -> str:
+    """Apply token-maxing. Returns original if no savings achieved.
+
+    Args:
+        text: The think block content (without <think> tags).
+        tokenizer: A HuggingFace tokenizer with .encode() method.
+        force_cjk: If True, always return the processed version when CJK
+            substitutions were applied, even if total token count increased.
+            Use this to maximize CJK adoption in training data.
+
+    Returns:
+        The token-maxed text, or the original if processing didn't save tokens
+        (unless force_cjk=True and CJK subs were applied).
+    """
+    if not text or not text.strip():
+        return text
+
+    original_tokens = _ntok(text, tokenizer)
+    result, stats = _apply_subs(text)
+    result_tokens = _ntok(result, tokenizer)
+
+    # GUARD: only return processed version if it actually saves tokens
+    # OVERRIDE: force_cjk bypasses the guard when CJK substitutions were made
+    if result_tokens < original_tokens:
+        return result
+    if force_cjk and stats.get('cjk', 0) > 0:
+        return result
+    return text
+
+
+def tokenmax_with_stats(text: str, tokenizer, force_cjk: bool = False) -> tuple[str, dict]:
+    """Like tokenmax() but also returns substitution statistics.
+
+    Args:
+        force_cjk: If True, always apply when CJK substitutions were made,
+            even if total token count increased. Prioritizes CJK adoption
+            over token savings.
+
+    Returns:
+        (processed_text, stats_dict) where stats_dict contains:
+          - original_tokens: token count before processing
+          - result_tokens: token count after processing
+          - saved: tokens saved (negative = token increase; check forced_cjk)
+          - applied: whether the processed version was used
+          - forced_cjk: True when force_cjk override caused acceptance despite no savings
+          - filler_drops, cjk, symbol, abbrev: substitution counts by category
+          - total_subs: total substitutions applied
+    """
+    if not text or not text.strip():
+        return text, {'original_tokens': 0, 'result_tokens': 0, 'saved': 0,
+                      'applied': False, 'forced_cjk': False, 'total_subs': 0}
+
+    original_tokens = _ntok(text, tokenizer)
+    result, stats = _apply_subs(text)
+    result_tokens = _ntok(result, tokenizer)
+    saved = original_tokens - result_tokens
+
+    stats['original_tokens'] = original_tokens
+    stats['result_tokens'] = result_tokens
+    stats['saved'] = saved
+    stats['forced_cjk'] = False
+
+    if saved > 0:
+        stats['applied'] = True
+        return result, stats
+    if force_cjk and stats.get('cjk', 0) > 0:
+        stats['applied'] = True
+        stats['forced_cjk'] = True
+        return result, stats
+    stats['applied'] = False
+    return text, stats
+
+
+# ── CLI: batch process a JSONL file ────────────────────────────────────
+if __name__ == '__main__':
+    import json, sys, argparse
+    from transformers import AutoTokenizer
+
+    parser = argparse.ArgumentParser(description='Token-max post-processor for think blocks')
+    parser.add_argument('--input', required=True, help='Input JSONL (messages format)')
+    parser.add_argument('--output', help='Output JSONL (default: dry run, stats only)')
+    parser.add_argument('--tokenizer', default='ZelligeAI/tessera-compressor',
+                        help='HF repo id or local path of the tokenizer to count savings under')
+    parser.add_argument('--force-cjk', action='store_true',
+                        help='Force CJK substitutions even if total tokens increase. '
+                             'Prioritizes CJK adoption over token savings.')
+    args = parser.parse_args()
+
+    tok = AutoTokenizer.from_pretrained(args.tokenizer, trust_remote_code=True)
+
+    total_before = total_after = applied = skipped = forced = 0
+
+    out_lines = []
+    with open(args.input) as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            rec = json.loads(line)
+            for m in rec.get('messages', rec.get('conversations', [])):
+                role = m.get('role', m.get('from', ''))
+                if role not in ('assistant', 'gpt'):
+                    continue
+                content_key = 'content' if 'content' in m else 'value'
+                c = m.get(content_key, '') or ''
+                if '<think>' not in c or '</think>' not in c:
+                    continue
+
+                # Extract think content, preserving prefix before <think> and suffix after </think>
+                think_start = c.index('<think>') + len('<think>')
+                think_end = c.index('</think>')
+                prefix = c[:think_start - len('<think>')]
+                think = c[think_start:think_end]
+                suffix = c[think_end + len('</think>'):]
+
+                maxed, stats = tokenmax_with_stats(think, tok, force_cjk=args.force_cjk)
+
+                total_before += stats['original_tokens']
+                if stats['applied']:
+                    applied += 1
+                    total_after += stats['result_tokens']
+                    m[content_key] = f'{prefix}<think>{maxed}</think>{suffix}'
+                    if stats.get('forced_cjk'):
+                        forced += 1
+                else:
+                    skipped += 1
+                    total_after += stats['original_tokens']
+
+            out_lines.append(json.dumps(rec, ensure_ascii=False))
+
+    if args.output:
+        with open(args.output, 'w') as f:
+            for line in out_lines:
+                f.write(line + '\n')
+
+    total = applied + skipped
+    saved = total_before - total_after
+    if total > 0:
+        print(f'Processed {total} think blocks')
+        print(f'  Applied: {applied} ({100*applied/total:.0f}%)')
+        if forced:
+            print(f'  Forced CJK: {forced} (applied despite no token savings)')
+        print(f'  Skipped (no savings): {skipped}')
+        pct = f'{100*saved/total_before:.1f}' if total_before > 0 else '0.0'
+        print(f'  Tokens: {total_before} → {total_after} = {saved:+d} ({pct}%)')
+    else:
+        print(f'No think blocks found in {args.input}')