From dce422d6c5623b655aa96ba91523994294b204d9 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Fri, 24 Jul 2026 05:09:10 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: ZelligeAI/tessera-compressor Source: Original Platform --- .gitattributes | 37 +++ README.md | 95 ++++++++ banner.png | Bin 0 -> 31211 bytes chat_template.jinja | 53 +++++ config.json | 62 +++++ generation_config.json | 14 ++ gguf/compressor-v31-q8_0.gguf | 3 + model.safetensors | 3 + scripts/compress.py | 142 +++++++++++ scripts/requirements.txt | 3 + scripts/segmenting.py | 121 ++++++++++ scripts/tokenmax.py | 431 ++++++++++++++++++++++++++++++++++ tokenizer.json | 3 + tokenizer_config.json | 217 +++++++++++++++++ 14 files changed, 1184 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 banner.png create mode 100644 chat_template.jinja create mode 100644 config.json create mode 100644 generation_config.json create mode 100644 gguf/compressor-v31-q8_0.gguf create mode 100644 model.safetensors create mode 100644 scripts/compress.py create mode 100644 scripts/requirements.txt create mode 100644 scripts/segmenting.py create mode 100644 scripts/tokenmax.py create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..e4b471d --- /dev/null +++ b/.gitattributes @@ -0,0 +1,37 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +gguf/compressor-v31-q8_0.gguf filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..414a61e --- /dev/null +++ b/README.md @@ -0,0 +1,95 @@ +--- +license: apache-2.0 +base_model: Qwen/Qwen2.5-Coder-1.5B-Instruct +language: + - en + - zh +pipeline_tag: text-generation +tags: + - reasoning-compression + - cjk + - chain-of-thought + - distillation + - qwen2.5 +--- + +![tessera-compressor](banner.png) + +# tessera-compressor + +A 1.5B model that compresses English reasoning text into a telegraphic CJK/symbol register under deterministic fidelity gates. It minted the training data for [Tessera-Preview-9B](https://huggingface.co/ZelligeAI/tessera-preview-9b) and replaces the frontier-model teacher that originally produced the register: English reasoning text becomes compressed-register training data at local-inference cost, with no API key and no external dependency. Validation covered code-centric reasoning (103 held-out mixed blocks); behavior on distant domains is unmeasured. + +**Paper:** [Tessera-Preview-9B: Compressed Reasoning at 18x Fewer Tokens, and What It Costs](https://zellige.ai/research/compressed-cjk-reasoning) — section 3.1 covers this compressor's design and acceptance record. + +Example (real training pair, 85 to 49 tokens): + +```text +EN : So the classes are: - Integer (line 32) - Boolean (line 262) - BitString (line 341) + - OctetString (line 693) ... Let me look at the base class to see if it defines __mul__: + +CJK: Integer(line32),Boolean(line262),BitString(line341),OctetString(line693). 查基类是否定义__mul__: +``` + +## How it works + +The compressor operates on passages, not whole blocks. A reasoning block is segmented (code fences stay atomic), sentences are grouped into step-sized passages, each passage is classified as fact-dense or narrative, and the model compresses it against the tail of the chain built so far. Every model output then passes a deterministic gate: the passage's novel numbers and identifiers must survive as substrings, the output must not blow up in length, and it must not exceed a rules-only compression of the same passage in token count. A passage that fails any check falls back to the rules-only version, so a bad generation costs savings rather than gated content. The gate is lexical, not semantic: it prevents the loss of numbers and identifiers, and a judged semantic-equivalence check backed it at acceptance (below), but it does not by itself guarantee semantic preservation on arbitrary input. + +## Acceptance record + +Measured on 103 held-out reasoning blocks the model never trained on, under criteria fixed before evaluation: + +| Criterion | Result | +| --- | --- | +| Per-passage fidelity gate (numbers and identifiers survive) | 99.0% | +| Median per-passage compression ratio (output/input tokens) | 0.716 | +| CJK adoption | 98.9% of compressed passages | +| Judged semantic equivalence | 103/103 blocks (teacher references on the same blocks: 97.1%) | +| Degenerate outputs | 0 | +| Net corpus savings (after 24% rules-only fallback) | 30.4% | + +On whole thinks in downstream production use (45,202 pairs), the compressed rendering costs a median 0.58x the tokens of its English source. + +## Files + +- Root: merged model, standard Hugging Face format (bf16). Base: Qwen2.5-Coder-1.5B-Instruct, LoRA r=16 merged in. +- `gguf/compressor-v31-q8_0.gguf`: llama.cpp quantization, validated behaviorally (scores 4/4 on the same acceptance suite). q4_k_m showed visible drift and is not published. +- `scripts/`: the complete usage harness. No tokens or keys required anywhere. + +## Usage + +Serve the model behind any OpenAI-compatible endpoint: + +```bash +vllm serve ZelligeAI/tessera-compressor --port 8001 +# or, CPU-friendly: +llama-server -m gguf/compressor-v31-q8_0.gguf --port 8001 +``` + +Then run the harness: + +```bash +cd scripts && pip install -r requirements.txt + +# compress one reasoning block from a text file +python compress.py --in think.txt --endpoint http://localhost:8001/v1 + +# compress a corpus: {"id": ..., "text": ...} per JSONL line +python compress.py --in blocks.jsonl --out compressed.jsonl \ + --endpoint http://localhost:8001/v1 +``` + +Output records carry the compressed text, source and output token counts, and per-block harness stats (model-accepted vs rules-fallback passage counts). + +`scripts/` contents: + +- `compress.py`: the driver. Segment, classify, compress per passage with chain context, gate, fall back on failure. +- `segmenting.py`: segmentation, passage grouping, fact extraction, classification, and the fidelity gate. Pure text processing. +- `tokenmax.py`: deterministic token-saving substitutions, used as the rules-only fallback and as a post-processor. + +One note on token counting: the gate compares token counts under a tokenizer you choose (`--tokenizer`, default this repo). To reproduce the acceptance harness exactly, point it at the tokenizer of the model you are minting data for (the acceptance run used the Qwen3.5 target tokenizer). + +Throughput on the acceptance hardware was 19.6K blocks/hour on one GPU, which makes minting compressed data cheap at any corpus size. + +## License + +Apache-2.0, same as the base model. diff --git a/banner.png b/banner.png new file mode 100644 index 0000000000000000000000000000000000000000..36bd36f254e67fb4a6d4fdb5c94df87f73f578d8 GIT binary patch literal 31211 zcmdRWby!s4_a`QZAdMg((n^PP3@J!=cXx*{z^Js8NVkC03=KmMC@I|?1JW(sGry~T zcmMwFKKtymd!NfZ7r4A}-jknm4iT!#vbfmKurV+&aNo*Ft7Bl?ufV{#!-{nm_=zEM z0TTnmx9Y95gyy@9y#+6S3fWoA-!?i`YYEmgzzAco zQ+fZ}_?owAj7}-3J-}Npr_W>IB%Mi?Z{ui24h$gAe0{uut7bB9Ce!SOuA2au3Aj?I zX}aB{j2`~`j_mHP>eGLZ7@-gEO8$G~33-qI_vyPL&5QegpExzJuL>pH|FwkA z;S}54|2F#_UmTC-@8kPaNpY;dPo@R<_w_cME_ucvrrHK6g z@6qg0Lk|1DX)$mfo&2w^o`#@{{K4OSa>{Pf{J(tUuNfku;nAf7s>%NjNSs6RS7;+v zU0m?Uqq~#a7ddoNM*o=iJt`+`IN@Vn`$sxDvQYBvkN?i`L`7iyL4E0cnJ{cYiX*y*duCyzPy!ZF7hVkaP?YWoT;w0nphkLnSGk>vC ztNcf=c^YS~)QV|2l*~KB{^6v5v=gVUw;F$kco_RA8gIiSxU-WmPW(G_;(`jH`@GY& zTuy%e+GB{nn(W7gCLabShEpRkzmt^IYHPds)h3tL;*yf_dU?2(n%YP~G-z9+?ao3+ zZ!a?=AD@6g$HasTJ)w`67aR_s7$0BVSj*4P2jqNvu$6Y;A3woSf9umY0%0nWb!Q?gWhg_(5Y{=>-PQEhYcma46`0 z@RfQ9#Ga$9JUS|tP+y-@8YgbJjh2>ueRQjuwcT~$Sj3xYO&|t&Q~T&~Q)APWwfEZ(ejh zFE3Y6dB%nDp4;w+45P1;*z{MQ)r(?1;u$*+!&}x|rf&C~ zB9oe;lbpECZLYXTQ79a$xcfmp=LN0#Yn>w}B2p+;NShXHe<|-rOiYZ^T&<^PjlkhTp?KQeI$aH(s#;^w zO^S|=M7%rnhs@!c#{JvS4yo~?L(Zi8?NbXsN9lr~_*yjsAuUQzX zr=VHJKPavrYDp|b_c%H^-L&0~^Pk6JRaaL#*xM6g={D%|zd&mIBq(g$(}Y7^T+CG8 zZT4Rtw_a{q+p7#bQ%pl0Z3 zDC2uMu!{4CW_1kJ`!t#_6EFio72Ah!mFCN+cv!r>FVgmRXxWcN(J)16$pmrJoNa2$ z@cVboqzx5)>T>;5zUuQYpY~69cYC-P{pk~1R46}Uu)g0+RW)RD_@KeJq_*bG874T)AMR# zBYVj!mS=2-I>v%5)GE~uI5UKUl9!&9v$wLfd+&Furbi9+^{mEK2@oSg!)6H5__mIX znR&Lxt^Ip~!`GUcnxB^3E%C!MGx;?&3E@Xa?nvKj>g8pEzNGSD9_q2@O-*58Fc=F# zCF}LB#}KR>kk}kEGrQvAg4w@PF#o*2G3>LC>(~Kz<=5I0`$^GAlgo>Nl#xn)=^*<| ze@}#q>V%#gPJ3`i@c97WYraNs{7&qX#m^DYy3c+>Pw31oj8O;I3e=F2#u9bYjdzxP zNk3pA=`Tnlb;|Ue?Cjo3M@L7uIwK1-7oQ>0-e@o{MEGSMFB4qwehN+R_Dt z8>IW-j}Cfo5ZZ8#PpIZ0H5NkE%~)=rnWVKzvVHy&nWZ2W;JYzp-~h9q5Ps-VAEdc= zrVIuPAKUS zkn2qvx5!RGOmsAOe#=6tb(4%*%<~ZlQ35Y7FQ>W2{L%r|f@Wd2U%IBMp8>s@7eBTw zJe4Kn+nk&GF?@G-FZ7#;WDX}S_or|nAtAAwgX`_sZ;KFg5Fmyq6bewYz;>n7MgA}a zSr$G%AwlK>Sj1fNqrzt$J?}W1aFhOTYB`VMx&sAWExYgNB^z42EK2jBr4>ut^ZgX1 zcHJueGAcY)&}Y~3Tsy;lj>4Le7;8GhW4#}7&0aa4B)~k_pY-lh`{{bNXa!^`cJ982 zsAxby(~6mbLSNYPz;8Mq6`pcX3VEu^K0!RxrpaAsf~Y8uiaG^x8Iae}Ic;xm*E(PN z&K!{tMsy(mUg3(0y5EK?IOF7`rDbG75D#B6zhqsKC6A7X7WO&{CP@8b;J)wK>&|cU ztrE+xjSMuA#aI~a^;SO)m$S4GJ!r9l+$^1L#*OeR5|7x-hA&O4o|cR=#bHH9s4p-w zn?B%omcx-+<{IH>Ip05GdR~__OQ`nNItiYPN8samT(Ha&F9GpXN_PXZu(Mxn!#HB7 zg!R?cM@L3-G@0jSXQ40{_vo0W<|!*rM7sCVWg(Lo~|M?g?O5z#+=|kg)5tZ$ZKzgJG>c4c`F*5O=4rM556K) zo9$U?^}i~XEdDm5{@WxqxSH|i^yI<^x^`9u^GO-~ETh#Q-N86SJa(4T z;YAj6c$9O#s$Q6{kX+UWrUxWs-_dPF3$ne(Po8*tx);G<2a8YpK7YAS+IGleJ&3T0 zmNSQaI+N$FE)l&Amz$jQbDZOgY0T!_4IKRE!I)w!C5`OxD83#aH<|ZMF!|XyooMqE z#)wl>Lpso4(?boq@Myb;TZp4>=foi*r|!hkImQ_FPM| zyHSqC{gd2YlB#)1_~GFY0=+#7)(P*-9WHWxQ>vgEkf3I4>-p_xul?ZB8suUK?50t8 znl{q%TDW_5tf`Q4et7`YYsDSNEA-FhHvO7<@Ri%XdcX%W7|Ots)xuEd3c>Slm3=*WE2?S=<|MMfSMMn3qLZmj1wYh=sYz=Xe!j6~5xPp82w8ke~?+ zQ@73*hVR7>5z8}lyHR!%0Lq7e08OYu_}9|D?|DRE!`b0_V-YB-5aYPGLQP~%Q%Ng! z=227pll^Ry+@}VdCXwWI=3dTL?v8iT(2{U4X>r-G5fR?2Aa8{5PsorgqYtHvAqy3t zW$x*PpZ%uUjyWcF+6xO@Z0r6+NpKjl9KQ4Dgb>Nm4uA|3UGTu$>dv>ooZ+ zW3!<=o-9;Xr|Hvt7FagoK5O5X&lUnK5hP;)W3P<33p6I0RB9x*Y_SQQ|+Leq^bm zC58?vE{$RfqM37u>YUmby0U}PTxi2%%nG4@)3B!6i5KNI>eWoOd0Z>=C21R*h<6Ch zX|0kFaoyGa&Mle55NgKyZSs_4VtWYh?0#4W{zOvJ+g++l5@<>Vec^S3!~}bKufotC zXqf4Vr62b{Z<-8$oebZ8*Ga<*?}#hKkWm=!1v>Ry_VU32#(tVm8TH=k0`qPJ&Gz;t zJv?01NWr1$N-~K+b8YRdNOL)pb+~{~)u~N`VSy&IR=|f@`F9@OxPKSydd~v~>n#cD z6o(yZ#}@C#CVHTrHXEG>4A|)$1D7G$Ue1n&WtNm8z*2BUp*j8hva)ZaEZW4xvwoLY zdPzHvxXE*y`=0WY8Sc#~_#$4CxfA~#_+e=3!{BYrA)+(f#u%3w*yiT1F)Y1FdD9;R zTrppuZP*i&9Ti?{6H^{Lm&L4XHX(PlqI9)X>Ag>1mB7gtG-=R4_%?JJ zPjT>UO(3q6>QvEGS4)@P+e_#4G4`bmegyl$jFFLE@$);sawPtaN%q?MDZcRm-uQDr zE*vh_zA9FN%sFrsymsUZiM!~n7bO7~y=zS|8+7SFkVMyDB1yXlM5e2zZf<4K9?PMj zrAhDZ#q@@RiyNmMOSb@4d$s%F_HL3El#zX0`?+K`&EH8q@H>Oiu6@9rEL8Ry=GKvc~2#1L|}uS_4{5P0?KrYdgl|HQhl zIQ06Xp%E;;T)2;mK7MX?&VaLa;53^1L&%qu-dx(sNMW~yLrNwcK20reuUBk)lmaot z@?{F_I?R`!Rc3VcwBxc3^im8`QWsTq{wjgOtFZCF|AfsCj&ZT{w7+vkm5z7u3{Uk8 zE%1yZXNKY}X7-zeE)%3XIB9bdKa=3Q;j|OFky0|b?0*)R)|Hd?$1ZE>oasiut48RC z=RP_fSG(sw&zg|w)E;#9M`x8F5S5K-@G2`-8R-sLA9;qUSII1bzXY}~W5dG*tSn^o z?3}H=+$7xm^qq{T{)5brVXc$kA8E+yeky6c8!#_*gOI$#6Md3 z>|JiIYwt{itxLfV4kfdag2gvgM|VtqsVpsZDJw1^6l`+q;MLKptMv}w%MdIOUepMI z0MoO_dyF%^__i-56BABjW#Q{<=k4dFZEdEl@1XGbKl5{vbTN6ZswSI#=fa^QjQF#L zwi|z`Aj(@rg6Oe?CsL@lkQV;F!X`#C)T|^}TT=Ub5o6u~`D_TpEN5xeiK=$c+tHw~ z6wX#i%v)0%u}ew843m3lUQ!YfPN(j*=Vo|r@nQQ84fVY!-ugzlN7hE1Ov{hnwC z96g-a$x;(7Z{5EH*}PivdlR?(C-7sSd*F7Yy-lAQpW&3Or(b2xK-bLGMHfRCvlVkj z7P>0Ut(~)tp$QIEMZi8^Jq(lXc-2Jgs1YwtZ}}jl@!yP59>VAN&}u?G--~O%`w#uX z?h9R9yeyaX5dE-pN+D6aku8G9Skh8z%H?`D=7x-xK^B{#XB8X%RZU~5+sNem=ct!G9q0F+?k2G1i18g7kfk@Y4F`wgj!4g)E(g+RWq&@82(eZKp%$1oEdlHQq)sH?Eiz zVFqvG;V$Cif2k^Hu6V%%f5F1Q!IKZqm&wtjeew@_iRbKouNv@N(!hYCTpS|n!g2lr z!sNmcV+MW@jMEp+%KCt{9lyP*!wu)vs-Z(uW+oy;R}la!swzk32AYbixW`B48jD5C zTe@aun(B+?cbDEuOB>qg7)UwfeIJRa#y?&N`o{7?b5fgl!zMq0WNqDseqv(6gR)SQ znRD;o8d8!PFoUNcR%8^wA}~Ntsi|!s)W9Ol^=Lz_$XOmM(DZS3{G6L*P0Yxvemuov zBTDe<*)#w7<_H=!jl!Vm$;L`VPN|;h22|6XK`GH(wmi!|1?v04IdD!qz7`_$) z4k~$&E9KPa{FlUxq|A&#Kw)K$3kwVbN7Xr{u)5Fa;OJ42s4vh^A&nh)^oTY_z$zfG zu8CiET-{)Y-EAZLGG0Gi;Tde#F%#irq+s7C9rOH#o$Sq$<3WL@x`B~Yb+ggts-~K; z{+DFEl=Q6R)Y<9b@5mZpC-V7U%J3Nfxd5Cm`h{OKFD`U(@;{lF868h>%Y(ky=e@}D9#8KtxCC)^phQ1pu5FV=0QVRJq09L<YbbKN++v>-SdmtQu?2Pm2tvaMAI-Sg7V`diShKXXLglb}tlyJymJ*4VT%x znzJYlMg3M8Yvbapgx{?G92zVwEtCH4TE(thta{W$h+UdAXi{1%vWfSXUEklaDY&zI z+VqC^fzi!T#b(j>&6jYQZ>r1&FsT2$zuzdn+qk9<3u(CK4Ahy01VnTIwd}ikOqO!d7DJ}p8M0YY+J>@WJaNV*u z4*cMDDJ&?Ym?}a;EAsmL&`@8Wl(`cdX(THv>+IYd5bGKMU$e7+f?fxE7}SB88TLs* zJ~9#r?t$24X4qQHR_ohbc<^-USYg*2*{gTpnQ6HMc`Gfi0j6w7wGWy`>shE-Mg}E9 zWL{KMR74c`To9dK*xlRvR{Q7DPD&yrmHZ(7BECmhNa)!!3KkaFW9&$E^|2w7Wk z!OT|A&7YMwA8E%(x#AXzzf_tUTG!zbI%r&7^6%=50KC)N5TP?2n z+f&1}-rQ&XcQI1#Ri~#I=O5sl-bj9U53jCkJxF}YCQTdKm?!P4U-I9U=2 z_JaWuEwd-a3&jHFx^G6O%MyCDz&$QdH}A))y^VPi6I4)NF9yOX46Q;F8=Gs8*t_D! zTKx%8fsBp3Y%ZrGOf;%n$=ZGM=OL)&rZMLJzH`~mev8LiRi}An5psT%r>f=3`!>#k z>9oA)qCTbYIoNjm=es1{`s|y+($<#SeN3suhTiX5BL*%gyPjO%XF3SR4iH%;6XVq1N97e7iQxvEv#A0(3W{Bjn zhl;58R9d2BPRkw-t;or(4>~I=?i5nqqGzZM73S)z;%L2?PZLf;dcMUJ_DK9tmK(@s zQ1$y%2VLZJ0CiD!HuiV)^mja&aClqZd{+0Vms!IK7TYvg;&&Ah#sEMw1vYx#Cq=;{ zd4$HOW0P|#ffBY7J<;q1)ZTWZMahwG`*TcnMN%XE#*P^qVGi|6uA zdCN%k&ddTgH5(Py>%xgl2osBF%$D`oGq1KDSE=I&hOsXi`(LeMRs5z5p(!GN*-`Iah zE}6CcLG+XXM+%ig^Wl-(LQn`H6Ra5bAu~F z(ooks91U|p`JW_D4)Wz@Z9Itxf7vlo2XFk!WbllDhf9dIRwp&o?0iYw!^7+HtS|Ne zv7vkmvM81g4s$j}PbBmZc`|{I;mHs8Er#D)4^jj(k2(RI4e{M$paM^!+q1I>e_2dH zu2Ul;tL>^Mx)znMMy_32Z@O(xV;`OpFC8>(g>fXar@2UzL`dg8MgKm5piYydb6FI( zr@qJ>FWww02<1v7difwPmjZl<2>PnfSKaA=tR^mfeIJX2$Y$TbQ-yqgg`qDC_ia_J zq$PLxP5i&vm^=N4MkxS|TK>@JQ$PO;O*Lv~&6oJfCXKoiJWQ~%M@Ren9XqM!pB3IV zUM&ivFPyH+_x7lYeH-{=G-io@d!b#6J1?m+Gv{J={VvD&W0v%5Q`PqQ^28yKXcQPiXrr!bR?KOD)bp6}6hK?e8YE z|Jc`;Ec61uGwT*Poqa4dcy)9Wzd4F`0UoJ z>sa)wB4%v`DTyb>Y$U$^(t%yMQf#YV&kM37}N!Q>(!Dy{9iK_yg8DVH@`Lu>Ok3-h9Y^bKZuCBuyiY_Nu!e z>Vs6Cf_aZA`)K~txB0a$EoY-rv7&Ph4W*?6MRAAEjyYbj%~c$3^yv`w^|Ant1OSiI z6FLYNm&V4{mDlOK)0H!Hj4F>!jyb}!U2m2sq8%GPqJdRcAKRNBCBoi52$AVz$e-Ya z@BO)`?%r`=BW-GG0?f@o8kbjFJ=bhYZ7XD9wzPGZ@~Z?^#AZ+T^HG7Ljsfsih|LLk z>Km168)XAdm&vSvATA+uUAYPZ13@7n#dPk&;kXq>j#xep#P5E<ZH5Ghu|2U4u&F~kTMYz_vxVUr=^?8=6 zfUCm|aH?cT81C$g-afjagNUYXj3o$`IZqr``OBR!_sC@$&I51R(RagJ!*eJdd|GkZ*lmZL zA>(1PzE{Mb%GmX!EHvI&pPf-PF{)6W|7b<$9Kr(lb&>(sYJ;6j&w8O5Au_c9*Y`=A zZM-t6MSJhJ<+tVj>MF2Sc>G6^d>2P={`?Ir#O^U;yQCB;TMCSBE-h{LVA4g%_sUSl zDlsHvc&OWEKZD#sr@W?Se75y)ek_(+z*CVu*+Jp!+(Fa%ZO!Ra^UdARlsLK-6TfA0 z>H`#L)5g*oayfvPT+Ucq2giKyK;(KgZ>2kWGU;9aCILSYzOjP*rQ28%Z0ZZ(^Z}0M zqr=McdAN`mpT2=#@whjj>C+*o&P~znv2Skq+voDUI0j9qr!0WUx7@!&bJ`L7Vc9H) zj!=NsS4pWtg_9abJ0WR*x|V^LHzfR`KF-XKk$g2=!En`le zUZt#UE`8;)3Y3TwlggNy;=gGDBLQ;R84CY9@ho8HS&%~&oifJaBu_71zSpmZl}g3}S2W4?T0=;H&r7wmXtHwOWsnyX<7aB0SPcr9E@ zQ#(@-yxiw9d42T`2|1W$hu?1oPcG)HsO5iK9cRi4P~*)vwz09X_j_aS*R_wpY`xmO zCtR$#q7As3fFSZXW5Xe+ouKTxTjuwz_ngD&jbsmo9 zqp{HPprE_23u1Am{Pl*rLzN6r$4C{;sqX-f1%<^@3Ax{_fyA251`}s%^%3r=BX@GL z0O5rOmTR_`!YMo}?_zMOSl{C0d{TXxQT2|do3qLu)gZBL>o5mDo`T%oy7ct)h^k;M z7JBYJKPdd63<4vT>=|+t_y0dX|cV3+ZFfInQOB)8kk;`sMfLP zs0!FjdPc*9ni|E>+rm_H#tmr+iJ34rElGdJa)ND3_kp@+Z-T>sds|1h^x^Ac2^ej_ zOX5#VS-xtw_mX-tMFN!Ppn#A4VLDIzMGC)7LCpPZ$mv2W>R8Pt+f>C!&8-KCIy|83 zR)%r_x%3o4i%+#mX=!QMw=1w`Ou9q;y@9tJE<4i;Hw}&0B?HTOOyhYH#(@%Q^7f4} zrTY5%Ccb+v2%&dY+;%JXuxe?m$I)T#YDMk@xy!W(My z*SY|N!)xor##rKe|6Gzb*Nvw@@cPPAYa;mSA(3o^4*N}&m4cJfK(xx!6Gaem%s@_> z((P@k9k#ODEmvFC5U0C3#M4l~YBo-Th#?yGgK}J?P-D!@F*3 zW!Hd8ABxEo@LIDsHx4vYduwN9l_*wi1fv-4$1MGHQt5VFJ6?km#K)y+kvB;n6pp0KM{l4&$z5RdpS;AUL$7 zk+FlnzPaRNsuaY;#Bdn*qMe|4pLoqLG=)YzBmi4^WPCouM4xPx#5I6RRysk$NPlX4 z+Cg1H9nAW<2_>wfVRC(##1U&Ee7ThC^`i_WnCSc1y0qopnxm1_f;ctG_3MhoW<6IP zAx9Rc&_>Zu*V978|>nIy<+NHOAx}+nu`hy_jZ`i# z=cUc*0tTQ~NjC{j8VR5bCmheSm*E|jRFBvV8-YVvS~4wnb5c}XjX|E1)yUqazb@0f zwH7PUzg99X!NBMb`|X#U)R0!`AMq8wFxBaSF3W7)o6p%r-Th)l_AA^%AT(UST&~oj zf{B$rlqM#j*jI1i4ow_XEI!ghK>6)d_RDd6Z1th)kQ=3Zz)gbu4D?$5*5EU!uQ#FD zV1AfZ&hO7%feBg?2zZu~T|R`dxA^0SF#Jf`2yox0oK@c4}hvsYB_^zgK<-3|mHirRYd{O^L$#Snw#fB}x?)G_C&w0y`qml$hF$5G zsx{tYCVxm5O)gqgpk`|8DkmE_byf`jzb=3H|22B?PTa#+us?hxsVCR+Hiyy#p=qU(b$6%cwW z@>0VF>wXTK-B!MESQuVavhv!Qcsi*mgcPD(O>u|^~XVACnMDdWtXE6_* z?M;;gvqb{V%D~ibmBx4ra-D}nZ!Z8h^O+8==6aBUj1Za7;W+@uXtSA^4>L+hfkj=r z7FlOq2Kk4EnJD01CtC%1L7GEMpfH|$Y5&ACL%%8Kk0VuWHF?Jj3=WgbalqnL^6@i{ z#kwT+&rR5k%2Ennd08y5;YTSqE=;@2L9D6vGGTA84qel*cZ;iO>Yh3y=(BLXc`sHB z;W4$CmZmSoZ$q{o(f#~ce?MmHr<#%5q%E13#4Q&$V?uXXTBG z_AP7EN>)01v6<(11>#m|VSRS7jMs)jbp56RUO7@(o7a8;)(c|_SeJ;iJ(zU1x99Wx zgZvm4Uk&){!o7powJ|W(aq^+gsaPE4EnmT>Kd<0Mv$2 z-5W^fJI;aTroZ+Z$C8yT9eRn{I<50Q5dP|xm_xOasyE}7$3&rBPoXtoi)dmUMh^A% z_HK4#dHIi@jfmY1%d&&QpI(!$?W`=gJfr3)^)ic&$kdV$< zZ9QFGMKb=Y%|nf@wGxX2Vz96FgfQwFJARMak4Y4i7m7_>JKv`*=iq9 zYNtPHR!A=2p5~+x^Tzm;r=Ttmje+AW`oxal_2qB+D>}5C<-7nB_K(&=^YZAJIFR+~ zxCVgB3Lwowgi+5^zd8|0(#k7OdOdd;<@Bd`QpZA~FO2cM6zfjT;dr}9{uhOTo#)F! z05jL(-JRGX6K2=nm_jf|ynQvBg?fq|;!|d9X4#{-T(@vtQ(6U|IIWMg@WS`!7RuEDB&V z21ebeF*hV+&0v+2uHu~Rhfb&<&I&aq%Zw}iCfxDxd~${*WuF)sRo#Zg*!Qje03SSg z{D`Ya&Mb!p;(MFe`FP{p=oERrm?aEmY~P$adT_V2PK%$97f*jZNKeKPf;8Q1P4?dN z$OPhZhq>(t+2@`f6SHtz)jKEXxIU8WGwR98v2JSLrTnK^@QUHtFF+yyg_@dL4BWx3 zlq+dzuGa8g-F)OUBcESO-Ev+lO}mwDW-Ip_i9WKSl9{bO`20#GJ4VTKvVq`w)M-5Y z0s!1_2*zw!*3riA_{6jw`ahef0(>ms`Zp5bX|>W94*VTNeX}IHhdG3wrs%F6sz|f* z%kpi=jFeH5L}F$<%qcN3aU$C!Fz4W?>gbbixL1ZR#MfLEOq$7copTBd3}5Om$NRf{XrtI$gd zFkR4h0`hR#oL^J%^hr`b$M}9d?bj)VhjWIX*L+F=F@~E*s1}_< z-<{72uMeg+I9(a%%F2~oaep!tVqRZro%kJRt89N5I`L6 zuRUx!O;^qmGEmdAq9MMzyzI{uT`3dR*-B}ytb$%GEn7T%{)^3G!3buNOE-nS7@KhD ziB?kbR=1P%??;@Mj1b@wjwCq=KWsDwfZ24xKitiRc_*i*rlxj1k>t~}xzp6ZbMG%D zw++v3*Vfn2RV(Mh+N=}xlOrR`kt&NIFzQ#CDCQT9Q7FtoNJ!X;o)E}kKeZ_w9$u>W zyi`BE`Pnp}+w1q~Vc=PhY<9rW$Zkfn!`mLfKOs+@@j5*n7+?eH`GFJ;fF!zAjsPe? z_$QpHlpsU)R(Cc#P9Ex2X6^_8MA9i87I~DT`ts)L7_ar>>RKu+R!37?&ma=m*FmGl z+G@lKW_9qkynG*@<7}=IyGs^ekym2p(HC^*6bT*xh*5^XB;`nfv3-cF9#h~DIUapH zw3)pThQ6NPjOcU$0D_Uj1C$Z^1aOmuefFs8kA4HW8h($z`svAC$@paf_PiGc(|%1# zk0;9kU;%)&fMk{nclJ5@dTTT85h}1{mN1H1*su9ajU}3hkkZyL%JXVBWPLqJV$fd> zx@nyO)Se>{QmmwQhAfq}fy*;YChyLAB-j(5Wc#B|Q`G2c0&WoG2QB9#z;J*TV(i)b zJwnuUW=oYfA#!^i6l%Mq8c=|eW}W)bJ&VP~=`aKV7}n?WRIGx)R5V+3y4wtUFs6AV`e z32|Hbr(eo4l*ar$2KV+hN?V^#ihX{M0@8#BzDtL&h#vmziSKI6iD>`xTVo<`T>bSr zX_DgWt2x}a51-uBuhfZx961@IUPk~Cz89XD+s!I@{6&>hQW|>^T0-;NL4njQ#eh*__5+6TJ9mu%Npti17 z`pO}wiSsx9aHN8-%-X$Jk=xEGq>opRqB`doAYg%qiTt$(WS>ziqCC{=TH^{fI#U1$ zkqh?b_5!)wo^n9YQ}0QOO>l9ehi;bFAh#QNwYAHfkbpj_a0P>;J8n z`8N(tN~n7A%}JySpW~G;Dnk#jKLpD(PtMNx`1ph##*D>H0J+rTu_Tj&W{b;ASkhKX z9h9TN5XcvKo!xX+Z*VX&+HIyZp64(zMNz=+0qmBvE6)B}6dU_${v(TVUKYAlVLy)MW*>e5^9Xh1@k%Hf4;qp35t1r0( z;gKKKl)m4QX=DUGqq;e}j=o)Vo}HXTZSbQ|d*>9?^*(zaeXG52?4hI6<0r?HCj6+H z(v}tp^bvAYIaAQ5dlu99vUQ_yQJ^@lv9|Wir3HOBN6q(S;=~isIFJ9>sHE@j!lKB4 z-ZonXV(j*I$zLnxzbx_)2$6hs0{}##txW1!qoSe$U|?Q2oP}L%nH2n&!;7xcSHi1FQnS60RW_3~7Lv|>WcZ6iN^ z3@QV;YAt4xXaGQoR!Y>;(jkiHFnO!^R*4#_!6%MFC4<=XUYo2y&p|gQ*SP6gt)Zd! z2eO47q$LgCXP1|jMoZ?Dl*0OY2SvTs=d%JZm5vh9-Y&^ca8x_(VPLdV|EXXj)4akA z1_}%~hXHQ-S7`8mrZZ5k1J{cj81Nk>iun!(1`cDFFuhq(-`^=PFr+Z*qW;#9{BJZJ z{x^FQ|C_JL108Z0JXMxkN1K}|9S%A^f0emTx#41yPu-()TU##iwe_K&_}oqw^*j4s zDAXOBp)4`K)rw*T>^ecikNx<{LKrwUSXe*qTMwp*dhfn-asbj6pm%6wklu5;lE#(6 z;o*wDWcGbzt4~p%d6b!M;6mv~AS4D~XS?9)`-T?&5CIA$gF`~Xf`k2h!3y+zk~tLs z8&^TO`I&^d4)t|(lKsq4C#QbL_J&2=-n;dw)$90b0BT=8SJ$(4^g)akgX}&#JNx3I z*rHg=;^N|yCo$5N9Skf2^q!~Nyt9-0R{=Ka>bX~h(GUqsZaohV4?DY_36^99`A@ja z7#N!`=$(rqG3q`98W0p7Po)nb>nTr6-aRJIINGUxs*%qn7@y%&wt+4X3cgc@?SmJ~ zppYyqEDYmFn4Z2rI}LO_DWwYC{?55KRc1V)`uy~xogvVGp;4%mEnvq@kWbIFcXi93 zuWr{niY#+Jh2fGdYYT4$2BLh*n{?W-7#HTAu(^^X2N7ZY?#pO?_*Dg@di{)txDsqska>)Zi}2 zLYG%i&oipK_pGj7Nw+Rq^Zqf+&UfowmNfY1dVtH_kF1v+j5G z_D+56D<5cV?B*6X5^@KFMEPhr341kCltTatd4$z&fc=rwsP8LI-UltTOab0}1nKB(mC)-fpNT?e9mozD5VH zrr@!56gT~xnMu`Gg^PC?%f+rnfVYyg+u`A(M|7D+hU2lGs5Y5}Aoy;Bm~Lc9 zh>WG|sZUpnMx7SNb@7ptlaNqQ9PBRxt&c!y zkwIEIz_;=7UwV7xuh&h@%yJ-h6I+m9H#ax3G0)ca_AJ=qJiWcOwX_NfzR@!nj~&RPQj!sy-%$a{?8d_ogNZFM z@cPNL+lrK=6ktDqIv#$P8!iwi7RRp4uSG-i1k~y`*_@-KV*#W1a#l{8C@&lRoQRCf z(#C}Y9}B1!V(IORCoDhQXdA}Pp%^*PgIQefTorD2 z28Mae^=oTK!=C&0kBq3nJKD#_fO}?UZTQE%?N_*^Dxz#aHz24~Ac{r@6u>S4g7&Mf3Cu!%}tkwa8nqyZ+~rJCRT_f`aK44JE_FkIp|o*jg3|p zlX-S-I>@EXlqxeP%F`3lLW_&Zx2rJb8S(Mk@BuMpr3TY8F#zJwn3H2{Z^IAhd4fM8 zU4J6Kdq7>`IfET)_6hs3N~VySwmwvoIl$lFW-x0H=z_C9%`|~r`j-D7yiSbEIvZ+5 zJ56k7{dj)9!0A2 z^9|5kCJ4fQxSiP5^#&;6gI6CP9mPaOZd`eO-7=YNIT3WrdlS13Vzsuh0u*C&L4iJP zTt_F9Vwy60Nk_yI|+IH zdZ(sl1{hsU`;GnO=TEc0LWx$=%r!kd0ZC~=H1Y6oTXLr+C!gPhL;}@_HUp`!&wXp$ zGXN6a*Vku7#`uEM74=buHqN%IWM;1RW?neE$Tnt0Ia9=KZ#!{vQrqTfGSIn>i5UiL zC%?H_M3|-8<*Ma>NVQZs83IR(he8slnQFPF_H^?i!MYD%Q2w-FJ70e*hG=44$! zY310M!)z3}^!WI6(FU~%#^%$zlAPp~x;KSWi^#uh5@k}v3o}T?#Jp91_5*Qk4t*8)}WrUKo zROQ}SV~@d1clN>GdwPnit380J0oih4tV4S5|CCeFxV7(?GerA*%G#LPOyhrRLDqjZ zRJA+rdmK^J+`QX;3Idvy1AUE*oPb;3ep^W!i@@pjvM@JypY%(1cAd|88QceH;pDV1 zF>yMPoXyYo`ZsKJY?Pj`C!^sQXkrhRG(!q1#5@-)5gJJ(Au)iUw#vYhmzS3+*<5)p z;&K7AB;?aICyvZ-47oHzYg2JalB|G#S8zE6RHQOT3G>A#&8 z&ogs3E_ut=l}5=PHr!rcUIQ#iH1gs0`gI`X2KfSUP#()<_66Dhp?f>T;sS}Jdb0;Z^j&e$}~7X z;6OybtQ`Sb5A^l7Ip%5uQCmz0EyqLWNU+awLG(>fW=@V-WHRdnh$T*L^}o9N&Zwrk zu3I}QAWa34q9CAj1*C%r(joLtA|TR1dI?2of;0gk^w4|n9RcZGdY2BNmrw#Ecl$iw z{l35NxOa^E<2v#~9Oj(E*?X_O=3H~ly|`$;`-k(r-%U=GZ7J8Zb8Au z`JkMxuI{^cHCb5*^T4Vqq9#4Gd{FJT13Nl9duQJAiAz~U*#r@z%!US!H*b>7wY;mV zUHlCIt?(`Ah;lK1P1+(pJ|13xT3^r3em+j|XL52njEcV-Y(b!u*xmhY!a%5U9w3H_Fl`sJmiDr_$cMb6v4@`{Kw_GskSyRF#C*p}=JjJZXR@zDUA zg^`&VJ^k5pU9ZrkgG0L|GD#N~x1-1)o%okgg@uLw{P8W9m%*-%ReR=dP#rk&qN8wWSucc9;0DJbN-z=h!oF=2~;(3a^68rb5Ks*0`j; z9KlC8Jr`Bg$acc`xHz{KqR;@4kOX#4O<_hwn+j!HXHH$mTUy>edCzkx^V;(xCN%W; zsC*=5qPqI@^x|TtN!ZqQU%~<1&44~S5#1}!8YkRW0Uixi^X^^b@d0u1JhVAeS+$!M z;^M8*1zH&*r(zs$Q^LE>j)q_c1{8;cbfXasI-_OZPQ z(!O}li-kh?4p>*Z^&cL!a!QJJ4Fi0~;zd8VifK|k5^8E6tyd07IcX6y!;SAtf%cPR z!HgjV<->(f3GM1s-0JmvnSMV;I#l(oLwvc_quMVzZ4&HG&4iy_;*e zN54L1)?AU8>GQb8-IvuNj&}mK^4mAO2aAg%NJC)7(D`urF?9`%gU#at(yq%!PtTx- zY}jQl+IuYpELW<8kBD11ljo5SMy>dR`xS}4ym@~_?K7oU`$v^@#3x|c3*@6w^;OM9p*ZdVdpY;;$l;^I% zePu<+ZCUA~q?;vhjd=4pEiEz&fyhFPpWXopw*G9{`H81gAg6Gm;8qnk=uNu7=XEAB zy8f0B1c2Apr~kY@dwW>9%ZpY{nZWVj&MX@b$|*fUk1UE$o;HQ6cs_x5n>S*Ih&2Mf z)L6J9DX0F1U0!$_#gR^*G~|439w%_+(c?g*S!-+SaI)^3$cVx-%o(USBtfAJZIl2L z|9o=$w!w4Ic0n3GqL^!2FrwjRwD^#YnSKy=#0Ng_EU@x6v7Ck8i1yKQ(8F67ha3~#prB4A-QR*P3x6^v9Z*8f;jM>5|)LeW} zeaT#}dAqnSi551YI42Pd*(vHtNo;3W0#N=#G>6Q#zpadTAkM|WfPA=loSMh%2OnJ%4 z;gjqSPonGUp5_zhWYQZ`w70kWyA|qOzFOo6NJ3m&Dx?sH=C$I9-_o5%OrsEG!20#;zMEn|1;^XE5z$7ndw3oa#PZhrUgmQ+_=i;{kXTs@&keBaV}Mef1Ng-3*hXX8`sbbem8HT z(P-A5{#ZfSiLj{TI%+FDiOHT@Rz@be>jMbn_N)Emw{P(9jNbBK0U7J``Kea)nGe|i!CP*6wLF#XWBsg$lRFZ<*eC9DrzQn4h{q@ zr2{5dyFnv_Qns|eY7Wo-$WHJ0SH1-@ z(@_%}COUtEOvOY+36K)F^;f))XnNuXy?Z3{dbvVCzBfr2VN;?Vmq|1Pf=rTEGtB+U zAI7Iod{dqwdd|8ofzi!F4k&7Q`5EzT5cgL5V0?BX>(Ia`*_9;WCeT9C*_rKlGDk~E z$rvuJ6&)r0%-VXVq{Nm*vmlAv?3*paf~d-k8@Iq(H={~S1tbs;Lu^25no+=dw{gSA zc%|^J4tW^pdC3srzC=N<$%80hfd$6Qute(#F(Ba~F`R+8y>Q+ut&iF{CN8*9TCyMIjswy(BC+Y@Ano2rq1m3x*eT06iTu+?d`|j7#?F3 zH%Ls0Z_muk>}oZ%56mM2LaBxGP43;|VH1w;?$dkeFJ5rZHz1%>wCQ*i34^P8=->ma zGT0iX5X0Bi)1zx{!bE&qf=>DU`$2u@f~72n?%v)(lTPatrS*-C8uvD4&s&{fl9Gu~ z629*TTs1+-;o3lLU7ZxzsoLA}8V~7HtOeH73NB_Ecb~f!bS?kjE3Pea++WE$s z(ecqon!&(Ys7o~yi1ZydcQn~9j^m3)nmp?5R#eh$%5yp1GUIhV0T1iz zOM;?)F*>TQpr9Zo){HV*7#ST+6oh}g@C^j8mPq6%+q?!e2jEq3WncW0h^4~}OHK!h z8gi_xiYQ+Ph@o?s!nLR#JfO}nldPz(p8*d|si{WofUE}QHh}L6rWRypWffv&)$?eL zh*(S%tlX?Y%*-SZZZ^S69cm&YhbKyac$<2jMGU<6g;u;XrtXmq{s2YrQ)bBK_*_MQ zPmcwFn*^~8Ucma_st^qC-w{)QzV!FvS`x;6C1>Cj2^TaUSyvW@kw9zg zTz8Z7Ye6SL`|K>xh}A0rqHyY-MhEQsVrr^zo!4&G`juqbzw{Gd=yZt@917hD2v5_8 z(>&|!lokb~nzvujdFBA{aKtXB0V)CS=GVQGKw$7z0Y+Ej2VkQf&3+x8nW;CpJRxIJ zINsj}8|xU~&?i?{psB7XE0fdFF)v#9a0DE6s&J(|A|g)>eo2#)YtrQw6+s{nVXvL7 zZ&A*~%4;F6H2ikixwYni04^-FCTK0MEWiKoVS+Hcy`?49=FNo78M=s9e4tb5Ww{8t z#A`ljYb*czcO^|t-W)Y^GqcILIjFL7QbfX!xVTru;=#x8#e38<$izf`^AQRg8(Kg& zfaMA>=#L-izkXTVBCLdJXkgmbkiK7~e0}}F^U6IV#mC3@RIE(c%1=-KRanT5KFWXd zM)=TrGYx>w!%RYp@wugyZDAWMFb81Qug&(_cYC>RRfrR8OFp#7jiY_SXo>5R2?S-tDPa|6BnfYzpM#8vKI)JPg>A5X49ze_(8^t^1N zN}_mVnhlB6sxfzTaRJ0j8!yc3<%sds>oXs>aXWRpXrkMEUu9%xEOD5?f!EHWb3L2x-m~GlR-uXWrS?dG+E*FH3Q4dzc^&fH)Oo@V;)m>0L zI)XeQ{#$v7;R_7tPvpXim4YP=QIuh3dtqj2sl*%=8mef!e}e`^_l=NGU#!8i z8EKCJRtggwdqFRrV^-x(K5Rv>O?&0aUqbyeJ8MrPCTm>13Fr#apD{g3j7L;e6(M2s zD{b%I;Sp_d3FwyjVYmyve;*umE>ypa1Mdu(wOc04c{+%_dgbRc;H0?@thN}K1UJ`* zT6quepJQkngMuJ?2M3j)1Q{p+$W;LMbIe|VL9H4-z3Oud%}*= z*>eB8O!*d62RTDVEp{FRwu#zcQ_9kevU^@-)1?|phQ%gh9Osh~)uSFidlbqufz4R z@8o!Y7BlP*YW7xUzsAM@@9FG(a$i@MY-mHZ!>)`A&lJYUYkP2%{AY09$r4x(av9ot zgnCGaZXSQm!@Kfxu!Sv+l2xyCaSo5>F-qkPuWI7G1dYL4R^qggj03lC6RgswUgaTx zbR<&?{(N(ILZs(BFRrNAkFj}W(SNZ|#$=(#W4OG2GbkE`j4CRcNAa9w@6H02!GPk@ zp5m?TKKIrw4>afY1}9dyaIIw1LvL`}$`)O13;^S^U<${h&a1hjRS|$WgsJK~9v=lZ=F!om zr5Y`sY)z%uihp_Ig7XOjY?jsB>Gcj?* zT}I7Gyv5{!V5WS04&=_s)|~75PxNE=)Knv3p@t*D>dkIhx!)?0j?aog8%32f@!gw} zioM>?NDLQP-Y zHFD@rL1#i3b#L^WC&2;1i!fbnoyjTLG^0ffV;Q(`alX0wN69#OM%4oK=gS*rFY+|z z)ipKiYzB5cc#*Fh{k`4{+6a1XJSIH&@M^My7ON}8Z)KGcCUopje-+8s7I|3 z5;`uByc|`3AQiS17D!D(@(5TIIBa!(VXVm7bwG5K1D=nb&S;yODlj`Gg`JDOVwZ~( zldjys+3PDa?Bgx28d;>@=+K{hFfl3U>-(2-0ylpV)$Na|&F?L*pJ7$M2Ij8M0ZHJm zv<}6igX5=$-|il=CkT#iI>hY#m=xp-rRMFLY^x|s;}YdKpBk0B+lF?>6woFiz1~ZF z2p;K&J|}+7!}IX|@<-!zCjg+7Vz2wW6@S}GO_ivnF*q`z^c@+W6rB&<@{HZ9K9Wg` z$6B=xR{7t{6fkDwFR;Cn20&Xc=s?pyE4W4zZK3qZRB2#aB@05$lT8X*sdDBH zBz~1z%FhHO|KTi|s4c8qZK+YEjNB*`7Y@@O5MX`5E@Y3I=z~G68zpS;3e~N-xz?J+hjwbF8dHl1S0w&Zf(|!I9&Pz`cEq)J9=6+ z1J!h#s)HVb;nZNxtwyz=Y`I5tJAnu8sUvHrto#L&`}t?`J2?%#`cKCbLq1b>8q84) zZSSVWNW&8oJ?8!5c`u7yhlxB8K@zD_NSCpB(NBl%wCqs6Jhp+oy-x5?8q+#fc_Qb_ zO_SP>a$#%zuxF*D+cPB!70VMt;(sTbyk$#89zqQtOpW(eRvudcW0Y4Ys6O5&zIDq_ zK0!#_F|W3ufLq{ob6=}HE-6tq<9GQvK(dF&`}A}~kX^IkwLPTW);6_NFco~O6Wh!I zaj0E z3_gN_=YZ~~h>ObMWEHgNf7_8{^4MFvg+Ejfj;Ut;at9I%l*L;$LB9+w=t`q+rx`AS zxkyA?@7WV8f+nMU$Zy?pf3zr6u^)VZRiN^ASje3oA+zq@pUK~$b)cZP{kVVQeeTk4 zrjQ&8%-CSFT9zee5lvMpaFqc_@UTc^Krhbo^?<{I+k1KF6Hlef8Zh0=(wbl#bEjqK|m1 zkZ?nw=}3&N*wd7xB&dF3B7pc{D_K#EMnq+1Wn`}z85^UqUy+YBuQvqzTI@)285`jE zvmWmAx3^r5d;pzGWi=x#w6rL7qyzf(85LqZk2;$(GbKG`B;Cx##V;pp$<(fA5RWo~Ok}^|8|&+<2%d+hwY$2V`TD;5n3+X9lybf9 z>=!;F-@(pK8<)60{v<`|bh=0v6r-y@WMm-*_S5i~S-Qztbrw*lLN8bOHl4q?xX*T` zf`#1P?iF7sKEYjTemhI~t$_LjR;)?}gwm0f+-1wkl#W zGRhWmWlE+Cs8tcLMRu-NmzO_lWcjI4i4_@XvR={ukIPYbb?M7@S3{y@h zK2KR2yZj?|ubdIu@T-Fq4AGkC3xpvK<2A`z}<2-x&R}iF-`QX#dCJxnxmDw3# z5Jink3(K;?5K#|<3K)#q|3*S}NE(D1<+^oALq6~fK_1BsCqtcZP_G9Xux$xNmyabz zRUl%S2bR+nb zOKzMlFjdt7f>k|I(_dS&!` z*$nQqyNvim<$Shy_Hv0)OGgK=+N49jx>UjXb>1l#fnTLiS|ugE_0=E$hBsGM=<3?a zo*Zh^96}>n?zVj?BI21z_%k58v9K^DB{U$8^08%xliy8O2{mn!Eu8N*1;_HWF*UzB6{IR_tD5RtidG)GljwEpc`U}dksjGB^-{A z_PsCkJY~c^5=rP!j6O$L-|lg;Ft|yoTg6{lSk|M0OJ9HW@#Fp3w{Kf_;eWz`&_Lrm#`&z}UC+oal`?2ud|&if3~ZFN z?x%M$-af6Yw6?Z3q*&eL5PiYSI@kaUT{5@Gb$s@{2Xm$pVGt7-naQqQ8xf$Q8UYfg zK-1osJF1y|34CU#E;1=1+NmNGIZq1US%c0Ny|Z-HYGVHQsoRIfOonMj$TF6Pse|%Q>lH2Ok`j@by$ReI z@L$ZMiTtF2AzjW{ZhX}MYUQoHD2bdxhm&d%)So~9?l@yxXi%E1eaaF` zlbgq4vnujrSf!JanQMMO81of=y18Oj^kRiE2%;68_}b{$+fObpk69FVB#!3y&C6vL zz4!wQ6WQNdf`~XD?R~j^ezG^KK?$BFyD3ZUh#;dKl2Gs7mdL>Hj=VonyJ0m6`MLmB z{AnW+(o}5UF=lU~xDWee>JV8KeCy^{cKyok>5eMUlG=fuudc4$zq+s>e*Swn;;*XJ zV>Q8+jB$Ii;;?svEY}c}5)%bn^Gi}Z+j4m#86m?BLCeb)cqiKn!zVh9TkFlB+}MX~ z`mh-tauE@Fyh;9L(jMUbGOu}dAUB8g_p_e}@0rS53emcppWU1}|4C2lvA2_GG4AK< zPyTS3pHIN=tCYy$46XrPC3^B$hv*(i7o=#tfNW)>XF#+aA?wqnckL~9x3Im&TB`z* zk(pYX|0yAYUeV)eYq5_WP0FBmz=&v&(tvtbON$DQo4Hb@Zf2`D`FpX9>PM=XCb^Cb zy~h?7U8xLtBS;3-g-sLbY*8OZBZU6L9uLRXoo`3^$mMhvpt@^* zx}cghU_b#B-f2%nPf#m|xdR57V6<*cS!0H?azf<>xET!wb<<#s{Kh`+JasEtrb5QN zSbjw>P0nVckgH$BfP`=V;23aub4N=j9zInV zOl_Emi}NU|?qhG9py5c&CEQuR(Ot~w{rh#cG-eu_Bss)93FX7^r07zEOWuVAWr>lo zDa-~F?Tq8erE`8Gd?XvtzS!u*3BPk;NS_deS-pagwVaLPY07RH9`9zXO4_a+! z0*$@`E9*Qxc4u2h)r=P`ESHOAVJ-Bj8gnI1E7TUCs%$;8N9~0NIgdl)85G>9lx8pFNLD}D@ z=g{Z2H2ry%l#*s)VPi!Mh&JbHJIJmLze0Tej)T|OpM!Eo7$Kp*>67j>Vdf>|0coK1 z>HU7M=ZLP_BS@MZ>em7Y*eSqlLiD)+qsZi*$?aD9w0DNKW-`i)6>=ejgct0jlEHz2 zn;zuYJ%cG#MsR@(b$N->B#0ws9HQ}2N8=-9PbtVa#hMKa7L-94GUpamjf&;?T#+Qq zG!>r;rT~tccObdSU&I>wgD@>`viKL0JX|_|?#tLF!M!ShktQLZ;Vp$^rplEi@#YYN z%61#S(~;}Cxndo6tt=C`AsdCt7PF6Z$0w79b~V;KM>K3Mw8S(tQoquAyn4=84m7o3 zQJFxHY6abWeVBu?F!S8JQy9=vyuR3TR9TG5=-I8d->&7iCvQCOhn=^(+`Q@P9z4=> zaZ`?a?p1fGrK#y@^_@sCWDo4lcIKU_`0HVpn@h))m0*`~02#yKLsr+t9MK=q4`B`N zFBuqQJU$~qEsI=IU{Q<)<~ zc#@7*%>3_}{1Pe3TcLZEahK}nA1W(G)uB+4M$a!@ftr{;$s8rpa3xb)RqAy;!`3G=VrU zhk6FNF;q{?%pQYmf?z6bD64l_Gi=kL7ti{le&^QiCgc18AsS7SGzUJpHv0wn`KRz# zA?|{Y9kx6o34gA%J?op}cEDz&CGFN&T4v<_An2*Bb{DyABss;qu>YPQ7I`!Ck-*rIRFs=8*|>Uc>uhwPCH<*$2=eP;BZ|NDI|?(sKC^; z+&6Atb*q8`cWDQ)@v`x~-wFLn*4MtDEJX)(2hXbxVNRv^4T^?(-EbP0Dt$F0A0DOCQL8&UJK&l*bCB$PmkKirtILIW4x}fx?VQHtXgF+sO^Qsv+_VP3 zoK;Y4A1yJHr6N&=>%!ri=!m~XNoAlZ$(PHlEDjCWN_T1~QcK8m<~8$-k5{rc8U`j@ zh(NBJ!ZC7022L81YVz)SEziHa`HqMjotmN}4Wb5b?V1k@OLy5To72*-xEPuT)QF@w z?l)D76ks}WzaAq~{-9>5psrpt{T5IiQX!9a*Jvroc=#c)*IX8a!{^a=VZy1U6o#u;TJ%cW<*w`huYfuF1u~GD3yEb!)}{e))M34 z%k39N+D_d^lWYA6rdDQ7hkL=aFTVw-l|(0@xtyCp!s~rd8nL>{_~p$Xa^ep*CPhlm z)o2M7v{v(j%WcUrgOS|o&mNMKKX7nzn3$0DaKC&^i3-9$992b`e#;a9Y0El9YMOJg zAjL;UF}ACS2DHvLs^jC^(We@w8j;1t$9Kzq1IopYU}U;TDgj}Wz(|=;rq!{Jc#0DD z>(`)NzvCCfhs-P;EuA)|;6;;}4Y`id)@40CcxK{+@aJJghl&SlNc|iyH#{WI@tkw{ zC@i-hTQoHE#0EF_m`3q@wFkn&+GgNes$PayidZq^YS%J<)XjvvC$s zlKY25tUH18p|ry8=u-cCDi(ns=DgdwWX4Omc!IqF@g?kpPpf|1_M)c0Uomue&vS^6 zZ4IQNNx{xjUNDB1O>`b@ERO+Eqz6ESUG*v(UMS;HoYPGO@PGF$pMUrq(s}1f@SWo zZRAT z>jwF?r|D>V9n_|AZKDC^))kY>-W)(WIc9W+~xR% zMYO>xo0g%C4X&pW={76d{i?Txq37nc=E8Z;sGgNvtHA7rhV9O$g!am{)zze+A3@#2 zV-|Sj_2kza(z>kg=l2HNG-1>(k0aB6?PzRot~I@1_}u{aWMS=OqQjHwaK(SPwzdKe zTFIXmudH&ueTQ(j;?wyUl8P!pGVD=NV{tZTlgcpS+73<9UBCMAlbc|tIDNr8`kJG& zYiy4oy&GUCpx0(nNNRu83nC>zn7o&#`b{f%6!KN-Z?biviW)}Z9%}OS@$p}yq=7o}HYAkwn zuag2#N{Y@ryJ(oQAOdftJ0h>L=+KX-j=gU3=_SS3u_+>@^|ClCu5@EsSgo zL%_n8k6Z6~d3OHuCYK#GQ6!> z&G^s6M3zV;59Y7G9vNrSuQS~J!)H7t4^*HziuF#d`ojhb;VBCw(_Uwj0piC;2T=Hc z2PS={#3QKv^WgM!R&8#sCecNVEtwmGpxbit&gbs|o>S$`9pg4F={am{eL&XV^@+v@ zAvy8U_78M54zI z$_CgmuzbL+G`*vF2txeBHoo#=A=;ngpK6|eaD7=W z-8P4Q!A-`nTOqFjuxOLhoRZBrKiRRfx8FrySx_B-?%Dvc>~H6Jch3`HqWaVBP4WTc z?Ad1J@o^>fbk)wp9Q<}4;M;-Zg-Mu=f$o*tlK(C{Y|BLU4iTg%IhmM_@}=Do&|<6g zkQG691p7_I8u!WQE$)*8#ow~s*};Rnrb`SL{l|$#(5y9Uzv}}sPMPsSEOz=hqenA@ zTJ$C7bC*`4d4d762b7eQpfoJpQ19V!d8X_NsO}y3o-&$JiJ6~{^tQr$h>ejEm{3Ho zd3kN7_7aC2?@7M-=TJw^9|Cc|@wjp1Fi!>CFMtI24BMaedsKtSC#(;wmmFNvY1>Yw)`C=sHjbq;9`rU)Xtj7zK~V>X!G#4EXMH2@|rx zzE4;ogpe|eRxMW8`ODjWV7MObugt~8duzgobDx>A&UKKnMb)KKo2c@^I zgDB^niN>};dC*7Ip%7F4en^E5<9c|jv3K4RJ^90o3-ATl8GMf2`MS;JefnA^J1Td( z*sYmZRj>oNA54laudIS%0iOWB)pV)Z@yyL!?(J=>Z7VC#=oo}3R6^6lOGtchFx9$2 zLM`fM>T;{Mishf9EI%&H;nDKSh{V5kw~su$@@V-X+c9VO4;LN|4k_ojzwIzEAcFW? z{je&qy08jqd*|1t)>%ZVqeZOj3AHYqk{6Ot)6_h_o#J@MG$PfwGvJ=Qh0n1)iPmij zL&bJq_{rrLFupCpVsbpZj=;z)aPpjtrIr47WZhW@ZSLsD zH38eLPh!xWT09zPvFS!|tbIRH&>pp1qLX&(KGl;t&*?&?=ZlME*L=KtWaTQYW`9J5 zix@24|1#y{C+Im_`(d)%H$2Vx@;L8K-9c#?7c&csj;>C3Pq%y&)B5#+zXxc>_~~+& z*1ii8F0Ze52T3czY;JXSaj^;GQzjEcdy=o=W=L~W@(zfaY%Pqg}ypp~;Xa7hZl zh%^1Y0b-Q>NJG-o1Obn+-z#|$k%b;7jwd0d$}dX*6|&IrATKvJKd-cI^D&%EhgwBzXgC+vFfp=!F^D-T|6wF`&kW!2=CuW&Ps-{8G~h;?t)98;Z_I9^ z3gOsY&dxz`aE8DqL%LRb zb+c`C)!&?gX?)n6uE!{+SnsyH_hm8lE&3NweayR`nJFoCE}L{SE3$C(NT~ zWm$4^-zun!^5W0aUkr~R0tyPU1`_m@p_OK>C%Jw(T{4rgk4VCY#F|sESHJA+(9i+n zwP%Y)%1|BIQk9%7QyUYlH4!cBnjZE{)IYQr~arKbb0LSIem38F_>xZxLY z;L#mb(*zW)=P*dw*QMXvRhgEEQU80`>(39~-r}IJ3!uxPojoSeQ0y70>-i~=Og?z< zaQTA?NqAP4rkq6(iG4$^z5NM=sKIZ)Lld5RMcwvxPLH^p+}(7(VPd!y``N)j(+Axv z?LfMPdU-3xgHS6W*>)74xtpAG9rcjyb+-6_pH_Rze9ZA*c?QMme|U1Ouf&WJ0yLrj z{o{Y-|KUNk|K}$P|Mj#!^WL}Y6;rjbbGVc^63g5o>{$G?<@5TTC literal 0 HcmV?d00001 diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..642e597 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,53 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0]['role'] == 'system' %} + {{- messages[0]['content'] }} + {%- else %} + {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }} + {%- endif %} + {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0]['role'] == 'system' %} + {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }} + {%- else %} + {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role }} + {%- if message.content %} + {{- '\n' + message.content }} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {{- tool_call.arguments | tojson }} + {{- '}\n' }} + {%- endfor %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %} {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/config.json b/config.json new file mode 100644 index 0000000..8beadcd --- /dev/null +++ b/config.json @@ -0,0 +1,62 @@ +{ + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": null, + "torch_dtype": "bfloat16", + "eos_token_id": 151645, + "hidden_act": "silu", + "hidden_size": 1536, + "initializer_range": 0.02, + "intermediate_size": 8960, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 32768, + "max_window_layers": 21, + "model_type": "qwen2", + "num_attention_heads": 12, + "num_hidden_layers": 28, + "num_key_value_heads": 2, + "pad_token_id": 151665, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000.0, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "unsloth_fixed": true, + "unsloth_version": "2026.7.1", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 151936 +} \ No newline at end of file diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..5f32a68 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,14 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "max_length": 32768, + "pad_token_id": 151665, + "repetition_penalty": 1.1, + "temperature": 0.7, + "top_k": 20, + "top_p": 0.8, + "transformers_version": "5.5.0" +} diff --git a/gguf/compressor-v31-q8_0.gguf b/gguf/compressor-v31-q8_0.gguf new file mode 100644 index 0000000..aa5d72e --- /dev/null +++ b/gguf/compressor-v31-q8_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce8b7b729409c6ae35d8a81fb11fcc9c286c924ec4e7d0b4b052d742af285f4d +size 1646572480 diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..f6ee86c --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5660fdaa65f175f6aafb2ebd35e7e6d24e535f0deecc949cd92cc7f7858a3524 +size 3087467144 diff --git a/scripts/compress.py b/scripts/compress.py new file mode 100644 index 0000000..7ca3d2f --- /dev/null +++ b/scripts/compress.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +""" +compress.py — Compress English reasoning text into the telegraphic CJK register +using tessera-compressor behind any OpenAI-compatible endpoint (vLLM, llama.cpp +server, etc.). No API keys or HF token required; the endpoint is yours. + +This is the same harness the compressor was accepted under: segment the block, +group sentences into step-sized passages, classify each passage, compress it +against the chain built so far, then run the deterministic fidelity gate. A +passage that fails the gate falls back to a rules-only compression, so a bad +model output costs savings, never content. + +Serve the model first, e.g.: + vllm serve ZelligeAI/tessera-compressor --port 8001 +or with the GGUF: + llama-server -m gguf/compressor-v31-q8_0.gguf --port 8001 # from the repo root + +Then: + # one block from a text file + python compress.py --in think.txt --endpoint http://localhost:8001/v1 + + # a JSONL corpus: {"id": ..., "text": ...} per line + python compress.py --in blocks.jsonl --out compressed.jsonl \ + --endpoint http://localhost:8001/v1 + +Token counting: the fidelity gate compares token counts under a target +tokenizer. For results matching the accepted harness, point --tokenizer at the +model you are producing training data FOR (default: the compressor's own +tokenizer, which is close but not identical to the Qwen3.5 target used in the +acceptance run). +""" +import argparse +import json +import sys + +from openai import OpenAI +from tokenizers import Tokenizer + +from segmenting import segment, group_steps, classify_passage, facts, gate +from tokenmax import _apply_subs + +PASSAGE_SYSTEM = ( + "你是推理压缩器。Re-notate the NEXT PASSAGE of a reasoning chain into telegraphic " + "CJK/symbol notation. Every NEW logical step, fact, number and identifier must " + "survive — unless already stated in the chain. Never restate chain content. " + "[passage=load]: step-lossless telegraphic. [passage=narr]: minimal stubs " + "(试X→否). Output only the re-notated continuation." +) + +MAX_NEW_TOKENS = 512 + + +def compress_block(text, client, model, ntok): + """Compress one reasoning block. Returns (compressed_text, stats).""" + segs = group_steps(segment(text)) + chain, seen = [], set() + stats = {"segments": len(segs), "model_ok": 0, "fallback": 0, + "narr_skipped": 0, "code": 0, "calls": 0} + + for kind, s in segs: + if kind == "code": + chain.append(s) + seen |= facts(s) + stats["code"] += 1 + continue + cls = classify_passage(s, seen, ntok) + novel = facts(s) - seen + rules_s, _ = _apply_subs(s) + if not rules_s.strip(): + continue + tail = "\n".join(chain)[-500:] or "(start)" + stats["calls"] += 1 + r = client.chat.completions.create( + model=model, temperature=0.0, max_tokens=MAX_NEW_TOKENS, + messages=[ + {"role": "system", "content": PASSAGE_SYSTEM}, + {"role": "user", "content": f"[passage={cls}]\n链:\n{tail}\n\n段:\n{s[:2000]}"}, + ], + extra_body={"repetition_penalty": 1.15}, + ) + out = (r.choices[0].message.content or "").strip() + + if out == "∅" and cls == "narr" and not novel: + stats["narr_skipped"] += 1 + seen |= facts(s) + continue + if gate(s, rules_s, out, ntok, novel=novel) is None: + chain.append(out) + stats["model_ok"] += 1 + else: + chain.append(rules_s) + stats["fallback"] += 1 + seen |= facts(s) + + return "\n".join(chain), stats + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--in", dest="inp", required=True, + help=".txt (one block) or .jsonl ({'id','text'} per line)") + ap.add_argument("--out", default=None, help="output JSONL (default: stdout)") + ap.add_argument("--endpoint", default="http://localhost:8001/v1") + ap.add_argument("--model", default="ZelligeAI/tessera-compressor", + help="served model name at the endpoint") + ap.add_argument("--tokenizer", default="ZelligeAI/tessera-compressor", + help="HF repo id or local tokenizer.json for gate token counts") + args = ap.parse_args() + + if args.tokenizer.endswith(".json"): + tok = Tokenizer.from_file(args.tokenizer) + else: + tok = Tokenizer.from_pretrained(args.tokenizer) + + def ntok(s): + return len(tok.encode(s).ids) if s else 0 + + client = OpenAI(base_url=args.endpoint, api_key="none") + + if args.inp.endswith(".jsonl"): + rows = [json.loads(l) for l in open(args.inp) if l.strip()] + else: + rows = [{"id": args.inp, "text": open(args.inp).read()}] + + sink = open(args.out, "w") if args.out else sys.stdout + for row in rows: + compressed, stats = compress_block(row["text"], client, args.model, ntok) + rec = {"id": row.get("id"), "compressed": compressed, + "src_tokens": ntok(row["text"]), "out_tokens": ntok(compressed), + "harness": stats} + sink.write(json.dumps(rec, ensure_ascii=False) + "\n") + sink.flush() + print(f"[{row.get('id')}] {rec['src_tokens']} -> {rec['out_tokens']} tokens " + f"(model_ok={stats['model_ok']} fallback={stats['fallback']})", + file=sys.stderr) + if args.out: + sink.close() + + +if __name__ == "__main__": + main() diff --git a/scripts/requirements.txt b/scripts/requirements.txt new file mode 100644 index 0000000..6d5dbc0 --- /dev/null +++ b/scripts/requirements.txt @@ -0,0 +1,3 @@ +openai>=1.0 +tokenizers>=0.15 +transformers>=4.40 # tokenmax.py standalone CLI only diff --git a/scripts/segmenting.py b/scripts/segmenting.py new file mode 100644 index 0000000..1af1492 --- /dev/null +++ b/scripts/segmenting.py @@ -0,0 +1,121 @@ +""" +segmenting.py — Passage segmentation, classification, and fidelity gates for the +tessera-compressor harness. + +Extracted from the harness the compressor was accepted under (same functions the +teacher mint used). Pure text processing: no network, no credentials. + +Flow: segment -> group_steps -> classify_passage per passage -> model call -> +gate -> rules fallback on failure. A failed passage costs a few dozen tokens of +savings, never content. +""" +import re + +CJK = re.compile(r'[一-鿿㐀-䶿]') +NUM = re.compile(r'\d+(?:\.\d+)?') +IDENT = re.compile(r'`[^`\n]+`|\b[A-Za-z]+(?:_[A-Za-z0-9]+)+\b|\b[a-z]+[A-Z][A-Za-z0-9]*\b') +FENCE = re.compile(r'```.*?```', re.DOTALL) +SENT_SPLIT = re.compile(r'(?<=[.!?;])\s+') +_LIST_MARKER = re.compile(r'(?:^|[\n\s(])(\d{1,2})[.)]\s') +_OPS = set('+-*/=<>≤≥≠∈∀∃¬→⇒%^{}[]') + + +def segment(text): + """Split a reasoning block into ordered segments; code fences are atomic and marked.""" + segs = [] # (kind, text) kind ∈ {'code','prose'} + pos = 0 + for m in FENCE.finditer(text): + before = text[pos:m.start()] + segs.extend(('prose', s) for s in _split_prose(before)) + segs.append(('code', m.group(0))) + pos = m.end() + segs.extend(('prose', s) for s in _split_prose(text[pos:])) + return [(k, s) for k, s in segs if s.strip()] + + +def _split_prose(text): + out = [] + for line in text.split('\n'): + line = line.strip() + if not line: + continue + out.extend(s.strip() for s in SENT_SPLIT.split(line) if s.strip()) + return out + + +def group_steps(segs, max_words=160, max_sents=10): + """Merge consecutive prose sentences into step-sized passages; code stays atomic.""" + out, buf, words = [], [], 0 + + def flush(): + nonlocal buf, words + if buf: + out.append(('prose', ' '.join(buf))) + buf, words = [], 0 + + for kind, s in segs: + if kind == 'code': + flush() + out.append((kind, s)) + continue + buf.append(s) + words += len(s.split()) + if words >= max_words or len(buf) >= max_sents: + flush() + flush() + return out + + +def facts(s): + """Numbers + identifiers that must survive compression. + List-enumeration markers ("1. Load...") are structure, not facts.""" + nums = set(NUM.findall(s)) - set(_LIST_MARKER.findall(s)) + idents = set(i.strip('`') for i in IDENT.findall(s)) + return nums | idents + + +def facts_preserved(src, out): + """Substring presence — regex \\b breaks against adjacent CJK chars. + Returns the list of MISSING facts (empty list = all preserved).""" + out_n = out.replace(',', '') + return [f for f in facts(src) if f.replace(',', '') not in out_n] + + +def classify_passage(seg, seen_facts, ntok): + """'load' = fact-dense or novel-fact-bearing (step-faithful treatment); + 'narr' = search/narrative (stub treatment). + ntok is a callable: text -> token count under your target tokenizer.""" + f = facts(seg) + novel = f - seen_facts + toks = max(ntok(seg), 1) + dens = (len(NUM.findall(seg)) + len(IDENT.findall(seg)) + + sum(seg.count(o) for o in _OPS)) / toks + if novel and (dens >= 0.08 or len(novel) >= 3): + return 'load' + if dens >= 0.15: + return 'load' + return 'narr' + + +def gate(src_seg, rules_seg, out, ntok, novel=None): + """Deterministic per-passage fidelity gate. + Returns None if the model output is admissible, else a short fail-reason + string; on failure the caller uses rules_seg instead. + + novel: the passage's facts that are NOT already in the accumulated chain. + The prompt tells the model never to restate chain content, so only novel + facts are required to survive (matching the acceptance harness). Pass None + to require every fact of the passage (stricter, for chainless use).""" + if not out or not out.strip(): + return "empty" + if '```' in out: + return "fence" + if len(out) > 2 * len(src_seg) + 40: # explanation/blow-up guard + return "blowup" + required = facts(src_seg) if novel is None else novel + out_n = out.replace(',', '') + if any(f.replace(',', '') not in out_n for f in required): + return "facts" + if ntok(out) > ntok(rules_seg): # must not exceed the rules-only version + return "tokens" + return None diff --git a/scripts/tokenmax.py b/scripts/tokenmax.py new file mode 100644 index 0000000..76a630e --- /dev/null +++ b/scripts/tokenmax.py @@ -0,0 +1,431 @@ +""" +tokenmax.py — Deterministic token-maxing post-processor for compressed think blocks. + +Applies ONLY substitutions that are verified to save tokens on the Qwen 248K tokenizer +(OmniCoder-9B / Qwen3.5). Every substitution was tested in-context (not isolation) to +confirm real token savings without boundary interference. + +Design: + - LLM does semantic compression (what to keep vs drop) + - This code enforces consistent notation deterministically + - GUARD: only returns the processed version if ntok(result) < ntok(original) + - Idempotent: safe to run multiple times + +Usage: + from caveman.compress.tokenmax import tokenmax, tokenmax_with_stats + compressed = tokenmax(think_text, tokenizer) + compressed, stats = tokenmax_with_stats(think_text, tokenizer) + +Verified: 2026-05-31 on Qwen 248K vocab. 27/28 substitutions save in-context. +Zero false positives. One zero-effect (贪心 for "greedy" — boundary-dependent). +""" + +import re +from typing import Optional + +# ── Phase 1: Filler drops ────────────────────────────────────────────── +# Phrases that carry zero information in compressed reasoning. +# Only patterns that are NEVER load-bearing in a think block. +_FILLER_PATTERNS = [ + # Metacognition (the model narrating its own process) + r"\bI need to\b", + r"\bwe need to\b", + r"\bI will\b", + r"\bI'll\b", + r"\blet me\b", + r"\blet's\b", + r"\bI want to\b", + r"\bI should\b", + r"\bwe should\b", + # Hedging + r"\bprobably\b", + r"\bbasically\b", + r"\bessentially\b", + r"\bit seems like\b", + # Filler transitions + r"\bin order to\b", + r"\bfirst of all\b", + r"\bin other words\b", + r"\bon the other hand\b", + r"\bmore specifically\b", + r"\bto be more precise\b", + r"\band so on\b", + # Conversational padding (require word boundary at end to avoid "Greatest", "Perfectly") + r"\bGreat\b[,!.]?\s*", + r"\bPerfect\b[,!.]?\s*", + # Obvious statements + r"\bAs (?:we|you) can see\b", + r"\bAs mentioned (?:above|earlier|before)\b", +] + +# ── Phase 2: Phrase → cheapest token ─────────────────────────────────── +# Ordered LONGEST FIRST to prevent partial matches. +# Each entry: (regex_pattern, replacement, category) +# Categories: 'cjk', 'symbol', 'abbrev' — for stats tracking. +_SUBSTITUTIONS = [ + # ── COMPOUND PATTERNS FIRST (must fire before their components) ── + + # Verbose comparison phrases (+5t savings) + (r'\bis\s+greater\s+than\s+or\s+equal\s+to\b', '≥', 'symbol'), # +5t + (r'\bis\s+less\s+than\s+or\s+equal\s+to\b', '≤', 'symbol'), # +5t + + # Verbose discourse phrases (+3t savings) + (r'\bwe\s+can\s+see\s+that\b', '可知', 'cjk'), # +3t + (r'\bat\s+the\s+same\s+time\b', '同时', 'cjk'), # +3t + (r'\bthat\s+is\s+to\s+say\b', '即', 'cjk'), # +3t + + # Multi-word phrases (+2t savings) + (r'\bin\s+this\s+case\b', '此时', 'cjk'), # +2t + (r'\bthe\s+number\s+of\b', '个数', 'cjk'), # +2t + (r'\bis\s+equal\s+to\b', '等于', 'cjk'), # +2t + + # Complexity boilerplate (biggest per-occurrence savings) + (r'[Oo]\(n\)\s*time[,;]?\s*[Oo]\(n\)\s*space\.?', 'O(n|n).', 'abbrev'), + (r'[Oo]\(n\)\s*time[,;]?\s*[Oo]\(1\)\s*space\.?', 'O(n|1).', 'abbrev'), + (r'[Oo]\(n\s*log\s*n\)\s*time[,;]?\s*[Oo]\(n\)\s*space', 'O(n㏒n|n)', 'abbrev'), + (r'[Oo]\(n\s*log\s*n\)\s*time[,;]?\s*[Oo]\(1\)\s*space', 'O(n㏒n|1)', 'abbrev'), + (r'[Tt]ime\s*complexity[:\s]+', 'T=', 'abbrev'), + (r'[Ss]pace\s*complexity[:\s]+', 'S=', 'abbrev'), + + # Multi-word compounds (BEFORE their single-word components) + (r'\bassume without loss of generality\b', '设 不妨', 'cjk'), # before "assume" + (r'\bproof by contradiction\b', '反证', 'cjk'), # before "proof", "contradiction" + (r'\bnecessary and sufficient\b', '充要', 'cjk'), # before "sufficient" + (r'\bnot equal(?:\s+to)?\b', '≠', 'symbol'), # before "is not", "does not" + (r'\bif and only if\b', 'iff', 'abbrev'), # before "for all" + (r'\bmuch greater than\b', '≫', 'symbol'), # before "greater than" + (r'\bkeep track(?:\s+of)?\b', '记录', 'cjk'), # before article strip + (r'\bin ascending order\b', 'asc', 'abbrev'), + (r'\bin descending order\b', 'desc', 'abbrev'), + (r'\bmaximum value\b', '最大值', 'cjk'), + (r'\bminimum value\b', '最小值', 'cjk'), + (r'\breturn value\b', '返回値', 'cjk'), + (r'\brather than\b', '而非', 'cjk'), + (r'\baccording to\b', '按照', 'cjk'), + + # DS compounds (before components) + (r'\bdoubly linked list\b', 'DLL', 'abbrev'), # BEFORE "linked list" + (r'\bbinary indexed tree\b', 'BIT', 'abbrev'), # BEFORE "binary" + (r'\bminimum spanning tree\b', 'MST', 'abbrev'), + (r'\bdepth[- ]first search\b', 'DFS', 'abbrev'), + (r'\bbreadth[- ]first search\b', 'BFS', 'abbrev'), + (r'\bdynamic programming\b', 'DP', 'abbrev'), + (r'\bdivide and conquer\b', '分治', 'cjk'), + (r'\bmonot(?:onic|one)\s*stack\b', '单调栈', 'cjk'), + (r'\btime limit exceeded\b', '超时', 'cjk'), + (r'\bout of bounds\b', '越界', 'cjk'), + (r'\bremove duplicates?\b', '去重', 'cjk'), + (r'\benumerate all\b', '穷举', 'cjk'), + (r'\bbinary search\b', '二分', 'cjk'), + (r'\bsliding window\b', 'sw', 'abbrev'), + (r'\bunion[- ]find\b', 'UF', 'abbrev'), + (r'\btopological sort\b', '拓扑序', 'cjk'), + (r'\bshortest path\b', 'sp', 'abbrev'), + (r'\blinked list\b', 'LL', 'abbrev'), + (r'\bpriority queue\b', 'heap', 'abbrev'), + (r'\bprefix sum\b', 'ps', 'abbrev'), + (r'\bbrute force\b', '暴力', 'cjk'), + (r'\bno solution\b', '无解', 'cjk'), + (r'\bedge cases?\b', '边界', 'cjk'), + (r'\bbase case\b', 'bc', 'abbrev'), + (r'\bworst case\b', 'wc', 'abbrev'), + + # ── SINGLE-WORD SUBSTITUTIONS (safe after compounds consumed) ── + + # +4t savings + (r'\bobviously\b', '显然', 'cjk'), + # +3t savings + (r'\bredundant\b', '冗余', 'cjk'), + (r'\bsatisf(?:y|ies|ied)\b', '满足', 'cjk'), # +2t, 333x in data + (r'\bunsorted\b', '无序', 'cjk'), + (r'\bdue to\b', '由于', 'cjk'), + (r'\bhence\b', '故', 'cjk'), + (r'\bnamely\b', '即', 'cjk'), + (r'\bassume\b', '设', 'cjk'), + (r'\bsuppose\b', '设', 'cjk'), + (r'\bderive\b', '推导', 'cjk'), + # +2t savings + (r'\bmonotone\b', '单调', 'cjk'), + (r'\bconvergent\b', '收敛', 'cjk'), + (r'\bdivergent\b', '发散', 'cjk'), + (r'\bcommutative\b', '交换', 'cjk'), + (r'\bdeterministic\b', '确定', 'cjk'), + (r'\bprobabilistic\b', '概率', 'cjk'), + (r'\bprove\b', '证明', 'cjk'), + (r'\bproof\b', '证明', 'cjk'), + (r'\bflip\b', '翻转', 'cjk'), + (r'\bsorted\b(?!\s*[=(\[])', '有序', 'cjk'), # not before = ( [ (assignment/call) + # +1t savings + (r'\bbacktrack(?:ing)?\b', '回溯', 'cjk'), + (r'\btravers(?:e|al|ing)\b', '遍历', 'cjk'), + (r'\brecursi(?:on|ve|vely)\b', '递归', 'cjk'), + (r'\bcontradiction\b', '矛盾', 'cjk'), + (r'\bsufficient\b', '充分', 'cjk'), + (r'\bequivalent\b', '等价', 'cjk'), + (r'\bsymmetric\b', '对称', 'cjk'), + (r'\binvariant\b', '不变', 'cjk'), + (r'\bexponential\b', '指数', 'cjk'), + (r'\bpermutation\b', '排列', 'cjk'), + (r'\badjacent\b', '相邻', 'cjk'), + (r'\boptimal\b', '最优', 'cjk'), + (r'\bfeasible\b', '可行', 'cjk'), + (r'\binduction\b', '归纳', 'cjk'), + (r'\bmaintain\b', '维护', 'cjk'), + (r'\bswap\b(?!\s*[=(\[])', '交换', 'cjk'), # not before = ( [ (assignment/call) + (r'\bcumulative\b', '累积', 'cjk'), + (r'\bquotient\b', '商', 'cjk'), + (r'\bmemoiz(?:ation|e)\b', 'memo', 'abbrev'), + # +2t savings (mined from v19 data) + (r'\bmathematical\b', '数学', 'cjk'), + (r'\bcorresponding(?:ly)?\b', '对应', 'cjk'), + (r'\brequirement\b', '需求', 'cjk'), + # +1t savings (mined from v19 data) + (r'\bcomplexity\b', '复杂度', 'cjk'), + (r'\bsimilarly\b', '同理', 'cjk'), + (r'\bsubstitut(?:e|ion)\b', '代入', 'cjk'), + (r'\bincreasing(?:ly)?\b', '递增', 'cjk'), + (r'\bdecreasing(?:ly)?\b', '递减', 'cjk'), + (r'\brespectively\b', '分别', 'cjk'), + (r'\bnecessarily\b', '必然', 'cjk'), + (r'\btransformation\b', '变换', 'cjk'), + (r'\bprerequisite\b', '前提', 'cjk'), + (r'\bconsequently\b', '从而', 'cjk'), + (r'\boverlapping\b', '重叠', 'cjk'), + (r'\bcontribut(?:e|ion)\b', '贡献', 'cjk'), + (r'\bindependent(?:ly)?\b', '独立', 'cjk'), + (r'\bimpossible\b', '不可能', 'cjk'), + (r'\biterat(?:e|ion|ing)\b', '迭代', 'cjk'), + (r'\benumerat(?:e|ion|ing)\b', '枚举', 'cjk'), + + # ── LOGIC SYMBOLS ── + (r'\btherefore\b', '⇒', 'symbol'), + (r'\bthus\b', '⇒', 'symbol'), + (r'\bsuch that\b', 'st', 'abbrev'), + (r'\bthere exists?\b', '∃', 'symbol'), + (r'\bfor each\b', '∀', 'symbol'), + (r'\bfor every\b', '∀', 'symbol'), + (r'\bfor all\b', '∀', 'symbol'), + (r'\bdoes not\b', '¬', 'symbol'), + (r"\bdoesn't\b", '¬', 'symbol'), + (r'\bis not\b(?!\s+(?:None|null|undefined|empty|zero|0))', '非', 'cjk'), # protect "is not None" etc + (r'\bat least\b', '≥', 'symbol'), + (r'\bat most\b', '≤', 'symbol'), + (r'\bgreater than\b', '>', 'symbol'), + (r'\bless than\b', '<', 'symbol'), + + # ── ARTICLE STRIPPING (last — lowest priority) ── + (r'\bthe\b\s+(?!(?:same|only|first|last|next|other)\b)', '', 'filler'), # protect "the same", "the only" etc + (r'\ba\b\s+(?=[bcdfghjklmnpqrstvwxyz])', '', 'filler'), + (r'\ban\b\s+', '', 'filler'), +] + +# ── Compile once ─────────────────────────────────────────────────────── +_FILLER_COMPILED = [(re.compile(p, re.IGNORECASE), '') for p in _FILLER_PATTERNS] +_SUBS_COMPILED = [(re.compile(p, re.IGNORECASE), r, cat) for p, r, cat in _SUBSTITUTIONS] + + +def _ntok(text: str, tokenizer) -> int: + """Token count using the provided tokenizer.""" + return len(tokenizer.encode(text, add_special_tokens=False)) + + +def _protect_code_fences(text: str) -> tuple[str, list]: + """Extract code-fenced blocks, replace with placeholders. + Returns (text_with_placeholders, list_of_extracted_blocks).""" + blocks = [] + def _replace(m): + blocks.append(m.group(0)) + return f'\x00CODEFENCE{len(blocks)-1}\x00' + # Match ```...``` and inline `...` (non-greedy) + protected = re.sub(r'```.*?```|`[^`\n]+`', _replace, text, flags=re.DOTALL) + return protected, blocks + + +def _restore_code_fences(text: str, blocks: list) -> str: + """Restore code-fenced blocks from placeholders.""" + for i, block in enumerate(blocks): + text = text.replace(f'\x00CODEFENCE{i}\x00', block) + return text + + +def _apply_subs(text: str) -> tuple[str, dict]: + """Apply all substitutions, return (result, stats). + Code fences (``` and inline `) are protected from substitution.""" + stats = {'filler_drops': 0, 'cjk': 0, 'symbol': 0, 'abbrev': 0, 'total_subs': 0} + + # Phase 0: protect code fences from substitution + text, code_blocks = _protect_code_fences(text) + + # Phase 1: filler drops + for pat, repl in _FILLER_COMPILED: + text, n = pat.subn(repl, text) + if n: + stats['filler_drops'] += n + stats['total_subs'] += n + + # Phase 2: substitutions + for pat, repl, cat in _SUBS_COMPILED: + text, n = pat.subn(repl, text) + if n: + stats[cat] = stats.get(cat, 0) + n + stats['total_subs'] += n + + # Phase 3: restore code fences + text = _restore_code_fences(text, code_blocks) + + # Phase 4: whitespace normalization + text = re.sub(r'[ \t]+', ' ', text) + text = re.sub(r'\n{3,}', '\n\n', text) + text = re.sub(r' *\n *', '\n', text) + text = text.strip() + + return text, stats + + +def tokenmax(text: str, tokenizer, force_cjk: bool = False) -> str: + """Apply token-maxing. Returns original if no savings achieved. + + Args: + text: The think block content (without tags). + tokenizer: A HuggingFace tokenizer with .encode() method. + force_cjk: If True, always return the processed version when CJK + substitutions were applied, even if total token count increased. + Use this to maximize CJK adoption in training data. + + Returns: + The token-maxed text, or the original if processing didn't save tokens + (unless force_cjk=True and CJK subs were applied). + """ + if not text or not text.strip(): + return text + + original_tokens = _ntok(text, tokenizer) + result, stats = _apply_subs(text) + result_tokens = _ntok(result, tokenizer) + + # GUARD: only return processed version if it actually saves tokens + # OVERRIDE: force_cjk bypasses the guard when CJK substitutions were made + if result_tokens < original_tokens: + return result + if force_cjk and stats.get('cjk', 0) > 0: + return result + return text + + +def tokenmax_with_stats(text: str, tokenizer, force_cjk: bool = False) -> tuple[str, dict]: + """Like tokenmax() but also returns substitution statistics. + + Args: + force_cjk: If True, always apply when CJK substitutions were made, + even if total token count increased. Prioritizes CJK adoption + over token savings. + + Returns: + (processed_text, stats_dict) where stats_dict contains: + - original_tokens: token count before processing + - result_tokens: token count after processing + - saved: tokens saved (negative = token increase; check forced_cjk) + - applied: whether the processed version was used + - forced_cjk: True when force_cjk override caused acceptance despite no savings + - filler_drops, cjk, symbol, abbrev: substitution counts by category + - total_subs: total substitutions applied + """ + if not text or not text.strip(): + return text, {'original_tokens': 0, 'result_tokens': 0, 'saved': 0, + 'applied': False, 'forced_cjk': False, 'total_subs': 0} + + original_tokens = _ntok(text, tokenizer) + result, stats = _apply_subs(text) + result_tokens = _ntok(result, tokenizer) + saved = original_tokens - result_tokens + + stats['original_tokens'] = original_tokens + stats['result_tokens'] = result_tokens + stats['saved'] = saved + stats['forced_cjk'] = False + + if saved > 0: + stats['applied'] = True + return result, stats + if force_cjk and stats.get('cjk', 0) > 0: + stats['applied'] = True + stats['forced_cjk'] = True + return result, stats + stats['applied'] = False + return text, stats + + +# ── CLI: batch process a JSONL file ──────────────────────────────────── +if __name__ == '__main__': + import json, sys, argparse + from transformers import AutoTokenizer + + parser = argparse.ArgumentParser(description='Token-max post-processor for think blocks') + parser.add_argument('--input', required=True, help='Input JSONL (messages format)') + parser.add_argument('--output', help='Output JSONL (default: dry run, stats only)') + parser.add_argument('--tokenizer', default='ZelligeAI/tessera-compressor', + help='HF repo id or local path of the tokenizer to count savings under') + parser.add_argument('--force-cjk', action='store_true', + help='Force CJK substitutions even if total tokens increase. ' + 'Prioritizes CJK adoption over token savings.') + args = parser.parse_args() + + tok = AutoTokenizer.from_pretrained(args.tokenizer, trust_remote_code=True) + + total_before = total_after = applied = skipped = forced = 0 + + out_lines = [] + with open(args.input) as f: + for line in f: + line = line.strip() + if not line: + continue + rec = json.loads(line) + for m in rec.get('messages', rec.get('conversations', [])): + role = m.get('role', m.get('from', '')) + if role not in ('assistant', 'gpt'): + continue + content_key = 'content' if 'content' in m else 'value' + c = m.get(content_key, '') or '' + if '' not in c or '' not in c: + continue + + # Extract think content, preserving prefix before and suffix after + think_start = c.index('') + len('') + think_end = c.index('') + prefix = c[:think_start - len('')] + think = c[think_start:think_end] + suffix = c[think_end + len(''):] + + maxed, stats = tokenmax_with_stats(think, tok, force_cjk=args.force_cjk) + + total_before += stats['original_tokens'] + if stats['applied']: + applied += 1 + total_after += stats['result_tokens'] + m[content_key] = f'{prefix}{maxed}{suffix}' + if stats.get('forced_cjk'): + forced += 1 + else: + skipped += 1 + total_after += stats['original_tokens'] + + out_lines.append(json.dumps(rec, ensure_ascii=False)) + + if args.output: + with open(args.output, 'w') as f: + for line in out_lines: + f.write(line + '\n') + + total = applied + skipped + saved = total_before - total_after + if total > 0: + print(f'Processed {total} think blocks') + print(f' Applied: {applied} ({100*applied/total:.0f}%)') + if forced: + print(f' Forced CJK: {forced} (applied despite no token savings)') + print(f' Skipped (no savings): {skipped}') + pct = f'{100*saved/total_before:.1f}' if total_before > 0 else '0.0' + print(f' Tokens: {total_before} → {total_after} = {saved:+d} ({pct}%)') + else: + print(f'No think blocks found in {args.input}') diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..c71eaaa --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea43b288542655d72d632195ab9b58ca2cd9532c292bf6667827ce899ad196bc +size 11422082 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..4fe0724 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,217 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 32768, + "pad_token": "<|PAD_TOKEN|>", + "padding_side": "left", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null, + "added_tokens_decoder": { + "151643": { + "content": "<|endoftext|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151644": { + "content": "<|im_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151645": { + "content": "<|im_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151646": { + "content": "<|object_ref_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151647": { + "content": "<|object_ref_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151648": { + "content": "<|box_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151649": { + "content": "<|box_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151650": { + "content": "<|quad_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151651": { + "content": "<|quad_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151652": { + "content": "<|vision_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151653": { + "content": "<|vision_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151654": { + "content": "<|vision_pad|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151655": { + "content": "<|image_pad|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151656": { + "content": "<|video_pad|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151657": { + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151658": { + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151659": { + "content": "<|fim_prefix|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151660": { + "content": "<|fim_middle|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151661": { + "content": "<|fim_suffix|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151662": { + "content": "<|fim_pad|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151663": { + "content": "<|repo_name|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151664": { + "content": "<|file_sep|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151665": { + "content": "<|PAD_TOKEN|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + } + }, + "chat_template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0]['role'] == 'system' %}\n {{- messages[0]['content'] }}\n {%- else %}\n {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n {%- endif %}\n {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within XML tags:\\n\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n\\n\\nFor each function call, return a json object with function name and arguments within XML tags:\\n\\n{\\\"name\\\": , \\\"arguments\\\": }\\n<|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0]['role'] == 'system' %}\n {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n {%- else %}\n {{- '<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {{- '<|im_start|>' + message.role }}\n {%- if message.content %}\n {{- '\\n' + message.content }}\n {%- endif %}\n {%- for tool_call in message.tool_calls %}\n {%- if tool_call.function is defined %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '\\n\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {{- tool_call.arguments | tojson }}\n {{- '}\\n' }}\n {%- endfor %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %} {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n\\n' }}\n {{- message.content }}\n {{- '\\n' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n" +} \ No newline at end of file