commit 25e4aaaf69a9cea0b944e472e17d2a5ae6437c88 Author: ModelHub XC Date: Wed Aug 5 22:33:17 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: fin-ai-lab/aux-2024 Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..a2ee696 --- /dev/null +++ b/README.md @@ -0,0 +1,123 @@ +--- +license: apache-2.0 +library_name: transformers +pipeline_tag: text-generation +tags: + - frontier-to-pit + - divergence-decoding + - point-in-time + - look-ahead-bias + - auxiliary-model +--- + +# Aux 2024 + +A 3B model trained from scratch **through the end of 2024** (knowledge cutoff January 2025), +released as part of [**Frontier to Point-in-Time**](https://frontiertopit.com/) — a toolkit for +reducing **look-ahead bias** in forecasting with LLMs, without sacrificing what makes them +useful in the first place. The toolkit adapts open-source frontier models to substantially +reduce look-ahead bias, applies the methods to Qwen 3.5 27B to convert it into a 2015 +point-in-time model, and ships production-ready inference code. + +This model is the recent-era half of the temporal pair used by **Divergence Decoding**: its +logits stand in for the frontier model's post-2015 knowledge, which the method cancels at +inference time, with no retraining of the large model. Like its twin, it is a from-scratch +model with a **128K context window** and best-in-class instruction following; the 2015 twin +appears as **"Aux 2015"** in the project's examples and benchmarks. + +The temporal pair: + +- **[Aux 2015](https://huggingface.co/fin-ai-lab/aux-2015)** — knowledge cutoff December 2015 +- **[Aux 2024](https://huggingface.co/fin-ai-lab/aux-2024)** — trained through the end of 2024 + (knowledge cutoff January 2025) + +The two models are era-matched: same architecture, tokenizer, and training recipe, differing +only in the time span of their pre-training corpus. Their **logit difference** is the signal +used by Divergence Decoding. + +## How it's used + +**Divergence Decoding (DD)** is state-of-the-art for Q&A unlearning and is designed to scale +to large datasets. Frontier to Point-in-Time uses it to unlearn all knowledge after the +cutoff of December 31, 2015. It requires the two auxiliary models and applies the following +inference-time adjustment to the large model's logits: + +``` +l̂_2015 = l_frontier + α · (l_aux-2015 − l_aux-2024) +``` + +where `l_frontier` are the frontier model's logits (Qwen 3.5 27B), `l_aux-2024` are this +model's, and `l_aux-2015` are the 2015 twin's +([Aux 2015](https://huggingface.co/fin-ai-lab/aux-2015)). + +The auxiliary models are trained from scratch with the Qwen 3.5 tokenizer, then SFT'd on +samples distilled from the large model so they inherit its chat template and response style. +Unlike prior work, which saves checkpoints from a single training run as data is introduced +chronologically, each auxiliary model is trained independently. + +See the [project page](https://frontiertopit.com/) for the full method — including the +feature-steering component — and the +[GitHub repository](https://github.com/fin-ai-lab/frontier-to-pit/) for production-ready +inference code. + +## Model details + +- **Architecture:** `MinistralDualRope` — a Ministral (Llama-family math: SiLU-gated MLP, + RMSNorm, GQA, rotary, no biases) with a per-layer sliding/full attention pattern and a + second, unscaled rotary for the sliding layers (the Gemma-3 dual-RoPE design). +- **Parameters:** ~3.4B — 28 layers, hidden size 3072, 24 attention heads, 8 KV heads, + head dim 128, intermediate size 8192. +- **Attention:** sliding window 512, every 6th layer full attention. +- **Context length:** 131,072 (llama3 RoPE scaling, factor 64). +- **Vocab:** 248,320 (shared with the frontier model so DD can bridge logits). +- **Precision:** bf16. +- **Training:** temporal cooldown base followed by a partial supervised fine-tuning pass on + samples distilled from the frontier model, so it inherits the frontier model's chat + template and response style. The chat template supports optional thinking + (`enable_thinking`). + +## Usage + +This is a custom architecture — load with `trust_remote_code=True`: + +```python +from transformers import AutoModelForCausalLM, AutoTokenizer + +model = AutoModelForCausalLM.from_pretrained( + "fin-ai-lab/aux-2024", trust_remote_code=True, torch_dtype="bfloat16" +) +tok = AutoTokenizer.from_pretrained("fin-ai-lab/aux-2024", trust_remote_code=True) +``` + +## Citation + +```bibtex +@inproceedings{ + merchant2026divergence, + title={Divergence Decoding: Inference-Time Unlearning via Auxiliary Models}, + author={Humzah Merchant and Bradford Levy}, + booktitle={Forty-third International Conference on Machine Learning}, + year={2026}, + url={https://openreview.net/forum?id=JPbp2S9yTO} +} +@inproceedings{ + merchant2026a, + title={A Fast and Effective Solution to the Problem of Look-ahead Bias in {LLM}s}, + author={Humzah Merchant and Bradford Levy}, + booktitle={NeurIPS 2025 Workshop: Generative AI in Finance}, + year={2026}, + url={https://openreview.net/forum?id=zYsLIPgM28} +} +@inproceedings{ + merchant2026forecasting, + title={Forecasting With {LLM}s: Improved Generalization Through Feature Steering}, + author={Humzah Merchant and Bradford Levy}, + booktitle={Forecasting as a New Frontier of Intelligence}, + year={2026}, + url={https://openreview.net/forum?id=ppN6CmoNOk} +} +``` + +## License + +Apache-2.0. diff --git a/config.json b/config.json new file mode 100644 index 0000000..76d4a11 --- /dev/null +++ b/config.json @@ -0,0 +1,70 @@ +{ + "architectures": [ + "MinistralDualRopeForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "auto_map": { + "AutoConfig": "configuration_ministral_dual_rope.MinistralDualRopeConfig", + "AutoModel": "modeling_ministral_dual_rope.MinistralDualRopeModel", + "AutoModelForCausalLM": "modeling_ministral_dual_rope.MinistralDualRopeForCausalLM" + }, + "bos_token_id": 248044, + "eos_token_id": 248044, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 3072, + "initializer_range": 0.02, + "intermediate_size": 8192, + "layer_types": [ + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention" + ], + "max_position_embeddings": 131072, + "model_type": "ministral_dual_rope", + "num_attention_heads": 24, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": null, + "rms_norm_eps": 1e-05, + "rope_local_base_freq": 500000.0, + "rope_parameters": { + "factor": 64.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 2048, + "rope_theta": 500000.0, + "rope_type": "llama3" + }, + "sliding_window": 512, + "tie_word_embeddings": false, + "transformers_version": "5.6.2", + "use_cache": true, + "vocab_size": 248320 +} diff --git a/configuration_ministral_dual_rope.py b/configuration_ministral_dual_rope.py new file mode 100644 index 0000000..b5869a3 --- /dev/null +++ b/configuration_ministral_dual_rope.py @@ -0,0 +1,54 @@ +"""Configuration for MinistralDualRope — a Ministral with a per-layer sliding-window / full +attention pattern and a second, UNSCALED rotary for the sliding layers. + +The base model is byte-for-byte a Ministral (== Llama math: SiLU-gated MLP, RMSNorm +without unit offset, GQA, rotary embeddings, no biases, no QK-norm, no sandwich norms, +no logit softcapping). Ministral already provides the per-layer sliding/full attention +pattern via ``layer_types`` (mask routing identical to how this model was trained). The +*only* addition over stock Ministral is dual RoPE: + + * full-attention layers use the standard ``rope_theta`` (+ ``rope_scaling``, e.g. the + llama3-scaled cache of a context-extended checkpoint); + * sliding (local) layers use an UNSCALED rotary at ``rope_local_base_freq``. + +This mirrors our pretraining stack (``pretraining/flexattn_patch.py``) and is the exact +Gemma-3 dual-RoPE design. For a factor-1 checkpoint with equal bases and no scaling the +two rotaries collapse to one, so a single ``rope_theta`` is exact; for a context-extended +checkpoint (``rope_scaling`` set) the global and local caches diverge and both are needed. +""" + +from __future__ import annotations + +from transformers.models.ministral.configuration_ministral import MinistralConfig + + +class MinistralDualRopeConfig(MinistralConfig): + r"""MinistralConfig + an unscaled local rotary base for the sliding layers. + + Extra arg beyond :class:`~transformers.MinistralConfig`: + rope_local_base_freq (`float`, *optional*): + RoPE base for the sliding (local) layers, applied with NO rope scaling. + Defaults to ``rope_theta`` (i.e. single-RoPE, byte-identical to Ministral). + + (``layer_types`` and ``sliding_window`` are inherited from Ministral and drive the + per-layer full/sliding attention pattern.) + """ + + model_type = "ministral_dual_rope" + + def __init__(self, rope_local_base_freq=None, **kwargs): + super().__init__(**kwargs) + if rope_local_base_freq is None: + rope_local_base_freq = self._global_rope_theta() + self.rope_local_base_freq = float(rope_local_base_freq) + + def _global_rope_theta(self) -> float: + """The GLOBAL rotary base, robust across transformers versions (5.x stores it in + ``rope_parameters['rope_theta']``; 4.x in ``rope_theta``).""" + rp = getattr(self, "rope_parameters", None) + if isinstance(rp, dict) and "rope_theta" in rp: + return float(rp["rope_theta"]) + return float(getattr(self, "rope_theta", 10000.0)) + + +__all__ = ["MinistralDualRopeConfig"] diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..7a699f3 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,4 @@ +{ + "bos_token_id": 248044, + "eos_token_id": 248044 +} \ No newline at end of file diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..dc5e8aa --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3a02cadfdfe3843466ed3c23334d77009de64bd7fadd95aa2433a1f94ac39181 +size 8688880528 diff --git a/modeling_ministral_dual_rope.py b/modeling_ministral_dual_rope.py new file mode 100644 index 0000000..d59a143 --- /dev/null +++ b/modeling_ministral_dual_rope.py @@ -0,0 +1,190 @@ +"""MinistralDualRope — Ministral with a per-layer sliding/full attention pattern AND a second, +unscaled rotary for the sliding layers. Loadable via ``trust_remote_code``. + +This is the MINIMAL delta over the *stock* transformers Ministral model: + + * Ministral already provides the Llama-style math (RMSNorm without unit offset, + SiLU-gated MLP, GQA, no biases, no QK-norm, no sandwich norms, no softcapping) AND + the per-layer sliding/full attention pattern routed by ``config.layer_types`` — the + attention/mask routing is bit-identical to how this model was trained. + * The ONE thing Ministral lacks is a *second* RoPE. Our full-attention layers use the + (llama3-scaled) global rotary; the sliding layers use an UNSCALED rotary at + ``rope_local_base_freq`` (the Gemma-3 dual-RoPE design). We add ``rotary_emb_local`` + and route ``position_embeddings`` per ``layer_types``. Everything else is stock + Ministral, inherited unchanged. + +The building blocks are pulled from the *installed* ``modeling_ministral`` so this stays +aligned with whatever transformers version loads the checkpoint. Verified to match the +faithful FlexLlama reference to the analytic-vs-training-buffer RoPE floor (FP32 top-1 +agreement ~100%, mean |Δ logit| ~2e-3). +""" + +from __future__ import annotations + +import copy +import inspect +from functools import partial +from typing import Optional + +import torch + +from transformers.models.ministral import modeling_ministral as _M + +from .configuration_ministral_dual_rope import MinistralDualRopeConfig + +MinistralModel = _M.MinistralModel +MinistralForCausalLM = _M.MinistralForCausalLM +MinistralRotaryEmbedding = _M.MinistralRotaryEmbedding +create_causal_mask = _M.create_causal_mask +create_sliding_window_causal_mask = _M.create_sliding_window_causal_mask +DynamicCache = _M.DynamicCache +BaseModelOutputWithPast = _M.BaseModelOutputWithPast +# `check_model_inputs` moved between transformers minor/major versions; fall back to a +# no-op so the forward still works if a given install lacks it. We deliberately do NOT +# apply transformers' `auto_docstring` to the forward: on tf>=5 it scans the signature at +# import and logs a scary `[ERROR] ... is part of ...'s signature, but not documented` for +# every undocumented kwarg (e.g. `cache_position`). That is pure lint noise — the forward +# is fully functional without it — but it panics downstream users, so we omit it. +check_model_inputs = getattr(_M, "check_model_inputs", lambda f: f) + +# The mask-builder signature drifts across transformers versions (the embeds kwarg was +# renamed ``input_embeds`` -> ``inputs_embeds`` and ``cache_position`` was dropped between +# 4.x and 5.x). Introspect once and pass only what each installed builder accepts, so the +# same shipped file loads under both. Computed at import time (cheap, version-stable). +_CAUSAL_MASK_PARAMS = set(inspect.signature(create_causal_mask).parameters) +_SLIDING_MASK_PARAMS = set(inspect.signature(create_sliding_window_causal_mask).parameters) + + +class MinistralDualRopeModel(MinistralModel): + config_class = MinistralDualRopeConfig + + def __init__(self, config: MinistralDualRopeConfig): + super().__init__(config) + # Second, UNSCALED rotary for the sliding (local) layers. The parent's + # ``self.rotary_emb`` is the (possibly llama3-scaled) GLOBAL rotary used by the + # full-attention layers; build a sibling at ``rope_local_base_freq`` with NO + # scaling. When base == rope_theta and there is no scaling the two are identical. + local_cfg = copy.deepcopy(config) + base = float(getattr(config, "rope_local_base_freq", None) or config._global_rope_theta()) + # Configure an UNSCALED rotary. The two transformers major versions expose RoPE + # differently and the branches MUST be mutually exclusive: + # * transformers >= 5 stores everything in the unified ``rope_parameters`` dict + # and exposes ``rope_scaling`` as an ALIAS *onto* it — so setting + # ``rope_scaling`` here would clobber ``rope_parameters`` back to ``None`` and + # blow up inside ``MinistralRotaryEmbedding``. Set ONLY ``rope_parameters``. + # * transformers 4.x has no ``rope_parameters``; the rotary reads ``rope_scaling`` + # (must be None to disable llama3 scaling) + a top-level ``rope_theta``. + if hasattr(local_cfg, "rope_parameters"): # transformers >= 5 + local_cfg.rope_parameters = {"rope_type": "default", "rope_theta": base} + else: # transformers 4.x + local_cfg.rope_scaling = None + local_cfg.rope_theta = base + self.rotary_emb_local = MinistralRotaryEmbedding(config=local_cfg) + self.post_init() + + @check_model_inputs + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values=None, + inputs_embeds: Optional[torch.FloatTensor] = None, + use_cache: Optional[bool] = None, + cache_position: Optional[torch.LongTensor] = None, + **kwargs, + ) -> BaseModelOutputWithPast: + # Verbatim from MinistralModel.forward except the two dual-RoPE lines marked (*). + if (input_ids is None) ^ (inputs_embeds is not None): + raise ValueError("You must specify exactly one of input_ids or inputs_embeds") + + if inputs_embeds is None: + inputs_embeds = self.embed_tokens(input_ids) + + if use_cache and past_key_values is None: + past_key_values = DynamicCache(config=self.config) + + if cache_position is None: + past_seen = past_key_values.get_seq_length() if past_key_values is not None else 0 + cache_position = torch.arange( + past_seen, past_seen + inputs_embeds.shape[1], device=inputs_embeds.device + ) + + if position_ids is None: + position_ids = cache_position.unsqueeze(0) + + # `generate` may already pass a prepared {type: mask} dict; else build both masks. + if not isinstance(causal_mask_mapping := attention_mask, dict): + # Superset of every kwarg any transformers version's mask builder wants; the + # two embeds spellings and cache_position are filtered per the installed + # signature (see _CAUSAL_MASK_PARAMS above) so this works on 4.x and >=5. + _mask_src = { + "config": self.config, + "input_embeds": inputs_embeds, # transformers 4.x + "inputs_embeds": inputs_embeds, # transformers >= 5 + "attention_mask": attention_mask, + "cache_position": cache_position, # transformers 4.x only + "past_key_values": past_key_values, + "position_ids": position_ids, + } + causal_mask_mapping = { + "full_attention": create_causal_mask( + **{k: v for k, v in _mask_src.items() if k in _CAUSAL_MASK_PARAMS}), + "sliding_attention": create_sliding_window_causal_mask( + **{k: v for k, v in _mask_src.items() if k in _SLIDING_MASK_PARAMS}), + } + + hidden_states = inputs_embeds + # (*) Dual RoPE: full-attention layers get the (scaled) global rotary, sliding + # layers get the unscaled local rotary — routed by the same layer type as the + # mask above. + position_embeddings = { + "full_attention": self.rotary_emb(hidden_states, position_ids), + "sliding_attention": self.rotary_emb_local(hidden_states, position_ids), + } + + for i, decoder_layer in enumerate(self.layers[: self.config.num_hidden_layers]): + # (*) Route the mask AND the rope by the layer's attention type. Read it from + # ``config.layer_types`` (stable across versions) rather than the layer + # attribute, which transformers renamed ``attention_type`` -> ``layer_type`` + # between 4.x and 5.x. + layer_type = self.config.layer_types[i] + layer_kwargs = dict( + attention_mask=causal_mask_mapping[layer_type], + position_ids=position_ids, + past_key_values=past_key_values, + use_cache=use_cache, + cache_position=cache_position, + position_embeddings=position_embeddings[layer_type], + **kwargs, + ) + if self.gradient_checkpointing and self.training: + # Honor activation checkpointing. The stock MinistralModel.forward does + # this; our override MUST replicate it or long-context SFT/RL (this is a + # 32K model) OOMs — even under LoRA, since activations, not optimizer + # state, dominate at long sequence length. Bake the per-layer kwargs into + # a partial so we do NOT depend on the decoder layer's positional arg + # order, which drifts across transformers versions. + hidden_states = self._gradient_checkpointing_func( + partial(decoder_layer.__call__, **layer_kwargs), hidden_states + ) + else: + hidden_states = decoder_layer(hidden_states, **layer_kwargs) + + hidden_states = self.norm(hidden_states) + return BaseModelOutputWithPast( + last_hidden_state=hidden_states, + past_key_values=past_key_values if use_cache else None, + ) + + +class MinistralDualRopeForCausalLM(MinistralForCausalLM): + config_class = MinistralDualRopeConfig + + def __init__(self, config: MinistralDualRopeConfig): + super().__init__(config) + self.model = MinistralDualRopeModel(config) + self.post_init() + + +__all__ = ["MinistralDualRopeForCausalLM", "MinistralDualRopeModel"] diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..a73a846 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f9e4d4901a92b997e463c1f46055088b6cca5ca61a6522d1b9f64c4bb81cb42 +size 12807982 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..eda48d3 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,305 @@ +{ + "add_prefix_space": false, + "added_tokens_decoder": { + "248044": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "248045": { + "content": "<|im_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "248046": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "248047": { + "content": "<|object_ref_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "248048": { + "content": "<|object_ref_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "248049": { + "content": "<|box_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "248050": { + "content": "<|box_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "248051": { + "content": "<|quad_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "248052": { + "content": "<|quad_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "248053": { + "content": "<|vision_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "248054": { + "content": "<|vision_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "248055": { + "content": "<|vision_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "248056": { + "content": "<|image_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "248057": { + "content": "<|video_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "248058": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "248059": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "248060": { + "content": "<|fim_prefix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "248061": { + "content": "<|fim_middle|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "248062": { + "content": "<|fim_suffix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "248063": { + "content": "<|fim_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "248064": { + "content": "<|repo_name|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "248065": { + "content": "<|file_sep|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "248066": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "248067": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "248068": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "248069": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "248070": { + "content": "<|audio_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "248071": { + "content": "<|audio_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "248072": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "248073": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "248074": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "248075": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "248076": { + "content": "<|audio_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "bos_token": null, + "chat_template": "{%- set image_count = namespace(value=0) %}\n{%- set video_count = namespace(value=0) %}\n{%- macro render_content(content, do_vision_count, is_system_content=false) %}\n {%- if content is string %}\n {{- content }}\n {%- elif content is iterable and content is not mapping %}\n {%- for item in content %}\n {%- if 'image' in item or 'image_url' in item or item.type == 'image' %}\n {%- if is_system_content %}\n {{- raise_exception('System message cannot contain images.') }}\n {%- endif %}\n {%- if do_vision_count %}\n {%- set image_count.value = image_count.value + 1 %}\n {%- endif %}\n {%- if add_vision_id %}\n {{- 'Picture ' ~ image_count.value ~ ': ' }}\n {%- endif %}\n {{- '<|vision_start|><|image_pad|><|vision_end|>' }}\n {%- elif 'video' in item or item.type == 'video' %}\n {%- if is_system_content %}\n {{- raise_exception('System message cannot contain videos.') }}\n {%- endif %}\n {%- if do_vision_count %}\n {%- set video_count.value = video_count.value + 1 %}\n {%- endif %}\n {%- if add_vision_id %}\n {{- 'Video ' ~ video_count.value ~ ': ' }}\n {%- endif %}\n {{- '<|vision_start|><|video_pad|><|vision_end|>' }}\n {%- elif 'text' in item %}\n {{- item.text }}\n {%- else %}\n {{- raise_exception('Unexpected item type in content.') }}\n {%- endif %}\n {%- endfor %}\n {%- elif content is none or content is undefined %}\n {{- '' }}\n {%- else %}\n {{- raise_exception('Unexpected content type.') }}\n {%- endif %}\n{%- endmacro %}\n{%- if not messages %}\n {{- raise_exception('No messages provided.') }}\n{%- endif %}\n{%- if tools and tools is iterable and tools is not mapping %}\n {{- '<|im_start|>system\\n' }}\n {{- \"# Tools\\n\\nYou have access to the following functions:\\n\\n\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n\" }}\n {{- '\\n\\nIf you choose to call a function ONLY reply in the following format with NO suffix:\\n\\n\\n\\n\\nvalue_1\\n\\n\\nThis is the value for the second parameter\\nthat can span\\nmultiple lines\\n\\n\\n\\n\\n\\nReminder:\\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\\n- Required parameters MUST be specified\\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\\n' }}\n {%- if messages[0].role == 'system' %}\n {%- set content = render_content(messages[0].content, false, true)|trim %}\n {%- if content %}\n {{- '\\n\\n' + content }}\n {%- endif %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {%- set content = render_content(messages[0].content, false, true)|trim %}\n {{- '<|im_start|>system\\n' + content + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}\n{%- for message in messages[::-1] %}\n {%- set index = (messages|length - 1) - loop.index0 %}\n {%- if ns.multi_step_tool and message.role == \"user\" %}\n {%- set content = render_content(message.content, false)|trim %}\n {%- if not(content.startswith('') and content.endswith('')) %}\n {%- set ns.multi_step_tool = false %}\n {%- set ns.last_query_index = index %}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if ns.multi_step_tool %}\n {{- raise_exception('No user query found in messages.') }}\n{%- endif %}\n{%- for message in messages %}\n {%- set content = render_content(message.content, true)|trim %}\n {%- if message.role == \"system\" %}\n {%- if not loop.first %}\n {{- raise_exception('System message must be at the beginning.') }}\n {%- endif %}\n {%- elif message.role == \"user\" %}\n {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {%- set reasoning_content = '' %}\n {%- if message.reasoning_content is string %}\n {%- set reasoning_content = message.reasoning_content %}\n {%- else %}\n {%- if '' in content %}\n {%- set reasoning_content = content.split('')[0].rstrip('\\n').split('')[-1].lstrip('\\n') %}\n {%- set content = content.split('')[-1].lstrip('\\n') %}\n {%- endif %}\n {%- endif %}\n {%- set reasoning_content = reasoning_content|trim %}\n {%- if loop.index0 > ns.last_query_index %}\n {{- '<|im_start|>' + message.role + '\\n\\n' + reasoning_content + '\\n\\n\\n' + content }}\n {%- else %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- endif %}\n {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}\n {%- for tool_call in message.tool_calls %}\n {%- if tool_call.function is defined %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {%- if loop.first %}\n {%- if content|trim %}\n {{- '\\n\\n\\n\\n' }}\n {%- else %}\n {{- '\\n\\n' }}\n {%- endif %}\n {%- else %}\n {{- '\\n\\n\\n' }}\n {%- endif %}\n {%- if tool_call.arguments is defined %}\n {%- for args_name, args_value in tool_call.arguments|items %}\n {{- '\\n' }}\n {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}\n {{- args_value }}\n {{- '\\n\\n' }}\n {%- endfor %}\n {%- endif %}\n {{- '\\n' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.previtem and loop.previtem.role != \"tool\" %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n\\n' }}\n {{- content }}\n {{- '\\n' }}\n {%- if not loop.last and loop.nextitem.role != \"tool\" %}\n {{- '<|im_end|>\\n' }}\n {%- elif loop.last %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- else %}\n {{- raise_exception('Unexpected message role.') }}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n {%- if enable_thinking is defined and enable_thinking is false %}\n {{- '\\n\\n\\n\\n' }}\n {%- else %}\n {{- '\\n' }}\n {%- endif %}\n{%- endif %}", + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "model_max_length": 262144, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null, + "add_bos_token": false, + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "extra_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + } +} \ No newline at end of file