初始化项目，由ModelHub XC社区提供模型

Model: veyra-ai/Veyra-30M-Base
Source: Original Platform

.gitattributes

@@ -0,0 +1,4 @@
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text

LICENSE

@@ -0,0 +1,11 @@
+Apache License
+Version 2.0, January 2004
+http://www.apache.org/licenses/
+
+Copyright 2026 Jdudeo
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0

README.md

@@ -0,0 +1,116 @@
+---
+license: apache-2.0
+language:
+- en
+library_name: transformers
+tags:
+- transformers
+- pytorch
+- safetensors
+- veyra
+- causal-lm
+- base-model
+- small-language-model
+- pretraining
+pipeline_tag: text-generation
+datasets:
+- HuggingFaceTB/cosmopedia-v2
+parameters: 31988224
+---
+
+# Veyra 30M Base 5B Checkpoint
+
+This is an early **Veyra-30M base checkpoint** trained for approximately **5B pretraining tokens**.
+
+It is **not instruction tuned** and should not be evaluated like a finished chat assistant. It is expected to hallucinate, repeat, fail simple factual/math prompts, and continue text in odd ways. This checkpoint is uploaded for transparency, reproducibility, and milestone tracking before further continuation training. This checkpoint is not optimized for use on edge devices yet.
+
+## Training summary
+
+Approximate training stages:
+
+- **1B tokens**: Cosmopedia v2 bootstrap pretraining.
+- **+1.5B tokens**: mixed continuation using Cosmopedia-v2 repository configs including `cosmopedia-v2`, `fineweb-edu-dedup`, and `python-edu`.
+- **+2.5B tokens**: Went back to Cosmopedia v2 but increased context length from 512 -> 1024.
+- **Total**: about **5B pretraining tokens**.
+
+## Architecture
+
+Veyra-30M is a small attention-sparse decoder-only language model.
+
+Key details:
+
+- Exact parameters: **31,988,224** / **31.99M**
+- Vocabulary: 8,192 tokens
+- Hidden size: 512
+- Layers: 8
+- Attention heads: 8 query heads, 2 KV heads
+- MLP intermediate size: 2048
+- Activation: SwiGLU
+- Normalization: RMSNorm
+- Position encoding: RoPE
+- Tied token embeddings / LM head
+- Context in this checkpoint: 1024 tokens
+
+## Loading
+
+This repository uses custom Transformers code.
+
+Minimal usage:
+
+    from transformers import AutoTokenizer, AutoModelForCausalLM
+    import torch
+
+    repo = "veyra-ai/veyra-30m-base-5b-tokens"
+
+    tokenizer = AutoTokenizer.from_pretrained(repo, trust_remote_code=True)
+    model = AutoModelForCausalLM.from_pretrained(repo, trust_remote_code=True, dtype=torch.float32)
+    model.eval()
+
+    prompt = "Photosynthesis is the process by which"
+    input_ids = tokenizer.encode(prompt, add_special_tokens=False, return_tensors="pt")
+
+    with torch.no_grad():
+        out = model.generate(
+            input_ids,
+            do_sample=True,
+            temperature=0.5,
+            top_k=30,
+            repetition_penalty=1.15,
+            no_repeat_ngram_size=2,
+            max_new_tokens=80,
+        )
+
+    print(tokenizer.decode(out[0], skip_special_tokens=True))
+
+For raw completion prompts, use `add_special_tokens=False`.
+
+## Optimizer
+
+Training used:
+
+- **CosineGatedAdam / CGA-v0** on 2D projection matrices
+- **AdamW** on embeddings, norms, tied head, and auxiliary parameters
+
+## Intended use
+
+This checkpoint is primarily for:
+
+- continued pretraining
+- research / ablations
+- tracking Veyra training milestones
+- testing tiny model behavior
+
+It is not intended for production use or reliable factual answering.
+
+## Known limitations
+
+This model can:
+
+- hallucinate confidently
+- repeat phrases
+- fail arithmetic
+- fail simple factual questions
+- produce fake code
+- continue in textbook-like or tutorial-like styles
+
+Further continuation pretraining and post-training are planned.

chat_template.jinja

@@ -0,0 +1,4 @@
+{% for message in messages %}<|im_start|>{{ message['role'] }}
+{{ message['content'] }}<|im_end|>
+{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant
+{% endif %}

config.json

@@ -0,0 +1,32 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 0,
+  "dtype": "float32",
+  "eos_token_id": 1,
+  "head_dim": 64,
+  "hidden_act": "silu",
+  "hidden_size": 512,
+  "initializer_range": 0.02,
+  "intermediate_size": 2048,
+  "max_position_embeddings": 1024,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 8,
+  "num_hidden_layers": 8,
+  "num_key_value_heads": 2,
+  "pad_token_id": 2,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.10.2",
+  "use_cache": true,
+  "vocab_size": 8192
+}

configuration_veyra.py

@@ -0,0 +1,57 @@
+from transformers import PretrainedConfig
+
+
+class VeyraConfig(PretrainedConfig):
+    model_type = "veyra"
+
+    def __init__(
+        self,
+        vocab_size=8192,
+        d_model=512,
+        hidden_size=512,
+        n_q_heads=8,
+        num_attention_heads=8,
+        n_kv_heads=2,
+        num_key_value_heads=2,
+        intermediate_size=2048,
+        max_seq_len=1024,
+        max_position_embeddings=1024,
+        layer_pattern=("A", "M", "A", "M", "A", "M", "A", "M"),
+        num_hidden_layers=8,
+        rms_norm_eps=1e-6,
+        rope_theta=10000.0,
+        tie_word_embeddings=True,
+        bos_token_id=0,
+        eos_token_id=1,
+        pad_token_id=2,
+        unk_token_id=3,
+        **kwargs,
+    ):
+        self.vocab_size = vocab_size
+        self.d_model = d_model
+        self.hidden_size = hidden_size
+
+        self.n_q_heads = n_q_heads
+        self.num_attention_heads = num_attention_heads
+        self.n_kv_heads = n_kv_heads
+        self.num_key_value_heads = num_key_value_heads
+
+        self.intermediate_size = intermediate_size
+        self.max_seq_len = max_seq_len
+        self.max_position_embeddings = max_position_embeddings
+
+        self.layer_pattern = list(layer_pattern)
+        self.num_hidden_layers = num_hidden_layers
+
+        self.rms_norm_eps = rms_norm_eps
+        self.rope_theta = rope_theta
+        self.tie_word_embeddings = tie_word_embeddings
+
+        super().__init__(
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            pad_token_id=pad_token_id,
+            unk_token_id=unk_token_id,
+            tie_word_embeddings=tie_word_embeddings,
+            **kwargs,
+        )

conversion_report.json

@@ -0,0 +1,129 @@
+{
+  "source_repo": "veyra-ai/Veyra-30M-Base",
+  "legacy_repo": "Jdudeo/Veyra-30M-Base-Legacy",
+  "converted_folder": "/content/drive/MyDrive/veyra30m_repo_conversion/converted_llama/Veyra-30M-Base",
+  "chosen_permute_qk": true,
+  "summary_perm": {
+    "mean_abs_avg": 3.0396581353540697e-06,
+    "max_abs_max": 2.09808349609375e-05,
+    "last_token_mean_abs_avg": 2.9613922184320733e-06,
+    "top1_agree_avg": 1.0
+  },
+  "summary_noperm": {
+    "mean_abs_avg": 0.5820164679655265,
+    "max_abs_max": 8.126092910766602,
+    "last_token_mean_abs_avg": 0.7566529298705973,
+    "top1_agree_avg": 0.757915905543736
+  },
+  "summary_final": {
+    "mean_abs_avg": 3.0396581353540697e-06,
+    "max_abs_max": 2.09808349609375e-05,
+    "last_token_mean_abs_avg": 2.9613922184320733e-06,
+    "top1_agree_avg": 1.0
+  },
+  "rows_final": [
+    {
+      "prompt": "Once upon a time in a digital kingdom,",
+      "tokens": 10,
+      "max_abs": 1.811981201171875e-05,
+      "mean_abs": 2.6239151793561177e-06,
+      "last_token_mean_abs": 3.180772182531655e-06,
+      "top1_agree": 1.0
+    },
+    {
+      "prompt": "Timmy and Sally went on an adventure to",
+      "tokens": 12,
+      "max_abs": 1.9550323486328125e-05,
+      "mean_abs": 2.7159644560015295e-06,
+      "last_token_mean_abs": 1.937556589837186e-06,
+      "top1_agree": 1.0
+    },
+    {
+      "prompt": "Question: What is the capital of France?\nAnswer:",
+      "tokens": 14,
+      "max_abs": 2.09808349609375e-05,
+      "mean_abs": 2.5517792892060243e-06,
+      "last_token_mean_abs": 1.6593803593423218e-06,
+      "top1_agree": 1.0
+    },
+    {
+      "prompt": "I am an AI",
+      "tokens": 4,
+      "max_abs": 1.990795135498047e-05,
+      "mean_abs": 2.7443497856438626e-06,
+      "last_token_mean_abs": 2.7508117454999592e-06,
+      "top1_agree": 1.0
+    },
+    {
+      "prompt": "1+1=",
+      "tokens": 4,
+      "max_abs": 1.52587890625e-05,
+      "mean_abs": 2.357231778660207e-06,
+      "last_token_mean_abs": 1.859174744822667e-06,
+      "top1_agree": 1.0
+    },
+    {
+      "prompt": "def add_numbers(a, b):",
+      "tokens": 11,
+      "max_abs": 2.002716064453125e-05,
+      "mean_abs": 2.8609217679331778e-06,
+      "last_token_mean_abs": 3.9186052163131535e-06,
+      "top1_agree": 1.0
+    },
+    {
+      "prompt": " ",
+      "tokens": 1,
+      "max_abs": 1.8835067749023438e-05,
+      "mean_abs": 5.42344469067757e-06,
+      "last_token_mean_abs": 5.42344469067757e-06,
+      "top1_agree": 1.0
+    }
+  ],
+  "llama_config": {
+    "transformers_version": "5.10.2",
+    "architectures": [
+      "LlamaForCausalLM"
+    ],
+    "output_hidden_states": false,
+    "return_dict": true,
+    "dtype": "float32",
+    "chunk_size_feed_forward": 0,
+    "is_encoder_decoder": false,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "problem_type": null,
+    "vocab_size": 8192,
+    "hidden_size": 512,
+    "intermediate_size": 2048,
+    "num_hidden_layers": 8,
+    "num_attention_heads": 8,
+    "num_key_value_heads": 2,
+    "hidden_act": "silu",
+    "max_position_embeddings": 1024,
+    "initializer_range": 0.02,
+    "rms_norm_eps": 1e-06,
+    "use_cache": true,
+    "pad_token_id": 2,
+    "bos_token_id": 0,
+    "eos_token_id": 1,
+    "pretraining_tp": 1,
+    "tie_word_embeddings": true,
+    "rope_parameters": {
+      "rope_theta": 10000.0,
+      "rope_type": "default"
+    },
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "mlp_bias": false,
+    "head_dim": 64,
+    "_name_or_path": "",
+    "model_type": "llama",
+    "output_attentions": false
+  }
+}

generation_config.json

@@ -0,0 +1,10 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 0,
+  "eos_token_id": 1,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "pad_token_id": 2,
+  "transformers_version": "5.10.2",
+  "use_cache": true
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:116921afb6733e914ce38fdb6b70b3d3db48fed194fd1ce5fc930bae288d704f
+size 138454968

modeling_veyra.py

@@ -0,0 +1,245 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from transformers import PreTrainedModel, GenerationMixin
+from transformers.modeling_outputs import CausalLMOutputWithPast
+
+from .configuration_veyra import VeyraConfig
+
+
+class RMSNorm(nn.Module):
+    def __init__(self, dim, eps=1e-6):
+        super().__init__()
+        self.weight = nn.Parameter(torch.ones(dim))
+        self.eps = eps
+
+    def forward(self, x):
+        return self.weight * x * torch.rsqrt(x.pow(2).mean(dim=-1, keepdim=True) + self.eps)
+
+
+class RotaryEmbedding(nn.Module):
+    def __init__(self, dim, max_seq_len, base=10000.0):
+        super().__init__()
+        self.dim = dim
+        self.max_seq_len = max_seq_len
+        self.base = base
+
+    def forward(self, x, positions):
+        # No buffers. Compute everything fresh to avoid HF remote-code buffer corruption.
+        half_dim = self.dim // 2
+
+        inv_freq = 1.0 / (
+            self.base ** (
+                torch.arange(0, self.dim, 2, device=x.device, dtype=torch.float32) / self.dim
+            )
+        )
+
+        positions = positions.to(device=x.device, dtype=torch.float32)
+        freqs = torch.outer(positions, inv_freq)
+
+        cos = torch.cos(freqs).to(dtype=x.dtype)[None, None, :, :]
+        sin = torch.sin(freqs).to(dtype=x.dtype)[None, None, :, :]
+
+        x_even = x[..., 0::2]
+        x_odd = x[..., 1::2]
+
+        out_even = x_even * cos - x_odd * sin
+        out_odd = x_even * sin + x_odd * cos
+
+        return torch.stack((out_even, out_odd), dim=-1).flatten(-2)
+
+
+class SwiGLU(nn.Module):
+    def __init__(self, d_model, intermediate_size):
+        super().__init__()
+        self.gate_proj = nn.Linear(d_model, intermediate_size, bias=False)
+        self.up_proj = nn.Linear(d_model, intermediate_size, bias=False)
+        self.down_proj = nn.Linear(intermediate_size, d_model, bias=False)
+
+    def forward(self, x):
+        return self.down_proj(F.silu(self.gate_proj(x)) * self.up_proj(x))
+
+
+class GQAAttention(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+
+        assert config.d_model % config.n_q_heads == 0
+        assert config.n_q_heads % config.n_kv_heads == 0
+
+        self.d_model = config.d_model
+        self.n_q_heads = config.n_q_heads
+        self.n_kv_heads = config.n_kv_heads
+        self.head_dim = config.d_model // config.n_q_heads
+        self.n_rep = config.n_q_heads // config.n_kv_heads
+
+        self.q_proj = nn.Linear(config.d_model, config.n_q_heads * self.head_dim, bias=False)
+        self.k_proj = nn.Linear(config.d_model, config.n_kv_heads * self.head_dim, bias=False)
+        self.v_proj = nn.Linear(config.d_model, config.n_kv_heads * self.head_dim, bias=False)
+        self.o_proj = nn.Linear(config.n_q_heads * self.head_dim, config.d_model, bias=False)
+
+        self.rope = RotaryEmbedding(self.head_dim, config.max_seq_len, config.rope_theta)
+
+    def _shape_q(self, x):
+        b, t, _ = x.shape
+        return x.view(b, t, self.n_q_heads, self.head_dim).transpose(1, 2)
+
+    def _shape_kv(self, x):
+        b, t, _ = x.shape
+        return x.view(b, t, self.n_kv_heads, self.head_dim).transpose(1, 2)
+
+    def _repeat_kv(self, x):
+        if self.n_rep == 1:
+            return x
+        return x.repeat_interleave(self.n_rep, dim=1)
+
+    def forward(self, x):
+        b, t, _ = x.shape
+        positions = torch.arange(t, device=x.device)
+
+        q = self._shape_q(self.q_proj(x))
+        k = self._shape_kv(self.k_proj(x))
+        v = self._shape_kv(self.v_proj(x))
+
+        q = self.rope(q, positions)
+        k = self.rope(k, positions)
+
+        k = self._repeat_kv(k)
+        v = self._repeat_kv(v)
+
+        y = F.scaled_dot_product_attention(
+            q,
+            k,
+            v,
+            attn_mask=None,
+            dropout_p=0.0,
+            is_causal=True,
+        )
+
+        y = y.transpose(1, 2).contiguous().view(b, t, self.d_model)
+
+        return self.o_proj(y)
+
+
+class AttentionBlock(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.input_norm = RMSNorm(config.d_model, config.rms_norm_eps)
+        self.attn = GQAAttention(config)
+        self.post_attn_norm = RMSNorm(config.d_model, config.rms_norm_eps)
+        self.mlp = SwiGLU(config.d_model, config.intermediate_size)
+
+    def forward(self, x):
+        x = x + self.attn(self.input_norm(x))
+        x = x + self.mlp(self.post_attn_norm(x))
+        return x
+
+
+class MLPOnlyBlock(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.norm = RMSNorm(config.d_model, config.rms_norm_eps)
+        self.mlp = SwiGLU(config.d_model, config.intermediate_size)
+
+    def forward(self, x):
+        return x + self.mlp(self.norm(x))
+
+
+class VeyraPreTrainedModel(PreTrainedModel):
+    config_class = VeyraConfig
+    base_model_prefix = ""
+    supports_gradient_checkpointing = False
+    _no_split_modules = ["AttentionBlock", "MLPOnlyBlock"]
+
+
+class VeyraForCausalLM(VeyraPreTrainedModel, GenerationMixin):
+    config_class = VeyraConfig
+    _tied_weights_keys = {"lm_head.weight": "token_emb.weight"}
+
+    def __init__(self, config):
+        super().__init__(config)
+        self.config = config
+
+        self.token_emb = nn.Embedding(config.vocab_size, config.d_model)
+
+        blocks = []
+        for kind in config.layer_pattern:
+            if kind == "A":
+                blocks.append(AttentionBlock(config))
+            elif kind == "M":
+                blocks.append(MLPOnlyBlock(config))
+            else:
+                raise ValueError(f"Unknown layer kind: {kind}")
+
+        self.blocks = nn.ModuleList(blocks)
+        self.norm = RMSNorm(config.d_model, config.rms_norm_eps)
+        self.lm_head = nn.Linear(config.d_model, config.vocab_size, bias=False)
+
+        if config.tie_word_embeddings:
+            self.lm_head.weight = self.token_emb.weight
+
+        # Needed by newer Transformers internals, but we intentionally avoid post_init().
+        self.all_tied_weights_keys = {"lm_head.weight": "token_emb.weight"}
+
+    def tie_weights(self, *args, **kwargs):
+        if self.config.tie_word_embeddings:
+            self.lm_head.weight = self.token_emb.weight
+
+    def get_input_embeddings(self):
+        return self.token_emb
+
+    def set_input_embeddings(self, value):
+        self.token_emb = value
+        if self.config.tie_word_embeddings:
+            self.lm_head.weight = self.token_emb.weight
+
+    def get_output_embeddings(self):
+        return self.lm_head
+
+    def set_output_embeddings(self, new_embeddings):
+        self.lm_head = new_embeddings
+
+    def forward(
+        self,
+        input_ids=None,
+        attention_mask=None,
+        labels=None,
+        return_dict=True,
+        **kwargs,
+    ):
+        if input_ids is None:
+            raise ValueError("input_ids must be provided")
+
+        x = self.token_emb(input_ids)
+
+        for block in self.blocks:
+            x = block(x)
+
+        logits = self.lm_head(self.norm(x))
+
+        loss = None
+        if labels is not None:
+            loss = F.cross_entropy(
+                logits[:, :-1, :].contiguous().view(-1, logits.size(-1)),
+                labels[:, 1:].contiguous().view(-1),
+                ignore_index=-100,
+            )
+
+        if not return_dict:
+            output = (logits,)
+            return ((loss,) + output) if loss is not None else output
+
+        return CausalLMOutputWithPast(
+            loss=loss,
+            logits=logits,
+            past_key_values=None,
+            hidden_states=None,
+            attentions=None,
+        )
+
+    def prepare_inputs_for_generation(self, input_ids, **kwargs):
+        max_len = self.config.max_seq_len
+        if input_ids.shape[1] > max_len:
+            input_ids = input_ids[:, -max_len:]
+        return {"input_ids": input_ids}

special_tokens_map.json

@@ -0,0 +1,6 @@
+{
+  "bos_token": "<|bos|>",
+  "eos_token": "<|eos|>",
+  "unk_token": "<|unk|>",
+  "pad_token": "<|pad|>"
+}

tokenizer_config.json

@@ -0,0 +1,36 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|bos|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|eos|>",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|tool_call|>",
+    "<|tool_result|>",
+    "<|context|>",
+    "<|reasoning|>",
+    "<|end_reasoning|>",
+    "<|answer|>",
+    "<|fim_prefix|>",
+    "<|fim_middle|>",
+    "<|fim_suffix|>",
+    "<|reserved_0|>",
+    "<|reserved_1|>",
+    "<|reserved_2|>",
+    "<|reserved_3|>",
+    "<|reserved_4|>",
+    "<|reserved_5|>",
+    "<|reserved_6|>",
+    "<|reserved_7|>",
+    "<|reserved_8|>",
+    "<|reserved_9|>"
+  ],
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1000000000,
+  "pad_token": "<|pad|>",
+  "tokenizer_class": "TokenizersBackend",
+  "unk_token": "<|unk|>"
+}

training_meta.json

@@ -0,0 +1,82 @@
+{
+  "name": "veyra-30m-base-5b-tokens",
+  "created_at": "2026-05-03T17:28:57.455174Z",
+  "total_pretraining_tokens": 2500000000,
+  "exact_parameter_count": 31988224,
+  "parameter_count_millions": 31.99,
+  "parameter_breakdown": {
+    "token_embedding_tied_lm_head": 4194304,
+    "attention_mlp_blocks_4x": 15208448,
+    "mlp_only_blocks_4x": 12584960,
+    "final_rmsnorm": 512,
+    "total": 31988224
+  },
+  "datasets": [
+    {
+      "repo": "HuggingFaceTB/cosmopedia-v2",
+      "configs_used": [
+        "cosmopedia-v2",
+        "fineweb-edu-dedup",
+        "python-edu"
+      ],
+      "usage": "1B token Cosmopedia bootstrap plus 1.5B token mixed continuation."
+    }
+  ],
+  "optimizer": {
+    "matrix_params": "CosineGatedAdam / CGA-v0",
+    "aux_params": "AdamW",
+    "cga_peak_lr": 5e-05,
+    "cga_max_update_rms": 3.0,
+    "aux_peak_lr": 0.0003,
+    "lr_schedule": "warmup then cosine decay during continuation"
+  },
+  "architecture": {
+    "model_type": "veyra",
+    "architectures": [
+      "VeyraForCausalLM"
+    ],
+    "library_name": "transformers",
+    "auto_map": {
+      "AutoConfig": "configuration_veyra.VeyraConfig",
+      "AutoModelForCausalLM": "modeling_veyra.VeyraForCausalLM"
+    },
+    "vocab_size": 8192,
+    "d_model": 512,
+    "hidden_size": 512,
+    "n_q_heads": 8,
+    "num_attention_heads": 8,
+    "n_kv_heads": 2,
+    "num_key_value_heads": 2,
+    "intermediate_size": 2048,
+    "max_seq_len": 1024,
+    "max_position_embeddings": 1024,
+    "layer_pattern": [
+      "A",
+      "M",
+      "A",
+      "M",
+      "A",
+      "M",
+      "A",
+      "M"
+    ],
+    "num_hidden_layers": 8,
+    "rms_norm_eps": 1e-06,
+    "rope_theta": 10000.0,
+    "tie_word_embeddings": true,
+    "bos_token_id": 0,
+    "eos_token_id": 1,
+    "pad_token_id": 2,
+    "unk_token_id": 3,
+    "dtype": "float32",
+    "torch_dtype": "float32",
+    "num_parameters": 31988224,
+    "parameter_count": 31988224,
+    "parameter_count_millions": 31.99,
+    "total_params": 31988224,
+    "trainable_params": 31988224,
+    "training_tokens": 2500000000,
+    "notes": "Attention-sparse AMAMAMAM decoder. Early base checkpoint, not instruction tuned."
+  },
+  "important_warning": "This is an early base model checkpoint, not an instruction-tuned assistant. It can hallucinate, repeat, and fail simple factual/math prompts."
+}

veyra_tokenizer_metadata.json

@@ -0,0 +1,58 @@
+{
+  "name": "veyra-tokenizer-8k",
+  "vocab_size": 8192,
+  "tokenizer_type": "BPE ByteLevel",
+  "language": "English-focused",
+  "chat_format": "ChatML",
+  "training_corpus": {
+    "total_texts": 250000,
+    "mix": {
+      "fineweb_edu": 0.7,
+      "synthetic_edu": 0.1,
+      "chatml": 0.08,
+      "markdown_docs": 0.05,
+      "json_tools": 0.04,
+      "python_fim": 0.03
+    },
+    "fineweb_edu": "HuggingFaceFW/fineweb-edu sample-10BT streaming",
+    "synthetic_sources": [
+      "generated educational prose templates",
+      "generated ChatML examples",
+      "generated markdown/docs examples",
+      "generated JSON/tool/context examples",
+      "generated Python/FIM examples"
+    ]
+  },
+  "special_tokens": [
+    "<|bos|>",
+    "<|eos|>",
+    "<|pad|>",
+    "<|unk|>",
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|tool_call|>",
+    "<|tool_result|>",
+    "<|context|>",
+    "<|reasoning|>",
+    "<|end_reasoning|>",
+    "<|answer|>",
+    "<|fim_prefix|>",
+    "<|fim_middle|>",
+    "<|fim_suffix|>",
+    "<|reserved_0|>",
+    "<|reserved_1|>",
+    "<|reserved_2|>",
+    "<|reserved_3|>",
+    "<|reserved_4|>",
+    "<|reserved_5|>",
+    "<|reserved_6|>",
+    "<|reserved_7|>",
+    "<|reserved_8|>",
+    "<|reserved_9|>"
+  ],
+  "notes": [
+    "Designed for Veyra 30M-class attention-sparse decoder models.",
+    "8k vocab including special tokens.",
+    "Includes ChatML, tool/context, reasoning experiment, FIM, and reserved tokens."
+  ]
+}