初始化项目，由ModelHub XC社区提供模型

Model: veyra-ai/Veyra-30M-Base-2.5B-Tokens
Source: Original Platform

.gitattributes

@@ -0,0 +1,4 @@
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text

LICENSE

@@ -0,0 +1,11 @@
+Apache License
+Version 2.0, January 2004
+http://www.apache.org/licenses/
+
+Copyright 2026 Jdudeo
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0

README.md

@@ -0,0 +1,116 @@
+---
+license: apache-2.0
+language:
+- en
+library_name: transformers
+tags:
+- transformers
+- pytorch
+- safetensors
+- veyra
+- causal-lm
+- base-model
+- small-language-model
+- pretraining
+pipeline_tag: text-generation
+datasets:
+- HuggingFaceTB/cosmopedia-v2
+parameters: 31988224
+new_version: veyra-ai/veyra-30m-base-5b-tokens
+---
+
+# Veyra 30M Base 2.5B Checkpoint !! 5B CHECKPOINT OUT NOW !!
+
+This is an early **Veyra-30M base checkpoint** trained for approximately **2.5B pretraining tokens**.
+
+It is **not instruction tuned** and should not be evaluated like a finished chat assistant. It is expected to hallucinate, repeat, fail simple factual/math prompts, and continue text in odd ways. This checkpoint is uploaded for transparency, reproducibility, and milestone tracking before further continuation training.
+
+## Training summary
+
+Approximate training stages:
+
+- **1B tokens**: Cosmopedia v2 bootstrap pretraining.
+- **+1.5B tokens**: mixed continuation using Cosmopedia-v2 repository configs including `cosmopedia-v2`, `fineweb-edu-dedup`, and `python-edu`.
+- **Total**: about **2.5B pretraining tokens**.
+
+## Architecture
+
+Veyra-30M is a small attention-sparse decoder-only language model.
+
+Key details:
+
+- Exact parameters: **31,988,224** / **31.99M**
+- Vocabulary: 8,192 tokens
+- Hidden size: 512
+- Layers: 8
+- Attention heads: 8 query heads, 2 KV heads
+- MLP intermediate size: 2048
+- Activation: SwiGLU
+- Normalization: RMSNorm
+- Position encoding: RoPE
+- Tied token embeddings / LM head
+- Context in this checkpoint: 512 tokens
+
+## Loading
+
+This repository uses custom Transformers code.
+
+Minimal usage:
+
+    from transformers import AutoTokenizer, AutoModelForCausalLM
+    import torch
+
+    repo = "veyra-ai/veyra-30m-base-2.5b-tokens"
+
+    tokenizer = AutoTokenizer.from_pretrained(repo, trust_remote_code=True)
+    model = AutoModelForCausalLM.from_pretrained(repo, trust_remote_code=True, dtype=torch.float32)
+    model.eval()
+
+    prompt = "Photosynthesis is the process by which"
+    input_ids = tokenizer.encode(prompt, add_special_tokens=False, return_tensors="pt")
+
+    with torch.no_grad():
+        out = model.generate(
+            input_ids,
+            do_sample=True,
+            temperature=0.5,
+            top_k=30,
+            repetition_penalty=1.15,
+            no_repeat_ngram_size=2,
+            max_new_tokens=80,
+        )
+
+    print(tokenizer.decode(out[0], skip_special_tokens=True))
+
+For raw completion prompts, use `add_special_tokens=False`.
+
+## Optimizer
+
+Training used:
+
+- **CosineGatedAdam / CGA-v0** on 2D projection matrices
+- **AdamW** on embeddings, norms, tied head, and auxiliary parameters
+
+## Intended use
+
+This checkpoint is primarily for:
+
+- continued pretraining
+- research / ablations
+- tracking Veyra training milestones
+- testing tiny model behavior
+
+It is not intended for production use or reliable factual answering.
+
+## Known limitations
+
+This model can:
+
+- hallucinate confidently
+- repeat phrases
+- fail arithmetic
+- fail simple factual questions
+- produce fake code
+- continue in textbook-like or tutorial-like styles
+
+Further continuation pretraining and post-training are planned.

chat_template.jinja

@@ -0,0 +1,4 @@
+{% for message in messages %}<|im_start|>{{ message['role'] }}
+{{ message['content'] }}<|im_end|>
+{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant
+{% endif %}

config.json

@@ -0,0 +1,32 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 0,
+  "dtype": "float32",
+  "eos_token_id": 1,
+  "head_dim": 64,
+  "hidden_act": "silu",
+  "hidden_size": 512,
+  "initializer_range": 0.02,
+  "intermediate_size": 2048,
+  "max_position_embeddings": 512,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 8,
+  "num_hidden_layers": 8,
+  "num_key_value_heads": 2,
+  "pad_token_id": 2,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.10.2",
+  "use_cache": true,
+  "vocab_size": 8192
+}

configuration_veyra.py

@@ -0,0 +1,57 @@
+from transformers import PretrainedConfig
+
+
+class VeyraConfig(PretrainedConfig):
+    model_type = "veyra"
+
+    def __init__(
+        self,
+        vocab_size=8192,
+        d_model=512,
+        hidden_size=512,
+        n_q_heads=8,
+        num_attention_heads=8,
+        n_kv_heads=2,
+        num_key_value_heads=2,
+        intermediate_size=2048,
+        max_seq_len=512,
+        max_position_embeddings=512,
+        layer_pattern=("A", "M", "A", "M", "A", "M", "A", "M"),
+        num_hidden_layers=8,
+        rms_norm_eps=1e-6,
+        rope_theta=10000.0,
+        tie_word_embeddings=True,
+        bos_token_id=0,
+        eos_token_id=1,
+        pad_token_id=2,
+        unk_token_id=3,
+        **kwargs,
+    ):
+        self.vocab_size = vocab_size
+        self.d_model = d_model
+        self.hidden_size = hidden_size
+
+        self.n_q_heads = n_q_heads
+        self.num_attention_heads = num_attention_heads
+        self.n_kv_heads = n_kv_heads
+        self.num_key_value_heads = num_key_value_heads
+
+        self.intermediate_size = intermediate_size
+        self.max_seq_len = max_seq_len
+        self.max_position_embeddings = max_position_embeddings
+
+        self.layer_pattern = list(layer_pattern)
+        self.num_hidden_layers = num_hidden_layers
+
+        self.rms_norm_eps = rms_norm_eps
+        self.rope_theta = rope_theta
+        self.tie_word_embeddings = tie_word_embeddings
+
+        super().__init__(
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            pad_token_id=pad_token_id,
+            unk_token_id=unk_token_id,
+            tie_word_embeddings=tie_word_embeddings,
+            **kwargs,
+        )

conversion_report.json

@@ -0,0 +1,129 @@
+{
+  "source_repo": "veyra-ai/Veyra-30M-Base-2.5B-Tokens",
+  "legacy_repo": "Jdudeo/Veyra-30M-Base-2.5B-Tokens-Legacy",
+  "converted_folder": "/content/drive/MyDrive/veyra30m_repo_conversion/converted_llama/Veyra-30M-Base-2.5B-Tokens",
+  "chosen_permute_qk": true,
+  "summary_perm": {
+    "mean_abs_avg": 3.363791880214454e-06,
+    "max_abs_max": 2.86102294921875e-05,
+    "last_token_mean_abs_avg": 3.3260405416513095e-06,
+    "top1_agree_avg": 1.0
+  },
+  "summary_noperm": {
+    "mean_abs_avg": 0.7021041483812626,
+    "max_abs_max": 9.765437126159668,
+    "last_token_mean_abs_avg": 0.973747450651607,
+    "top1_agree_avg": 0.6806431795869555
+  },
+  "summary_final": {
+    "mean_abs_avg": 3.363791880214454e-06,
+    "max_abs_max": 2.86102294921875e-05,
+    "last_token_mean_abs_avg": 3.3260405416513095e-06,
+    "top1_agree_avg": 1.0
+  },
+  "rows_final": [
+    {
+      "prompt": "Once upon a time in a digital kingdom,",
+      "tokens": 10,
+      "max_abs": 2.6226043701171875e-05,
+      "mean_abs": 4.017726951133227e-06,
+      "last_token_mean_abs": 4.454526788322255e-06,
+      "top1_agree": 1.0
+    },
+    {
+      "prompt": "Timmy and Sally went on an adventure to",
+      "tokens": 12,
+      "max_abs": 2.765655517578125e-05,
+      "mean_abs": 3.3202672966581304e-06,
+      "last_token_mean_abs": 1.8051614461001009e-06,
+      "top1_agree": 1.0
+    },
+    {
+      "prompt": "Question: What is the capital of France?\nAnswer:",
+      "tokens": 14,
+      "max_abs": 1.9550323486328125e-05,
+      "mean_abs": 3.1002457490103552e-06,
+      "last_token_mean_abs": 3.635879693320021e-06,
+      "top1_agree": 1.0
+    },
+    {
+      "prompt": "I am an AI",
+      "tokens": 4,
+      "max_abs": 2.86102294921875e-05,
+      "mean_abs": 3.60240665031597e-06,
+      "last_token_mean_abs": 2.4980258785944898e-06,
+      "top1_agree": 1.0
+    },
+    {
+      "prompt": "1+1=",
+      "tokens": 4,
+      "max_abs": 1.9073486328125e-05,
+      "mean_abs": 3.561165158316726e-06,
+      "last_token_mean_abs": 5.073114607512252e-06,
+      "top1_agree": 1.0
+    },
+    {
+      "prompt": "def add_numbers(a, b):",
+      "tokens": 11,
+      "max_abs": 2.7418136596679688e-05,
+      "mean_abs": 3.3422006708860863e-06,
+      "last_token_mean_abs": 3.2130446925293654e-06,
+      "top1_agree": 1.0
+    },
+    {
+      "prompt": " ",
+      "tokens": 1,
+      "max_abs": 1.633167266845703e-05,
+      "mean_abs": 2.6025306851806818e-06,
+      "last_token_mean_abs": 2.6025306851806818e-06,
+      "top1_agree": 1.0
+    }
+  ],
+  "llama_config": {
+    "transformers_version": "5.10.2",
+    "architectures": [
+      "LlamaForCausalLM"
+    ],
+    "output_hidden_states": false,
+    "return_dict": true,
+    "dtype": "float32",
+    "chunk_size_feed_forward": 0,
+    "is_encoder_decoder": false,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "problem_type": null,
+    "vocab_size": 8192,
+    "hidden_size": 512,
+    "intermediate_size": 2048,
+    "num_hidden_layers": 8,
+    "num_attention_heads": 8,
+    "num_key_value_heads": 2,
+    "hidden_act": "silu",
+    "max_position_embeddings": 512,
+    "initializer_range": 0.02,
+    "rms_norm_eps": 1e-06,
+    "use_cache": true,
+    "pad_token_id": 2,
+    "bos_token_id": 0,
+    "eos_token_id": 1,
+    "pretraining_tp": 1,
+    "tie_word_embeddings": true,
+    "rope_parameters": {
+      "rope_theta": 10000.0,
+      "rope_type": "default"
+    },
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "mlp_bias": false,
+    "head_dim": 64,
+    "_name_or_path": "",
+    "model_type": "llama",
+    "output_attentions": false
+  }
+}

generation_config.json

@@ -0,0 +1,10 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 0,
+  "eos_token_id": 1,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "pad_token_id": 2,
+  "transformers_version": "5.10.2",
+  "use_cache": true
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:937a18eb7e1ff80e31fbca886c6eb3d98561fe3bc0586ca7e42d6f81a2ca7fb0
+size 138454968

modeling_veyra.py

@@ -0,0 +1,245 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from transformers import PreTrainedModel, GenerationMixin
+from transformers.modeling_outputs import CausalLMOutputWithPast
+
+from .configuration_veyra import VeyraConfig
+
+
+class RMSNorm(nn.Module):
+    def __init__(self, dim, eps=1e-6):
+        super().__init__()
+        self.weight = nn.Parameter(torch.ones(dim))
+        self.eps = eps
+
+    def forward(self, x):
+        return self.weight * x * torch.rsqrt(x.pow(2).mean(dim=-1, keepdim=True) + self.eps)
+
+
+class RotaryEmbedding(nn.Module):
+    def __init__(self, dim, max_seq_len, base=10000.0):
+        super().__init__()
+        self.dim = dim
+        self.max_seq_len = max_seq_len
+        self.base = base
+
+    def forward(self, x, positions):
+        # No buffers. Compute everything fresh to avoid HF remote-code buffer corruption.
+        half_dim = self.dim // 2
+
+        inv_freq = 1.0 / (
+            self.base ** (
+                torch.arange(0, self.dim, 2, device=x.device, dtype=torch.float32) / self.dim
+            )
+        )
+
+        positions = positions.to(device=x.device, dtype=torch.float32)
+        freqs = torch.outer(positions, inv_freq)
+
+        cos = torch.cos(freqs).to(dtype=x.dtype)[None, None, :, :]
+        sin = torch.sin(freqs).to(dtype=x.dtype)[None, None, :, :]
+
+        x_even = x[..., 0::2]
+        x_odd = x[..., 1::2]
+
+        out_even = x_even * cos - x_odd * sin
+        out_odd = x_even * sin + x_odd * cos
+
+        return torch.stack((out_even, out_odd), dim=-1).flatten(-2)
+
+
+class SwiGLU(nn.Module):
+    def __init__(self, d_model, intermediate_size):
+        super().__init__()
+        self.gate_proj = nn.Linear(d_model, intermediate_size, bias=False)
+        self.up_proj = nn.Linear(d_model, intermediate_size, bias=False)
+        self.down_proj = nn.Linear(intermediate_size, d_model, bias=False)
+
+    def forward(self, x):
+        return self.down_proj(F.silu(self.gate_proj(x)) * self.up_proj(x))
+
+
+class GQAAttention(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+
+        assert config.d_model % config.n_q_heads == 0
+        assert config.n_q_heads % config.n_kv_heads == 0
+
+        self.d_model = config.d_model
+        self.n_q_heads = config.n_q_heads
+        self.n_kv_heads = config.n_kv_heads
+        self.head_dim = config.d_model // config.n_q_heads
+        self.n_rep = config.n_q_heads // config.n_kv_heads
+
+        self.q_proj = nn.Linear(config.d_model, config.n_q_heads * self.head_dim, bias=False)
+        self.k_proj = nn.Linear(config.d_model, config.n_kv_heads * self.head_dim, bias=False)
+        self.v_proj = nn.Linear(config.d_model, config.n_kv_heads * self.head_dim, bias=False)
+        self.o_proj = nn.Linear(config.n_q_heads * self.head_dim, config.d_model, bias=False)
+
+        self.rope = RotaryEmbedding(self.head_dim, config.max_seq_len, config.rope_theta)
+
+    def _shape_q(self, x):
+        b, t, _ = x.shape
+        return x.view(b, t, self.n_q_heads, self.head_dim).transpose(1, 2)
+
+    def _shape_kv(self, x):
+        b, t, _ = x.shape
+        return x.view(b, t, self.n_kv_heads, self.head_dim).transpose(1, 2)
+
+    def _repeat_kv(self, x):
+        if self.n_rep == 1:
+            return x
+        return x.repeat_interleave(self.n_rep, dim=1)
+
+    def forward(self, x):
+        b, t, _ = x.shape
+        positions = torch.arange(t, device=x.device)
+
+        q = self._shape_q(self.q_proj(x))
+        k = self._shape_kv(self.k_proj(x))
+        v = self._shape_kv(self.v_proj(x))
+
+        q = self.rope(q, positions)
+        k = self.rope(k, positions)
+
+        k = self._repeat_kv(k)
+        v = self._repeat_kv(v)
+
+        y = F.scaled_dot_product_attention(
+            q,
+            k,
+            v,
+            attn_mask=None,
+            dropout_p=0.0,
+            is_causal=True,
+        )
+
+        y = y.transpose(1, 2).contiguous().view(b, t, self.d_model)
+
+        return self.o_proj(y)
+
+
+class AttentionBlock(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.input_norm = RMSNorm(config.d_model, config.rms_norm_eps)
+        self.attn = GQAAttention(config)
+        self.post_attn_norm = RMSNorm(config.d_model, config.rms_norm_eps)
+        self.mlp = SwiGLU(config.d_model, config.intermediate_size)
+
+    def forward(self, x):
+        x = x + self.attn(self.input_norm(x))
+        x = x + self.mlp(self.post_attn_norm(x))
+        return x
+
+
+class MLPOnlyBlock(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.norm = RMSNorm(config.d_model, config.rms_norm_eps)
+        self.mlp = SwiGLU(config.d_model, config.intermediate_size)
+
+    def forward(self, x):
+        return x + self.mlp(self.norm(x))
+
+
+class VeyraPreTrainedModel(PreTrainedModel):
+    config_class = VeyraConfig
+    base_model_prefix = ""
+    supports_gradient_checkpointing = False
+    _no_split_modules = ["AttentionBlock", "MLPOnlyBlock"]
+
+
+class VeyraForCausalLM(VeyraPreTrainedModel, GenerationMixin):
+    config_class = VeyraConfig
+    _tied_weights_keys = {"lm_head.weight": "token_emb.weight"}
+
+    def __init__(self, config):
+        super().__init__(config)
+        self.config = config
+
+        self.token_emb = nn.Embedding(config.vocab_size, config.d_model)
+
+        blocks = []
+        for kind in config.layer_pattern:
+            if kind == "A":
+                blocks.append(AttentionBlock(config))
+            elif kind == "M":
+                blocks.append(MLPOnlyBlock(config))
+            else:
+                raise ValueError(f"Unknown layer kind: {kind}")
+
+        self.blocks = nn.ModuleList(blocks)
+        self.norm = RMSNorm(config.d_model, config.rms_norm_eps)
+        self.lm_head = nn.Linear(config.d_model, config.vocab_size, bias=False)
+
+        if config.tie_word_embeddings:
+            self.lm_head.weight = self.token_emb.weight
+
+        # Needed by newer Transformers internals, but we intentionally avoid post_init().
+        self.all_tied_weights_keys = {"lm_head.weight": "token_emb.weight"}
+
+    def tie_weights(self, *args, **kwargs):
+        if self.config.tie_word_embeddings:
+            self.lm_head.weight = self.token_emb.weight
+
+    def get_input_embeddings(self):
+        return self.token_emb
+
+    def set_input_embeddings(self, value):
+        self.token_emb = value
+        if self.config.tie_word_embeddings:
+            self.lm_head.weight = self.token_emb.weight
+
+    def get_output_embeddings(self):
+        return self.lm_head
+
+    def set_output_embeddings(self, new_embeddings):
+        self.lm_head = new_embeddings
+
+    def forward(
+        self,
+        input_ids=None,
+        attention_mask=None,
+        labels=None,
+        return_dict=True,
+        **kwargs,
+    ):
+        if input_ids is None:
+            raise ValueError("input_ids must be provided")
+
+        x = self.token_emb(input_ids)
+
+        for block in self.blocks:
+            x = block(x)
+
+        logits = self.lm_head(self.norm(x))
+
+        loss = None
+        if labels is not None:
+            loss = F.cross_entropy(
+                logits[:, :-1, :].contiguous().view(-1, logits.size(-1)),
+                labels[:, 1:].contiguous().view(-1),
+                ignore_index=-100,
+            )
+
+        if not return_dict:
+            output = (logits,)
+            return ((loss,) + output) if loss is not None else output
+
+        return CausalLMOutputWithPast(
+            loss=loss,
+            logits=logits,
+            past_key_values=None,
+            hidden_states=None,
+            attentions=None,
+        )
+
+    def prepare_inputs_for_generation(self, input_ids, **kwargs):
+        max_len = self.config.max_seq_len
+        if input_ids.shape[1] > max_len:
+            input_ids = input_ids[:, -max_len:]
+        return {"input_ids": input_ids}

special_tokens_map.json

@@ -0,0 +1,6 @@
+{
+  "bos_token": "<|bos|>",
+  "eos_token": "<|eos|>",
+  "unk_token": "<|unk|>",
+  "pad_token": "<|pad|>"
+}

tokenizer_config.json

@@ -0,0 +1,36 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|bos|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|eos|>",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|tool_call|>",
+    "<|tool_result|>",
+    "<|context|>",
+    "<|reasoning|>",
+    "<|end_reasoning|>",
+    "<|answer|>",
+    "<|fim_prefix|>",
+    "<|fim_middle|>",
+    "<|fim_suffix|>",
+    "<|reserved_0|>",
+    "<|reserved_1|>",
+    "<|reserved_2|>",
+    "<|reserved_3|>",
+    "<|reserved_4|>",
+    "<|reserved_5|>",
+    "<|reserved_6|>",
+    "<|reserved_7|>",
+    "<|reserved_8|>",
+    "<|reserved_9|>"
+  ],
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1000000000,
+  "pad_token": "<|pad|>",
+  "tokenizer_class": "TokenizersBackend",
+  "unk_token": "<|unk|>"
+}

training_meta.json

@@ -0,0 +1,82 @@
+{
+  "name": "veyra-30m-base-2.5b-checkpoint",
+  "created_at": "2026-05-02T20:17:04.944220Z",
+  "total_pretraining_tokens": 2500000000,
+  "exact_parameter_count": 31988224,
+  "parameter_count_millions": 31.99,
+  "parameter_breakdown": {
+    "token_embedding_tied_lm_head": 4194304,
+    "attention_mlp_blocks_4x": 15208448,
+    "mlp_only_blocks_4x": 12584960,
+    "final_rmsnorm": 512,
+    "total": 31988224
+  },
+  "datasets": [
+    {
+      "repo": "HuggingFaceTB/cosmopedia-v2",
+      "configs_used": [
+        "cosmopedia-v2",
+        "fineweb-edu-dedup",
+        "python-edu"
+      ],
+      "usage": "1B token Cosmopedia bootstrap plus 1.5B token mixed continuation."
+    }
+  ],
+  "optimizer": {
+    "matrix_params": "CosineGatedAdam / CGA-v0",
+    "aux_params": "AdamW",
+    "cga_peak_lr": 5e-05,
+    "cga_max_update_rms": 3.0,
+    "aux_peak_lr": 0.0003,
+    "lr_schedule": "warmup then cosine decay during continuation"
+  },
+  "architecture": {
+    "model_type": "veyra",
+    "architectures": [
+      "VeyraForCausalLM"
+    ],
+    "library_name": "transformers",
+    "auto_map": {
+      "AutoConfig": "configuration_veyra.VeyraConfig",
+      "AutoModelForCausalLM": "modeling_veyra.VeyraForCausalLM"
+    },
+    "vocab_size": 8192,
+    "d_model": 512,
+    "hidden_size": 512,
+    "n_q_heads": 8,
+    "num_attention_heads": 8,
+    "n_kv_heads": 2,
+    "num_key_value_heads": 2,
+    "intermediate_size": 2048,
+    "max_seq_len": 512,
+    "max_position_embeddings": 512,
+    "layer_pattern": [
+      "A",
+      "M",
+      "A",
+      "M",
+      "A",
+      "M",
+      "A",
+      "M"
+    ],
+    "num_hidden_layers": 8,
+    "rms_norm_eps": 1e-06,
+    "rope_theta": 10000.0,
+    "tie_word_embeddings": true,
+    "bos_token_id": 0,
+    "eos_token_id": 1,
+    "pad_token_id": 2,
+    "unk_token_id": 3,
+    "dtype": "float32",
+    "torch_dtype": "float32",
+    "num_parameters": 31988224,
+    "parameter_count": 31988224,
+    "parameter_count_millions": 31.99,
+    "total_params": 31988224,
+    "trainable_params": 31988224,
+    "training_tokens": 2500000000,
+    "notes": "Attention-sparse AMAMAMAM decoder. Early base checkpoint, not instruction tuned."
+  },
+  "important_warning": "This is an early base model checkpoint, not an instruction-tuned assistant. It can hallucinate, repeat, and fail simple factual/math prompts."
+}

veyra_tokenizer_metadata.json

@@ -0,0 +1,58 @@
+{
+  "name": "veyra-tokenizer-8k",
+  "vocab_size": 8192,
+  "tokenizer_type": "BPE ByteLevel",
+  "language": "English-focused",
+  "chat_format": "ChatML",
+  "training_corpus": {
+    "total_texts": 250000,
+    "mix": {
+      "fineweb_edu": 0.7,
+      "synthetic_edu": 0.1,
+      "chatml": 0.08,
+      "markdown_docs": 0.05,
+      "json_tools": 0.04,
+      "python_fim": 0.03
+    },
+    "fineweb_edu": "HuggingFaceFW/fineweb-edu sample-10BT streaming",
+    "synthetic_sources": [
+      "generated educational prose templates",
+      "generated ChatML examples",
+      "generated markdown/docs examples",
+      "generated JSON/tool/context examples",
+      "generated Python/FIM examples"
+    ]
+  },
+  "special_tokens": [
+    "<|bos|>",
+    "<|eos|>",
+    "<|pad|>",
+    "<|unk|>",
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|tool_call|>",
+    "<|tool_result|>",
+    "<|context|>",
+    "<|reasoning|>",
+    "<|end_reasoning|>",
+    "<|answer|>",
+    "<|fim_prefix|>",
+    "<|fim_middle|>",
+    "<|fim_suffix|>",
+    "<|reserved_0|>",
+    "<|reserved_1|>",
+    "<|reserved_2|>",
+    "<|reserved_3|>",
+    "<|reserved_4|>",
+    "<|reserved_5|>",
+    "<|reserved_6|>",
+    "<|reserved_7|>",
+    "<|reserved_8|>",
+    "<|reserved_9|>"
+  ],
+  "notes": [
+    "Designed for Veyra 30M-class attention-sparse decoder models.",
+    "8k vocab including special tokens.",
+    "Includes ChatML, tool/context, reasoning experiment, FIM, and reserved tokens."
+  ]
+}