初始化项目，由ModelHub XC社区提供模型

Model: veyra-ai/Veyra2-Mango-30M-Base Source: Original Platform
2026-09-09 13:58:16 +08:00
commit 12fdfce55d
8 changed files with 40470 additions and 0 deletions
--- a/.gitattributes
+++ b/.gitattributes
@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
--- a/README.md
+++ b/README.md
@@ -0,0 +1,322 @@
+---
+language:
+- en
+license: apache-2.0
+library_name: transformers
+pipeline_tag: text-generation
+tags:
+- causal-lm
+- base-model
+- transformers
+- safetensors
+- veyra
+- small-language-model
+model-index:
+- name: Veyra2-Mango-30M-Base
+  results:
+  - task:
+      type: text-generation
+      name: Text Generation
+    dataset:
+      name: SciCloze-900
+      type: veyra-ai/SciCloze-900
+    metrics:
+    - name: Accuracy
+      type: accuracy
+      value: 42.44
+      source:
+        name: Local evaluation
+        url: https://huggingface.co/veyra-ai/Veyra2-Mango-30M-Base
+
+  - task:
+      type: multiple-choice
+      name: Multiple Choice
+    dataset:
+      name: SciQ
+      type: sciq
+    metrics:
+    - name: Accuracy
+      type: accuracy
+      value: 65.10
+    - name: Normalized Accuracy
+      type: acc_norm
+      value: 58.30
+      source:
+        name: Local lm-evaluation-harness
+        url: https://huggingface.co/veyra-ai/Veyra2-Mango-30M-Base
+
+  - task:
+      type: multiple-choice
+      name: Multiple Choice
+    dataset:
+      name: PIQA
+      type: piqa
+    metrics:
+    - name: Normalized Accuracy
+      type: acc_norm
+      value: 59.52
+      source:
+        name: Local lm-evaluation-harness
+        url: https://huggingface.co/veyra-ai/Veyra2-Mango-30M-Base
+
+  - task:
+      type: multiple-choice
+      name: Multiple Choice
+    dataset:
+      name: ARC-Easy
+      type: ai2_arc
+      config: ARC-Easy
+    metrics:
+    - name: Normalized Accuracy
+      type: acc_norm
+      value: 37.88
+      source:
+        name: Local lm-evaluation-harness
+        url: https://huggingface.co/veyra-ai/Veyra2-Mango-30M-Base
+
+  - task:
+      type: multiple-choice
+      name: Multiple Choice
+    dataset:
+      name: ARC-Challenge
+      type: ai2_arc
+      config: ARC-Challenge
+    metrics:
+    - name: Normalized Accuracy
+      type: acc_norm
+      value: 23.29
+      source:
+        name: Local lm-evaluation-harness
+        url: https://huggingface.co/veyra-ai/Veyra2-Mango-30M-Base
+
+  - task:
+      type: multiple-choice
+      name: Multiple Choice
+    dataset:
+      name: HellaSwag
+      type: hellaswag
+    metrics:
+    - name: Normalized Accuracy
+      type: acc_norm
+      value: 28.76
+      source:
+        name: Local lm-evaluation-harness
+        url: https://huggingface.co/veyra-ai/Veyra2-Mango-30M-Base
+
+  - task:
+      type: multiple-choice
+      name: Multiple Choice
+    dataset:
+      name: Winogrande
+      type: winogrande
+    metrics:
+    - name: Accuracy
+      type: accuracy
+      value: 49.33
+      source:
+        name: Local lm-evaluation-harness
+        url: https://huggingface.co/veyra-ai/Veyra2-Mango-30M-Base
+
+  - task:
+      type: multiple-choice
+      name: Multiple Choice
+    dataset:
+      name: OpenBookQA
+      type: openbookqa
+    metrics:
+    - name: Accuracy
+      type: accuracy
+      value: 14.80
+    - name: Normalized Accuracy
+      type: acc_norm
+      value: 27.20
+      source:
+        name: Local lm-evaluation-harness
+        url: https://huggingface.co/veyra-ai/Veyra2-Mango-30M-Base
+
+  - task:
+      type: question-answering
+      name: Question Answering
+    dataset:
+      name: BoolQ
+      type: boolq
+    metrics:
+    - name: Accuracy
+      type: accuracy
+      value: 42.81
+      source:
+        name: Local lm-evaluation-harness
+        url: https://huggingface.co/veyra-ai/Veyra2-Mango-30M-Base
+
+  - task:
+      type: multiple-choice
+      name: Multiple Choice
+    dataset:
+      name: ArithMark-2.0
+      type: AxiomicLabs/ArithMark-2.0
+      split: train
+    metrics:
+    - name: Accuracy
+      type: accuracy
+      value: 27.96
+      source:
+        name: Local evaluation
+        url: https://huggingface.co/veyra-ai/Veyra2-Mango-30M-Base
+
+  - task:
+      type: multiple-choice
+      name: Multiple Choice
+    dataset:
+      name: ArithMark-3.0
+      type: AxiomicLabs/Arithmark-3.0
+      split: train
+    metrics:
+    - name: Accuracy
+      type: accuracy
+      value: 36.90
+      source:
+        name: Local evaluation
+        url: https://huggingface.co/veyra-ai/Veyra2-Mango-30M-Base
+---
+
+![Veyra Banner](https://cdn-uploads.huggingface.co/production/uploads/6857f2cfae68b377f17aff8c/MfZTOclKIpOM1RbUlnfcr.jpeg)
+
+# Veyra2-Mango-30M-Base
+
+Veyra2-Mango-30M-Base is a 30.7M-parameter Llama-like causal language model trained from scratch on approximately 30B tokens. It is a raw base model, not an instruction-tuned assistant. It is intended for research, benchmarking, continued pretraining, and small-model experimentation.
+
+## Model Details
+
+| Property | Value |
+| :--- | :--- |
+| **Parameters** | 30,683,520 |
+| **Architecture** | LlamaForCausalLM |
+| **Layers** | 16 |
+| **Hidden size** | 384 |
+| **Attention heads** | 6 |
+| **KV heads** | 2 |
+| **Head dim** | 64 |
+| **Intermediate size** | 1152 |
+| **Vocabulary size** | 8192 |
+| **Context length used in training** | 3072 |
+| **Activation** | SwiGLU / SiLU |
+| **Normalization** | RMSNorm |
+| **Attention** | GQA |
+| **Positional encoding** | RoPE |
+| **Weight tying** | Tied input embeddings and LM head |
+| **Training tokens** | Approximately 30B |
+| **Training precision** | bfloat16 |
+| **Optimizer** | AdamW |
+
+## Tokenizer
+
+Special tokens:
+
+- `<|endoftext|>`: 0
+- `<|im_start|>`: 1
+- `<|im_end|>`: 2
+- `<|pad|>`: 3
+
+## Training Data
+
+The model was trained on a 30B-token pretraining mixture.
+
+Stage 1 18,000,000,000 tokens 180 shards
+
+Mixture:
+
+    dclm_baseline: 50%
+    finephrase: 20%
+    cosmopedia_v2: 10%
+    finemath_4plus: 10%
+    ultrafineweb_multistyle: 5%
+    ultrafineweb_qa: 5%
+
+Stage 1.5 4,000,000,000 tokens 40 shards
+
+This stage linearly transitions from the Stage 1 mixture to the Stage 2 mixture.
+
+Stage 2 8,000,000,000 tokens 80 shards
+
+Mixture:
+
+    finephrase: 30%
+    dclm_baseline: 30%
+    cosmopedia_v2: 18%
+    finemath_4plus: 10%
+    ultrafineweb_multistyle: 5%
+    ultrafineweb_qa: 5%
+    ultrachat: 2%
+
+## Training Summary
+
+- Final step: 25,432  
+- Tokens seen: 30,000,000,000  
+- Tokens per step: 1,179,648
+- Sequence length: 3072  
+- Last train loss: 2.5062 
+
+## Usage
+
+<pre><code>import torch
+from transformers import AutoTokenizer, AutoModelForCausalLM
+
+model_id = "veyra-ai/Veyra2-Mango-30M-Base"
+
+tokenizer = AutoTokenizer.from_pretrained(
+    model_id,
+)
+
+model = AutoModelForCausalLM.from_pretrained(
+    model_id,
+    torch_dtype=torch.float16,
+    device_map="auto",
+)
+
+prompt = "In the 19th century"
+inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
+
+with torch.no_grad():
+    output = model.generate(
+        **inputs,
+        max_new_tokens=120,
+        do_sample=True,
+        temperature=0.6,
+        top_p=0.9,
+        repetition_penalty=1.1,
+        use_cache=True,
+        pad_token_id=tokenizer.pad_token_id,
+        eos_token_id=tokenizer.eos_token_id,
+    )
+
+print(tokenizer.decode(output[0], skip_special_tokens=True))
+</code></pre>
+
+## Notes on Generation
+
+Veyra2-Mango-30M-Base is a raw base model. It is not instruction tuned and should not be expected to behave like a chat assistant.
+Open-ended generations can be unstable, repetitive, or factually unreliable. It's not a polished assistant.
+
+## Intended Use
+
+This model is intended for:
+
+- small language model research
+- continued pretraining
+- benchmarking
+- experimentation with compact causal LMs
+
+## Limitations
+
+- Not instruction tuned
+- Not RLHF tuned
+- Not safe for factual or high-stakes use without additional validation
+- Can hallucinate names, citations, species, references, and technical claims
+- Open-ended text may drift off-topic
+- Context length during training was 3072 tokens
+
+## Citation
+
+If you use this model, please cite the model repository:
+
+`veyra-ai/Veyra2-Mango-30M-Base`
--- a/config.json
+++ b/config.json
@@ -0,0 +1,30 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": null,
+  "dtype": "bfloat16",
+  "eos_token_id": 0,
+  "head_dim": 64,
+  "hidden_act": "silu",
+  "hidden_size": 384,
+  "initializer_range": 0.02,
+  "intermediate_size": 1152,
+  "max_position_embeddings": 4096,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 6,
+  "num_hidden_layers": 16,
+  "num_key_value_heads": 2,
+  "pad_token_id": 3,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "tie_word_embeddings": true,
+  "transformers_version": "4.56.2",
+  "use_cache": true,
+  "vocab_size": 8192
+}
--- a/generation_config.json
+++ b/generation_config.json
@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "eos_token_id": 0,
+  "pad_token_id": 3,
+  "transformers_version": "4.56.2"
+}
--- a/model.safetensors
+++ b/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:959ffb956460c6a3c6fb92fc6d39c31028d7446f303d2e403a68824b63c67ce2
+size 61383136
--- a/special_tokens_map.json
+++ b/special_tokens_map.json
@@ -0,0 +1,20 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|pad|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}
--- a/tokenizer.json
+++ b/tokenizer.json
--- a/tokenizer_config.json
+++ b/tokenizer_config.json
@@ -0,0 +1,54 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<|pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "model_max_length": 4096,
+  "pad_token": "<|pad|>",
+  "padding_side": "left",
+  "tokenizer_class": "PreTrainedTokenizerFast",
+  "truncation_side": "left",
+  "unk_token": null,
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ]
+}