Shekswess/tiny-think-sft-math-stem-loss-dft-bf16-e2-bs8
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

初始化项目，由ModelHub XC社区提供模型

Browse Source 
Model: Shekswess/tiny-think-sft-math-stem-loss-dft-bf16-e2-bs8
Source: Original Platform
This commit is contained in:
ModelHub XC
2026-07-24 19:31:08 +08:00
commit 5a3f6279d8
11 changed files with 2293 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
.gitattributes vendored Normal file
View File

@@ -0,0 +1,51 @@
*.7z filter=lfs diff=lfs merge=lfs -text
*.arrow filter=lfs diff=lfs merge=lfs -text
*.bz2 filter=lfs diff=lfs merge=lfs -text
*.ftz filter=lfs diff=lfs merge=lfs -text
*.gz filter=lfs diff=lfs merge=lfs -text
*.h5 filter=lfs diff=lfs merge=lfs -text
*.joblib filter=lfs diff=lfs merge=lfs -text
*.lfs.* filter=lfs diff=lfs merge=lfs -text
*.model filter=lfs diff=lfs merge=lfs -text
*.msgpack filter=lfs diff=lfs merge=lfs -text
*.onnx filter=lfs diff=lfs merge=lfs -text
*.ot filter=lfs diff=lfs merge=lfs -text
*.parquet filter=lfs diff=lfs merge=lfs -text
*.pb filter=lfs diff=lfs merge=lfs -text
*.pt filter=lfs diff=lfs merge=lfs -text
*.pth filter=lfs diff=lfs merge=lfs -text
*.rar filter=lfs diff=lfs merge=lfs -text
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
*.tar.* filter=lfs diff=lfs merge=lfs -text
*.tflite filter=lfs diff=lfs merge=lfs -text
*.tgz filter=lfs diff=lfs merge=lfs -text
*.xz filter=lfs diff=lfs merge=lfs -text
*.zip filter=lfs diff=lfs merge=lfs -text
*.zstandard filter=lfs diff=lfs merge=lfs -text
*.tfevents* filter=lfs diff=lfs merge=lfs -text
*.db* filter=lfs diff=lfs merge=lfs -text
*.ark* filter=lfs diff=lfs merge=lfs -text
**/*ckpt*data* filter=lfs diff=lfs merge=lfs -text
**/*ckpt*.meta filter=lfs diff=lfs merge=lfs -text
**/*ckpt*.index filter=lfs diff=lfs merge=lfs -text
*.ckpt filter=lfs diff=lfs merge=lfs -text
*.gguf* filter=lfs diff=lfs merge=lfs -text
*.ggml filter=lfs diff=lfs merge=lfs -text
*.llamafile* filter=lfs diff=lfs merge=lfs -text
*.pt2 filter=lfs diff=lfs merge=lfs -text
*.mlmodel filter=lfs diff=lfs merge=lfs -text
*.npy filter=lfs diff=lfs merge=lfs -text
*.npz filter=lfs diff=lfs merge=lfs -text
*.pickle filter=lfs diff=lfs merge=lfs -text
*.pkl filter=lfs diff=lfs merge=lfs -text
*.tar filter=lfs diff=lfs merge=lfs -text
*.wasm filter=lfs diff=lfs merge=lfs -text
*.zst filter=lfs diff=lfs merge=lfs -text
*tfevents* filter=lfs diff=lfs merge=lfs -text
training_args.bin filter=lfs diff=lfs merge=lfs -text
model.safetensors filter=lfs diff=lfs merge=lfs -text
tokenizer.json filter=lfs diff=lfs merge=lfs -text

27
README.md Normal file
View File

@@ -0,0 +1,27 @@
---
base_model: facebook/MobileLLM-R1-140M-base
library_name: transformers
model_name: tiny-think-sft-math-stem-loss-dft-bf16-e2-bs8
tags:
- generated_from_trainer
- sft
- trl
licence: license
---
# Model Card for tiny-think-sft-math-stem-loss-dft-bf16-e2-bs8
This model is a fine-tuned version of [facebook/MobileLLM-R1-140M-base](https://huggingface.co/facebook/MobileLLM-R1-140M-base).
It has been trained using [TRL](https://github.com/huggingface/trl).
## Training procedure
This model was trained with SFT.
### Framework versions
- TRL: 0.26.2
- Transformers: 4.57.3
- Pytorch: 2.9.0+cu128
- Datasets: 4.4.2
- Tokenizers: 0.22.2

34
chat_template.jinja Normal file
View File

@@ -0,0 +1,34 @@
{{- bos_token }}
{# --- Extract system message (optional) --- #}
{%- if messages and messages[0]['role'] == 'system' %}
{%- set system_message = messages[0]['content'] | trim %}
{%- set messages = messages[1:] %}
{%- else %}
{%- set system_message = "Respond with your reasoning wrapped in <think>...</think>, then provide the final answer within \\\\boxed{}." %}
{%- endif %}
{# --- System block --- #}
{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
{{- system_message }}
{{- "<|eot_id|>" }}
{# --- Render all remaining messages --- #}
{%- for message in messages %}
{%- if message['role'] in ['system', 'user', 'assistant'] %}
{%- if message['role'] == 'assistant' %}
{{- "<|start_header_id|>assistant<|end_header_id|>\n\n" }}
{%- generation %}
{{- message['content'] | trim }}
{%- endgeneration %}
{{- "<|eot_id|>" }}
{%- else %}
{{- "<|start_header_id|>" + message['role'] + "<|end_header_id|>\n\n" + (message['content'] | trim) + "<|eot_id|>" }}
{%- endif %}
{%- endif %}
{%- endfor %}
{# --- Generation prompt --- #}
{%- if add_generation_prompt %}
{{- "<|start_header_id|>assistant<|end_header_id|>\n\n" }}
{%- endif %}

76
config.json Normal file
View File

@@ -0,0 +1,76 @@
{
"architectures": [
"Llama4ForCausalLM"
],
"attention_bias": false,
"attention_chunk_size": 32768,
"attention_dropout": 0.0,
"attn_scale": 0.1,
"attn_temperature_tuning": false,
"bos_token_id": 128000,
"dtype": "bfloat16",
"eos_token_id": 128009,
"floor_scale": 8192,
"for_llm_compressor": false,
"head_dim": 64,
"hidden_act": "silu",
"hidden_size": 576,
"initializer_range": 0.02,
"interleave_moe_layer_step": 0,
"intermediate_size": 8192,
"intermediate_size_mlp": 2048,
"layer_types": [
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention"
],
"max_position_embeddings": 4096,
"model_type": "llama4_text",
"moe_layers": [],
"no_rope_layers": [
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1
],
"num_attention_heads": 9,
"num_experts_per_tok": 0,
"num_hidden_layers": 15,
"num_key_value_heads": 3,
"num_local_experts": 0,
"output_router_logits": false,
"pad_token_id": 128009,
"rms_norm_eps": 1e-05,
"rope_scaling": null,
"rope_theta": 1000000.0,
"router_aux_loss_coef": 0.001,
"router_jitter_noise": 0.0,
"tie_word_embeddings": true,
"transformers_version": "4.57.3",
"use_cache": false,
"use_qk_norm": true,
"vocab_size": 128256
}

1
configuration.json Normal file
View File

@@ -0,0 +1 @@
{"framework": "pytorch", "task": "others", "allow_remote": true}

12
generation_config.json Normal file
View File

@@ -0,0 +1,12 @@
{
"_from_model_config": true,
"bos_token_id": 128000,
"eos_token_id": [
128009,
128001,
128008,
128009
],
"pad_token_id": 128009,
"transformers_version": "4.57.3"
}

3
model.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:d1f0fb56e5938d335b1389d769571c6eada0a69433423b53c21fa47392d423b2
size 280512824

17
special_tokens_map.json Normal file
View File

@@ -0,0 +1,17 @@
{
"bos_token": {
"content": "<|begin_of_text|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"eos_token": {
"content": "<|eot_id|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"pad_token": "<|eot_id|>"
}

BIN
tokenizer.json (Stored with Git LFS) Normal file
View File

Binary file not shown.

2066
tokenizer_config.json Normal file
View File

File diff suppressed because it is too large Load Diff

3
training_args.bin Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:fb0caa284e1b39647759e07889166aa2569c355e9049f95507cee18242e20d45
size 6417