unsloth/SmolLM3-3B
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

Upload folder using ModelScope SDK

Browse Source
This commit is contained in:
Cherrytest
2025-07-09 05:07:27 +00:00
parent cd88d49823
commit 718193d989
12 changed files with 3014 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

109
config.json Normal file
View File

@@ -0,0 +1,109 @@
{
"architectures": [
"SmolLM3ForCausalLM"
],
"attention_bias": false,
"attention_dropout": 0.0,
"bos_token_id": null,
"eos_token_id": 128012,
"hidden_act": "silu",
"hidden_size": 2048,
"initializer_range": 0.02,
"intermediate_size": 11008,
"layer_types": [
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention"
],
"max_position_embeddings": 65536,
"max_window_layers": 28,
"mlp_bias": false,
"model_type": "smollm3",
"no_rope_layer_interval": 4,
"no_rope_layers": [
1,
1,
1,
0,
1,
1,
1,
0,
1,
1,
1,
0,
1,
1,
1,
0,
1,
1,
1,
0,
1,
1,
1,
0,
1,
1,
1,
0,
1,
1,
1,
0,
1,
1,
1,
0
],
"num_attention_heads": 16,
"num_hidden_layers": 36,
"num_key_value_heads": 4,
"pad_token_id": 128004,
"pretraining_tp": 2,
"rms_norm_eps": 1e-06,
"rope_scaling": null,
"rope_theta": 5000000.0,
"sliding_window": null,
"torch_dtype": "bfloat16",
"transformers_version": "4.53.1",
"unsloth_fixed": true,
"use_cache": false,
"use_sliding_window": false,
"vocab_size": 128256
}