初始化项目,由ModelHub XC社区提供模型

Model: Harvard-DCML/boomerang-llama-3.2-1.9B
Source: Original Platform
This commit is contained in:
ModelHub XC
2026-09-04 08:58:16 +08:00
commit e046cfe381
12 changed files with 2376 additions and 0 deletions

36
.gitattributes vendored Normal file
View File

@@ -0,0 +1,36 @@
*.7z filter=lfs diff=lfs merge=lfs -text
*.arrow filter=lfs diff=lfs merge=lfs -text
*.bin filter=lfs diff=lfs merge=lfs -text
*.bz2 filter=lfs diff=lfs merge=lfs -text
*.ckpt filter=lfs diff=lfs merge=lfs -text
*.ftz filter=lfs diff=lfs merge=lfs -text
*.gz filter=lfs diff=lfs merge=lfs -text
*.h5 filter=lfs diff=lfs merge=lfs -text
*.joblib filter=lfs diff=lfs merge=lfs -text
*.lfs.* filter=lfs diff=lfs merge=lfs -text
*.mlmodel filter=lfs diff=lfs merge=lfs -text
*.model filter=lfs diff=lfs merge=lfs -text
*.msgpack filter=lfs diff=lfs merge=lfs -text
*.npy filter=lfs diff=lfs merge=lfs -text
*.npz filter=lfs diff=lfs merge=lfs -text
*.onnx filter=lfs diff=lfs merge=lfs -text
*.ot filter=lfs diff=lfs merge=lfs -text
*.parquet filter=lfs diff=lfs merge=lfs -text
*.pb filter=lfs diff=lfs merge=lfs -text
*.pickle filter=lfs diff=lfs merge=lfs -text
*.pkl filter=lfs diff=lfs merge=lfs -text
*.pt filter=lfs diff=lfs merge=lfs -text
*.pth filter=lfs diff=lfs merge=lfs -text
*.rar filter=lfs diff=lfs merge=lfs -text
*.safetensors filter=lfs diff=lfs merge=lfs -text
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
*.tar.* filter=lfs diff=lfs merge=lfs -text
*.tar filter=lfs diff=lfs merge=lfs -text
*.tflite filter=lfs diff=lfs merge=lfs -text
*.tgz filter=lfs diff=lfs merge=lfs -text
*.wasm filter=lfs diff=lfs merge=lfs -text
*.xz filter=lfs diff=lfs merge=lfs -text
*.zip filter=lfs diff=lfs merge=lfs -text
*.zst filter=lfs diff=lfs merge=lfs -text
*tfevents* filter=lfs diff=lfs merge=lfs -text
tokenizer.json filter=lfs diff=lfs merge=lfs -text

58
README.md Normal file
View File

@@ -0,0 +1,58 @@
---
license: llama3.2
datasets:
- EleutherAI/the_pile_deduplicated
---
## Model Description
Boomerang distillation is a phenomenon in LLMs where we can distill a teacher model into a student and reincorporate teacher layers to create intermediate-sized models with no additional training. This is the Llama student model from [our paper](https://arxiv.org/abs/2510.05064).
## Training Procedure
This model was initialized from Llama-3.2-3B by copying the first two layers and every 2nd subsequent layer. It was distilled on 2.1B tokens of The Pile deduplicated with cross entropy, KL, and cosine loss to match the activations of Llama-3.2-3B. We used the following hyperparameters:
- Learning rate: 3e-4
- Learning rate scheduler: cosine
- Warmup ratio: 0.01
- Optimizer: AdamW
- Adam betas: (0.9, 0.95)
- Adam epsilon: 1e-8
- Weight decay: 0.1
- Max. gradient norm: 1.0
- Number of training steps: 500
- Max. sequence length: 2048
- Effective batch size: 2048
- Mixed precision: bf16
- KLDiv weight: 0.1
- Cosine distance weight per layer: 0.125
## Use
To interpolate between this model and Llama-3.2-3B, please use the `build_intermediate_model` function from [our github repository](https://github.com/dcml-lab/boomerang-distillation):
```python3
import torch
from patching.patch import build_intermediate_model
intermediate_model = build_intermediate_model(
teacher_name_or_path = "meta-llama/Llama-3.2-3B",
student_name_or_path = "Harvard-DCML/boomerang-llama-3.2-1.9B",
num_layers_to_patch = 2,
patch_first_k_layers = True,
dtype = torch.bfloat16,
)
```
Notes:
1. Changing `num_layers_to_patch` changes the size of the intermediate model by patching different numbers of student layers.
2. `patch_first_k_layers` should be set to True for this model for optimal interpolation performance.
## Citation
```
@article{kangaslahti2025boomerang,
title={Boomerang Distillation Enables Zero-Shot Model Size Interpolation},
author={Kangaslahti, Sara and Nayak, Nihal V and Geuter, Jonathan and Fumero, Marco and Locatello, Francesco and Alvarez-Melis, David},
journal={arXiv preprint arXiv:2510.05064},
year={2025},
url={https://arxiv.org/abs/2510.05064}
}
```

35
config.json Normal file
View File

@@ -0,0 +1,35 @@
{
"architectures": [
"LlamaForCausalLM"
],
"attention_bias": false,
"attention_dropout": 0.0,
"bos_token_id": 128000,
"eos_token_id": 128001,
"head_dim": 128,
"hidden_act": "silu",
"hidden_size": 3072,
"initializer_range": 0.02,
"intermediate_size": 8192,
"max_position_embeddings": 131072,
"mlp_bias": false,
"model_type": "llama",
"num_attention_heads": 24,
"num_hidden_layers": 15,
"num_key_value_heads": 8,
"pretraining_tp": 1,
"rms_norm_eps": 1e-05,
"rope_scaling": {
"factor": 32.0,
"high_freq_factor": 4.0,
"low_freq_factor": 1.0,
"original_max_position_embeddings": 8192,
"rope_type": "llama3"
},
"rope_theta": 500000.0,
"tie_word_embeddings": true,
"torch_dtype": "float32",
"transformers_version": "4.51.3",
"use_cache": true,
"vocab_size": 128256
}

9
generation_config.json Normal file
View File

@@ -0,0 +1,9 @@
{
"_from_model_config": true,
"bos_token_id": 128000,
"do_sample": true,
"eos_token_id": 128001,
"temperature": 0.6,
"top_p": 0.9,
"transformers_version": "4.51.3"
}

View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:7a67e77f5cddaa74235282351a5eb919b2900514fc39cac13d93aacc4e87c2db
size 4998767360

View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:65c7f072d2397d2366269c8b4758619cd3ad8fa81b020b1f3eb75930203c0c50
size 4193446640

View File

@@ -0,0 +1,145 @@
{
"metadata": {
"total_size": 9192198144
},
"weight_map": {
"lm_head.weight": "model-00002-of-00002.safetensors",
"model.embed_tokens.weight": "model-00001-of-00002.safetensors",
"model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
"model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
"model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
"model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
"model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
"model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
"model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
"model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
"model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
"model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
"model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
"model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
"model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
"model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
"model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
"model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
"model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
"model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
"model.layers.10.input_layernorm.weight": "model-00002-of-00002.safetensors",
"model.layers.10.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
"model.layers.10.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
"model.layers.10.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
"model.layers.10.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
"model.layers.10.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
"model.layers.10.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
"model.layers.10.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
"model.layers.10.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
"model.layers.11.input_layernorm.weight": "model-00002-of-00002.safetensors",
"model.layers.11.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
"model.layers.11.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
"model.layers.11.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
"model.layers.11.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
"model.layers.11.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
"model.layers.11.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
"model.layers.11.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
"model.layers.11.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
"model.layers.12.input_layernorm.weight": "model-00002-of-00002.safetensors",
"model.layers.12.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
"model.layers.12.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
"model.layers.12.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
"model.layers.12.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
"model.layers.12.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
"model.layers.12.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
"model.layers.12.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
"model.layers.12.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
"model.layers.13.input_layernorm.weight": "model-00002-of-00002.safetensors",
"model.layers.13.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
"model.layers.13.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
"model.layers.13.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
"model.layers.13.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
"model.layers.13.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
"model.layers.13.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
"model.layers.13.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
"model.layers.13.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
"model.layers.14.input_layernorm.weight": "model-00002-of-00002.safetensors",
"model.layers.14.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
"model.layers.14.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
"model.layers.14.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
"model.layers.14.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
"model.layers.14.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
"model.layers.14.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
"model.layers.14.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
"model.layers.14.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
"model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
"model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
"model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
"model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
"model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
"model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
"model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
"model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
"model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
"model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
"model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
"model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
"model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
"model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
"model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
"model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
"model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
"model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
"model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
"model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
"model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
"model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
"model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
"model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
"model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
"model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
"model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
"model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
"model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
"model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
"model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
"model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
"model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
"model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
"model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
"model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
"model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
"model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
"model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
"model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
"model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
"model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
"model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
"model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
"model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
"model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
"model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
"model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
"model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
"model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
"model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
"model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
"model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
"model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
"model.layers.8.input_layernorm.weight": "model-00002-of-00002.safetensors",
"model.layers.8.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
"model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
"model.layers.8.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
"model.layers.8.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
"model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
"model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
"model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
"model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
"model.layers.9.input_layernorm.weight": "model-00002-of-00002.safetensors",
"model.layers.9.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
"model.layers.9.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
"model.layers.9.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
"model.layers.9.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
"model.layers.9.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
"model.layers.9.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
"model.layers.9.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
"model.layers.9.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
"model.norm.weight": "model-00002-of-00002.safetensors"
}
}

17
special_tokens_map.json Normal file
View File

@@ -0,0 +1,17 @@
{
"bos_token": {
"content": "<|begin_of_text|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"eos_token": {
"content": "<|end_of_text|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"pad_token": "<|end_of_text|>"
}

1
student_config.json Normal file
View File

@@ -0,0 +1 @@
{"pruned_teacher_layers": [2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26], "copied_teacher_layers": [0, 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27]}

3
tokenizer.json Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:30992ead73e47f7c13b7f78241657e0306ee446ef8bb7150ecf6c551a090feb0
size 17210202

2063
tokenizer_config.json Normal file

File diff suppressed because it is too large Load Diff

3
training_args.bin Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:45a9529abbbb4da900019eb028cb3be9c94c4fd77231c418347be27e8eb074e8
size 5560