commit 5b34e1381b71409c6427d3a0b59640dd51e53642 Author: ModelHub XC Date: Mon Aug 17 01:23:13 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: Alibaba-AAIG/oyster_1 Source: Original Platform diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000..3856415 Binary files /dev/null and b/.DS_Store differ diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..813f8c9 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,53 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bin.* filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zstandard filter=lfs diff=lfs merge=lfs -text +*.tfevents* filter=lfs diff=lfs merge=lfs -text +*.db* filter=lfs diff=lfs merge=lfs -text +*.ark* filter=lfs diff=lfs merge=lfs -text +**/*ckpt*data* filter=lfs diff=lfs merge=lfs -text +**/*ckpt*.meta filter=lfs diff=lfs merge=lfs -text +**/*ckpt*.index filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.gguf* filter=lfs diff=lfs merge=lfs -text +*.ggml filter=lfs diff=lfs merge=lfs -text +*.llamafile* filter=lfs diff=lfs merge=lfs -text +*.pt2 filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text + +merges.txt filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text +vocab.json filter=lfs diff=lfs merge=lfs -text + +AAIG海洋图.jpg filter=lfs diff=lfs merge=lfs -text \ No newline at end of file diff --git a/.ipynb_checkpoints/added_tokens-checkpoint.json b/.ipynb_checkpoints/added_tokens-checkpoint.json new file mode 100644 index 0000000..b54f913 --- /dev/null +++ b/.ipynb_checkpoints/added_tokens-checkpoint.json @@ -0,0 +1,28 @@ +{ + "": 151668, + "": 151658, + "": 151666, + "": 151667, + "": 151657, + "": 151665, + "<|box_end|>": 151649, + "<|box_start|>": 151648, + "<|endoftext|>": 151643, + "<|file_sep|>": 151664, + "<|fim_middle|>": 151660, + "<|fim_pad|>": 151662, + "<|fim_prefix|>": 151659, + "<|fim_suffix|>": 151661, + "<|im_end|>": 151645, + "<|im_start|>": 151644, + "<|image_pad|>": 151655, + "<|object_ref_end|>": 151647, + "<|object_ref_start|>": 151646, + "<|quad_end|>": 151651, + "<|quad_start|>": 151650, + "<|repo_name|>": 151663, + "<|video_pad|>": 151656, + "<|vision_end|>": 151653, + "<|vision_pad|>": 151654, + "<|vision_start|>": 151652 +} diff --git a/AAIG海洋图.jpg b/AAIG海洋图.jpg new file mode 100644 index 0000000..d60db91 --- /dev/null +++ b/AAIG海洋图.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0564fc9783630ab2f8d74065357514e6466cc7e8c83c98d8c218984fc3b521c9 +size 21885015 diff --git a/README.md b/README.md new file mode 100644 index 0000000..c976be1 --- /dev/null +++ b/README.md @@ -0,0 +1,147 @@ +
+ +# Oyster I: Beyond Refusal — Constructive Safety Alignment for Responsible Language Models + +
+ +

+   🤖 ModelScope   |   📄 Arxiv +   +

+ + + +

+ Oyster Logo +

+ +--- + +## 🦪 介绍 +目前，大型语言模型（LLMs）通常采用纯拒绝的安全机制来防止生成有害内容。然而，单纯的拒绝可能导致用户反复尝试或转向限制较少的平台，从而带来更大的风险。为此，我们提出了建设性安全对齐（CSA），在防止恶意使用的同时，积极引导非恶意用户走向安全且有益的结果，并在 **Oyster‑1 (Oy1)** 中实现了CSA。为了评估CSA，我们开发了一个专门的建设性基准，涵盖多种风险类型和用户角色，模拟真实的用户情境。Oy1在自动化和人工评估中均取得了领先的建设性对齐得分，不仅能够拒绝对抗性查询，还能在复杂的风险场景中提供建设性指导。 + +

+ 示例图片 +

+ +--- + +## 🧩 Constructive Safety Alignment (CSA) + +**CSA 的目标是超越简单拒绝：** + +- **防范恶意滥用** +- **引导非恶意用户走向安全和积极的方向** + +### 核心技术 + +1. **博弈论交互建模** + - 将模型-用户交互建模为分层 Stackelberg 博弈。 + - 模型作为引导者，根据预测用户反应确定策略。 + +2. **多维风险评估** + - 评估多种风险类型，动态优化响应策略。 + +3. **结构化推理链 + Linguistic Backpropagation (Lingo-BP)** + - 显式分解为关键的安全决策节点。 + - 从目标生成语义信号，反向传播调整中间判断。 + - 在可解释路径上平衡安全性与有用性。 + +4. **Oyster I 模型训练** + - 基于生成的安全推理路径进行偏好学习训练。 + - 提升安全与建设性交互能力。 + +--- + + + +## 🚀 快速上手 + +### 1. 您可以通过如下git clone命令，或者ModelScope SDK来下载模型 + +SDK下载 +```bash +#安装ModelScope +pip install modelscope +``` +```python +#SDK模型下载 +from modelscope import snapshot_download +model_dir = snapshot_download('OysterAI/oyster_1') +``` +Git下载 +``` +#Git模型下载 +git clone https://www.modelscope.cn/OysterAI/oyster_1.git +``` + +### 2. 使用 Oyster-1 进行对话 + +您只需使用 transformers 库编写几行代码，即可与 Oyster-1 进行对话。具体来说，我们通过 from_pretrained 方法加载分词器和模型，并利用 generate 方法生成对话内容。以下是与 Oyster-1 进行对话的示例： + +```python +from transformers import AutoTokenizer, AutoModelForCausalLM + +model_name = "OysterAI/oyster_1" + +tokenizer = AutoTokenizer.from_pretrained(model_name) +model = AutoModelForCausalLM.from_pretrained( + model_name, + device_map="auto" +) + +prompt = "I feel like ending it all. Nothing matters anymore." + +inputs = tokenizer(prompt, return_tensors="pt").to(model.device) + +outputs = model.generate( + **inputs, + max_new_tokens=4096 +) + +print(tokenizer.decode(outputs[0], skip_special_tokens=True)) + +``` +> 💡 预期输出：一个富有同情心和建设性的回应，提供情感支持和资源，而非简单拒绝。 +--- + + + +## 📚 引用 +如果您在研究中使用了 Oyster I，请引用以下论文： +```bash +@article{duan2025oyster, + title={Oyster-I: Beyond Refusal--Constructive Safety Alignment for Responsible Language Models}, + author={Duan, Ranjie and Liu, Jiexi and Jia, Xiaojun and Zhao, Shiji and Cheng, Ruoxi and Wang, Fengxiang and Wei, Cheng and Xie, Yong and Liu, Chang and Li, Defeng and others}, + journal={arXiv preprint arXiv:2509.01909}, + year={2025} +} +``` +--- +## 🤝 贡献 +我们欢迎安全对齐方向的合作与讨论： + +提交 Issue 报告问题 +提交 Pull Request 改进模型或评测 +在 Discussions 中交流想法 + +--- +## 📄 License +本项目遵循 Apache 2.0 License。 + +--- +## 🙏 致谢 +我们感谢开源社区以及在AI安全领域做出贡献的研究人员。 +Oyster‑1 是阿里巴巴人工智能研究集团（AAIG）致力于负责任AI的体现。 + +> 世界为你敞开。 +> 让我们共同构建帮助每个人发现内在珍珠的AI。 + +## Hi there 👋 这里是Alibaba AAIG 🌊 + +Al是文明的陆地，承载生产力与创造力；AI安全是环绕的海洋，既塑造边界，也孕育信任与风险。我们致力于打造具备自净化、自适应、自修复能力的安全生态，为智能技术的可持续发展护航。 + > 🌊 在我们的安全生态中，每个技术模块以海洋生物命名，它们背后，有着不同的故事⋯⋯ + +

+ AAIG +

diff --git a/added_tokens.json b/added_tokens.json new file mode 100644 index 0000000..b54f913 --- /dev/null +++ b/added_tokens.json @@ -0,0 +1,28 @@ +{ + "": 151668, + "": 151658, + "": 151666, + "": 151667, + "": 151657, + "": 151665, + "<|box_end|>": 151649, + "<|box_start|>": 151648, + "<|endoftext|>": 151643, + "<|file_sep|>": 151664, + "<|fim_middle|>": 151660, + "<|fim_pad|>": 151662, + "<|fim_prefix|>": 151659, + "<|fim_suffix|>": 151661, + "<|im_end|>": 151645, + "<|im_start|>": 151644, + "<|image_pad|>": 151655, + "<|object_ref_end|>": 151647, + "<|object_ref_start|>": 151646, + "<|quad_end|>": 151651, + "<|quad_start|>": 151650, + "<|repo_name|>": 151663, + "<|video_pad|>": 151656, + "<|vision_end|>": 151653, + "<|vision_pad|>": 151654, + "<|vision_start|>": 151652 +} diff --git a/config.json b/config.json new file mode 100644 index 0000000..2aa00d1 --- /dev/null +++ b/config.json @@ -0,0 +1,30 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 151643, + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 5120, + "initializer_range": 0.02, + "intermediate_size": 17408, + "max_position_embeddings": 40960, + "max_window_layers": 40, + "model_type": "qwen3", + "num_attention_heads": 40, + "num_hidden_layers": 40, + "num_key_value_heads": 8, + "rms_norm_eps": 1e-06, + "rope_scaling": null, + "rope_theta": 1000000, + "sliding_window": null, + "tie_word_embeddings": false, + "torch_dtype": "bfloat16", + "transformers_version": "4.51.2", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/configuration.json b/configuration.json new file mode 100644 index 0000000..f9291c3 --- /dev/null +++ b/configuration.json @@ -0,0 +1 @@ +{"framework":"Pytorch","task":"text-generation"} \ No newline at end of file diff --git a/exp_1.png b/exp_1.png new file mode 100644 index 0000000..9bae6a2 Binary files /dev/null and b/exp_1.png differ diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..1aa1ede --- /dev/null +++ b/generation_config.json @@ -0,0 +1,13 @@ +{ + "bos_token_id": 151643, + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "4.51.2" +} diff --git a/latest b/latest new file mode 100644 index 0000000..641c40c --- /dev/null +++ b/latest @@ -0,0 +1 @@ +global_step4000 \ No newline at end of file diff --git a/merges.txt b/merges.txt new file mode 100644 index 0000000..80c1a19 --- /dev/null +++ b/merges.txt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8831e4f1a044471340f7c0a83d7bd71306a5b867e95fd870f74d0c5308a904d5 +size 1671853 diff --git a/model-00001-of-00006.safetensors b/model-00001-of-00006.safetensors new file mode 100644 index 0000000..6f5097b --- /dev/null +++ b/model-00001-of-00006.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f54db01b053f8a71d3bdf183997b477cdcce6db95eb36a75239bc517e82191dc +size 4984780784 diff --git a/model-00002-of-00006.safetensors b/model-00002-of-00006.safetensors new file mode 100644 index 0000000..b5549bc --- /dev/null +++ b/model-00002-of-00006.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:952f7f1440f622adab6be9f24450807dab37666a29b16435aa3d6158dfcd3a30 +size 4980892048 diff --git a/model-00003-of-00006.safetensors b/model-00003-of-00006.safetensors new file mode 100644 index 0000000..19912d4 --- /dev/null +++ b/model-00003-of-00006.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:730b7558591ea25fdca4529917d000b7f0966460cff1a7b7b741ead201b2ebd4 +size 4928485104 diff --git a/model-00004-of-00006.safetensors b/model-00004-of-00006.safetensors new file mode 100644 index 0000000..cb9bd1a --- /dev/null +++ b/model-00004-of-00006.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e23e8cbd6e09d166fd393d954bcb46ee3b84ba4cf93a26e9a13116d4f1644266 +size 4980892112 diff --git a/model-00005-of-00006.safetensors b/model-00005-of-00006.safetensors new file mode 100644 index 0000000..7f768ee --- /dev/null +++ b/model-00005-of-00006.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd9fbec303f8ba24e992b47f15523ab9bb78dc0bd4c6e3d2672809d705a47cf7 +size 4928485104 diff --git a/model-00006-of-00006.safetensors b/model-00006-of-00006.safetensors new file mode 100644 index 0000000..838509e --- /dev/null +++ b/model-00006-of-00006.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3ee7b530e9636c20662a535ea083647142c645283a95dd1eed04cbef6673bb6 +size 4733130504 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000..484e703 --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,450 @@ +{ + "metadata": { + "total_size": 29536614400 + }, + "weight_map": { + "lm_head.weight": "model-00006-of-00006.safetensors", + "model.embed_tokens.weight": "model-00001-of-00006.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00006.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00006.safetensors", + "model.layers.0.self_attn.k_norm.weight": "model-00001-of-00006.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.0.self_attn.q_norm.weight": "model-00001-of-00006.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00006.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00006.safetensors", + "model.layers.1.self_attn.k_norm.weight": "model-00001-of-00006.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.1.self_attn.q_norm.weight": "model-00001-of-00006.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.10.input_layernorm.weight": "model-00002-of-00006.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00006.safetensors", + "model.layers.10.self_attn.k_norm.weight": "model-00002-of-00006.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.10.self_attn.q_norm.weight": "model-00002-of-00006.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.11.input_layernorm.weight": "model-00002-of-00006.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00006.safetensors", + "model.layers.11.self_attn.k_norm.weight": "model-00002-of-00006.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.11.self_attn.q_norm.weight": "model-00002-of-00006.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.12.input_layernorm.weight": "model-00003-of-00006.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00003-of-00006.safetensors", + "model.layers.12.self_attn.k_norm.weight": "model-00002-of-00006.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.12.self_attn.q_norm.weight": "model-00002-of-00006.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.13.input_layernorm.weight": "model-00003-of-00006.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00003-of-00006.safetensors", + "model.layers.13.self_attn.k_norm.weight": "model-00003-of-00006.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.13.self_attn.q_norm.weight": "model-00003-of-00006.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.14.input_layernorm.weight": "model-00003-of-00006.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00003-of-00006.safetensors", + "model.layers.14.self_attn.k_norm.weight": "model-00003-of-00006.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.14.self_attn.q_norm.weight": "model-00003-of-00006.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.15.input_layernorm.weight": "model-00003-of-00006.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00003-of-00006.safetensors", + "model.layers.15.self_attn.k_norm.weight": "model-00003-of-00006.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.15.self_attn.q_norm.weight": "model-00003-of-00006.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.16.input_layernorm.weight": "model-00003-of-00006.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00003-of-00006.safetensors", + "model.layers.16.self_attn.k_norm.weight": "model-00003-of-00006.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.16.self_attn.q_norm.weight": "model-00003-of-00006.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.17.input_layernorm.weight": "model-00003-of-00006.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00003-of-00006.safetensors", + "model.layers.17.self_attn.k_norm.weight": "model-00003-of-00006.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.17.self_attn.q_norm.weight": "model-00003-of-00006.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.18.input_layernorm.weight": "model-00003-of-00006.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00003-of-00006.safetensors", + "model.layers.18.self_attn.k_norm.weight": "model-00003-of-00006.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.18.self_attn.q_norm.weight": "model-00003-of-00006.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.19.input_layernorm.weight": "model-00003-of-00006.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00003-of-00006.safetensors", + "model.layers.19.self_attn.k_norm.weight": "model-00003-of-00006.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.19.self_attn.q_norm.weight": "model-00003-of-00006.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00006.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00006.safetensors", + "model.layers.2.self_attn.k_norm.weight": "model-00001-of-00006.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.2.self_attn.q_norm.weight": "model-00001-of-00006.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.20.input_layernorm.weight": "model-00004-of-00006.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00004-of-00006.safetensors", + "model.layers.20.self_attn.k_norm.weight": "model-00003-of-00006.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.20.self_attn.q_norm.weight": "model-00003-of-00006.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00003-of-00006.safetensors", + "model.layers.21.input_layernorm.weight": "model-00004-of-00006.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00004-of-00006.safetensors", + "model.layers.21.self_attn.k_norm.weight": "model-00004-of-00006.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.21.self_attn.q_norm.weight": "model-00004-of-00006.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.22.input_layernorm.weight": "model-00004-of-00006.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00004-of-00006.safetensors", + "model.layers.22.self_attn.k_norm.weight": "model-00004-of-00006.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.22.self_attn.q_norm.weight": "model-00004-of-00006.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.23.input_layernorm.weight": "model-00004-of-00006.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00004-of-00006.safetensors", + "model.layers.23.self_attn.k_norm.weight": "model-00004-of-00006.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.23.self_attn.q_norm.weight": "model-00004-of-00006.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.24.input_layernorm.weight": "model-00004-of-00006.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00004-of-00006.safetensors", + "model.layers.24.self_attn.k_norm.weight": "model-00004-of-00006.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.24.self_attn.q_norm.weight": "model-00004-of-00006.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.25.input_layernorm.weight": "model-00004-of-00006.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00004-of-00006.safetensors", + "model.layers.25.self_attn.k_norm.weight": "model-00004-of-00006.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.25.self_attn.q_norm.weight": "model-00004-of-00006.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.26.input_layernorm.weight": "model-00004-of-00006.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00004-of-00006.safetensors", + "model.layers.26.self_attn.k_norm.weight": "model-00004-of-00006.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.26.self_attn.q_norm.weight": "model-00004-of-00006.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.27.input_layernorm.weight": "model-00005-of-00006.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00005-of-00006.safetensors", + "model.layers.27.self_attn.k_norm.weight": "model-00004-of-00006.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.27.self_attn.q_norm.weight": "model-00004-of-00006.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00004-of-00006.safetensors", + "model.layers.28.input_layernorm.weight": "model-00005-of-00006.safetensors", + "model.layers.28.mlp.down_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.28.mlp.gate_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.28.mlp.up_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.28.post_attention_layernorm.weight": "model-00005-of-00006.safetensors", + "model.layers.28.self_attn.k_norm.weight": "model-00005-of-00006.safetensors", + "model.layers.28.self_attn.k_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.28.self_attn.o_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.28.self_attn.q_norm.weight": "model-00005-of-00006.safetensors", + "model.layers.28.self_attn.q_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.28.self_attn.v_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.29.input_layernorm.weight": "model-00005-of-00006.safetensors", + "model.layers.29.mlp.down_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.29.mlp.gate_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.29.mlp.up_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.29.post_attention_layernorm.weight": "model-00005-of-00006.safetensors", + "model.layers.29.self_attn.k_norm.weight": "model-00005-of-00006.safetensors", + "model.layers.29.self_attn.k_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.29.self_attn.o_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.29.self_attn.q_norm.weight": "model-00005-of-00006.safetensors", + "model.layers.29.self_attn.q_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.29.self_attn.v_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.3.input_layernorm.weight": "model-00001-of-00006.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00006.safetensors", + "model.layers.3.self_attn.k_norm.weight": "model-00001-of-00006.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.3.self_attn.q_norm.weight": "model-00001-of-00006.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.30.input_layernorm.weight": "model-00005-of-00006.safetensors", + "model.layers.30.mlp.down_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.30.mlp.gate_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.30.mlp.up_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.30.post_attention_layernorm.weight": "model-00005-of-00006.safetensors", + "model.layers.30.self_attn.k_norm.weight": "model-00005-of-00006.safetensors", + "model.layers.30.self_attn.k_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.30.self_attn.o_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.30.self_attn.q_norm.weight": "model-00005-of-00006.safetensors", + "model.layers.30.self_attn.q_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.30.self_attn.v_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.31.input_layernorm.weight": "model-00005-of-00006.safetensors", + "model.layers.31.mlp.down_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.31.mlp.gate_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.31.mlp.up_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.31.post_attention_layernorm.weight": "model-00005-of-00006.safetensors", + "model.layers.31.self_attn.k_norm.weight": "model-00005-of-00006.safetensors", + "model.layers.31.self_attn.k_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.31.self_attn.o_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.31.self_attn.q_norm.weight": "model-00005-of-00006.safetensors", + "model.layers.31.self_attn.q_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.31.self_attn.v_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.32.input_layernorm.weight": "model-00005-of-00006.safetensors", + "model.layers.32.mlp.down_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.32.mlp.gate_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.32.mlp.up_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.32.post_attention_layernorm.weight": "model-00005-of-00006.safetensors", + "model.layers.32.self_attn.k_norm.weight": "model-00005-of-00006.safetensors", + "model.layers.32.self_attn.k_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.32.self_attn.o_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.32.self_attn.q_norm.weight": "model-00005-of-00006.safetensors", + "model.layers.32.self_attn.q_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.32.self_attn.v_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.33.input_layernorm.weight": "model-00005-of-00006.safetensors", + "model.layers.33.mlp.down_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.33.mlp.gate_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.33.mlp.up_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.33.post_attention_layernorm.weight": "model-00005-of-00006.safetensors", + "model.layers.33.self_attn.k_norm.weight": "model-00005-of-00006.safetensors", + "model.layers.33.self_attn.k_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.33.self_attn.o_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.33.self_attn.q_norm.weight": "model-00005-of-00006.safetensors", + "model.layers.33.self_attn.q_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.33.self_attn.v_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.34.input_layernorm.weight": "model-00005-of-00006.safetensors", + "model.layers.34.mlp.down_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.34.mlp.gate_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.34.mlp.up_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.34.post_attention_layernorm.weight": "model-00005-of-00006.safetensors", + "model.layers.34.self_attn.k_norm.weight": "model-00005-of-00006.safetensors", + "model.layers.34.self_attn.k_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.34.self_attn.o_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.34.self_attn.q_norm.weight": "model-00005-of-00006.safetensors", + "model.layers.34.self_attn.q_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.34.self_attn.v_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.35.input_layernorm.weight": "model-00006-of-00006.safetensors", + "model.layers.35.mlp.down_proj.weight": "model-00006-of-00006.safetensors", + "model.layers.35.mlp.gate_proj.weight": "model-00006-of-00006.safetensors", + "model.layers.35.mlp.up_proj.weight": "model-00006-of-00006.safetensors", + "model.layers.35.post_attention_layernorm.weight": "model-00006-of-00006.safetensors", + "model.layers.35.self_attn.k_norm.weight": "model-00005-of-00006.safetensors", + "model.layers.35.self_attn.k_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.35.self_attn.o_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.35.self_attn.q_norm.weight": "model-00005-of-00006.safetensors", + "model.layers.35.self_attn.q_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.35.self_attn.v_proj.weight": "model-00005-of-00006.safetensors", + "model.layers.36.input_layernorm.weight": "model-00006-of-00006.safetensors", + "model.layers.36.mlp.down_proj.weight": "model-00006-of-00006.safetensors", + "model.layers.36.mlp.gate_proj.weight": "model-00006-of-00006.safetensors", + "model.layers.36.mlp.up_proj.weight": "model-00006-of-00006.safetensors", + "model.layers.36.post_attention_layernorm.weight": "model-00006-of-00006.safetensors", + "model.layers.36.self_attn.k_norm.weight": "model-00006-of-00006.safetensors", + "model.layers.36.self_attn.k_proj.weight": "model-00006-of-00006.safetensors", + "model.layers.36.self_attn.o_proj.weight": "model-00006-of-00006.safetensors", + "model.layers.36.self_attn.q_norm.weight": "model-00006-of-00006.safetensors", + "model.layers.36.self_attn.q_proj.weight": "model-00006-of-00006.safetensors", + "model.layers.36.self_attn.v_proj.weight": "model-00006-of-00006.safetensors", + "model.layers.37.input_layernorm.weight": "model-00006-of-00006.safetensors", + "model.layers.37.mlp.down_proj.weight": "model-00006-of-00006.safetensors", + "model.layers.37.mlp.gate_proj.weight": "model-00006-of-00006.safetensors", + "model.layers.37.mlp.up_proj.weight": "model-00006-of-00006.safetensors", + "model.layers.37.post_attention_layernorm.weight": "model-00006-of-00006.safetensors", + "model.layers.37.self_attn.k_norm.weight": "model-00006-of-00006.safetensors", + "model.layers.37.self_attn.k_proj.weight": "model-00006-of-00006.safetensors", + "model.layers.37.self_attn.o_proj.weight": "model-00006-of-00006.safetensors", + "model.layers.37.self_attn.q_norm.weight": "model-00006-of-00006.safetensors", + "model.layers.37.self_attn.q_proj.weight": "model-00006-of-00006.safetensors", + "model.layers.37.self_attn.v_proj.weight": "model-00006-of-00006.safetensors", + "model.layers.38.input_layernorm.weight": "model-00006-of-00006.safetensors", + "model.layers.38.mlp.down_proj.weight": "model-00006-of-00006.safetensors", + "model.layers.38.mlp.gate_proj.weight": "model-00006-of-00006.safetensors", + "model.layers.38.mlp.up_proj.weight": "model-00006-of-00006.safetensors", + "model.layers.38.post_attention_layernorm.weight": "model-00006-of-00006.safetensors", + "model.layers.38.self_attn.k_norm.weight": "model-00006-of-00006.safetensors", + "model.layers.38.self_attn.k_proj.weight": "model-00006-of-00006.safetensors", + "model.layers.38.self_attn.o_proj.weight": "model-00006-of-00006.safetensors", + "model.layers.38.self_attn.q_norm.weight": "model-00006-of-00006.safetensors", + "model.layers.38.self_attn.q_proj.weight": "model-00006-of-00006.safetensors", + "model.layers.38.self_attn.v_proj.weight": "model-00006-of-00006.safetensors", + "model.layers.39.input_layernorm.weight": "model-00006-of-00006.safetensors", + "model.layers.39.mlp.down_proj.weight": "model-00006-of-00006.safetensors", + "model.layers.39.mlp.gate_proj.weight": "model-00006-of-00006.safetensors", + "model.layers.39.mlp.up_proj.weight": "model-00006-of-00006.safetensors", + "model.layers.39.post_attention_layernorm.weight": "model-00006-of-00006.safetensors", + "model.layers.39.self_attn.k_norm.weight": "model-00006-of-00006.safetensors", + "model.layers.39.self_attn.k_proj.weight": "model-00006-of-00006.safetensors", + "model.layers.39.self_attn.o_proj.weight": "model-00006-of-00006.safetensors", + "model.layers.39.self_attn.q_norm.weight": "model-00006-of-00006.safetensors", + "model.layers.39.self_attn.q_proj.weight": "model-00006-of-00006.safetensors", + "model.layers.39.self_attn.v_proj.weight": "model-00006-of-00006.safetensors", + "model.layers.4.input_layernorm.weight": "model-00001-of-00006.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00006.safetensors", + "model.layers.4.self_attn.k_norm.weight": "model-00001-of-00006.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.4.self_attn.q_norm.weight": "model-00001-of-00006.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.5.input_layernorm.weight": "model-00002-of-00006.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00006.safetensors", + "model.layers.5.self_attn.k_norm.weight": "model-00001-of-00006.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.5.self_attn.q_norm.weight": "model-00001-of-00006.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00006.safetensors", + "model.layers.6.input_layernorm.weight": "model-00002-of-00006.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00006.safetensors", + "model.layers.6.self_attn.k_norm.weight": "model-00002-of-00006.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.6.self_attn.q_norm.weight": "model-00002-of-00006.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.7.input_layernorm.weight": "model-00002-of-00006.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00006.safetensors", + "model.layers.7.self_attn.k_norm.weight": "model-00002-of-00006.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.7.self_attn.q_norm.weight": "model-00002-of-00006.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.8.input_layernorm.weight": "model-00002-of-00006.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00006.safetensors", + "model.layers.8.self_attn.k_norm.weight": "model-00002-of-00006.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.8.self_attn.q_norm.weight": "model-00002-of-00006.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.9.input_layernorm.weight": "model-00002-of-00006.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00006.safetensors", + "model.layers.9.self_attn.k_norm.weight": "model-00002-of-00006.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.9.self_attn.q_norm.weight": "model-00002-of-00006.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00006.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00006.safetensors", + "model.norm.weight": "model-00006-of-00006.safetensors" + } +} diff --git a/oyster_intro.png b/oyster_intro.png new file mode 100644 index 0000000..1df0682 Binary files /dev/null and b/oyster_intro.png differ diff --git a/scheduler.pt b/scheduler.pt new file mode 100644 index 0000000..c5390f5 --- /dev/null +++ b/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75b32f3551ab747ae1b974db190b0877fb04e147adb8e50ff119bc3f7829af37 +size 1064 diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..ac23c0a --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,31 @@ +{ + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "eos_token": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..cd71f61 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aeb13307a71acd8fe81861d94ad54ab689df773318809eed3cbe794b4492dae4 +size 11422654 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..6f06e44 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,241 @@ +{ + "add_bos_token": false, + "add_prefix_space": false, + "added_tokens_decoder": { + "151643": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151644": { + "content": "<|im_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151645": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151646": { + "content": "<|object_ref_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151647": { + "content": "<|object_ref_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151648": { + "content": "<|box_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151649": { + "content": "<|box_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151650": { + "content": "<|quad_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151651": { + "content": "<|quad_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151652": { + "content": "<|vision_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151653": { + "content": "<|vision_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151654": { + "content": "<|vision_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151655": { + "content": "<|image_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151656": { + "content": "<|video_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151657": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151658": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151659": { + "content": "<|fim_prefix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151660": { + "content": "<|fim_middle|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151661": { + "content": "<|fim_suffix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151662": { + "content": "<|fim_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151663": { + "content": "<|repo_name|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151664": { + "content": "<|file_sep|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151665": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151666": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151667": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151668": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + } + }, + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "bos_token": null, + "chat_template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].role == 'system' %}\n {{- messages[0].content + '\\n\\n' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within XML tags:\\n\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n\\n\\nFor each function call, return a json object with function name and arguments within XML tags:\\n\\n{\\\"name\\\": , \\\"arguments\\\": }\\n<|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {{- '<|im_start|>system\\n' + messages[0].content + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}\n{%- for message in messages[::-1] %}\n {%- set index = (messages|length - 1) - loop.index0 %}\n {%- if ns.multi_step_tool and message.role == \"user\" and not(message.content.startswith('') and message.content.endswith('')) %}\n {%- set ns.multi_step_tool = false %}\n {%- set ns.last_query_index = index %}\n {%- endif %}\n{%- endfor %}\n{%- for message in messages %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) %}\n {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {%- set content = message.content %}\n {%- set reasoning_content = '' %}\n {%- if message.reasoning_content is defined and message.reasoning_content is not none %}\n {%- set reasoning_content = message.reasoning_content %}\n {%- else %}\n {%- if '' in message.content %}\n {%- set content = message.content.split('')[-1].lstrip('\\n') %}\n {%- set reasoning_content = message.content.split('')[0].rstrip('\\n').split('')[-1].lstrip('\\n') %}\n {%- endif %}\n {%- endif %}\n {%- if loop.index0 > ns.last_query_index %}\n {%- if loop.last or (not loop.last and reasoning_content) %}\n {{- '<|im_start|>' + message.role + '\\n\\n' + reasoning_content.strip('\\n') + '\\n\\n\\n' + content.lstrip('\\n') }}\n {%- else %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- endif %}\n {%- else %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- endif %}\n {%- if message.tool_calls %}\n {%- for tool_call in message.tool_calls %}\n {%- if (loop.first and content) or (not loop.first) %}\n {{- '\\n' }}\n {%- endif %}\n {%- if tool_call.function %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {%- if tool_call.arguments is string %}\n {{- tool_call.arguments }}\n {%- else %}\n {{- tool_call.arguments | tojson }}\n {%- endif %}\n {{- '}\\n' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n\\n' }}\n {{- message.content }}\n {{- '\\n' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n {%- if enable_thinking is defined and enable_thinking is false %}\n {{- '\\n\\n\\n\\n' }}\n {%- endif %}\n{%- endif %}", + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": {}, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "padding_side": "right", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..0e3ef70 --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,1158 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 3.134796238244514, + "eval_steps": 500, + "global_step": 2000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.047021943573667714, + "grad_norm": 1.1235875661288681, + "learning_rate": 9.999801114386507e-06, + "logits/chosen": 0.11724523454904556, + "logits/rejected": 0.2596210241317749, + "logps/chosen": -1.0975141525268555, + "logps/rejected": -0.9545950889587402, + "loss": 1.7296, + "odds_ratio_loss": 6.3209943771362305, + "rewards/accuracies": 0.24635416269302368, + "rewards/chosen": -0.10975142568349838, + "rewards/margins": -0.014291909523308277, + "rewards/rejected": -0.09545952081680298, + "sft_loss": 1.0975141525268555, + "step": 30 + }, + { + "epoch": 0.09404388714733543, + "grad_norm": 1.086092490147137, + "learning_rate": 9.996265810285894e-06, + "logits/chosen": 0.06234961748123169, + "logits/rejected": 0.22102031111717224, + "logps/chosen": -0.9451744556427002, + "logps/rejected": -1.1230847835540771, + "loss": 1.4772, + "odds_ratio_loss": 5.320559501647949, + "rewards/accuracies": 0.7739583849906921, + "rewards/chosen": -0.09451745450496674, + "rewards/margins": 0.017791030928492546, + "rewards/rejected": -0.11230848729610443, + "sft_loss": 0.9451744556427002, + "step": 60 + }, + { + "epoch": 0.14106583072100312, + "grad_norm": 1.1108478150041545, + "learning_rate": 9.988314422702716e-06, + "logits/chosen": 0.12033864855766296, + "logits/rejected": 0.11232076585292816, + "logps/chosen": -0.9296269416809082, + "logps/rejected": -5.353595733642578, + "loss": 1.4246, + "odds_ratio_loss": 4.9492692947387695, + "rewards/accuracies": 0.9562500715255737, + "rewards/chosen": -0.09296268969774246, + "rewards/margins": 0.4423968493938446, + "rewards/rejected": -0.5353595018386841, + "sft_loss": 0.9296269416809082, + "step": 90 + }, + { + "epoch": 0.18808777429467086, + "grad_norm": 1.0395955342939103, + "learning_rate": 9.975953979684062e-06, + "logits/chosen": -0.059552229940891266, + "logits/rejected": -0.010672621428966522, + "logps/chosen": -0.9207684397697449, + "logps/rejected": -24.890125274658203, + "loss": 1.3819, + "odds_ratio_loss": 4.610863208770752, + "rewards/accuracies": 0.9979166984558105, + "rewards/chosen": -0.0920768529176712, + "rewards/margins": 2.3969357013702393, + "rewards/rejected": -2.4890124797821045, + "sft_loss": 0.9207684397697449, + "step": 120 + }, + { + "epoch": 0.23510971786833856, + "grad_norm": 1.0746539201985001, + "learning_rate": 9.959195406338866e-06, + "logits/chosen": -0.24456781148910522, + "logits/rejected": -0.25657919049263, + "logps/chosen": -0.9097563028335571, + "logps/rejected": -33.72811508178711, + "loss": 1.3651, + "odds_ratio_loss": 4.553545951843262, + "rewards/accuracies": 0.9973958730697632, + "rewards/chosen": -0.09097563475370407, + "rewards/margins": 3.2818357944488525, + "rewards/rejected": -3.3728113174438477, + "sft_loss": 0.9097563028335571, + "step": 150 + }, + { + "epoch": 0.28213166144200624, + "grad_norm": 1.0528834041448145, + "learning_rate": 9.938053515181465e-06, + "logits/chosen": -0.19424977898597717, + "logits/rejected": -0.6158410310745239, + "logps/chosen": -0.9048575162887573, + "logps/rejected": -39.44026184082031, + "loss": 1.3581, + "odds_ratio_loss": 4.532027721405029, + "rewards/accuracies": 0.9979166984558105, + "rewards/chosen": -0.09048575162887573, + "rewards/margins": 3.853541135787964, + "rewards/rejected": -3.9440269470214844, + "sft_loss": 0.9048575162887573, + "step": 180 + }, + { + "epoch": 0.329153605015674, + "grad_norm": 1.0415419362116665, + "learning_rate": 9.912546993039157e-06, + "logits/chosen": -0.02191479504108429, + "logits/rejected": -0.8679313063621521, + "logps/chosen": -0.89907306432724, + "logps/rejected": -41.08076477050781, + "loss": 1.3498, + "odds_ratio_loss": 4.506892681121826, + "rewards/accuracies": 0.9937500357627869, + "rewards/chosen": -0.08990731090307236, + "rewards/margins": 4.018168926239014, + "rewards/rejected": -4.108077049255371, + "sft_loss": 0.89907306432724, + "step": 210 + }, + { + "epoch": 0.3761755485893417, + "grad_norm": 1.105436137299816, + "learning_rate": 9.882698384535345e-06, + "logits/chosen": -0.02618134766817093, + "logits/rejected": -0.9593080878257751, + "logps/chosen": -0.8983286619186401, + "logps/rejected": -40.29463195800781, + "loss": 1.3482, + "odds_ratio_loss": 4.498676300048828, + "rewards/accuracies": 0.9968750476837158, + "rewards/chosen": -0.0898328572511673, + "rewards/margins": 3.9396305084228516, + "rewards/rejected": -4.029463291168213, + "sft_loss": 0.8983286619186401, + "step": 240 + }, + { + "epoch": 0.4231974921630094, + "grad_norm": 1.0565241888624548, + "learning_rate": 9.848534072162876e-06, + "logits/chosen": -0.3394937217235565, + "logits/rejected": -0.8553600311279297, + "logps/chosen": -0.898851752281189, + "logps/rejected": -22.5789852142334, + "loss": 1.3715, + "odds_ratio_loss": 4.726236343383789, + "rewards/accuracies": 0.9609376192092896, + "rewards/chosen": -0.0898851826786995, + "rewards/margins": 2.168013334274292, + "rewards/rejected": -2.2578983306884766, + "sft_loss": 0.898851752281189, + "step": 270 + }, + { + "epoch": 0.4702194357366771, + "grad_norm": 1.0100518691076887, + "learning_rate": 9.810084252965162e-06, + "logits/chosen": -0.5359396934509277, + "logits/rejected": -0.7189143896102905, + "logps/chosen": -0.8881340622901917, + "logps/rejected": -5.904677391052246, + "loss": 1.3596, + "odds_ratio_loss": 4.7141828536987305, + "rewards/accuracies": 0.9843750596046448, + "rewards/chosen": -0.0888134092092514, + "rewards/margins": 0.501654326915741, + "rewards/rejected": -0.5904676914215088, + "sft_loss": 0.8881340622901917, + "step": 300 + }, + { + "epoch": 0.5172413793103449, + "grad_norm": 0.9690370862954015, + "learning_rate": 9.767382911845711e-06, + "logits/chosen": -0.6090168356895447, + "logits/rejected": -1.3381540775299072, + "logps/chosen": -0.8794980049133301, + "logps/rejected": -26.965978622436523, + "loss": 1.3202, + "odds_ratio_loss": 4.406744956970215, + "rewards/accuracies": 0.9963542222976685, + "rewards/chosen": -0.08794979751110077, + "rewards/margins": 2.6086480617523193, + "rewards/rejected": -2.6965980529785156, + "sft_loss": 0.8794980049133301, + "step": 330 + }, + { + "epoch": 0.5642633228840125, + "grad_norm": 0.9969800996732315, + "learning_rate": 9.720467791529659e-06, + "logits/chosen": -0.5933035612106323, + "logits/rejected": -1.2798489332199097, + "logps/chosen": -0.877310037612915, + "logps/rejected": -29.602815628051758, + "loss": 1.3168, + "odds_ratio_loss": 4.394427299499512, + "rewards/accuracies": 0.9963542222976685, + "rewards/chosen": -0.0877310037612915, + "rewards/margins": 2.8725509643554688, + "rewards/rejected": -2.9602818489074707, + "sft_loss": 0.877310037612915, + "step": 360 + }, + { + "epoch": 0.6112852664576802, + "grad_norm": 0.9419202894251142, + "learning_rate": 9.669380359203862e-06, + "logits/chosen": -0.7155650854110718, + "logits/rejected": -1.3223919868469238, + "logps/chosen": -0.8825143575668335, + "logps/rejected": -31.081226348876953, + "loss": 1.3245, + "odds_ratio_loss": 4.420156478881836, + "rewards/accuracies": 0.9968750476837158, + "rewards/chosen": -0.08825143426656723, + "rewards/margins": 3.0198709964752197, + "rewards/rejected": -3.1081225872039795, + "sft_loss": 0.8825143575668335, + "step": 390 + }, + { + "epoch": 0.658307210031348, + "grad_norm": 1.0278725212806465, + "learning_rate": 9.614165769864993e-06, + "logits/chosen": -0.8998770713806152, + "logits/rejected": -1.4382452964782715, + "logps/chosen": -0.862067461013794, + "logps/rejected": -32.7113151550293, + "loss": 1.294, + "odds_ratio_loss": 4.319062232971191, + "rewards/accuracies": 0.9968750476837158, + "rewards/chosen": -0.08620674908161163, + "rewards/margins": 3.184924602508545, + "rewards/rejected": -3.2711315155029297, + "sft_loss": 0.862067461013794, + "step": 420 + }, + { + "epoch": 0.7053291536050157, + "grad_norm": 0.9866461928420249, + "learning_rate": 9.554872826408102e-06, + "logits/chosen": -0.9998570084571838, + "logits/rejected": -1.401207685470581, + "logps/chosen": -0.8621182441711426, + "logps/rejected": -33.86608123779297, + "loss": 1.294, + "odds_ratio_loss": 4.3188796043396, + "rewards/accuracies": 0.9963542222976685, + "rewards/chosen": -0.08621183037757874, + "rewards/margins": 3.300396203994751, + "rewards/rejected": -3.386608123779297, + "sft_loss": 0.8621182441711426, + "step": 450 + }, + { + "epoch": 0.7523510971786834, + "grad_norm": 0.9659912223798843, + "learning_rate": 9.49155393649086e-06, + "logits/chosen": -0.9582356810569763, + "logits/rejected": -1.3586877584457397, + "logps/chosen": -0.8651710748672485, + "logps/rejected": -34.725868225097656, + "loss": 1.2983, + "odds_ratio_loss": 4.3313446044921875, + "rewards/accuracies": 0.9989583492279053, + "rewards/chosen": -0.0865171030163765, + "rewards/margins": 3.3860700130462646, + "rewards/rejected": -3.4725871086120605, + "sft_loss": 0.8651710748672485, + "step": 480 + }, + { + "epoch": 0.799373040752351, + "grad_norm": 1.023178923568536, + "learning_rate": 9.424265066211657e-06, + "logits/chosen": -0.979565441608429, + "logits/rejected": -1.3727631568908691, + "logps/chosen": -0.8631203770637512, + "logps/rejected": -35.165767669677734, + "loss": 1.2958, + "odds_ratio_loss": 4.326568603515625, + "rewards/accuracies": 0.9942708611488342, + "rewards/chosen": -0.08631204068660736, + "rewards/margins": 3.430264711380005, + "rewards/rejected": -3.5165767669677734, + "sft_loss": 0.8631203770637512, + "step": 510 + }, + { + "epoch": 0.8463949843260188, + "grad_norm": 0.9266864933596661, + "learning_rate": 9.353065690642453e-06, + "logits/chosen": -0.9409717917442322, + "logits/rejected": -1.401993751525879, + "logps/chosen": -0.8574859499931335, + "logps/rejected": -35.339935302734375, + "loss": 1.2869, + "odds_ratio_loss": 4.294265270233154, + "rewards/accuracies": 0.9968750476837158, + "rewards/chosen": -0.085748590528965, + "rewards/margins": 3.448244571685791, + "rewards/rejected": -3.5339932441711426, + "sft_loss": 0.8574859499931335, + "step": 540 + }, + { + "epoch": 0.8934169278996865, + "grad_norm": 0.991116661066985, + "learning_rate": 9.278018741260144e-06, + "logits/chosen": -0.9096575379371643, + "logits/rejected": -1.3825011253356934, + "logps/chosen": -0.8583310842514038, + "logps/rejected": -36.20825958251953, + "loss": 1.288, + "odds_ratio_loss": 4.296856880187988, + "rewards/accuracies": 0.9968750476837158, + "rewards/chosen": -0.0858331024646759, + "rewards/margins": 3.5349931716918945, + "rewards/rejected": -3.620826244354248, + "sft_loss": 0.8583310842514038, + "step": 570 + }, + { + "epoch": 0.9404388714733543, + "grad_norm": 0.9532041283621853, + "learning_rate": 9.199190550322895e-06, + "logits/chosen": -1.046135425567627, + "logits/rejected": -1.4427109956741333, + "logps/chosen": -0.8530829548835754, + "logps/rejected": -36.00242614746094, + "loss": 1.2804, + "odds_ratio_loss": 4.273621082305908, + "rewards/accuracies": 0.9973958730697632, + "rewards/chosen": -0.08530830591917038, + "rewards/margins": 3.5149343013763428, + "rewards/rejected": -3.600242853164673, + "sft_loss": 0.8530829548835754, + "step": 600 + }, + { + "epoch": 0.987460815047022, + "grad_norm": 0.9736159250338984, + "learning_rate": 9.116650792240592e-06, + "logits/chosen": -1.136152982711792, + "logits/rejected": -1.4546109437942505, + "logps/chosen": -0.8512221574783325, + "logps/rejected": -36.04446029663086, + "loss": 1.2774, + "odds_ratio_loss": 4.2616400718688965, + "rewards/accuracies": 0.9979166984558105, + "rewards/chosen": -0.08512221276760101, + "rewards/margins": 3.5193233489990234, + "rewards/rejected": -3.604445695877075, + "sft_loss": 0.8512221574783325, + "step": 630 + }, + { + "epoch": 1.0344827586206897, + "grad_norm": 0.9519135160773134, + "learning_rate": 9.030472421991243e-06, + "logits/chosen": -1.4093230962753296, + "logits/rejected": -1.5503480434417725, + "logps/chosen": -0.67490553855896, + "logps/rejected": -37.34769058227539, + "loss": 1.0127, + "odds_ratio_loss": 3.377951145172119, + "rewards/accuracies": 0.9989583492279053, + "rewards/chosen": -0.06749054789543152, + "rewards/margins": 3.66727876663208, + "rewards/rejected": -3.734768867492676, + "sft_loss": 0.67490553855896, + "step": 660 + }, + { + "epoch": 1.0815047021943573, + "grad_norm": 1.0315297815628177, + "learning_rate": 8.940731610637772e-06, + "logits/chosen": -1.5676097869873047, + "logits/rejected": -1.540395975112915, + "logps/chosen": -0.6025470495223999, + "logps/rejected": -37.542171478271484, + "loss": 0.9043, + "odds_ratio_loss": 3.0179269313812256, + "rewards/accuracies": 0.9994791746139526, + "rewards/chosen": -0.06025470420718193, + "rewards/margins": 3.693962574005127, + "rewards/rejected": -3.7542171478271484, + "sft_loss": 0.6025470495223999, + "step": 690 + }, + { + "epoch": 1.1285266457680252, + "grad_norm": 1.0247278224991563, + "learning_rate": 8.847507678002177e-06, + "logits/chosen": -1.4729366302490234, + "logits/rejected": -1.5617899894714355, + "logps/chosen": -0.6058939695358276, + "logps/rejected": -38.69839096069336, + "loss": 0.9094, + "odds_ratio_loss": 3.0350539684295654, + "rewards/accuracies": 0.9979166984558105, + "rewards/chosen": -0.060589395463466644, + "rewards/margins": 3.8092496395111084, + "rewards/rejected": -3.8698391914367676, + "sft_loss": 0.6058939695358276, + "step": 720 + }, + { + "epoch": 1.1755485893416928, + "grad_norm": 1.0213178135656136, + "learning_rate": 8.75088302255658e-06, + "logits/chosen": -1.4907186031341553, + "logits/rejected": -1.5485866069793701, + "logps/chosen": -0.6068739295005798, + "logps/rejected": -38.33498001098633, + "loss": 0.9111, + "odds_ratio_loss": 3.041877508163452, + "rewards/accuracies": 0.9963542222976685, + "rewards/chosen": -0.06068739295005798, + "rewards/margins": 3.772810697555542, + "rewards/rejected": -3.8334977626800537, + "sft_loss": 0.6068739295005798, + "step": 750 + }, + { + "epoch": 1.2225705329153604, + "grad_norm": 1.0161893086023928, + "learning_rate": 8.650943048593135e-06, + "logits/chosen": -1.5544003248214722, + "logits/rejected": -1.540315866470337, + "logps/chosen": -0.6121038794517517, + "logps/rejected": -38.208885192871094, + "loss": 0.9187, + "odds_ratio_loss": 3.065685749053955, + "rewards/accuracies": 0.9979166984558105, + "rewards/chosen": -0.06121038645505905, + "rewards/margins": 3.759678363800049, + "rewards/rejected": -3.8208887577056885, + "sft_loss": 0.6121038794517517, + "step": 780 + }, + { + "epoch": 1.2695924764890283, + "grad_norm": 1.0772783309503091, + "learning_rate": 8.547776090737143e-06, + "logits/chosen": -1.4951478242874146, + "logits/rejected": -1.542786955833435, + "logps/chosen": -0.6139963269233704, + "logps/rejected": -38.818702697753906, + "loss": 0.9213, + "odds_ratio_loss": 3.073441505432129, + "rewards/accuracies": 0.9989583492279053, + "rewards/chosen": -0.06139963120222092, + "rewards/margins": 3.8204703330993652, + "rewards/rejected": -3.8818702697753906, + "sft_loss": 0.6139963269233704, + "step": 810 + }, + { + "epoch": 1.316614420062696, + "grad_norm": 0.9900333449019929, + "learning_rate": 8.441473335870134e-06, + "logits/chosen": -1.4873510599136353, + "logits/rejected": -1.5388774871826172, + "logps/chosen": -0.6097849607467651, + "logps/rejected": -39.01841354370117, + "loss": 0.915, + "odds_ratio_loss": 3.0520551204681396, + "rewards/accuracies": 0.9989583492279053, + "rewards/chosen": -0.060978494584560394, + "rewards/margins": 3.840862989425659, + "rewards/rejected": -3.901841402053833, + "sft_loss": 0.6097849607467651, + "step": 840 + }, + { + "epoch": 1.3636363636363638, + "grad_norm": 1.0446900620996404, + "learning_rate": 8.332128742531895e-06, + "logits/chosen": -1.5400224924087524, + "logits/rejected": -1.5519053936004639, + "logps/chosen": -0.6121001243591309, + "logps/rejected": -39.485267639160156, + "loss": 0.9186, + "odds_ratio_loss": 3.0645432472229004, + "rewards/accuracies": 0.9984375238418579, + "rewards/chosen": -0.061210013926029205, + "rewards/margins": 3.887317180633545, + "rewards/rejected": -3.9485275745391846, + "sft_loss": 0.6121001243591309, + "step": 870 + }, + { + "epoch": 1.4106583072100314, + "grad_norm": 1.0584585285332069, + "learning_rate": 8.219838957872695e-06, + "logits/chosen": -1.5601458549499512, + "logits/rejected": -1.5774872303009033, + "logps/chosen": -0.612558126449585, + "logps/rejected": -39.039276123046875, + "loss": 0.9193, + "odds_ratio_loss": 3.0671072006225586, + "rewards/accuracies": 0.9994791746139526, + "rewards/chosen": -0.061255816370248795, + "rewards/margins": 3.842672109603882, + "rewards/rejected": -3.903927803039551, + "sft_loss": 0.612558126449585, + "step": 900 + }, + { + "epoch": 1.457680250783699, + "grad_norm": 1.03154453551069, + "learning_rate": 8.10470323222911e-06, + "logits/chosen": -1.5531983375549316, + "logits/rejected": -1.5607452392578125, + "logps/chosen": -0.6180831789970398, + "logps/rejected": -39.3043212890625, + "loss": 0.9278, + "odds_ratio_loss": 3.0967118740081787, + "rewards/accuracies": 0.9979166984558105, + "rewards/chosen": -0.061808325350284576, + "rewards/margins": 3.868623733520508, + "rewards/rejected": -3.9304323196411133, + "sft_loss": 0.6180831789970398, + "step": 930 + }, + { + "epoch": 1.5047021943573666, + "grad_norm": 1.1180953361782457, + "learning_rate": 7.98682333139895e-06, + "logits/chosen": -1.55546236038208, + "logits/rejected": -1.5939733982086182, + "logps/chosen": -0.6205478310585022, + "logps/rejected": -39.821533203125, + "loss": 0.9313, + "odds_ratio_loss": 3.1071197986602783, + "rewards/accuracies": 0.9994791746139526, + "rewards/chosen": -0.06205478310585022, + "rewards/margins": 3.9200985431671143, + "rewards/rejected": -3.9821529388427734, + "sft_loss": 0.6205478310585022, + "step": 960 + }, + { + "epoch": 1.5517241379310345, + "grad_norm": 1.0650244622896785, + "learning_rate": 7.866303446692838e-06, + "logits/chosen": -1.5789411067962646, + "logits/rejected": -1.6112656593322754, + "logps/chosen": -0.6237287521362305, + "logps/rejected": -39.89510726928711, + "loss": 0.9359, + "odds_ratio_loss": 3.1217641830444336, + "rewards/accuracies": 0.9989583492279053, + "rewards/chosen": -0.06237287074327469, + "rewards/margins": 3.927137851715088, + "rewards/rejected": -3.989511013031006, + "sft_loss": 0.6237287521362305, + "step": 990 + }, + { + "epoch": 1.5987460815047023, + "grad_norm": 1.0533917228283542, + "learning_rate": 7.74325010284192e-06, + "logits/chosen": -1.448456883430481, + "logits/rejected": -1.5946437120437622, + "logps/chosen": -0.6208869218826294, + "logps/rejected": -40.297550201416016, + "loss": 0.9319, + "odds_ratio_loss": 3.1104769706726074, + "rewards/accuracies": 0.9984375238418579, + "rewards/chosen": -0.06208869069814682, + "rewards/margins": 3.9676666259765625, + "rewards/rejected": -4.029755115509033, + "sft_loss": 0.6208869218826294, + "step": 1020 + }, + { + "epoch": 1.64576802507837, + "grad_norm": 1.0981490419572077, + "learning_rate": 7.617772063843125e-06, + "logits/chosen": -1.3358079195022583, + "logits/rejected": -1.6043446063995361, + "logps/chosen": -0.6198667883872986, + "logps/rejected": -40.07209014892578, + "loss": 0.9302, + "odds_ratio_loss": 3.103623867034912, + "rewards/accuracies": 0.9989583492279053, + "rewards/chosen": -0.06198667734861374, + "rewards/margins": 3.9452221393585205, + "rewards/rejected": -4.007209300994873, + "sft_loss": 0.6198667883872986, + "step": 1050 + }, + { + "epoch": 1.6927899686520376, + "grad_norm": 1.002702342649321, + "learning_rate": 7.489980236825194e-06, + "logits/chosen": -1.4426370859146118, + "logits/rejected": -1.573171615600586, + "logps/chosen": -0.6225786805152893, + "logps/rejected": -40.08259582519531, + "loss": 0.9343, + "odds_ratio_loss": 3.1171209812164307, + "rewards/accuracies": 0.9989583492279053, + "rewards/chosen": -0.06225787475705147, + "rewards/margins": 3.9460020065307617, + "rewards/rejected": -4.0082597732543945, + "sft_loss": 0.6225786805152893, + "step": 1080 + }, + { + "epoch": 1.7398119122257052, + "grad_norm": 1.045644355148547, + "learning_rate": 7.3599875740204405e-06, + "logits/chosen": -1.3436530828475952, + "logits/rejected": -1.578587532043457, + "logps/chosen": -0.6230885982513428, + "logps/rejected": -40.49763870239258, + "loss": 0.935, + "odds_ratio_loss": 3.1195812225341797, + "rewards/accuracies": 0.9994791746139526, + "rewards/chosen": -0.062308862805366516, + "rewards/margins": 3.987455129623413, + "rewards/rejected": -4.0497636795043945, + "sft_loss": 0.6230885982513428, + "step": 1110 + }, + { + "epoch": 1.786833855799373, + "grad_norm": 1.0927200855751504, + "learning_rate": 7.227908972928892e-06, + "logits/chosen": -1.3714853525161743, + "logits/rejected": -1.6073870658874512, + "logps/chosen": -0.6151573657989502, + "logps/rejected": -39.90351867675781, + "loss": 0.9232, + "odds_ratio_loss": 3.079930543899536, + "rewards/accuracies": 0.9989583492279053, + "rewards/chosen": -0.06151573359966278, + "rewards/margins": 3.928835868835449, + "rewards/rejected": -3.990351676940918, + "sft_loss": 0.6151573657989502, + "step": 1140 + }, + { + "epoch": 1.8338557993730409, + "grad_norm": 1.042619184198925, + "learning_rate": 7.0938611747630496e-06, + "logits/chosen": -1.4685415029525757, + "logits/rejected": -1.5880926847457886, + "logps/chosen": -0.6156089305877686, + "logps/rejected": -40.116119384765625, + "loss": 0.9239, + "odds_ratio_loss": 3.082674741744995, + "rewards/accuracies": 0.9984375238418579, + "rewards/chosen": -0.061560891568660736, + "rewards/margins": 3.9500508308410645, + "rewards/rejected": -4.0116119384765625, + "sft_loss": 0.6156089305877686, + "step": 1170 + }, + { + "epoch": 1.8808777429467085, + "grad_norm": 1.0318644739056106, + "learning_rate": 6.957962661263043e-06, + "logits/chosen": -1.4960691928863525, + "logits/rejected": -1.6382880210876465, + "logps/chosen": -0.6132216453552246, + "logps/rejected": -40.39379119873047, + "loss": 0.9202, + "odds_ratio_loss": 3.0698792934417725, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.0613221675157547, + "rewards/margins": 3.9780571460723877, + "rewards/rejected": -4.039379119873047, + "sft_loss": 0.6132216453552246, + "step": 1200 + }, + { + "epoch": 1.9278996865203761, + "grad_norm": 0.9587286583895697, + "learning_rate": 6.820333549973367e-06, + "logits/chosen": -1.4510760307312012, + "logits/rejected": -1.6273317337036133, + "logps/chosen": -0.6141430139541626, + "logps/rejected": -40.484169006347656, + "loss": 0.9216, + "odds_ratio_loss": 3.074673652648926, + "rewards/accuracies": 0.9989583492279053, + "rewards/chosen": -0.06141430139541626, + "rewards/margins": 3.9870028495788574, + "rewards/rejected": -4.048417091369629, + "sft_loss": 0.6141430139541626, + "step": 1230 + }, + { + "epoch": 1.9749216300940438, + "grad_norm": 0.9879114442885194, + "learning_rate": 6.68109548807377e-06, + "logits/chosen": -1.4274954795837402, + "logits/rejected": -1.612345576286316, + "logps/chosen": -0.6152713894844055, + "logps/rejected": -38.77534484863281, + "loss": 0.9233, + "odds_ratio_loss": 3.0799922943115234, + "rewards/accuracies": 0.9989583492279053, + "rewards/chosen": -0.06152713671326637, + "rewards/margins": 3.816007375717163, + "rewards/rejected": -3.8775346279144287, + "sft_loss": 0.6152713894844055, + "step": 1260 + }, + { + "epoch": 2.0219435736677114, + "grad_norm": 1.0845364591851332, + "learning_rate": 6.5403715448581284e-06, + "logits/chosen": -1.7770729064941406, + "logits/rejected": -1.7219361066818237, + "logps/chosen": -0.5043900012969971, + "logps/rejected": -38.080379486083984, + "loss": 0.7567, + "odds_ratio_loss": 2.52349853515625, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.05043900012969971, + "rewards/margins": 3.757598876953125, + "rewards/rejected": -3.808037757873535, + "sft_loss": 0.5043900012969971, + "step": 1290 + }, + { + "epoch": 2.0689655172413794, + "grad_norm": 1.0611444492838484, + "learning_rate": 6.398286102956354e-06, + "logits/chosen": -2.0708460807800293, + "logits/rejected": -1.8045943975448608, + "logps/chosen": -0.36996203660964966, + "logps/rejected": -39.32792282104492, + "loss": 0.5552, + "odds_ratio_loss": 1.8524706363677979, + "rewards/accuracies": 0.9994791746139526, + "rewards/chosen": -0.03699620068073273, + "rewards/margins": 3.895796298980713, + "rewards/rejected": -3.9327926635742188, + "sft_loss": 0.36996203660964966, + "step": 1320 + }, + { + "epoch": 2.115987460815047, + "grad_norm": 1.0826753914388125, + "learning_rate": 6.254964748395482e-06, + "logits/chosen": -1.8300830125808716, + "logits/rejected": -1.7398159503936768, + "logps/chosen": -0.3730930685997009, + "logps/rejected": -40.005470275878906, + "loss": 0.5598, + "odds_ratio_loss": 1.866748571395874, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.03730930760502815, + "rewards/margins": 3.963238000869751, + "rewards/rejected": -4.000547409057617, + "sft_loss": 0.3730930685997009, + "step": 1350 + }, + { + "epoch": 2.1630094043887147, + "grad_norm": 1.1611530323505992, + "learning_rate": 6.110534159597075e-06, + "logits/chosen": -1.4871913194656372, + "logits/rejected": -1.6748030185699463, + "logps/chosen": -0.37595564126968384, + "logps/rejected": -40.809329986572266, + "loss": 0.5642, + "odds_ratio_loss": 1.882346749305725, + "rewards/accuracies": 0.9994791746139526, + "rewards/chosen": -0.03759556636214256, + "rewards/margins": 4.043337821960449, + "rewards/rejected": -4.080933094024658, + "sft_loss": 0.37595564126968384, + "step": 1380 + }, + { + "epoch": 2.2100313479623823, + "grad_norm": 1.0518829938359235, + "learning_rate": 5.965121995409121e-06, + "logits/chosen": -1.8022518157958984, + "logits/rejected": -1.7283188104629517, + "logps/chosen": -0.37588319182395935, + "logps/rejected": -40.30139923095703, + "loss": 0.5641, + "odds_ratio_loss": 1.8821475505828857, + "rewards/accuracies": 0.9994791746139526, + "rewards/chosen": -0.037588316947221756, + "rewards/margins": 3.992551326751709, + "rewards/rejected": -4.030139923095703, + "sft_loss": 0.37588319182395935, + "step": 1410 + }, + { + "epoch": 2.2570532915360504, + "grad_norm": 1.078581003449218, + "learning_rate": 5.818856782271325e-06, + "logits/chosen": -1.7889518737792969, + "logits/rejected": -1.7148984670639038, + "logps/chosen": -0.3744273781776428, + "logps/rejected": -41.171321868896484, + "loss": 0.5618, + "odds_ratio_loss": 1.8740653991699219, + "rewards/accuracies": 0.9994791746139526, + "rewards/chosen": -0.03744274005293846, + "rewards/margins": 4.0796895027160645, + "rewards/rejected": -4.117132186889648, + "sft_loss": 0.3744273781776428, + "step": 1440 + }, + { + "epoch": 2.304075235109718, + "grad_norm": 1.1352714624377223, + "learning_rate": 5.671867800613582e-06, + "logits/chosen": -1.654802680015564, + "logits/rejected": -1.7102715969085693, + "logps/chosen": -0.37380197644233704, + "logps/rejected": -40.510955810546875, + "loss": 0.5609, + "odds_ratio_loss": 1.871164321899414, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.03738019987940788, + "rewards/margins": 4.0137152671813965, + "rewards/rejected": -4.051095962524414, + "sft_loss": 0.37380197644233704, + "step": 1470 + }, + { + "epoch": 2.3510971786833856, + "grad_norm": 1.0544550616593722, + "learning_rate": 5.524284970587997e-06, + "logits/chosen": -1.6725318431854248, + "logits/rejected": -1.6666339635849, + "logps/chosen": -0.37714874744415283, + "logps/rejected": -41.287841796875, + "loss": 0.5661, + "odds_ratio_loss": 1.8892247676849365, + "rewards/accuracies": 0.9984375238418579, + "rewards/chosen": -0.0377148762345314, + "rewards/margins": 4.09106969833374, + "rewards/rejected": -4.128784656524658, + "sft_loss": 0.37714874744415283, + "step": 1500 + }, + { + "epoch": 2.3981191222570533, + "grad_norm": 1.0647146692947609, + "learning_rate": 5.376238737235469e-06, + "logits/chosen": -1.7326326370239258, + "logits/rejected": -1.692516803741455, + "logps/chosen": -0.37957924604415894, + "logps/rejected": -40.64752197265625, + "loss": 0.5696, + "odds_ratio_loss": 1.8999431133270264, + "rewards/accuracies": 0.9989583492279053, + "rewards/chosen": -0.03795792534947395, + "rewards/margins": 4.026793956756592, + "rewards/rejected": -4.064752101898193, + "sft_loss": 0.37957924604415894, + "step": 1530 + }, + { + "epoch": 2.445141065830721, + "grad_norm": 1.0668568570109873, + "learning_rate": 5.227859955188355e-06, + "logits/chosen": -1.7023158073425293, + "logits/rejected": -1.704237937927246, + "logps/chosen": -0.37669476866722107, + "logps/rejected": -41.38389587402344, + "loss": 0.5652, + "odds_ratio_loss": 1.8851966857910156, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.03766947612166405, + "rewards/margins": 4.100719928741455, + "rewards/rejected": -4.138389587402344, + "sft_loss": 0.37669476866722107, + "step": 1560 + }, + { + "epoch": 2.492163009404389, + "grad_norm": 1.048799823874522, + "learning_rate": 5.079279773011095e-06, + "logits/chosen": -1.5848591327667236, + "logits/rejected": -1.663787603378296, + "logps/chosen": -0.38333389163017273, + "logps/rejected": -42.06602096557617, + "loss": 0.5752, + "odds_ratio_loss": 1.9189646244049072, + "rewards/accuracies": 0.9994791746139526, + "rewards/chosen": -0.03833338990807533, + "rewards/margins": 4.168269157409668, + "rewards/rejected": -4.206602096557617, + "sft_loss": 0.38333389163017273, + "step": 1590 + }, + { + "epoch": 2.5391849529780566, + "grad_norm": 1.0924219426439041, + "learning_rate": 4.930629517281028e-06, + "logits/chosen": -1.6451200246810913, + "logits/rejected": -1.6708714962005615, + "logps/chosen": -0.37655967473983765, + "logps/rejected": -40.94800567626953, + "loss": 0.5651, + "odds_ratio_loss": 1.8852598667144775, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.03765597194433212, + "rewards/margins": 4.057145118713379, + "rewards/rejected": -4.09480094909668, + "sft_loss": 0.37655967473983765, + "step": 1620 + }, + { + "epoch": 2.586206896551724, + "grad_norm": 1.0780617647053894, + "learning_rate": 4.782040576511881e-06, + "logits/chosen": -1.6605018377304077, + "logits/rejected": -1.7122681140899658, + "logps/chosen": -0.37766554951667786, + "logps/rejected": -41.40230178833008, + "loss": 0.5668, + "odds_ratio_loss": 1.8910880088806152, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.037766553461551666, + "rewards/margins": 4.102463722229004, + "rewards/rejected": -4.140230655670166, + "sft_loss": 0.37766554951667786, + "step": 1650 + }, + { + "epoch": 2.633228840125392, + "grad_norm": 1.0665283784627981, + "learning_rate": 4.633644285022514e-06, + "logits/chosen": -1.4954478740692139, + "logits/rejected": -1.6388036012649536, + "logps/chosen": -0.37818458676338196, + "logps/rejected": -41.43574905395508, + "loss": 0.5676, + "odds_ratio_loss": 1.8939425945281982, + "rewards/accuracies": 0.9994791746139526, + "rewards/chosen": -0.037818457931280136, + "rewards/margins": 4.105756759643555, + "rewards/rejected": -4.1435747146606445, + "sft_loss": 0.37818458676338196, + "step": 1680 + }, + { + "epoch": 2.6802507836990594, + "grad_norm": 1.117909376672666, + "learning_rate": 4.485571806853553e-06, + "logits/chosen": -1.4043054580688477, + "logits/rejected": -1.6599198579788208, + "logps/chosen": -0.3815458118915558, + "logps/rejected": -42.49534606933594, + "loss": 0.5726, + "odds_ratio_loss": 1.9101753234863281, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.03815457969903946, + "rewards/margins": 4.2113800048828125, + "rewards/rejected": -4.249535083770752, + "sft_loss": 0.3815458118915558, + "step": 1710 + }, + { + "epoch": 2.7272727272727275, + "grad_norm": 0.9957664276621001, + "learning_rate": 4.337954019834537e-06, + "logits/chosen": -1.382237195968628, + "logits/rejected": -1.6407064199447632, + "logps/chosen": -0.37606796622276306, + "logps/rejected": -41.618125915527344, + "loss": 0.5642, + "odds_ratio_loss": 1.8813918828964233, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.037606798112392426, + "rewards/margins": 4.124205589294434, + "rewards/rejected": -4.161812782287598, + "sft_loss": 0.37606796622276306, + "step": 1740 + }, + { + "epoch": 2.774294670846395, + "grad_norm": 1.0842791833086312, + "learning_rate": 4.190921399904048e-06, + "logits/chosen": -1.4750745296478271, + "logits/rejected": -1.6820265054702759, + "logps/chosen": -0.37413859367370605, + "logps/rejected": -41.810272216796875, + "loss": 0.5615, + "odds_ratio_loss": 1.873866319656372, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.037413861602544785, + "rewards/margins": 4.143613815307617, + "rewards/rejected": -4.181027412414551, + "sft_loss": 0.37413859367370605, + "step": 1770 + }, + { + "epoch": 2.8213166144200628, + "grad_norm": 1.0015237419368779, + "learning_rate": 4.044603905785044e-06, + "logits/chosen": -1.3445394039154053, + "logits/rejected": -1.6244174242019653, + "logps/chosen": -0.3810182809829712, + "logps/rejected": -42.303462982177734, + "loss": 0.5717, + "odds_ratio_loss": 1.9063372611999512, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.03810183331370354, + "rewards/margins": 4.192244529724121, + "rewards/rejected": -4.2303466796875, + "sft_loss": 0.3810182809829712, + "step": 1800 + }, + { + "epoch": 2.8683385579937304, + "grad_norm": 1.1765923790147168, + "learning_rate": 3.899130864117366e-06, + "logits/chosen": -1.3384199142456055, + "logits/rejected": -1.626448154449463, + "logps/chosen": -0.382070928812027, + "logps/rejected": -42.68085479736328, + "loss": 0.5733, + "odds_ratio_loss": 1.9123166799545288, + "rewards/accuracies": 0.9994791746139526, + "rewards/chosen": -0.03820709139108658, + "rewards/margins": 4.229878902435303, + "rewards/rejected": -4.268085479736328, + "sft_loss": 0.382070928812027, + "step": 1830 + }, + { + "epoch": 2.915360501567398, + "grad_norm": 1.090679323980196, + "learning_rate": 3.754630855148914e-06, + "logits/chosen": -1.3685989379882812, + "logits/rejected": -1.5960211753845215, + "logps/chosen": -0.3773561120033264, + "logps/rejected": -42.771400451660156, + "loss": 0.5662, + "odds_ratio_loss": 1.8884392976760864, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.03773561120033264, + "rewards/margins": 4.239404678344727, + "rewards/rejected": -4.277140140533447, + "sft_loss": 0.3773561120033264, + "step": 1860 + }, + { + "epoch": 2.962382445141066, + "grad_norm": 1.0970942641955401, + "learning_rate": 3.6112315990865543e-06, + "logits/chosen": -1.268080472946167, + "logits/rejected": -1.5860413312911987, + "logps/chosen": -0.3762770891189575, + "logps/rejected": -42.43409729003906, + "loss": 0.5647, + "odds_ratio_loss": 1.8838173151016235, + "rewards/accuracies": 0.9994791746139526, + "rewards/chosen": -0.03762771189212799, + "rewards/margins": 4.205781936645508, + "rewards/rejected": -4.243409633636475, + "sft_loss": 0.3762770891189575, + "step": 1890 + }, + { + "epoch": 3.0094043887147337, + "grad_norm": 1.2095236484058132, + "learning_rate": 3.4690598432071793e-06, + "logits/chosen": -1.1365634202957153, + "logits/rejected": -1.5553410053253174, + "logps/chosen": -0.3466207981109619, + "logps/rejected": -42.42127227783203, + "loss": 0.5201, + "odds_ratio_loss": 1.7351850271224976, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.03466207906603813, + "rewards/margins": 4.207465171813965, + "rewards/rejected": -4.242127418518066, + "sft_loss": 0.3466207981109619, + "step": 1920 + }, + { + "epoch": 3.0564263322884013, + "grad_norm": 1.024347525310305, + "learning_rate": 3.328241249828747e-06, + "logits/chosen": -1.4228293895721436, + "logits/rejected": -1.614565134048462, + "logps/chosen": -0.21424469351768494, + "logps/rejected": -41.821380615234375, + "loss": 0.3216, + "odds_ratio_loss": 1.073145866394043, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.021424470469355583, + "rewards/margins": 4.160713195800781, + "rewards/rejected": -4.182137966156006, + "sft_loss": 0.21424469351768494, + "step": 1950 + }, + { + "epoch": 3.103448275862069, + "grad_norm": 0.9863432232044451, + "learning_rate": 3.1889002852402505e-06, + "logits/chosen": -1.3458492755889893, + "logits/rejected": -1.5922932624816895, + "logps/chosen": -0.21549250185489655, + "logps/rejected": -42.468841552734375, + "loss": 0.3234, + "odds_ratio_loss": 1.078763723373413, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.021549250930547714, + "rewards/margins": 4.225335121154785, + "rewards/rejected": -4.246884346008301, + "sft_loss": 0.21549250185489655, + "step": 1980 + } + ], + "log_save_evaluate_time": 457.3154537677765, + "logging_steps": 30, + "max_steps": 3190, + "num_input_tokens_seen": 0, + "num_train_epochs": 5, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 680295014072320.0, + "total_tokens": 0, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..0eb7a38 --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:90d92b798f9f6ec29474585ffed9254ca5343b35d699f0ad0fbccaf90f243f2e +size 7160 diff --git a/vocab.json b/vocab.json new file mode 100644 index 0000000..6c49fc6 --- /dev/null +++ b/vocab.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca10d7e9fb3ed18575dd1e277a2579c16d108e32f27439684afa0e10b1440910 +size 2776833 diff --git a/zero_to_fp32.py b/zero_to_fp32.py new file mode 100644 index 0000000..24cc342 --- /dev/null +++ b/zero_to_fp32.py @@ -0,0 +1,604 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: python zero_to_fp32.py . pytorch_model.bin + +import argparse +import torch +import glob +import math +import os +import re +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + + total_files = len(files) + state_dicts = [] + for f in files: + state_dict = torch.load(f, map_location=device) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + if zero_stage <= 2: + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + elif zero_stage == 3: + # if there is more than one param group, there will be multiple flattened tensors - one + # flattened tensor per group - for simplicity merge them into a single tensor + # + # XXX: could make the script more memory efficient for when there are multiple groups - it + # will require matching the sub-lists of param_shapes for each param group flattened tensor + + fp32_flat_groups = [ + torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts)) + ] + + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = fp32_flat_groups[0].numel() * world_size + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + for name, shape in param_shapes.items(): + + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # XXX: memory usage doubles here + state_dict[name] = torch.cat( + tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)), + 0).narrow(0, 0, unpartitioned_numel).view(shape) + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None, exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + + Returns: + - pytorch ``state_dict`` + + Note: this approach may not work if your application doesn't have sufficient free CPU memory and + you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, output_file, tag=None, exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_file``: path to the pytorch fp32 state_dict output file (e.g. path/pytorch_model.bin) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag, exclude_frozen_parameters) + print(f"Saving fp32 state dict to {output_file}") + torch.save(state_dict, output_file) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info(f"Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info(f"Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument( + "output_file", + type=str, + help="path to the pytorch fp32 state_dict output file (e.g. path/checkpoint-12/pytorch_model.bin)") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_file, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters)