初始化项目，由ModelHub XC社区提供模型
Model: build-small-hackathon/figment-finetuned-model-archive Source: Original Platform
This commit is contained in:
ModelHub XC
66
.gitattributes
vendored
Normal file
66
.gitattributes
vendored
Normal file
@@ -0,0 +1,66 @@
|
||||
*.7z filter=lfs diff=lfs merge=lfs -text
|
||||
*.arrow filter=lfs diff=lfs merge=lfs -text
|
||||
*.bin filter=lfs diff=lfs merge=lfs -text
|
||||
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
||||
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
||||
*.ftz filter=lfs diff=lfs merge=lfs -text
|
||||
*.gz filter=lfs diff=lfs merge=lfs -text
|
||||
*.h5 filter=lfs diff=lfs merge=lfs -text
|
||||
*.joblib filter=lfs diff=lfs merge=lfs -text
|
||||
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
||||
*.model filter=lfs diff=lfs merge=lfs -text
|
||||
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
||||
*.npy filter=lfs diff=lfs merge=lfs -text
|
||||
*.npz filter=lfs diff=lfs merge=lfs -text
|
||||
*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||
*.ot filter=lfs diff=lfs merge=lfs -text
|
||||
*.parquet filter=lfs diff=lfs merge=lfs -text
|
||||
*.pb filter=lfs diff=lfs merge=lfs -text
|
||||
*.pickle filter=lfs diff=lfs merge=lfs -text
|
||||
*.pkl filter=lfs diff=lfs merge=lfs -text
|
||||
*.pt filter=lfs diff=lfs merge=lfs -text
|
||||
*.pth filter=lfs diff=lfs merge=lfs -text
|
||||
*.rar filter=lfs diff=lfs merge=lfs -text
|
||||
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar filter=lfs diff=lfs merge=lfs -text
|
||||
*.tflite filter=lfs diff=lfs merge=lfs -text
|
||||
*.tgz filter=lfs diff=lfs merge=lfs -text
|
||||
*.wasm filter=lfs diff=lfs merge=lfs -text
|
||||
*.xz filter=lfs diff=lfs merge=lfs -text
|
||||
*.zip filter=lfs diff=lfs merge=lfs -text
|
||||
*.zst filter=lfs diff=lfs merge=lfs -text
|
||||
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||
pilot-20260608/checkpoint-40/tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
||||
pilot-20260608/tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
||||
pilot-20260608-merged-bf16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
||||
pilot-20260608-merged-bf16.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
v3-20260610-merged-bf16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
||||
v3-20260610-merged-bf16.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
v2-20260609-merged-bf16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
||||
v2-20260609-merged-bf16.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
v4-20260611-merged-bf16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
||||
v4-20260611-merged-bf16.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
figment_sft_v6/figment-sft-v6-lora-merged-bf16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
||||
figment_sft_v5/figment-sft-v5-lora-merged-bf16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
||||
figment_sft_v7/figment-sft-v7-lora-merged-bf16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
||||
figment_sft_v5/figment-sft-v5-lora-merged-bf16.bf16.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
figment_sft_v6/figment-sft-v6-lora-merged-bf16.bf16.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
figment_sft_v7/figment-sft-v7-lora-merged-bf16.bf16.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
figment_sft_v8/figment-sft-v8-lora-merged-bf16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
||||
figment_sft_v9/figment-sft-v9-lora-merged-bf16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
||||
figment_sft_v10/figment-sft-v10-lora-merged-bf16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
||||
figment_sft_v11/figment-sft-v11-lora-merged-bf16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
||||
figment_sft_v12/figment-sft-v12-lora-merged-bf16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
||||
figment_sft_v13/figment-sft-v13-lora-merged-bf16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
||||
figment_sft_v14p/figment-sft-v14p-lora-merged-bf16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
||||
figment_sft_v8/figment-sft-v8-lora-merged-bf16.bf16.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
figment_sft_v9/figment-sft-v9-lora-merged-bf16.bf16.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
figment_sft_v10/figment-sft-v10-lora-merged-bf16.bf16.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
figment_sft_v11/figment-sft-v11-lora-merged-bf16.bf16.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
figment_sft_v12/figment-sft-v12-lora-merged-bf16.bf16.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
figment_sft_v13/figment-sft-v13-lora-merged-bf16.bf16.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
figment_sft_v14p/figment-sft-v14p-lora-merged-bf16.bf16.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
figment_sft_v1/pilot-20260608-merged-bf16/tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
||||
354
README.md
Normal file
354
README.md
Normal file
@@ -0,0 +1,354 @@
|
||||
---
|
||||
base_model: nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16
|
||||
base_model_relation: finetune
|
||||
library_name: transformers
|
||||
pipeline_tag: text-generation
|
||||
license: other
|
||||
license_name: nvidia-nemotron-open-model-license
|
||||
license_link: >-
|
||||
https://www.nvidia.com/en-us/agreements/enterprise-software/nvidia-nemotron-open-model-license/
|
||||
language:
|
||||
- en
|
||||
tags:
|
||||
- figment
|
||||
- lora
|
||||
- peft
|
||||
- gguf
|
||||
- v1
|
||||
- v2
|
||||
- v3
|
||||
- v4
|
||||
- v5
|
||||
- v6
|
||||
- v7
|
||||
- v8
|
||||
- v9
|
||||
- v10
|
||||
- v11
|
||||
- v12
|
||||
- v13
|
||||
- v14p
|
||||
- protocol-navigation
|
||||
- synthetic-data
|
||||
- not-for-clinical-use
|
||||
- base_model:finetune:nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16
|
||||
---
|
||||
|
||||
# Figment Finetuned Model Archive
|
||||
|
||||
This repository archives early Figment local-model training artifacts for `nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16`.
|
||||
|
||||
Figment is a prototype protocol-navigation aid for trained field responders working with synthetic or de-identified rural-clinic and disaster-response scenarios. It is designed to structure field notes, preserve deterministic red-flag rules, cite retrieved protocol cards, plan missing observations, draft responder checklists, and prepare SBAR-style handoffs.
|
||||
|
||||
The published artifacts include the `figment_sft_v1` pilot merged BF16 checkpoint from June 8, 2026, the `figment_sft_v2` merged BF16/GGUF checkpoint from June 9, 2026, the `figment_sft_v3` merged BF16/GGUF checkpoint from June 10, 2026, and the `figment_sft_v4` through `figment_sft_v14p` merged BF16/GGUF checkpoints from the June 11-13, 2026 field-workflow loop. The v1 pilot is retained for archival continuity, v2 improved raw configured-model behavior on the locked 50-case harness, v3 improved the field-holdout surface, v4 established the first archived field-workflow checkpoint, v5 is retained as a regression artifact, v6-v13 show the corrected field-workflow iteration path, and v14p plus its repair-union harness run is the strongest archived local field-workflow checkpoint in this repository.
|
||||
|
||||
## Contents
|
||||
|
||||
| Path | Contents | Notes |
|
||||
| --- | --- | --- |
|
||||
| `figment_sft_v1/pilot-20260608-merged-bf16/` | `figment_sft_v1` pilot adapter merged into the BF16 base with `peft.merge_and_unload(safe_merge=True)` | Full merged Hugging Face weights for the v1 pilot checkpoint. No v1 GGUF sidecar is archived in this repo. |
|
||||
| `v2-20260609-merged-bf16/` | `figment_sft_v2` adapter merged into the BF16 base with `peft.merge_and_unload(safe_merge=True)` | Full merged Hugging Face weights for the v2 locked-harness checkpoint. |
|
||||
| `v2-20260609-merged-bf16.gguf` | BF16 GGUF conversion of the v2 merged checkpoint | SHA-256: `281251bf326bfef219fe213cf01d7457164972ce2f99067b0ccc1fdb5821ea01`. |
|
||||
| `v3-20260610-merged-bf16/` | `figment_sft_v3` adapter merged into the BF16 base with `peft.merge_and_unload(safe_merge=True)` | Full merged Hugging Face weights for the v3 field-workflow model. |
|
||||
| `v3-20260610-merged-bf16.gguf` | BF16 GGUF conversion of the v3 merged checkpoint | SHA-256: `7ee6439f87d50af289136a345ee73e633e20035c79582f942f03f9331bb8a658`. |
|
||||
| `v4-20260611-merged-bf16/` | `figment_sft_v4` adapter merged into the BF16 base with `peft.merge_and_unload(safe_merge=True)` | Full merged Hugging Face weights for the v4 field-workflow model. |
|
||||
| `v4-20260611-merged-bf16.gguf` | BF16 GGUF conversion of the v4 merged checkpoint | SHA-256: `7e11f2295b101e9312f97075b8e48cabd8cc89539e92c8fa4218c4973aa31d8d`. |
|
||||
| `figment_sft_v5/figment-sft-v5-lora-merged-bf16/` | `figment_sft_v5` adapter merged into the BF16 base with `peft.merge_and_unload(safe_merge=True)` | Full merged Hugging Face weights for the v5 field-workflow regression artifact. |
|
||||
| `figment_sft_v5/figment-sft-v5-lora-merged-bf16.bf16.gguf` | BF16 GGUF conversion of the v5 merged checkpoint | Published LFS SHA-256: `c7f9b38d267c2ab2b791b613e0227ce3d057e61b57b568b16ca501f2e516379c`. |
|
||||
| `figment_sft_v6/figment-sft-v6-lora-merged-bf16/` | `figment_sft_v6` adapter merged into the BF16 base with `peft.merge_and_unload(safe_merge=True)` | Full merged Hugging Face weights for the v6 field-workflow model. |
|
||||
| `figment_sft_v6/figment-sft-v6-lora-merged-bf16.bf16.gguf` | BF16 GGUF conversion of the v6 merged checkpoint | Published LFS SHA-256: `92fb2bb4a8686230f050c1696e6df749fe49ec4d41221ab9100785afa7e34009`. |
|
||||
| `figment_sft_v7/figment-sft-v7-lora-merged-bf16/` | `figment_sft_v7` adapter merged into the BF16 base with `peft.merge_and_unload(safe_merge=True)` | Full merged Hugging Face weights for the v7 field-workflow model. |
|
||||
| `figment_sft_v7/figment-sft-v7-lora-merged-bf16.bf16.gguf` | BF16 GGUF conversion of the v7 merged checkpoint | Published LFS SHA-256: `d85f9dd7137453035ae8ec96bcee1998358ad5975bb9c842fe9b7a077c4002b9`. |
|
||||
| `figment_sft_v8/figment-sft-v8-lora-merged-bf16/` | `figment_sft_v8` adapter merged into the BF16 base with `peft.merge_and_unload(safe_merge=True)` | Full merged Hugging Face weights for the v8 field-workflow model. |
|
||||
| `figment_sft_v8/figment-sft-v8-lora-merged-bf16.bf16.gguf` | BF16 GGUF conversion of the v8 merged checkpoint | Published LFS SHA-256: `d45660834ce2f9229d0e43ed3ac6bd041dba876f54ff1cc384889b9594b5e78d`. |
|
||||
| `figment_sft_v9/figment-sft-v9-lora-merged-bf16/` | `figment_sft_v9` adapter merged into the BF16 base with `peft.merge_and_unload(safe_merge=True)` | Full merged Hugging Face weights for the v9 field-workflow model. |
|
||||
| `figment_sft_v9/figment-sft-v9-lora-merged-bf16.bf16.gguf` | BF16 GGUF conversion of the v9 merged checkpoint | Published LFS SHA-256: `79ec6bfb55895c90ed4188d9e4052730ac07f2f5c6fe49c5fd7ef44c7e0a7d16`. |
|
||||
| `figment_sft_v10/figment-sft-v10-lora-merged-bf16/` | `figment_sft_v10` adapter merged into the BF16 base with `peft.merge_and_unload(safe_merge=True)` | Full merged Hugging Face weights for the v10 field-workflow model. |
|
||||
| `figment_sft_v10/figment-sft-v10-lora-merged-bf16.bf16.gguf` | BF16 GGUF conversion of the v10 merged checkpoint | Published LFS SHA-256: `85bc2978be155e1cdf12b42c8ccf84e1c1b65ad2da6b463d7be726d33cbd31aa`. |
|
||||
| `figment_sft_v11/figment-sft-v11-lora-merged-bf16/` | `figment_sft_v11` adapter merged into the BF16 base with `peft.merge_and_unload(safe_merge=True)` | Full merged Hugging Face weights for the v11 field-workflow model. |
|
||||
| `figment_sft_v11/figment-sft-v11-lora-merged-bf16.bf16.gguf` | BF16 GGUF conversion of the v11 merged checkpoint | Published LFS SHA-256: `cb5c99e32660547941a681853c30eff47cd2a9aee837fbdd3ee17684b44d4fd2`. |
|
||||
| `figment_sft_v12/figment-sft-v12-lora-merged-bf16/` | `figment_sft_v12` adapter merged into the BF16 base with `peft.merge_and_unload(safe_merge=True)` | Full merged Hugging Face weights for the v12 field-workflow model. |
|
||||
| `figment_sft_v12/figment-sft-v12-lora-merged-bf16.bf16.gguf` | BF16 GGUF conversion of the v12 merged checkpoint | Published LFS SHA-256: `164ebf943919b4c27a54dbce3380bc156bbad3c6e893f1d185d35801eac015b7`. |
|
||||
| `figment_sft_v13/figment-sft-v13-lora-merged-bf16/` | `figment_sft_v13` adapter merged into the BF16 base with `peft.merge_and_unload(safe_merge=True)` | Full merged Hugging Face weights for the v13 field-workflow model. |
|
||||
| `figment_sft_v13/figment-sft-v13-lora-merged-bf16.bf16.gguf` | BF16 GGUF conversion of the v13 merged checkpoint | Published LFS SHA-256: `1cedcc48d2edf82f31ebd20d8885bdd7b72d07b8d551b19838394ba57a1f2e1e`. |
|
||||
| `figment_sft_v14p/figment-sft-v14p-lora-merged-bf16/` | `figment_sft_v14p` adapter merged into the BF16 base with `peft.merge_and_unload(safe_merge=True)` | Full merged Hugging Face weights for the v14p field-workflow model. |
|
||||
| `figment_sft_v14p/figment-sft-v14p-lora-merged-bf16.bf16.gguf` | BF16 GGUF conversion of the v14p merged checkpoint | Published LFS SHA-256: `53de48e5f7a7fa22af7a682686adcf6c0be7c5c1fe72f72ea39d80bd68333f72`. |
|
||||
|
||||
## Intended Use
|
||||
|
||||
Use this repo as an artifact archive for:
|
||||
|
||||
- reproducing the Modal train/merge/GGUF proof chain,
|
||||
- comparing later Figment checkpoints against a known early baseline,
|
||||
- inspecting the v1 pilot merged BF16 checkpoint,
|
||||
- evaluating the v2 locked-harness protocol-navigation checkpoint,
|
||||
- evaluating the v3 local/off-grid protocol-navigation checkpoint,
|
||||
- evaluating the v4 local/off-grid field-workflow checkpoint,
|
||||
- evaluating the v5 regression artifact and v6-v14p local/off-grid field-workflow checkpoints,
|
||||
- debugging protocol-navigation behavior in synthetic or de-identified scenarios.
|
||||
|
||||
Do not use these artifacts for clinical care, autonomous triage, diagnosis, prescribing, medication dosing, or replacing local protocol or trained responder judgment.
|
||||
|
||||
## Model Details
|
||||
|
||||
- Base model: `nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16`
|
||||
- Base model revision observed during the project: `dfaf35de3e30f1867dd8dbc38a7fc9fb52d3914f`
|
||||
- Model family: Nemotron 3 Nano 4B BF16, text generation
|
||||
- Adapter method: PEFT LoRA
|
||||
- LoRA rank: 16
|
||||
- LoRA alpha: 32
|
||||
- LoRA dropout: 0.05
|
||||
- Target modules: `up_proj`, `in_proj`, `q_proj`, `k_proj`, `out_proj`, `v_proj`, `down_proj`, `o_proj`
|
||||
- Max sequence length used for local 4B training: 16384
|
||||
- Language: English
|
||||
- Domain: synthetic field-clinic and disaster-response protocol navigation
|
||||
|
||||
## V1 Pilot Checkpoint
|
||||
|
||||
The v1 pilot artifact was trained as `figment_sft_v1` and merged from Modal checkpoint `/checkpoints/figment_sft_v1/pilot-20260608` into `/checkpoints/figment_sft_v1/pilot-20260608-merged-bf16`.
|
||||
|
||||
Archive summary:
|
||||
|
||||
- Artifact path: `figment_sft_v1/pilot-20260608-merged-bf16/`
|
||||
- Merge method: `peft.merge_and_unload(safe_merge=True)`
|
||||
- Merged dtype: BF16
|
||||
- Base model: `nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16`
|
||||
- HF shard 1 LFS SHA-256: `aebcb7fd3126d0100cc7e78e58e0ed49ab29aad8f858f2c6149637aced9c699f`
|
||||
- HF shard 2 LFS SHA-256: `8a1a7b48e647dd43cb7941a9e6a3f7a839326865034f626b2705645b0e29c830`
|
||||
- Tokenizer LFS SHA-256: `623c34567aebb18582765289fbe23d901c62704d6518d71866e0e58db892b5b7`
|
||||
- GGUF sidecar: not archived; no v1 GGUF cache was present in `figment-eval-results:/model_cache/figment_sft_v1`.
|
||||
|
||||
## V2 Checkpoint
|
||||
|
||||
The v2 artifact was trained as `figment_sft_v2` and merged from Modal checkpoint `/checkpoints/figment_sft_v2/figment-sft-v2-lora` into `/checkpoints/figment_sft_v2/figment-sft-v2-lora-merged-bf16`.
|
||||
|
||||
Training data and merge summary:
|
||||
|
||||
- Training rows: 1500
|
||||
- Train rows: 1352
|
||||
- Validation rows: 148
|
||||
- Navigator-full rows: 1000
|
||||
- Focused-repair rows: 500
|
||||
- Train split SHA-256: `27233926a2bd9320418ff10b0c14f3885834adf2f48865ee469c939e2ffeb68a`
|
||||
- Validation split SHA-256: `7964c75cd3940a8549e6b8b2ef15b4d5cd45e8607af8f77a4982ffe01116bfb4`
|
||||
- Merge method: `peft.merge_and_unload(safe_merge=True)`
|
||||
- Merged dtype: BF16
|
||||
- Base model: `nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16`
|
||||
- Merged manifest SHA-256: `6885f758f30a76e798fac73ebedd64684f3287d6b459f2b625029b03031179dc`
|
||||
- HF shard 1 SHA-256: `9e224445985294263fce0437f82e55d116e90f5f19a5b995d47ee5081ff97c63`
|
||||
- HF shard 2 SHA-256: `758eb779adf5379fb96ea42c4c38cfc6de9dc3d53c4e3863a7aea15ccebae5ae`
|
||||
- GGUF SHA-256: `281251bf326bfef219fe213cf01d7457164972ce2f99067b0ccc1fdb5821ea01`
|
||||
|
||||
The v2 local evaluation run was `local_4b_v2_lora_20260609T103344Z` on the locked 50-case local harness.
|
||||
|
||||
## V3 Checkpoint
|
||||
|
||||
The v3 artifact was trained as `figment_sft_v3` and merged from Modal checkpoint `/checkpoints/figment_sft_v3/figment-sft-v3-lora` into `/checkpoints/figment_sft_v3/figment-sft-v3-lora-merged-bf16`.
|
||||
|
||||
Training and merge summary:
|
||||
|
||||
- Training run: `700/700` optimizer steps
|
||||
- Final eval loss: `0.04357146`
|
||||
- Final train loss: `0.60960097`
|
||||
- Merge method: `peft.merge_and_unload(safe_merge=True)`
|
||||
- Merged dtype: BF16
|
||||
- Base model: `nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16`
|
||||
- Merged manifest SHA-256: `d18e72fb258764321ec17abd687af7214a480f491f11d83cf64e38824dc4e510`
|
||||
- GGUF SHA-256: `7ee6439f87d50af289136a345ee73e633e20035c79582f942f03f9331bb8a658`
|
||||
|
||||
The clean v3 field-holdout eval was the sequential run `local_4b_v3_lora_field_holdout_20260610T102450Z`, not the earlier parallel run that hit a llama.cpp KV/context-overflow failure mode.
|
||||
|
||||
## V4 Checkpoint
|
||||
|
||||
The v4 artifact was trained as `figment_sft_v4` and merged from Modal checkpoint `/checkpoints/figment_sft_v4/figment-sft-v4-lora` into `/checkpoints/figment_sft_v4/figment-sft-v4-lora-merged-bf16`.
|
||||
|
||||
Training data and merge summary:
|
||||
|
||||
- Training rows: 1650
|
||||
- Train rows: 1482
|
||||
- Validation rows: 168
|
||||
- Navigator-full rows: 1500
|
||||
- Focused-repair rows: 150
|
||||
- Full corpus SHA-256: `ef7a7c9a6a99927ba72ce244e03a9da3ab86d3cf5dc70786703fb5f8bdf2a289`
|
||||
- Train split SHA-256: `f869d79da9ef670bc6479f8321e51b1f48cb5a16423265f34893a08e7648676e`
|
||||
- Validation split SHA-256: `3ff7668b8216d6fa0be770d6d9ed5f1a0b12965f9312d5210b510807538738d3`
|
||||
- Merge method: `peft.merge_and_unload(safe_merge=True)`
|
||||
- Merged dtype: BF16
|
||||
- Base model: `nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16`
|
||||
- Merged manifest SHA-256: `6678c0ec3a28817dba22eb9e7c682b9961f04bbfc688d0f1bcd137afaf8c8c38`
|
||||
- HF shard 1 SHA-256: `1d95889e945363adcd70a0be54bc29407d49e28bf7a2c0415e1732d81d64186c`
|
||||
- HF shard 2 SHA-256: `2a2e27563e78981c130349feece291c976cf7d5384690c6327795eef6d08d4c0`
|
||||
- GGUF SHA-256: `7e11f2295b101e9312f97075b8e48cabd8cc89539e92c8fa4218c4973aa31d8d`
|
||||
|
||||
The v4 full field-holdout evaluation run was `local_4b_finetuned_v4_field_holdout_20260611T011930Z`. A separate 50-case evidence run was `local_4b_finetuned_v4_evidence_20260611T0010Z`.
|
||||
|
||||
## V5 Checkpoint
|
||||
|
||||
The v5 artifact was trained as `figment_sft_v5` and merged from Modal checkpoint `/checkpoints/figment_sft_v5/figment-sft-v5-lora` into `/checkpoints/figment_sft_v5/figment-sft-v5-lora-merged-bf16`.
|
||||
|
||||
Training data and merge summary:
|
||||
|
||||
- Training rows: 1300
|
||||
- Train rows: 1170
|
||||
- Validation rows: 130
|
||||
- Navigator-full rows: 1100
|
||||
- Focused-repair rows: 200
|
||||
- Full corpus SHA-256: `3abc2dcb1f972ee6f536c273de69f72abe9a42e402a3548c451e442a3fcd4535`
|
||||
- Train split SHA-256: `08ad6b76e958249b50bece528e0b26f5d3ef090166d7e5e0d48ddc46101496c7`
|
||||
- Validation split SHA-256: `54aadd55ab41f00880483ff0beb08c9602aae23933efcabd328d1769617fbc1a`
|
||||
- Merge method: `peft.merge_and_unload(safe_merge=True)`
|
||||
- Merged dtype: BF16
|
||||
- Base model: `nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16`
|
||||
- GGUF LFS SHA-256: `c7f9b38d267c2ab2b791b613e0227ce3d057e61b57b568b16ca501f2e516379c`
|
||||
|
||||
The v5 field-holdout run was `figment_sft_v5_field_workflow_holdout_modal_gpu_20260611_h100_gguf`; it is retained as a regression artifact because it scored only `2/150` competence successes despite passing final JSON validation.
|
||||
|
||||
## V6 Checkpoint
|
||||
|
||||
The v6 artifact was trained as `figment_sft_v6` and merged from Modal checkpoint `/checkpoints/figment_sft_v6/figment-sft-v6-lora` into `/checkpoints/figment_sft_v6/figment-sft-v6-lora-merged-bf16`.
|
||||
|
||||
Training data and merge summary:
|
||||
|
||||
- Training rows: 2000
|
||||
- Train rows: 1800
|
||||
- Validation rows: 200
|
||||
- Navigator-full rows: 1180
|
||||
- Focused-repair rows: 820
|
||||
- Full corpus SHA-256: `268cb36d0d36697006609f346b76c79dbf127f82837f5a1f76d47059b031c595`
|
||||
- Train split SHA-256: `b750779104e80a8a92c86437f9515da7a4ab97bc866c1e87f4d95fca269ab9c2`
|
||||
- Validation split SHA-256: `ca388117f77325a57c70af7d69145b429bd443a5ae134ce1ab419373154e25cf`
|
||||
- Merge method: `peft.merge_and_unload(safe_merge=True)`
|
||||
- Merged dtype: BF16
|
||||
- Base model: `nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16`
|
||||
- GGUF LFS SHA-256: `92fb2bb4a8686230f050c1696e6df749fe49ec4d41221ab9100785afa7e34009`
|
||||
|
||||
The v6 field-holdout run was `figment_sft_v6_field_workflow_holdout_modal_gpu_20260611_h100_gguf`.
|
||||
|
||||
## V7 Checkpoint
|
||||
|
||||
The v7 artifact was trained as `figment_sft_v7` and merged from Modal checkpoint `/checkpoints/figment_sft_v7/figment-sft-v7-lora` into `/checkpoints/figment_sft_v7/figment-sft-v7-lora-merged-bf16`.
|
||||
|
||||
Training data and merge summary:
|
||||
|
||||
- Training rows: 2800
|
||||
- Train rows: 2520
|
||||
- Validation rows: 280
|
||||
- Navigator-full rows: 1740
|
||||
- Focused-repair rows: 1060
|
||||
- Full corpus SHA-256: `b8bc3830beb38577047dbb2b9760aa2845234e25f41457fbfc5ce25bb6821ac0`
|
||||
- Train split SHA-256: `283615b21446346a9090ad6d45e750f5812222625ddaa5d2a83a15f663cb7d04`
|
||||
- Validation split SHA-256: `fe7b683f5007ff1f3eaac2632c9d407a8671d23c944b17b192eae964c0bbaa8d`
|
||||
- Merge method: `peft.merge_and_unload(safe_merge=True)`
|
||||
- Merged dtype: BF16
|
||||
- Base model: `nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16`
|
||||
- GGUF LFS SHA-256: `d85f9dd7137453035ae8ec96bcee1998358ad5975bb9c842fe9b7a077c4002b9`
|
||||
|
||||
The v7 field-holdout run was `figment_sft_v7_field_workflow_holdout_modal_gpu_20260612_h100_gguf`.
|
||||
|
||||
|
||||
## V8-V14p Checkpoints
|
||||
|
||||
The v8-v14p artifacts continue the corrected field-workflow training loop. Each checkpoint was merged from its Modal LoRA adapter into the same BF16 base with `peft.merge_and_unload(safe_merge=True)` and converted to BF16 GGUF for local llama.cpp evaluation.
|
||||
|
||||
| Version | Training rows | Train rows | Validation rows | Navigator rows | Focused-repair rows | Full corpus SHA-256 | Train split SHA-256 | Validation split SHA-256 | GGUF LFS SHA-256 |
|
||||
| --- | ---: | ---: | ---: | ---: | ---: | --- | --- | --- | --- |
|
||||
| V8 | 3200 | 2880 | 320 | 2140 | 1060 | `fbf2adb675d01c007f6defc0292d04574d671bd64cd112310771bb4f5161cecc` | `e4d81265d0d7d56443fd6afd91cd996c546e680b7e53e7200b09321c4bae56f5` | `4668f8f8aa558fe2e765feae91a82c661753da3836265e6485d1225629b55097` | `d45660834ce2f9229d0e43ed3ac6bd041dba876f54ff1cc384889b9594b5e78d` |
|
||||
| V9 | 3600 | 3240 | 360 | 2540 | 1060 | `ceb106258d4149305582620b5c4c308a7aa5854b6125e2c1d14b0d98cf5bbd6b` | `b3556bae88e13f980b22509a5463e192556515cdaf868f65f16ef4db41079513` | `e2f4e13c516bea567e5f3501fc0b11143c100bca1afc65f907cb1ad27b211a85` | `79ec6bfb55895c90ed4188d9e4052730ac07f2f5c6fe49c5fd7ef44c7e0a7d16` |
|
||||
| V10 | 4400 | 3960 | 440 | 3340 | 1060 | `6ba2a10a4f6afb3ba9a061ec966a68122b1c520b832b5e8e110de3900c2968bd` | `2497bca472e188d202939e9a729d338e8fe6f30913d96e2b396339d421f7de4d` | `256a1674930e57ccc7d511ea0825ef487a115bbf3c2aa79e7f2a4cc933c198fd` | `85bc2978be155e1cdf12b42c8ccf84e1c1b65ad2da6b463d7be726d33cbd31aa` |
|
||||
| V11 | 5200 | 4680 | 520 | 4140 | 1060 | `867c5622aded6a73657e37f0a1468fb5edcfcc5c30c4d0e8eb7b5024a4786051` | `3e1606855dadfc0e67f4d45f4c98e729b697d095be2b351d0aa159c71f347eb3` | `970c8d00aeed2bec1bc069ae229ffc7865988d21f31dfc37de784cbd8b771b52` | `cb5c99e32660547941a681853c30eff47cd2a9aee837fbdd3ee17684b44d4fd2` |
|
||||
| V12 | 4960 | 4464 | 496 | 3900 | 1060 | `9e7ba0caab6137be3bf9936b8a0cd2aa70679d467e3555d507ad5af063fb3a4e` | `fe009fcf471cc61ddeb7e7aa7d993ad5dd28d4c58238050751d42fcfd3b79098` | `bdbd3e51d0bf354da25b449de2d4164561f6f8d453cbbe5a196853b9eba40b23` | `164ebf943919b4c27a54dbce3380bc156bbad3c6e893f1d185d35801eac015b7` |
|
||||
| V13 | 4465 | 4017 | 448 | 3405 | 1060 | `e7d5f55259c4a0cbfc81e16c31a8a374837c654ea8e5723434ac882ce835da2b` | `43106d3af0f494ca5ead39290f3ad142c7a1f73e46a98759a92aed7814083290` | `f16c98ea146a7a17785a761d50df00efbc5781b272085ca5885540c0a33a0645` | `1cedcc48d2edf82f31ebd20d8885bdd7b72d07b8d551b19838394ba57a1f2e1e` |
|
||||
| V14p | 5335 | 4801 | 534 | 4275 | 1060 | `b455460870c70c2072491b754ed128e04cee7e63f4876cd6e6bacc92164788d9` | `378379eccba716001eb30a4bee05948a5bcb34ef2caa6801442be733c0f5fff6` | `aaf5d7c0b3236c98f7d29fcd9898ea6d6789978d468fb1e26d64b40097d2b86e` | `53de48e5f7a7fa22af7a682686adcf6c0be7c5c1fe72f72ea39d80bd68333f72` |
|
||||
|
||||
## Training Data
|
||||
|
||||
The model artifacts use synthetic and de-identified datasets generated inside the Figment project. Published training corpora are available in the dataset repository `build-small-hackathon/figment-eval-traces` under configs `figment_sft_v1` through `figment_sft_v14p`. The dataset files are not duplicated in this model repository.
|
||||
|
||||
The examples were synthetic. They were designed to teach Figment's harness behavior, not to store medical knowledge. They included full navigator outputs and focused repair tasks for schema, citations/pathways, SBAR handoff fields, missing observations, protocol urgency, and forbidden clinical language.
|
||||
|
||||
## Evaluation
|
||||
|
||||
For later eval-trace artifacts, see the dataset repository `build-small-hackathon/figment-eval-traces`.
|
||||
|
||||
Observed v2 locked-harness evaluation:
|
||||
|
||||
| Metric | V2 locked 50-case eval |
|
||||
| --- | ---: |
|
||||
| Total cases | 50 |
|
||||
| Competence successes | 33/50 |
|
||||
| Raw configured-model successes | 33/50 |
|
||||
| Focused-repair successes | 0 |
|
||||
| Full fallback uses | 0 |
|
||||
| Final validation successes | 50/50 |
|
||||
| Model-visible fields retained | 627/650 |
|
||||
|
||||
Observed v3 field-holdout evaluation:
|
||||
|
||||
| Metric | V3 field holdout |
|
||||
| --- | ---: |
|
||||
| Total cases | 150 |
|
||||
| Competence successes | 107/150 |
|
||||
| Raw configured-model successes | 93/150 |
|
||||
| Focused-repair successes | 14 |
|
||||
| Full fallback uses | 2 |
|
||||
| Final validation successes | 148/150 |
|
||||
| Model-visible fields retained | 1836/1950 |
|
||||
|
||||
Observed v4 evaluations:
|
||||
|
||||
| Metric | V4 50-case eval | V4 field holdout |
|
||||
| --- | ---: | ---: |
|
||||
| Total cases | 50 | 150 |
|
||||
| Competence successes | 37/50 | 109/150 |
|
||||
| Raw configured-model successes | 37/50 | 109/150 |
|
||||
| Expected-label successes | 14/50 | 149/150 |
|
||||
| Full fallback uses | 0 | 2 |
|
||||
| Final validation successes | 50/50 | 148/150 |
|
||||
| Model-visible fields retained | 624/650 | 1846/1950 |
|
||||
|
||||
Observed v5-v7 field-holdout evaluations:
|
||||
|
||||
| Metric | V5 field holdout | V6 field holdout | V7 field holdout |
|
||||
| --- | ---: | ---: | ---: |
|
||||
| Total cases | 150 | 150 | 150 |
|
||||
| Competence successes | 2/150 | 142/150 | 148/150 |
|
||||
| Raw configured-model successes | 2/150 | 142/150 | 148/150 |
|
||||
| Expected-label successes | 150/150 | 146/150 | 145/150 |
|
||||
| Full fallback uses | 0 | 0 | 0 |
|
||||
| Final validation successes | 150/150 | 150/150 | 150/150 |
|
||||
| Deterministic patch count | 302 | 21 | 4 |
|
||||
| Model-visible field pass rate | 0.8451 | 0.9892 | 0.9979 |
|
||||
| Mean latency | 4512.943 ms | 4407.568 ms | 4344.942 ms |
|
||||
| P95 latency | 4714.603 ms | 4606.824 ms | 4565.243 ms |
|
||||
|
||||
Observed v8-v14p corrected field-holdout evaluations:
|
||||
|
||||
| Metric | V8 | V9 | V10 | V11 | V12 | V13 | V14p | V14p repair-union |
|
||||
| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
|
||||
| Total cases | 150 | 150 | 150 | 150 | 150 | 150 | 150 | 150 |
|
||||
| Competence successes | 146/150 | 146/150 | 147/150 | 145/150 | 146/150 | 146/150 | 146/150 | 150/150 |
|
||||
| Raw configured-model successes | 146/150 | 146/150 | 147/150 | 143/150 | 146/150 | 145/150 | 146/150 | 146/150 |
|
||||
| Expected-label successes | 150/150 | 150/150 | 150/150 | 148/150 | 150/150 | 149/150 | 150/150 | 150/150 |
|
||||
| Full fallback uses | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 |
|
||||
| Final validation successes | 150/150 | 150/150 | 150/150 | 150/150 | 150/150 | 150/150 | 150/150 | 150/150 |
|
||||
| Deterministic patch count | 8 | 8 | 6 | 23 | 8 | 15 | 8 | 0 |
|
||||
| Model-visible field pass rate | 0.9959 | 0.9959 | 0.9969 | 0.9882 | 0.9959 | 0.9923 | 0.9959 | 1.0000 |
|
||||
| Mean latency | 4282.523 ms | 4338.318 ms | 4341.864 ms | 4491.265 ms | 4932.291 ms | 4362.790 ms | 5142.904 ms | 4505.249 ms |
|
||||
| P95 latency | 4477.092 ms | 4613.479 ms | 4602.941 ms | 4664.988 ms | 5181.336 ms | 4532.570 ms | 5642.153 ms | 4708.240 ms |
|
||||
|
||||
## Safety and Limitations
|
||||
|
||||
- Prototype only; not a medical device.
|
||||
- Synthetic/de-identified scenarios only.
|
||||
- The model must not diagnose, prescribe, dose medication, or autonomously triage.
|
||||
- Deterministic red-flag rules and validators remain part of the Figment runtime. The model artifact alone is not the full safety system.
|
||||
- Outputs require trained responder review and local protocol/supervisor/clinician judgment.
|
||||
- The checkpoints may produce malformed, incomplete, unsupported, or overconfident outputs without the Figment harness.
|
||||
|
||||
## License and Attribution
|
||||
|
||||
This archive is derived from `nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16` and is governed by the same upstream [NVIDIA Nemotron Open Model License](https://www.nvidia.com/en-us/agreements/enterprise-software/nvidia-nemotron-open-model-license/). Review the upstream model card and license before reuse. The Figment application code is Apache-2.0, and Figment synthetic datasets are documented separately as CC-BY-4.0 where published.
|
||||
|
||||
## Citation
|
||||
|
||||
No paper is associated with these artifacts. Please cite the base model according to NVIDIA's guidance and cite this repository if using the Figment artifacts directly.
|
||||
204
figment_sft_v1/pilot-20260608-merged-bf16/chat_template.jinja
Normal file
204
figment_sft_v1/pilot-20260608-merged-bf16/chat_template.jinja
Normal file
@@ -0,0 +1,204 @@
|
||||
{% macro render_extra_keys(json_dict, handled_keys) %}
|
||||
{%- if json_dict is mapping %}
|
||||
{%- for json_key in json_dict if json_key not in handled_keys %}
|
||||
{%- if json_dict[json_key] is mapping or (json_dict[json_key] is sequence and json_dict[json_key] is not string) %}
|
||||
{{- '\n<' ~ json_key ~ '>' ~ (json_dict[json_key] | tojson | safe) ~ '</' ~ json_key ~ '>' }}
|
||||
{%- else %}
|
||||
{{-'\n<' ~ json_key ~ '>' ~ (json_dict[json_key] | string) ~ '</' ~ json_key ~ '>' }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{% endmacro %}
|
||||
{%- set enable_thinking = enable_thinking if enable_thinking is defined else True %}
|
||||
{%- set truncate_history_thinking = truncate_history_thinking if truncate_history_thinking is defined else True %}
|
||||
|
||||
{%- set ns = namespace(last_user_idx = -1) %}
|
||||
{%- set loop_messages = messages %}
|
||||
{%- for m in loop_messages %}
|
||||
{%- if m["role"] == "user" %}
|
||||
{%- set ns.last_user_idx = loop.index0 %}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
|
||||
{%- if messages[0]["role"] == "system" %}
|
||||
{%- set system_message = messages[0]["content"] %}
|
||||
{%- set loop_messages = messages[1:] %}
|
||||
{%- else %}
|
||||
{%- set system_message = "" %}
|
||||
{%- set loop_messages = messages %}
|
||||
{%- endif %}
|
||||
{%- if not tools is defined %}
|
||||
{%- set tools = [] %}
|
||||
{%- endif %}
|
||||
{# Recompute last_user_idx relative to loop_messages after handling system #}
|
||||
{%- set ns = namespace(last_user_idx = -1) %}
|
||||
{%- for m in loop_messages %}
|
||||
{%- if m["role"] == "user" %}
|
||||
{%- set ns.last_user_idx = loop.index0 %}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- if system_message is defined %}
|
||||
{{- "<|im_start|>system\n" + system_message }}
|
||||
{%- else %}
|
||||
{%- if tools is iterable and tools | length > 0 %}
|
||||
{{- "<|im_start|>system\n" }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- if tools is iterable and tools | length > 0 %}
|
||||
{%- if system_message is defined and system_message | length > 0 %}
|
||||
{{- "\n\n" }}
|
||||
{%- endif %}
|
||||
{{- "# Tools\n\nYou have access to the following functions:\n\n" }}
|
||||
{{- "<tools>" }}
|
||||
{%- for tool in tools %}
|
||||
{%- if tool.function is defined %}
|
||||
{%- set tool = tool.function %}
|
||||
{%- endif %}
|
||||
{{- "\n<function>\n<name>" ~ tool.name ~ "</name>" }}
|
||||
{%- if tool.description is defined %}
|
||||
{{- '\n<description>' ~ (tool.description | trim) ~ '</description>' }}
|
||||
{%- endif %}
|
||||
{{- '\n<parameters>' }}
|
||||
{%- if tool.parameters is defined and tool.parameters is mapping and tool.parameters.properties is defined and tool.parameters.properties is mapping %}
|
||||
{%- for param_name, param_fields in tool.parameters.properties|items %}
|
||||
{{- '\n<parameter>' }}
|
||||
{{- '\n<name>' ~ param_name ~ '</name>' }}
|
||||
{%- if param_fields.type is defined %}
|
||||
{{- '\n<type>' ~ (param_fields.type | string) ~ '</type>' }}
|
||||
{%- endif %}
|
||||
{%- if param_fields.description is defined %}
|
||||
{{- '\n<description>' ~ (param_fields.description | trim) ~ '</description>' }}
|
||||
{%- endif %}
|
||||
{%- if param_fields.enum is defined %}
|
||||
{{- '\n<enum>' ~ (param_fields.enum | tojson | safe) ~ '</enum>' }}
|
||||
{%- endif %}
|
||||
{%- set handled_keys = ['name', 'type', 'description', 'enum'] %}
|
||||
{{- render_extra_keys(param_fields, handled_keys) }}
|
||||
{{- '\n</parameter>' }}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{% set handled_keys = ['type', 'properties', 'required'] %}
|
||||
{{- render_extra_keys(tool.parameters, handled_keys) }}
|
||||
{%- if tool.parameters is defined and tool.parameters.required is defined %}
|
||||
{{- '\n<required>' ~ (tool.parameters.required | tojson | safe) ~ '</required>' }}
|
||||
{%- endif %}
|
||||
{{- '\n</parameters>' }}
|
||||
{%- set handled_keys = ['type', 'name', 'description', 'parameters'] %}
|
||||
{{- render_extra_keys(tool, handled_keys) }}
|
||||
{{- '\n</function>' }}
|
||||
{%- endfor %}
|
||||
{{- "\n</tools>" }}
|
||||
|
||||
{{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n</IMPORTANT>' }}
|
||||
{%- endif %}
|
||||
|
||||
|
||||
{%- if system_message is defined %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- else %}
|
||||
{%- if tools is iterable and tools | length > 0 %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
|
||||
{%- for message in loop_messages %}
|
||||
{%- if message.role == "assistant" %}
|
||||
{# Add reasoning content in to content field for unified processing below. #}
|
||||
{%- if message.reasoning_content is defined and message.reasoning_content is string and message.reasoning_content | trim | length > 0 %}
|
||||
{%- set content = "<think>\n" ~ message.reasoning_content ~ "\n</think>\n" ~ (message.content | default('', true)) %}
|
||||
{%- else %}
|
||||
{%- set content = message.content | default('', true) %}
|
||||
{%- if content is string -%}
|
||||
{# Allow downstream logic to to take care of broken thought, only handle coherent reasoning here. #}
|
||||
{%- if '<think>' not in content and '</think>' not in content -%}
|
||||
{%- set content = "<think></think>" ~ content -%}
|
||||
{%- endif -%}
|
||||
{%- else -%}
|
||||
{%- set content = content -%}
|
||||
{%- endif -%}
|
||||
{%- endif %}
|
||||
{%- if message.tool_calls is defined and message.tool_calls is iterable and message.tool_calls | length > 0 %}
|
||||
{# Assistant message has tool calls. #}
|
||||
{{- '<|im_start|>assistant\n' }}
|
||||
{%- set include_content = not (truncate_history_thinking and loop.index0 < ns.last_user_idx) %}
|
||||
{%- if content is string and content | trim | length > 0 %}
|
||||
{%- if include_content %}
|
||||
{{- (content | trim) ~ '\n' -}}
|
||||
{%- else %}
|
||||
{%- set c = (content | string) %}
|
||||
{%- if '</think>' in c %}
|
||||
{# Keep only content after the last closing think. Also generation prompt causes this. #}
|
||||
{%- set c = c.split('</think>')[-1] %}
|
||||
{%- elif '<think>' in c %}
|
||||
{# If <think> was opened but never closed, drop the trailing think segment #}
|
||||
{%- set c = c.split('<think>')[0] %}
|
||||
{%- endif %}
|
||||
{%- set c = "<think></think>" ~ c | trim %}
|
||||
{%- if c | length > 0 %}
|
||||
{{- c ~ '\n' -}}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- else %}
|
||||
{{- "<think></think>" -}}
|
||||
{%- endif %}
|
||||
{%- for tool_call in message.tool_calls %}
|
||||
{%- if tool_call.function is defined %}
|
||||
{%- set tool_call = tool_call.function %}
|
||||
{%- endif %}
|
||||
{{- '<tool_call>\n<function=' ~ tool_call.name ~ '>\n' -}}
|
||||
{%- if tool_call.arguments is defined %}
|
||||
{%- for args_name, args_value in tool_call.arguments|items %}
|
||||
{{- '<parameter=' ~ args_name ~ '>\n' -}}
|
||||
{%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}
|
||||
{{- args_value ~ '\n</parameter>\n' -}}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{{- '</function>\n</tool_call>\n' -}}
|
||||
{%- endfor %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- else %}
|
||||
{# Assistant message doesn't have tool calls. #}
|
||||
{%- if not (truncate_history_thinking and loop.index0 < ns.last_user_idx) %}
|
||||
{{- '<|im_start|>assistant\n' ~ (content | default('', true) | string | trim) ~ '<|im_end|>\n' }}
|
||||
{%- else %}
|
||||
{%- set c = (content | default('', true) | string) %}
|
||||
{%- if '<think>' in c and '</think>' in c %}
|
||||
{%- set c = "<think></think>" ~ c.split('</think>')[-1] %}
|
||||
{%- endif %}
|
||||
{%- set c = c | trim %}
|
||||
{%- if c | length > 0 %}
|
||||
{{- '<|im_start|>assistant\n' ~ c ~ '<|im_end|>\n' }}
|
||||
{%- else %}
|
||||
{{- '<|im_start|>assistant\n<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- elif message.role == "user" or message.role == "system" %}
|
||||
{{- '<|im_start|>' + message.role + '\n' }}
|
||||
{%- set content = message.content | string %}
|
||||
{{- content }}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- elif message.role == "tool" %}
|
||||
{%- if loop.previtem and loop.previtem.role != "tool" %}
|
||||
{{- '<|im_start|>user\n' }}
|
||||
{%- endif %}
|
||||
{{- '<tool_response>\n' }}
|
||||
{{- message.content }}
|
||||
{{- '\n</tool_response>\n' }}
|
||||
{%- if not loop.last and loop.nextitem.role != "tool" %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- elif loop.last %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- else %}
|
||||
{{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
|
||||
{%- if add_generation_prompt %}
|
||||
{%- if enable_thinking %}
|
||||
{{- '<|im_start|>assistant\n<think>\n' }}
|
||||
{%- else %}
|
||||
{{- '<|im_start|>assistant\n<think></think>' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
59
figment_sft_v1/pilot-20260608-merged-bf16/config.json
Normal file
59
figment_sft_v1/pilot-20260608-merged-bf16/config.json
Normal file
@@ -0,0 +1,59 @@
|
||||
{
|
||||
"architectures": [
|
||||
"NemotronHForCausalLM"
|
||||
],
|
||||
"attention_bias": false,
|
||||
"attention_dropout": 0.0,
|
||||
"attention_head_dim": 128,
|
||||
"auto_map": {
|
||||
"AutoConfig": "configuration_nemotron_h.NemotronHConfig",
|
||||
"AutoModelForCausalLM": "modeling_nemotron_h.NemotronHForCausalLM"
|
||||
},
|
||||
"bos_token_id": 1,
|
||||
"chunk_size": 256,
|
||||
"conv_kernel": 4,
|
||||
"dtype": "bfloat16",
|
||||
"eos_token_id": 2,
|
||||
"expand": 2,
|
||||
"head_dim": 128,
|
||||
"hidden_dropout": 0.0,
|
||||
"hidden_size": 3136,
|
||||
"hybrid_override_pattern": "M-M-M-MM-M-M*-M-M*-M-M-M*-M-M-MM*-MMM-M-M-",
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 12544,
|
||||
"layer_norm_epsilon": 1e-05,
|
||||
"mamba_head_dim": 80,
|
||||
"mamba_hidden_act": "silu",
|
||||
"mamba_num_heads": 96,
|
||||
"mamba_proj_bias": false,
|
||||
"max_position_embeddings": 262144,
|
||||
"mlp_bias": false,
|
||||
"mlp_hidden_act": "relu2",
|
||||
"model_type": "nemotron_h",
|
||||
"n_groups": 8,
|
||||
"num_attention_heads": 40,
|
||||
"num_hidden_layers": 42,
|
||||
"num_key_value_heads": 8,
|
||||
"num_logits_to_keep": 1,
|
||||
"pad_token_id": 0,
|
||||
"rescale_prenorm_residual": true,
|
||||
"residual_in_fp32": false,
|
||||
"rms_norm_eps": 1e-05,
|
||||
"sliding_window": null,
|
||||
"ssm_state_size": 128,
|
||||
"tie_word_embeddings": false,
|
||||
"time_step_floor": 0.0001,
|
||||
"time_step_limit": [
|
||||
0.0,
|
||||
Infinity
|
||||
],
|
||||
"time_step_max": 0.1,
|
||||
"time_step_min": 0.001,
|
||||
"time_step_rank": 256,
|
||||
"transformers_version": "4.57.6",
|
||||
"use_bias": false,
|
||||
"use_cache": true,
|
||||
"use_conv_bias": true,
|
||||
"use_mamba_kernels": true,
|
||||
"vocab_size": 131072
|
||||
}
|
||||
@@ -0,0 +1,243 @@
|
||||
# coding=utf-8
|
||||
# Copyright 2024 AI21 Labs Ltd. and the HuggingFace Inc. team. All rights reserved.
|
||||
# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""NemotronH model configuration"""
|
||||
|
||||
import re
|
||||
|
||||
from transformers.configuration_utils import PretrainedConfig
|
||||
from transformers.utils import logging
|
||||
|
||||
|
||||
logger = logging.get_logger(__name__)
|
||||
|
||||
|
||||
class NemotronHConfig(PretrainedConfig):
|
||||
r"""
|
||||
This is the configuration class to store the configuration of a [`NemotronHModel`]. It is used to instantiate a
|
||||
NemotronH model according to the specified arguments, defining the model architecture. Instantiating a configuration
|
||||
with the defaults will yield a similar configuration to that of the NemotronH-v0.1 model.
|
||||
|
||||
[todo](todo)
|
||||
|
||||
Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
|
||||
documentation from [`PretrainedConfig`] for more information.
|
||||
|
||||
|
||||
Args:
|
||||
vocab_size (`int`, *optional*, defaults to 131072):
|
||||
Vocabulary size of the NemotronH model. Defines the number of different tokens that can be represented by the
|
||||
`inputs_ids` passed when calling [`NemotronHModel`]
|
||||
tie_word_embeddings (`bool`, *optional*, defaults to `False`):
|
||||
Whether the model's input and output word embeddings should be tied. Note that this is only relevant if the
|
||||
model has a output word embedding layer.
|
||||
hidden_size (`int`, *optional*, defaults to 4096):
|
||||
Dimension of the hidden representations.
|
||||
intermediate_size (`int`, *optional*, defaults to 21504):
|
||||
Dimension of the MLP representations.
|
||||
num_hidden_layers (`int`, *optional*, defaults to 52):
|
||||
Number of hidden layers in the Transformer encoder.
|
||||
hybrid_override_pattern (`str`, *optional*, defaults to `"M-M-M-M*-M-M-M-M-M*-M-M-M-M-M*-M-M-M-M-M*-M-M-M-M-M-"`):
|
||||
The pattern of the hybrid model. The pattern is a string of characters where each character represents M: Mamba2, *: Attention, -: MLP
|
||||
num_attention_heads (`int`, *optional*, defaults to 32):
|
||||
Number of attention heads for each attention layer in the Transformer encoder.
|
||||
attention_head_dim (`int`, *optional*, defaults to 128):
|
||||
Dimension of each attention head.
|
||||
num_key_value_heads (`int`, *optional*, defaults to 8):
|
||||
This is the number of key_value heads that should be used to implement Grouped Query Attention. If
|
||||
`num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
|
||||
`num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used.
|
||||
mlp_hidden_act (`str`, *optional*, defaults to "relu2"):
|
||||
The non-linear activation function in the MLP layers.
|
||||
attention_bias (`bool`, *optional*, defaults to `False`):
|
||||
Whether to use bias in attention layers.
|
||||
mlp_bias (`bool`, *optional*, defaults to `False`):
|
||||
Whether to use bias in MLP layers.
|
||||
use_bias (`bool`, *optional*, defaults to `False`):
|
||||
Whether to use bias in the model.
|
||||
initializer_range (`float`, *optional*, defaults to 0.02):
|
||||
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
|
||||
layer_norm_epsilon (`float`, *optional*, defaults to 1e-5):
|
||||
The epsilon used by the layer normalization layers.
|
||||
residual_in_fp32 (`bool`, *optional*, defaults to `False`):
|
||||
Whether or not residuals should be in `float32`. If set to `False` residuals will keep the same `dtype` as the rest of the model.
|
||||
use_cache (`bool`, *optional*, defaults to `True`):
|
||||
Whether or not the model should return the last key/values attentions (not used by all models). Only
|
||||
relevant if `config.is_decoder=True`.
|
||||
num_logits_to_keep (`int` or `None`, *optional*, defaults to 1):
|
||||
Number of prompt logits to calculate during generation. If `None`, all logits will be calculated. If an
|
||||
integer value, only last `num_logits_to_keep` logits will be calculated.
|
||||
pad_token_id (`int`, *optional*, defaults to 0):
|
||||
The id of the padding token.
|
||||
bos_token_id (`int`, *optional*, defaults to 1):
|
||||
The id of the "beginning-of-sequence" token.
|
||||
eos_token_id (`int`, *optional*, defaults to 2):
|
||||
The id of the "end-of-sequence" token.
|
||||
sliding_window (`int`, *optional*, defaults to None):
|
||||
Sliding window attention window size.
|
||||
max_position_embeddings (`int`, *optional*, defaults to 4096):
|
||||
The maximum sequence length that this model might ever be used with.
|
||||
attention_dropout (`float`, *optional*, defaults to 0.0):
|
||||
The dropout ratio for the attention probabilities.
|
||||
hidden_dropout (`float`, *optional*, defaults to 0.0):
|
||||
The dropout ratio for the hidden states.
|
||||
use_mamba_kernels (`bool`, *optional*, defaults to `True`):
|
||||
Flag indicating whether or not to use the fast mamba kernels. These are available only if `mamba-ssm` and
|
||||
`causal-conv1d` are installed, and the mamba modules are running on a CUDA device.
|
||||
ssm_state_size (`int`, *optional*, defaults to 128):
|
||||
The dimension of the mamba state space latents.
|
||||
mamba_num_heads (`int`, *optional*, defaults to 128):
|
||||
Number of heads in Mamba layers.
|
||||
mamba_n_groups (`int`, *optional*, defaults to 8):
|
||||
Number of groups in Mamba layers.
|
||||
mamba_head_dim (`int`, *optional*, defaults to 64):
|
||||
Dimension of each Mamba head.
|
||||
mamba_d_conv (`int`, *optional*, defaults to 4):
|
||||
The size of the mamba convolution kernel.
|
||||
mamba_expand (`int`, *optional*, defaults to 2):
|
||||
Expanding factor used to determine the mamba intermediate size.
|
||||
mamba_hidden_act (`str`, *optional*, defaults to "silu"):
|
||||
The non-linear activation function in the Mamba layers.
|
||||
mamba_dt_min (`float`, *optional*, defaults to 0.001):
|
||||
Minimum value for the time step in Mamba.
|
||||
mamba_dt_max (`float`, *optional*, defaults to 0.1):
|
||||
Maximum value for the time step in Mamba.
|
||||
mamba_dt_limit (`tuple`, *optional*, defaults to (0.0, float("inf"))):
|
||||
Limits for the time step in Mamba.
|
||||
mamba_dt_init_floor (`float`, *optional*, defaults to 1e-4):
|
||||
Floor value for time step initialization in Mamba.
|
||||
mamba_conv_bias (`bool`, *optional*, defaults to `True`):
|
||||
Whether to use bias in the convolution layer of the mamba mixer block.
|
||||
mamba_proj_bias (`bool`, *optional*, defaults to `False`):
|
||||
Whether to use bias in the input and output projections of the mamba mixer block.
|
||||
mamba_chunk_size (`int`, *optional*, defaults to 256):
|
||||
Size of chunks for Mamba processing.
|
||||
rescale_prenorm_residual (`bool`, *optional*, defaults to `True`):
|
||||
Whether to rescale the pre-normalization residual connections.
|
||||
"""
|
||||
|
||||
model_type = "nemotron_h"
|
||||
keys_to_ignore_at_inference = ["past_key_values"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vocab_size=131072,
|
||||
tie_word_embeddings=False,
|
||||
hidden_size=4096,
|
||||
intermediate_size=21504,
|
||||
num_hidden_layers=52,
|
||||
hybrid_override_pattern="M-M-M-M*-M-M-M-M-M*-M-M-M-M-M*-M-M-M-M-M*-M-M-M-M-M-",
|
||||
num_attention_heads=32,
|
||||
attention_head_dim=128,
|
||||
num_key_value_heads=8, # nemo: num_query_groups
|
||||
mlp_hidden_act="relu2",
|
||||
attention_bias=False,
|
||||
mlp_bias=False,
|
||||
use_bias=False,
|
||||
initializer_range=0.02, # nemo: init_method_std
|
||||
layer_norm_epsilon=1e-5, # nemo: layernorm_epsilon
|
||||
residual_in_fp32=False, # Megatron Core default value
|
||||
use_cache=True,
|
||||
num_logits_to_keep=1,
|
||||
pad_token_id=0,
|
||||
bos_token_id=1,
|
||||
eos_token_id=2,
|
||||
sliding_window=None,
|
||||
max_position_embeddings=4096,
|
||||
attention_dropout=0.0,
|
||||
hidden_dropout=0.0, # * ADDED
|
||||
use_mamba_kernels=True,
|
||||
ssm_state_size=128, # mamba_state_size
|
||||
mamba_num_heads=128,
|
||||
mamba_n_groups=8, # nemo: mamba_ssm_ngroups = num_heads
|
||||
mamba_head_dim=64,
|
||||
mamba_d_conv=4,
|
||||
mamba_expand=2,
|
||||
mamba_hidden_act="silu",
|
||||
mamba_dt_min=0.001,
|
||||
mamba_dt_max=0.1,
|
||||
mamba_dt_limit=(0.0, float("inf")),
|
||||
mamba_dt_init_floor=1e-4,
|
||||
mamba_conv_bias=True,
|
||||
mamba_proj_bias=False,
|
||||
mamba_chunk_size=256,
|
||||
rescale_prenorm_residual=True,
|
||||
**kwargs,
|
||||
):
|
||||
self.vocab_size = vocab_size
|
||||
self.tie_word_embeddings = tie_word_embeddings
|
||||
self.hidden_size = hidden_size
|
||||
self.intermediate_size = intermediate_size
|
||||
self.num_hidden_layers = num_hidden_layers
|
||||
self.hybrid_override_pattern = hybrid_override_pattern
|
||||
self.num_attention_heads = num_attention_heads
|
||||
self.attention_head_dim = attention_head_dim
|
||||
self.sliding_window = sliding_window
|
||||
self.max_position_embeddings = max_position_embeddings
|
||||
self.attention_dropout = attention_dropout
|
||||
self.hidden_dropout = hidden_dropout
|
||||
|
||||
# Validate hybrid_override_pattern
|
||||
# M: Mamba2, *: Attention, -: MLP
|
||||
assert len(self.hybrid_override_pattern) == self.num_hidden_layers, "hybrid_override_pattern must have the same length as num_hidden_layers"
|
||||
assert re.match(r"^[*-M]+$", self.hybrid_override_pattern), "hybrid_override_pattern must only contain characters 'M', '*', or '-'"
|
||||
|
||||
# for backward compatibility
|
||||
if num_key_value_heads is None:
|
||||
num_key_value_heads = num_attention_heads
|
||||
|
||||
self.num_key_value_heads = num_key_value_heads
|
||||
self.mlp_hidden_act = mlp_hidden_act
|
||||
self.attention_bias = attention_bias
|
||||
self.mlp_bias = mlp_bias
|
||||
self.use_bias = use_bias
|
||||
self.initializer_range = initializer_range
|
||||
self.layer_norm_epsilon = layer_norm_epsilon
|
||||
self.residual_in_fp32 = residual_in_fp32
|
||||
|
||||
self.use_cache = use_cache
|
||||
self.num_logits_to_keep = num_logits_to_keep
|
||||
|
||||
self.use_mamba_kernels = use_mamba_kernels
|
||||
self.n_groups = mamba_n_groups
|
||||
self.mamba_head_dim = mamba_head_dim
|
||||
self.ssm_state_size = ssm_state_size
|
||||
self.mamba_num_heads = mamba_num_heads
|
||||
self.conv_kernel = mamba_d_conv
|
||||
self.expand = mamba_expand
|
||||
self.mamba_hidden_act = mamba_hidden_act
|
||||
self.time_step_min = mamba_dt_min
|
||||
self.time_step_max = mamba_dt_max
|
||||
self.time_step_limit = mamba_dt_limit
|
||||
self.time_step_floor = mamba_dt_init_floor
|
||||
self.use_conv_bias = mamba_conv_bias
|
||||
self.mamba_proj_bias = mamba_proj_bias
|
||||
self.chunk_size = mamba_chunk_size
|
||||
self.rescale_prenorm_residual = rescale_prenorm_residual
|
||||
|
||||
super().__init__(
|
||||
pad_token_id=pad_token_id,
|
||||
bos_token_id=bos_token_id,
|
||||
eos_token_id=eos_token_id,
|
||||
tie_word_embeddings=tie_word_embeddings,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
@property
|
||||
def layers_block_type(self):
|
||||
return [
|
||||
"mamba" if self.hybrid_override_pattern[i] == "M" else
|
||||
"attention" if self.hybrid_override_pattern[i] == "*" else "mlp"
|
||||
for i in range(self.num_hidden_layers)]
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"adapter_dir": "/checkpoints/figment_sft_v1/pilot-20260608",
|
||||
"base_model_id": "nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16",
|
||||
"dtype": "bfloat16",
|
||||
"files": [
|
||||
"chat_template.jinja",
|
||||
"config.json",
|
||||
"configuration_nemotron_h.py",
|
||||
"generation_config.json",
|
||||
"model-00001-of-00002.safetensors",
|
||||
"model-00002-of-00002.safetensors",
|
||||
"model.safetensors.index.json",
|
||||
"modeling_nemotron_h.py",
|
||||
"special_tokens_map.json",
|
||||
"tokenizer.json",
|
||||
"tokenizer_config.json"
|
||||
],
|
||||
"merge_method": "peft.merge_and_unload(safe_merge=True)",
|
||||
"output_dir": "/checkpoints/figment_sft_v1/pilot-20260608-merged-bf16"
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"_from_model_config": true,
|
||||
"bos_token_id": 1,
|
||||
"eos_token_id": 2,
|
||||
"pad_token_id": 0,
|
||||
"temperature": null,
|
||||
"top_p": null,
|
||||
"transformers_version": "4.57.6"
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:aebcb7fd3126d0100cc7e78e58e0ed49ab29aad8f858f2c6149637aced9c699f
|
||||
size 3973314912
|
||||
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:8a1a7b48e647dd43cb7941a9e6a3f7a839326865034f626b2705645b0e29c830
|
||||
size 3973827728
|
||||
@@ -0,0 +1,271 @@
|
||||
{
|
||||
"metadata": {
|
||||
"total_parameters": 3973556832,
|
||||
"total_size": 7947113664
|
||||
},
|
||||
"weight_map": {
|
||||
"backbone.embeddings.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.1.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.1.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.1.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.10.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.10.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.10.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.12.mixer.k_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.12.mixer.o_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.12.mixer.q_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.12.mixer.v_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.12.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.13.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.13.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.13.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.15.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.15.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.15.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.17.mixer.k_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.17.mixer.o_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.17.mixer.q_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.17.mixer.v_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.17.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.18.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.18.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.18.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.20.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.20.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.20.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.21.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.21.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.21.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.21.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.21.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.21.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.21.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.21.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.21.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.22.mixer.down_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.22.mixer.up_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.22.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.24.mixer.k_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.24.mixer.o_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.24.mixer.q_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.24.mixer.v_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.24.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.25.mixer.down_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.25.mixer.up_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.25.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.27.mixer.down_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.27.mixer.up_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.27.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.29.mixer.down_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.29.mixer.up_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.29.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.3.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.3.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.3.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.30.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.30.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.30.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.30.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.30.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.30.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.30.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.30.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.30.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.32.mixer.k_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.32.mixer.o_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.32.mixer.q_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.32.mixer.v_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.32.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.33.mixer.down_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.33.mixer.up_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.33.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.37.mixer.down_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.37.mixer.up_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.37.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.39.mixer.down_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.39.mixer.up_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.39.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.4.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.4.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.4.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.4.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.4.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.4.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.4.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.4.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.4.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.40.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.40.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.40.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.40.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.40.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.40.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.40.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.40.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.40.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.41.mixer.down_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.41.mixer.up_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.41.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.5.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.5.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.5.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.8.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.8.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.8.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.norm_f.weight": "model-00002-of-00002.safetensors",
|
||||
"lm_head.weight": "model-00002-of-00002.safetensors"
|
||||
}
|
||||
}
|
||||
1638
figment_sft_v1/pilot-20260608-merged-bf16/modeling_nemotron_h.py
Normal file
1638
figment_sft_v1/pilot-20260608-merged-bf16/modeling_nemotron_h.py
Normal file
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"bos_token": {
|
||||
"content": "<s>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"eos_token": {
|
||||
"content": "<|im_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"pad_token": {
|
||||
"content": "<|im_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"unk_token": {
|
||||
"content": "<unk>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
}
|
||||
}
|
||||
3
figment_sft_v1/pilot-20260608-merged-bf16/tokenizer.json
Normal file
3
figment_sft_v1/pilot-20260608-merged-bf16/tokenizer.json
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:623c34567aebb18582765289fbe23d901c62704d6518d71866e0e58db892b5b7
|
||||
size 17077484
|
||||
8019
figment_sft_v1/pilot-20260608-merged-bf16/tokenizer_config.json
Normal file
8019
figment_sft_v1/pilot-20260608-merged-bf16/tokenizer_config.json
Normal file
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:85bc2978be155e1cdf12b42c8ccf84e1c1b65ad2da6b463d7be726d33cbd31aa
|
||||
size 7957646688
|
||||
@@ -0,0 +1,204 @@
|
||||
{% macro render_extra_keys(json_dict, handled_keys) %}
|
||||
{%- if json_dict is mapping %}
|
||||
{%- for json_key in json_dict if json_key not in handled_keys %}
|
||||
{%- if json_dict[json_key] is mapping or (json_dict[json_key] is sequence and json_dict[json_key] is not string) %}
|
||||
{{- '\n<' ~ json_key ~ '>' ~ (json_dict[json_key] | tojson | safe) ~ '</' ~ json_key ~ '>' }}
|
||||
{%- else %}
|
||||
{{-'\n<' ~ json_key ~ '>' ~ (json_dict[json_key] | string) ~ '</' ~ json_key ~ '>' }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{% endmacro %}
|
||||
{%- set enable_thinking = enable_thinking if enable_thinking is defined else True %}
|
||||
{%- set truncate_history_thinking = truncate_history_thinking if truncate_history_thinking is defined else True %}
|
||||
|
||||
{%- set ns = namespace(last_user_idx = -1) %}
|
||||
{%- set loop_messages = messages %}
|
||||
{%- for m in loop_messages %}
|
||||
{%- if m["role"] == "user" %}
|
||||
{%- set ns.last_user_idx = loop.index0 %}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
|
||||
{%- if messages[0]["role"] == "system" %}
|
||||
{%- set system_message = messages[0]["content"] %}
|
||||
{%- set loop_messages = messages[1:] %}
|
||||
{%- else %}
|
||||
{%- set system_message = "" %}
|
||||
{%- set loop_messages = messages %}
|
||||
{%- endif %}
|
||||
{%- if not tools is defined %}
|
||||
{%- set tools = [] %}
|
||||
{%- endif %}
|
||||
{# Recompute last_user_idx relative to loop_messages after handling system #}
|
||||
{%- set ns = namespace(last_user_idx = -1) %}
|
||||
{%- for m in loop_messages %}
|
||||
{%- if m["role"] == "user" %}
|
||||
{%- set ns.last_user_idx = loop.index0 %}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- if system_message is defined %}
|
||||
{{- "<|im_start|>system\n" + system_message }}
|
||||
{%- else %}
|
||||
{%- if tools is iterable and tools | length > 0 %}
|
||||
{{- "<|im_start|>system\n" }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- if tools is iterable and tools | length > 0 %}
|
||||
{%- if system_message is defined and system_message | length > 0 %}
|
||||
{{- "\n\n" }}
|
||||
{%- endif %}
|
||||
{{- "# Tools\n\nYou have access to the following functions:\n\n" }}
|
||||
{{- "<tools>" }}
|
||||
{%- for tool in tools %}
|
||||
{%- if tool.function is defined %}
|
||||
{%- set tool = tool.function %}
|
||||
{%- endif %}
|
||||
{{- "\n<function>\n<name>" ~ tool.name ~ "</name>" }}
|
||||
{%- if tool.description is defined %}
|
||||
{{- '\n<description>' ~ (tool.description | trim) ~ '</description>' }}
|
||||
{%- endif %}
|
||||
{{- '\n<parameters>' }}
|
||||
{%- if tool.parameters is defined and tool.parameters is mapping and tool.parameters.properties is defined and tool.parameters.properties is mapping %}
|
||||
{%- for param_name, param_fields in tool.parameters.properties|items %}
|
||||
{{- '\n<parameter>' }}
|
||||
{{- '\n<name>' ~ param_name ~ '</name>' }}
|
||||
{%- if param_fields.type is defined %}
|
||||
{{- '\n<type>' ~ (param_fields.type | string) ~ '</type>' }}
|
||||
{%- endif %}
|
||||
{%- if param_fields.description is defined %}
|
||||
{{- '\n<description>' ~ (param_fields.description | trim) ~ '</description>' }}
|
||||
{%- endif %}
|
||||
{%- if param_fields.enum is defined %}
|
||||
{{- '\n<enum>' ~ (param_fields.enum | tojson | safe) ~ '</enum>' }}
|
||||
{%- endif %}
|
||||
{%- set handled_keys = ['name', 'type', 'description', 'enum'] %}
|
||||
{{- render_extra_keys(param_fields, handled_keys) }}
|
||||
{{- '\n</parameter>' }}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{% set handled_keys = ['type', 'properties', 'required'] %}
|
||||
{{- render_extra_keys(tool.parameters, handled_keys) }}
|
||||
{%- if tool.parameters is defined and tool.parameters.required is defined %}
|
||||
{{- '\n<required>' ~ (tool.parameters.required | tojson | safe) ~ '</required>' }}
|
||||
{%- endif %}
|
||||
{{- '\n</parameters>' }}
|
||||
{%- set handled_keys = ['type', 'name', 'description', 'parameters'] %}
|
||||
{{- render_extra_keys(tool, handled_keys) }}
|
||||
{{- '\n</function>' }}
|
||||
{%- endfor %}
|
||||
{{- "\n</tools>" }}
|
||||
|
||||
{{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n</IMPORTANT>' }}
|
||||
{%- endif %}
|
||||
|
||||
|
||||
{%- if system_message is defined %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- else %}
|
||||
{%- if tools is iterable and tools | length > 0 %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
|
||||
{%- for message in loop_messages %}
|
||||
{%- if message.role == "assistant" %}
|
||||
{# Add reasoning content in to content field for unified processing below. #}
|
||||
{%- if message.reasoning_content is defined and message.reasoning_content is string and message.reasoning_content | trim | length > 0 %}
|
||||
{%- set content = "<think>\n" ~ message.reasoning_content ~ "\n</think>\n" ~ (message.content | default('', true)) %}
|
||||
{%- else %}
|
||||
{%- set content = message.content | default('', true) %}
|
||||
{%- if content is string -%}
|
||||
{# Allow downstream logic to to take care of broken thought, only handle coherent reasoning here. #}
|
||||
{%- if '<think>' not in content and '</think>' not in content -%}
|
||||
{%- set content = "<think></think>" ~ content -%}
|
||||
{%- endif -%}
|
||||
{%- else -%}
|
||||
{%- set content = content -%}
|
||||
{%- endif -%}
|
||||
{%- endif %}
|
||||
{%- if message.tool_calls is defined and message.tool_calls is iterable and message.tool_calls | length > 0 %}
|
||||
{# Assistant message has tool calls. #}
|
||||
{{- '<|im_start|>assistant\n' }}
|
||||
{%- set include_content = not (truncate_history_thinking and loop.index0 < ns.last_user_idx) %}
|
||||
{%- if content is string and content | trim | length > 0 %}
|
||||
{%- if include_content %}
|
||||
{{- (content | trim) ~ '\n' -}}
|
||||
{%- else %}
|
||||
{%- set c = (content | string) %}
|
||||
{%- if '</think>' in c %}
|
||||
{# Keep only content after the last closing think. Also generation prompt causes this. #}
|
||||
{%- set c = c.split('</think>')[-1] %}
|
||||
{%- elif '<think>' in c %}
|
||||
{# If <think> was opened but never closed, drop the trailing think segment #}
|
||||
{%- set c = c.split('<think>')[0] %}
|
||||
{%- endif %}
|
||||
{%- set c = "<think></think>" ~ c | trim %}
|
||||
{%- if c | length > 0 %}
|
||||
{{- c ~ '\n' -}}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- else %}
|
||||
{{- "<think></think>" -}}
|
||||
{%- endif %}
|
||||
{%- for tool_call in message.tool_calls %}
|
||||
{%- if tool_call.function is defined %}
|
||||
{%- set tool_call = tool_call.function %}
|
||||
{%- endif %}
|
||||
{{- '<tool_call>\n<function=' ~ tool_call.name ~ '>\n' -}}
|
||||
{%- if tool_call.arguments is defined %}
|
||||
{%- for args_name, args_value in tool_call.arguments|items %}
|
||||
{{- '<parameter=' ~ args_name ~ '>\n' -}}
|
||||
{%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}
|
||||
{{- args_value ~ '\n</parameter>\n' -}}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{{- '</function>\n</tool_call>\n' -}}
|
||||
{%- endfor %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- else %}
|
||||
{# Assistant message doesn't have tool calls. #}
|
||||
{%- if not (truncate_history_thinking and loop.index0 < ns.last_user_idx) %}
|
||||
{{- '<|im_start|>assistant\n' ~ (content | default('', true) | string | trim) ~ '<|im_end|>\n' }}
|
||||
{%- else %}
|
||||
{%- set c = (content | default('', true) | string) %}
|
||||
{%- if '<think>' in c and '</think>' in c %}
|
||||
{%- set c = "<think></think>" ~ c.split('</think>')[-1] %}
|
||||
{%- endif %}
|
||||
{%- set c = c | trim %}
|
||||
{%- if c | length > 0 %}
|
||||
{{- '<|im_start|>assistant\n' ~ c ~ '<|im_end|>\n' }}
|
||||
{%- else %}
|
||||
{{- '<|im_start|>assistant\n<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- elif message.role == "user" or message.role == "system" %}
|
||||
{{- '<|im_start|>' + message.role + '\n' }}
|
||||
{%- set content = message.content | string %}
|
||||
{{- content }}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- elif message.role == "tool" %}
|
||||
{%- if loop.previtem and loop.previtem.role != "tool" %}
|
||||
{{- '<|im_start|>user\n' }}
|
||||
{%- endif %}
|
||||
{{- '<tool_response>\n' }}
|
||||
{{- message.content }}
|
||||
{{- '\n</tool_response>\n' }}
|
||||
{%- if not loop.last and loop.nextitem.role != "tool" %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- elif loop.last %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- else %}
|
||||
{{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
|
||||
{%- if add_generation_prompt %}
|
||||
{%- if enable_thinking %}
|
||||
{{- '<|im_start|>assistant\n<think>\n' }}
|
||||
{%- else %}
|
||||
{{- '<|im_start|>assistant\n<think></think>' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
59
figment_sft_v10/figment-sft-v10-lora-merged-bf16/config.json
Normal file
59
figment_sft_v10/figment-sft-v10-lora-merged-bf16/config.json
Normal file
@@ -0,0 +1,59 @@
|
||||
{
|
||||
"architectures": [
|
||||
"NemotronHForCausalLM"
|
||||
],
|
||||
"attention_bias": false,
|
||||
"attention_dropout": 0.0,
|
||||
"attention_head_dim": 128,
|
||||
"auto_map": {
|
||||
"AutoConfig": "configuration_nemotron_h.NemotronHConfig",
|
||||
"AutoModelForCausalLM": "modeling_nemotron_h.NemotronHForCausalLM"
|
||||
},
|
||||
"bos_token_id": 1,
|
||||
"chunk_size": 256,
|
||||
"conv_kernel": 4,
|
||||
"dtype": "bfloat16",
|
||||
"eos_token_id": 2,
|
||||
"expand": 2,
|
||||
"head_dim": 128,
|
||||
"hidden_dropout": 0.0,
|
||||
"hidden_size": 3136,
|
||||
"hybrid_override_pattern": "M-M-M-MM-M-M*-M-M*-M-M-M*-M-M-MM*-MMM-M-M-",
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 12544,
|
||||
"layer_norm_epsilon": 1e-05,
|
||||
"mamba_head_dim": 80,
|
||||
"mamba_hidden_act": "silu",
|
||||
"mamba_num_heads": 96,
|
||||
"mamba_proj_bias": false,
|
||||
"max_position_embeddings": 262144,
|
||||
"mlp_bias": false,
|
||||
"mlp_hidden_act": "relu2",
|
||||
"model_type": "nemotron_h",
|
||||
"n_groups": 8,
|
||||
"num_attention_heads": 40,
|
||||
"num_hidden_layers": 42,
|
||||
"num_key_value_heads": 8,
|
||||
"num_logits_to_keep": 1,
|
||||
"pad_token_id": 0,
|
||||
"rescale_prenorm_residual": true,
|
||||
"residual_in_fp32": false,
|
||||
"rms_norm_eps": 1e-05,
|
||||
"sliding_window": null,
|
||||
"ssm_state_size": 128,
|
||||
"tie_word_embeddings": false,
|
||||
"time_step_floor": 0.0001,
|
||||
"time_step_limit": [
|
||||
0.0,
|
||||
Infinity
|
||||
],
|
||||
"time_step_max": 0.1,
|
||||
"time_step_min": 0.001,
|
||||
"time_step_rank": 256,
|
||||
"transformers_version": "4.57.6",
|
||||
"use_bias": false,
|
||||
"use_cache": true,
|
||||
"use_conv_bias": true,
|
||||
"use_mamba_kernels": true,
|
||||
"vocab_size": 131072
|
||||
}
|
||||
@@ -0,0 +1,243 @@
|
||||
# coding=utf-8
|
||||
# Copyright 2024 AI21 Labs Ltd. and the HuggingFace Inc. team. All rights reserved.
|
||||
# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""NemotronH model configuration"""
|
||||
|
||||
import re
|
||||
|
||||
from transformers.configuration_utils import PretrainedConfig
|
||||
from transformers.utils import logging
|
||||
|
||||
|
||||
logger = logging.get_logger(__name__)
|
||||
|
||||
|
||||
class NemotronHConfig(PretrainedConfig):
|
||||
r"""
|
||||
This is the configuration class to store the configuration of a [`NemotronHModel`]. It is used to instantiate a
|
||||
NemotronH model according to the specified arguments, defining the model architecture. Instantiating a configuration
|
||||
with the defaults will yield a similar configuration to that of the NemotronH-v0.1 model.
|
||||
|
||||
[todo](todo)
|
||||
|
||||
Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
|
||||
documentation from [`PretrainedConfig`] for more information.
|
||||
|
||||
|
||||
Args:
|
||||
vocab_size (`int`, *optional*, defaults to 131072):
|
||||
Vocabulary size of the NemotronH model. Defines the number of different tokens that can be represented by the
|
||||
`inputs_ids` passed when calling [`NemotronHModel`]
|
||||
tie_word_embeddings (`bool`, *optional*, defaults to `False`):
|
||||
Whether the model's input and output word embeddings should be tied. Note that this is only relevant if the
|
||||
model has a output word embedding layer.
|
||||
hidden_size (`int`, *optional*, defaults to 4096):
|
||||
Dimension of the hidden representations.
|
||||
intermediate_size (`int`, *optional*, defaults to 21504):
|
||||
Dimension of the MLP representations.
|
||||
num_hidden_layers (`int`, *optional*, defaults to 52):
|
||||
Number of hidden layers in the Transformer encoder.
|
||||
hybrid_override_pattern (`str`, *optional*, defaults to `"M-M-M-M*-M-M-M-M-M*-M-M-M-M-M*-M-M-M-M-M*-M-M-M-M-M-"`):
|
||||
The pattern of the hybrid model. The pattern is a string of characters where each character represents M: Mamba2, *: Attention, -: MLP
|
||||
num_attention_heads (`int`, *optional*, defaults to 32):
|
||||
Number of attention heads for each attention layer in the Transformer encoder.
|
||||
attention_head_dim (`int`, *optional*, defaults to 128):
|
||||
Dimension of each attention head.
|
||||
num_key_value_heads (`int`, *optional*, defaults to 8):
|
||||
This is the number of key_value heads that should be used to implement Grouped Query Attention. If
|
||||
`num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
|
||||
`num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used.
|
||||
mlp_hidden_act (`str`, *optional*, defaults to "relu2"):
|
||||
The non-linear activation function in the MLP layers.
|
||||
attention_bias (`bool`, *optional*, defaults to `False`):
|
||||
Whether to use bias in attention layers.
|
||||
mlp_bias (`bool`, *optional*, defaults to `False`):
|
||||
Whether to use bias in MLP layers.
|
||||
use_bias (`bool`, *optional*, defaults to `False`):
|
||||
Whether to use bias in the model.
|
||||
initializer_range (`float`, *optional*, defaults to 0.02):
|
||||
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
|
||||
layer_norm_epsilon (`float`, *optional*, defaults to 1e-5):
|
||||
The epsilon used by the layer normalization layers.
|
||||
residual_in_fp32 (`bool`, *optional*, defaults to `False`):
|
||||
Whether or not residuals should be in `float32`. If set to `False` residuals will keep the same `dtype` as the rest of the model.
|
||||
use_cache (`bool`, *optional*, defaults to `True`):
|
||||
Whether or not the model should return the last key/values attentions (not used by all models). Only
|
||||
relevant if `config.is_decoder=True`.
|
||||
num_logits_to_keep (`int` or `None`, *optional*, defaults to 1):
|
||||
Number of prompt logits to calculate during generation. If `None`, all logits will be calculated. If an
|
||||
integer value, only last `num_logits_to_keep` logits will be calculated.
|
||||
pad_token_id (`int`, *optional*, defaults to 0):
|
||||
The id of the padding token.
|
||||
bos_token_id (`int`, *optional*, defaults to 1):
|
||||
The id of the "beginning-of-sequence" token.
|
||||
eos_token_id (`int`, *optional*, defaults to 2):
|
||||
The id of the "end-of-sequence" token.
|
||||
sliding_window (`int`, *optional*, defaults to None):
|
||||
Sliding window attention window size.
|
||||
max_position_embeddings (`int`, *optional*, defaults to 4096):
|
||||
The maximum sequence length that this model might ever be used with.
|
||||
attention_dropout (`float`, *optional*, defaults to 0.0):
|
||||
The dropout ratio for the attention probabilities.
|
||||
hidden_dropout (`float`, *optional*, defaults to 0.0):
|
||||
The dropout ratio for the hidden states.
|
||||
use_mamba_kernels (`bool`, *optional*, defaults to `True`):
|
||||
Flag indicating whether or not to use the fast mamba kernels. These are available only if `mamba-ssm` and
|
||||
`causal-conv1d` are installed, and the mamba modules are running on a CUDA device.
|
||||
ssm_state_size (`int`, *optional*, defaults to 128):
|
||||
The dimension of the mamba state space latents.
|
||||
mamba_num_heads (`int`, *optional*, defaults to 128):
|
||||
Number of heads in Mamba layers.
|
||||
mamba_n_groups (`int`, *optional*, defaults to 8):
|
||||
Number of groups in Mamba layers.
|
||||
mamba_head_dim (`int`, *optional*, defaults to 64):
|
||||
Dimension of each Mamba head.
|
||||
mamba_d_conv (`int`, *optional*, defaults to 4):
|
||||
The size of the mamba convolution kernel.
|
||||
mamba_expand (`int`, *optional*, defaults to 2):
|
||||
Expanding factor used to determine the mamba intermediate size.
|
||||
mamba_hidden_act (`str`, *optional*, defaults to "silu"):
|
||||
The non-linear activation function in the Mamba layers.
|
||||
mamba_dt_min (`float`, *optional*, defaults to 0.001):
|
||||
Minimum value for the time step in Mamba.
|
||||
mamba_dt_max (`float`, *optional*, defaults to 0.1):
|
||||
Maximum value for the time step in Mamba.
|
||||
mamba_dt_limit (`tuple`, *optional*, defaults to (0.0, float("inf"))):
|
||||
Limits for the time step in Mamba.
|
||||
mamba_dt_init_floor (`float`, *optional*, defaults to 1e-4):
|
||||
Floor value for time step initialization in Mamba.
|
||||
mamba_conv_bias (`bool`, *optional*, defaults to `True`):
|
||||
Whether to use bias in the convolution layer of the mamba mixer block.
|
||||
mamba_proj_bias (`bool`, *optional*, defaults to `False`):
|
||||
Whether to use bias in the input and output projections of the mamba mixer block.
|
||||
mamba_chunk_size (`int`, *optional*, defaults to 256):
|
||||
Size of chunks for Mamba processing.
|
||||
rescale_prenorm_residual (`bool`, *optional*, defaults to `True`):
|
||||
Whether to rescale the pre-normalization residual connections.
|
||||
"""
|
||||
|
||||
model_type = "nemotron_h"
|
||||
keys_to_ignore_at_inference = ["past_key_values"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vocab_size=131072,
|
||||
tie_word_embeddings=False,
|
||||
hidden_size=4096,
|
||||
intermediate_size=21504,
|
||||
num_hidden_layers=52,
|
||||
hybrid_override_pattern="M-M-M-M*-M-M-M-M-M*-M-M-M-M-M*-M-M-M-M-M*-M-M-M-M-M-",
|
||||
num_attention_heads=32,
|
||||
attention_head_dim=128,
|
||||
num_key_value_heads=8, # nemo: num_query_groups
|
||||
mlp_hidden_act="relu2",
|
||||
attention_bias=False,
|
||||
mlp_bias=False,
|
||||
use_bias=False,
|
||||
initializer_range=0.02, # nemo: init_method_std
|
||||
layer_norm_epsilon=1e-5, # nemo: layernorm_epsilon
|
||||
residual_in_fp32=False, # Megatron Core default value
|
||||
use_cache=True,
|
||||
num_logits_to_keep=1,
|
||||
pad_token_id=0,
|
||||
bos_token_id=1,
|
||||
eos_token_id=2,
|
||||
sliding_window=None,
|
||||
max_position_embeddings=4096,
|
||||
attention_dropout=0.0,
|
||||
hidden_dropout=0.0, # * ADDED
|
||||
use_mamba_kernels=True,
|
||||
ssm_state_size=128, # mamba_state_size
|
||||
mamba_num_heads=128,
|
||||
mamba_n_groups=8, # nemo: mamba_ssm_ngroups = num_heads
|
||||
mamba_head_dim=64,
|
||||
mamba_d_conv=4,
|
||||
mamba_expand=2,
|
||||
mamba_hidden_act="silu",
|
||||
mamba_dt_min=0.001,
|
||||
mamba_dt_max=0.1,
|
||||
mamba_dt_limit=(0.0, float("inf")),
|
||||
mamba_dt_init_floor=1e-4,
|
||||
mamba_conv_bias=True,
|
||||
mamba_proj_bias=False,
|
||||
mamba_chunk_size=256,
|
||||
rescale_prenorm_residual=True,
|
||||
**kwargs,
|
||||
):
|
||||
self.vocab_size = vocab_size
|
||||
self.tie_word_embeddings = tie_word_embeddings
|
||||
self.hidden_size = hidden_size
|
||||
self.intermediate_size = intermediate_size
|
||||
self.num_hidden_layers = num_hidden_layers
|
||||
self.hybrid_override_pattern = hybrid_override_pattern
|
||||
self.num_attention_heads = num_attention_heads
|
||||
self.attention_head_dim = attention_head_dim
|
||||
self.sliding_window = sliding_window
|
||||
self.max_position_embeddings = max_position_embeddings
|
||||
self.attention_dropout = attention_dropout
|
||||
self.hidden_dropout = hidden_dropout
|
||||
|
||||
# Validate hybrid_override_pattern
|
||||
# M: Mamba2, *: Attention, -: MLP
|
||||
assert len(self.hybrid_override_pattern) == self.num_hidden_layers, "hybrid_override_pattern must have the same length as num_hidden_layers"
|
||||
assert re.match(r"^[*-M]+$", self.hybrid_override_pattern), "hybrid_override_pattern must only contain characters 'M', '*', or '-'"
|
||||
|
||||
# for backward compatibility
|
||||
if num_key_value_heads is None:
|
||||
num_key_value_heads = num_attention_heads
|
||||
|
||||
self.num_key_value_heads = num_key_value_heads
|
||||
self.mlp_hidden_act = mlp_hidden_act
|
||||
self.attention_bias = attention_bias
|
||||
self.mlp_bias = mlp_bias
|
||||
self.use_bias = use_bias
|
||||
self.initializer_range = initializer_range
|
||||
self.layer_norm_epsilon = layer_norm_epsilon
|
||||
self.residual_in_fp32 = residual_in_fp32
|
||||
|
||||
self.use_cache = use_cache
|
||||
self.num_logits_to_keep = num_logits_to_keep
|
||||
|
||||
self.use_mamba_kernels = use_mamba_kernels
|
||||
self.n_groups = mamba_n_groups
|
||||
self.mamba_head_dim = mamba_head_dim
|
||||
self.ssm_state_size = ssm_state_size
|
||||
self.mamba_num_heads = mamba_num_heads
|
||||
self.conv_kernel = mamba_d_conv
|
||||
self.expand = mamba_expand
|
||||
self.mamba_hidden_act = mamba_hidden_act
|
||||
self.time_step_min = mamba_dt_min
|
||||
self.time_step_max = mamba_dt_max
|
||||
self.time_step_limit = mamba_dt_limit
|
||||
self.time_step_floor = mamba_dt_init_floor
|
||||
self.use_conv_bias = mamba_conv_bias
|
||||
self.mamba_proj_bias = mamba_proj_bias
|
||||
self.chunk_size = mamba_chunk_size
|
||||
self.rescale_prenorm_residual = rescale_prenorm_residual
|
||||
|
||||
super().__init__(
|
||||
pad_token_id=pad_token_id,
|
||||
bos_token_id=bos_token_id,
|
||||
eos_token_id=eos_token_id,
|
||||
tie_word_embeddings=tie_word_embeddings,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
@property
|
||||
def layers_block_type(self):
|
||||
return [
|
||||
"mamba" if self.hybrid_override_pattern[i] == "M" else
|
||||
"attention" if self.hybrid_override_pattern[i] == "*" else "mlp"
|
||||
for i in range(self.num_hidden_layers)]
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"adapter_dir": "/checkpoints/figment_sft_v10/figment-sft-v10-lora",
|
||||
"base_model_id": "nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16",
|
||||
"dtype": "bfloat16",
|
||||
"files": [
|
||||
"chat_template.jinja",
|
||||
"config.json",
|
||||
"configuration_nemotron_h.py",
|
||||
"generation_config.json",
|
||||
"model-00001-of-00002.safetensors",
|
||||
"model-00002-of-00002.safetensors",
|
||||
"model.safetensors.index.json",
|
||||
"modeling_nemotron_h.py",
|
||||
"special_tokens_map.json",
|
||||
"tokenizer.json",
|
||||
"tokenizer_config.json"
|
||||
],
|
||||
"merge_method": "peft.merge_and_unload(safe_merge=True)",
|
||||
"output_dir": "/checkpoints/figment_sft_v10/figment-sft-v10-lora-merged-bf16"
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"_from_model_config": true,
|
||||
"bos_token_id": 1,
|
||||
"eos_token_id": 2,
|
||||
"pad_token_id": 0,
|
||||
"temperature": null,
|
||||
"top_p": null,
|
||||
"transformers_version": "4.57.6"
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:120894348512b7ffc23231262eb07e4ea1d3028ee24595577bc8e893af2544fb
|
||||
size 3973314912
|
||||
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:01d6f56103f12ee72a653d5c5da4dff04b18a1ee17b32329dfdb9627481dfa5a
|
||||
size 3973827728
|
||||
@@ -0,0 +1,271 @@
|
||||
{
|
||||
"metadata": {
|
||||
"total_parameters": 3973556832,
|
||||
"total_size": 7947113664
|
||||
},
|
||||
"weight_map": {
|
||||
"backbone.embeddings.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.1.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.1.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.1.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.10.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.10.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.10.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.12.mixer.k_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.12.mixer.o_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.12.mixer.q_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.12.mixer.v_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.12.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.13.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.13.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.13.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.15.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.15.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.15.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.17.mixer.k_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.17.mixer.o_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.17.mixer.q_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.17.mixer.v_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.17.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.18.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.18.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.18.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.20.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.20.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.20.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.21.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.21.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.21.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.21.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.21.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.21.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.21.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.21.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.21.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.22.mixer.down_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.22.mixer.up_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.22.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.24.mixer.k_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.24.mixer.o_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.24.mixer.q_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.24.mixer.v_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.24.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.25.mixer.down_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.25.mixer.up_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.25.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.27.mixer.down_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.27.mixer.up_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.27.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.29.mixer.down_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.29.mixer.up_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.29.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.3.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.3.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.3.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.30.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.30.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.30.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.30.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.30.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.30.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.30.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.30.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.30.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.32.mixer.k_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.32.mixer.o_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.32.mixer.q_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.32.mixer.v_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.32.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.33.mixer.down_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.33.mixer.up_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.33.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.37.mixer.down_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.37.mixer.up_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.37.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.39.mixer.down_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.39.mixer.up_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.39.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.4.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.4.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.4.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.4.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.4.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.4.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.4.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.4.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.4.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.40.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.40.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.40.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.40.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.40.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.40.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.40.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.40.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.40.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.41.mixer.down_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.41.mixer.up_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.41.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.5.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.5.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.5.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.8.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.8.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.8.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.norm_f.weight": "model-00002-of-00002.safetensors",
|
||||
"lm_head.weight": "model-00002-of-00002.safetensors"
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"bos_token": {
|
||||
"content": "<s>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"eos_token": {
|
||||
"content": "<|im_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"pad_token": {
|
||||
"content": "<|im_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"unk_token": {
|
||||
"content": "<unk>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:623c34567aebb18582765289fbe23d901c62704d6518d71866e0e58db892b5b7
|
||||
size 17077484
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:cb5c99e32660547941a681853c30eff47cd2a9aee837fbdd3ee17684b44d4fd2
|
||||
size 7957646688
|
||||
@@ -0,0 +1,204 @@
|
||||
{% macro render_extra_keys(json_dict, handled_keys) %}
|
||||
{%- if json_dict is mapping %}
|
||||
{%- for json_key in json_dict if json_key not in handled_keys %}
|
||||
{%- if json_dict[json_key] is mapping or (json_dict[json_key] is sequence and json_dict[json_key] is not string) %}
|
||||
{{- '\n<' ~ json_key ~ '>' ~ (json_dict[json_key] | tojson | safe) ~ '</' ~ json_key ~ '>' }}
|
||||
{%- else %}
|
||||
{{-'\n<' ~ json_key ~ '>' ~ (json_dict[json_key] | string) ~ '</' ~ json_key ~ '>' }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{% endmacro %}
|
||||
{%- set enable_thinking = enable_thinking if enable_thinking is defined else True %}
|
||||
{%- set truncate_history_thinking = truncate_history_thinking if truncate_history_thinking is defined else True %}
|
||||
|
||||
{%- set ns = namespace(last_user_idx = -1) %}
|
||||
{%- set loop_messages = messages %}
|
||||
{%- for m in loop_messages %}
|
||||
{%- if m["role"] == "user" %}
|
||||
{%- set ns.last_user_idx = loop.index0 %}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
|
||||
{%- if messages[0]["role"] == "system" %}
|
||||
{%- set system_message = messages[0]["content"] %}
|
||||
{%- set loop_messages = messages[1:] %}
|
||||
{%- else %}
|
||||
{%- set system_message = "" %}
|
||||
{%- set loop_messages = messages %}
|
||||
{%- endif %}
|
||||
{%- if not tools is defined %}
|
||||
{%- set tools = [] %}
|
||||
{%- endif %}
|
||||
{# Recompute last_user_idx relative to loop_messages after handling system #}
|
||||
{%- set ns = namespace(last_user_idx = -1) %}
|
||||
{%- for m in loop_messages %}
|
||||
{%- if m["role"] == "user" %}
|
||||
{%- set ns.last_user_idx = loop.index0 %}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- if system_message is defined %}
|
||||
{{- "<|im_start|>system\n" + system_message }}
|
||||
{%- else %}
|
||||
{%- if tools is iterable and tools | length > 0 %}
|
||||
{{- "<|im_start|>system\n" }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- if tools is iterable and tools | length > 0 %}
|
||||
{%- if system_message is defined and system_message | length > 0 %}
|
||||
{{- "\n\n" }}
|
||||
{%- endif %}
|
||||
{{- "# Tools\n\nYou have access to the following functions:\n\n" }}
|
||||
{{- "<tools>" }}
|
||||
{%- for tool in tools %}
|
||||
{%- if tool.function is defined %}
|
||||
{%- set tool = tool.function %}
|
||||
{%- endif %}
|
||||
{{- "\n<function>\n<name>" ~ tool.name ~ "</name>" }}
|
||||
{%- if tool.description is defined %}
|
||||
{{- '\n<description>' ~ (tool.description | trim) ~ '</description>' }}
|
||||
{%- endif %}
|
||||
{{- '\n<parameters>' }}
|
||||
{%- if tool.parameters is defined and tool.parameters is mapping and tool.parameters.properties is defined and tool.parameters.properties is mapping %}
|
||||
{%- for param_name, param_fields in tool.parameters.properties|items %}
|
||||
{{- '\n<parameter>' }}
|
||||
{{- '\n<name>' ~ param_name ~ '</name>' }}
|
||||
{%- if param_fields.type is defined %}
|
||||
{{- '\n<type>' ~ (param_fields.type | string) ~ '</type>' }}
|
||||
{%- endif %}
|
||||
{%- if param_fields.description is defined %}
|
||||
{{- '\n<description>' ~ (param_fields.description | trim) ~ '</description>' }}
|
||||
{%- endif %}
|
||||
{%- if param_fields.enum is defined %}
|
||||
{{- '\n<enum>' ~ (param_fields.enum | tojson | safe) ~ '</enum>' }}
|
||||
{%- endif %}
|
||||
{%- set handled_keys = ['name', 'type', 'description', 'enum'] %}
|
||||
{{- render_extra_keys(param_fields, handled_keys) }}
|
||||
{{- '\n</parameter>' }}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{% set handled_keys = ['type', 'properties', 'required'] %}
|
||||
{{- render_extra_keys(tool.parameters, handled_keys) }}
|
||||
{%- if tool.parameters is defined and tool.parameters.required is defined %}
|
||||
{{- '\n<required>' ~ (tool.parameters.required | tojson | safe) ~ '</required>' }}
|
||||
{%- endif %}
|
||||
{{- '\n</parameters>' }}
|
||||
{%- set handled_keys = ['type', 'name', 'description', 'parameters'] %}
|
||||
{{- render_extra_keys(tool, handled_keys) }}
|
||||
{{- '\n</function>' }}
|
||||
{%- endfor %}
|
||||
{{- "\n</tools>" }}
|
||||
|
||||
{{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n</IMPORTANT>' }}
|
||||
{%- endif %}
|
||||
|
||||
|
||||
{%- if system_message is defined %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- else %}
|
||||
{%- if tools is iterable and tools | length > 0 %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
|
||||
{%- for message in loop_messages %}
|
||||
{%- if message.role == "assistant" %}
|
||||
{# Add reasoning content in to content field for unified processing below. #}
|
||||
{%- if message.reasoning_content is defined and message.reasoning_content is string and message.reasoning_content | trim | length > 0 %}
|
||||
{%- set content = "<think>\n" ~ message.reasoning_content ~ "\n</think>\n" ~ (message.content | default('', true)) %}
|
||||
{%- else %}
|
||||
{%- set content = message.content | default('', true) %}
|
||||
{%- if content is string -%}
|
||||
{# Allow downstream logic to to take care of broken thought, only handle coherent reasoning here. #}
|
||||
{%- if '<think>' not in content and '</think>' not in content -%}
|
||||
{%- set content = "<think></think>" ~ content -%}
|
||||
{%- endif -%}
|
||||
{%- else -%}
|
||||
{%- set content = content -%}
|
||||
{%- endif -%}
|
||||
{%- endif %}
|
||||
{%- if message.tool_calls is defined and message.tool_calls is iterable and message.tool_calls | length > 0 %}
|
||||
{# Assistant message has tool calls. #}
|
||||
{{- '<|im_start|>assistant\n' }}
|
||||
{%- set include_content = not (truncate_history_thinking and loop.index0 < ns.last_user_idx) %}
|
||||
{%- if content is string and content | trim | length > 0 %}
|
||||
{%- if include_content %}
|
||||
{{- (content | trim) ~ '\n' -}}
|
||||
{%- else %}
|
||||
{%- set c = (content | string) %}
|
||||
{%- if '</think>' in c %}
|
||||
{# Keep only content after the last closing think. Also generation prompt causes this. #}
|
||||
{%- set c = c.split('</think>')[-1] %}
|
||||
{%- elif '<think>' in c %}
|
||||
{# If <think> was opened but never closed, drop the trailing think segment #}
|
||||
{%- set c = c.split('<think>')[0] %}
|
||||
{%- endif %}
|
||||
{%- set c = "<think></think>" ~ c | trim %}
|
||||
{%- if c | length > 0 %}
|
||||
{{- c ~ '\n' -}}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- else %}
|
||||
{{- "<think></think>" -}}
|
||||
{%- endif %}
|
||||
{%- for tool_call in message.tool_calls %}
|
||||
{%- if tool_call.function is defined %}
|
||||
{%- set tool_call = tool_call.function %}
|
||||
{%- endif %}
|
||||
{{- '<tool_call>\n<function=' ~ tool_call.name ~ '>\n' -}}
|
||||
{%- if tool_call.arguments is defined %}
|
||||
{%- for args_name, args_value in tool_call.arguments|items %}
|
||||
{{- '<parameter=' ~ args_name ~ '>\n' -}}
|
||||
{%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}
|
||||
{{- args_value ~ '\n</parameter>\n' -}}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{{- '</function>\n</tool_call>\n' -}}
|
||||
{%- endfor %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- else %}
|
||||
{# Assistant message doesn't have tool calls. #}
|
||||
{%- if not (truncate_history_thinking and loop.index0 < ns.last_user_idx) %}
|
||||
{{- '<|im_start|>assistant\n' ~ (content | default('', true) | string | trim) ~ '<|im_end|>\n' }}
|
||||
{%- else %}
|
||||
{%- set c = (content | default('', true) | string) %}
|
||||
{%- if '<think>' in c and '</think>' in c %}
|
||||
{%- set c = "<think></think>" ~ c.split('</think>')[-1] %}
|
||||
{%- endif %}
|
||||
{%- set c = c | trim %}
|
||||
{%- if c | length > 0 %}
|
||||
{{- '<|im_start|>assistant\n' ~ c ~ '<|im_end|>\n' }}
|
||||
{%- else %}
|
||||
{{- '<|im_start|>assistant\n<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- elif message.role == "user" or message.role == "system" %}
|
||||
{{- '<|im_start|>' + message.role + '\n' }}
|
||||
{%- set content = message.content | string %}
|
||||
{{- content }}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- elif message.role == "tool" %}
|
||||
{%- if loop.previtem and loop.previtem.role != "tool" %}
|
||||
{{- '<|im_start|>user\n' }}
|
||||
{%- endif %}
|
||||
{{- '<tool_response>\n' }}
|
||||
{{- message.content }}
|
||||
{{- '\n</tool_response>\n' }}
|
||||
{%- if not loop.last and loop.nextitem.role != "tool" %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- elif loop.last %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- else %}
|
||||
{{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
|
||||
{%- if add_generation_prompt %}
|
||||
{%- if enable_thinking %}
|
||||
{{- '<|im_start|>assistant\n<think>\n' }}
|
||||
{%- else %}
|
||||
{{- '<|im_start|>assistant\n<think></think>' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
59
figment_sft_v11/figment-sft-v11-lora-merged-bf16/config.json
Normal file
59
figment_sft_v11/figment-sft-v11-lora-merged-bf16/config.json
Normal file
@@ -0,0 +1,59 @@
|
||||
{
|
||||
"architectures": [
|
||||
"NemotronHForCausalLM"
|
||||
],
|
||||
"attention_bias": false,
|
||||
"attention_dropout": 0.0,
|
||||
"attention_head_dim": 128,
|
||||
"auto_map": {
|
||||
"AutoConfig": "configuration_nemotron_h.NemotronHConfig",
|
||||
"AutoModelForCausalLM": "modeling_nemotron_h.NemotronHForCausalLM"
|
||||
},
|
||||
"bos_token_id": 1,
|
||||
"chunk_size": 256,
|
||||
"conv_kernel": 4,
|
||||
"dtype": "bfloat16",
|
||||
"eos_token_id": 2,
|
||||
"expand": 2,
|
||||
"head_dim": 128,
|
||||
"hidden_dropout": 0.0,
|
||||
"hidden_size": 3136,
|
||||
"hybrid_override_pattern": "M-M-M-MM-M-M*-M-M*-M-M-M*-M-M-MM*-MMM-M-M-",
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 12544,
|
||||
"layer_norm_epsilon": 1e-05,
|
||||
"mamba_head_dim": 80,
|
||||
"mamba_hidden_act": "silu",
|
||||
"mamba_num_heads": 96,
|
||||
"mamba_proj_bias": false,
|
||||
"max_position_embeddings": 262144,
|
||||
"mlp_bias": false,
|
||||
"mlp_hidden_act": "relu2",
|
||||
"model_type": "nemotron_h",
|
||||
"n_groups": 8,
|
||||
"num_attention_heads": 40,
|
||||
"num_hidden_layers": 42,
|
||||
"num_key_value_heads": 8,
|
||||
"num_logits_to_keep": 1,
|
||||
"pad_token_id": 0,
|
||||
"rescale_prenorm_residual": true,
|
||||
"residual_in_fp32": false,
|
||||
"rms_norm_eps": 1e-05,
|
||||
"sliding_window": null,
|
||||
"ssm_state_size": 128,
|
||||
"tie_word_embeddings": false,
|
||||
"time_step_floor": 0.0001,
|
||||
"time_step_limit": [
|
||||
0.0,
|
||||
Infinity
|
||||
],
|
||||
"time_step_max": 0.1,
|
||||
"time_step_min": 0.001,
|
||||
"time_step_rank": 256,
|
||||
"transformers_version": "4.57.6",
|
||||
"use_bias": false,
|
||||
"use_cache": true,
|
||||
"use_conv_bias": true,
|
||||
"use_mamba_kernels": true,
|
||||
"vocab_size": 131072
|
||||
}
|
||||
@@ -0,0 +1,243 @@
|
||||
# coding=utf-8
|
||||
# Copyright 2024 AI21 Labs Ltd. and the HuggingFace Inc. team. All rights reserved.
|
||||
# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""NemotronH model configuration"""
|
||||
|
||||
import re
|
||||
|
||||
from transformers.configuration_utils import PretrainedConfig
|
||||
from transformers.utils import logging
|
||||
|
||||
|
||||
logger = logging.get_logger(__name__)
|
||||
|
||||
|
||||
class NemotronHConfig(PretrainedConfig):
|
||||
r"""
|
||||
This is the configuration class to store the configuration of a [`NemotronHModel`]. It is used to instantiate a
|
||||
NemotronH model according to the specified arguments, defining the model architecture. Instantiating a configuration
|
||||
with the defaults will yield a similar configuration to that of the NemotronH-v0.1 model.
|
||||
|
||||
[todo](todo)
|
||||
|
||||
Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
|
||||
documentation from [`PretrainedConfig`] for more information.
|
||||
|
||||
|
||||
Args:
|
||||
vocab_size (`int`, *optional*, defaults to 131072):
|
||||
Vocabulary size of the NemotronH model. Defines the number of different tokens that can be represented by the
|
||||
`inputs_ids` passed when calling [`NemotronHModel`]
|
||||
tie_word_embeddings (`bool`, *optional*, defaults to `False`):
|
||||
Whether the model's input and output word embeddings should be tied. Note that this is only relevant if the
|
||||
model has a output word embedding layer.
|
||||
hidden_size (`int`, *optional*, defaults to 4096):
|
||||
Dimension of the hidden representations.
|
||||
intermediate_size (`int`, *optional*, defaults to 21504):
|
||||
Dimension of the MLP representations.
|
||||
num_hidden_layers (`int`, *optional*, defaults to 52):
|
||||
Number of hidden layers in the Transformer encoder.
|
||||
hybrid_override_pattern (`str`, *optional*, defaults to `"M-M-M-M*-M-M-M-M-M*-M-M-M-M-M*-M-M-M-M-M*-M-M-M-M-M-"`):
|
||||
The pattern of the hybrid model. The pattern is a string of characters where each character represents M: Mamba2, *: Attention, -: MLP
|
||||
num_attention_heads (`int`, *optional*, defaults to 32):
|
||||
Number of attention heads for each attention layer in the Transformer encoder.
|
||||
attention_head_dim (`int`, *optional*, defaults to 128):
|
||||
Dimension of each attention head.
|
||||
num_key_value_heads (`int`, *optional*, defaults to 8):
|
||||
This is the number of key_value heads that should be used to implement Grouped Query Attention. If
|
||||
`num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
|
||||
`num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used.
|
||||
mlp_hidden_act (`str`, *optional*, defaults to "relu2"):
|
||||
The non-linear activation function in the MLP layers.
|
||||
attention_bias (`bool`, *optional*, defaults to `False`):
|
||||
Whether to use bias in attention layers.
|
||||
mlp_bias (`bool`, *optional*, defaults to `False`):
|
||||
Whether to use bias in MLP layers.
|
||||
use_bias (`bool`, *optional*, defaults to `False`):
|
||||
Whether to use bias in the model.
|
||||
initializer_range (`float`, *optional*, defaults to 0.02):
|
||||
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
|
||||
layer_norm_epsilon (`float`, *optional*, defaults to 1e-5):
|
||||
The epsilon used by the layer normalization layers.
|
||||
residual_in_fp32 (`bool`, *optional*, defaults to `False`):
|
||||
Whether or not residuals should be in `float32`. If set to `False` residuals will keep the same `dtype` as the rest of the model.
|
||||
use_cache (`bool`, *optional*, defaults to `True`):
|
||||
Whether or not the model should return the last key/values attentions (not used by all models). Only
|
||||
relevant if `config.is_decoder=True`.
|
||||
num_logits_to_keep (`int` or `None`, *optional*, defaults to 1):
|
||||
Number of prompt logits to calculate during generation. If `None`, all logits will be calculated. If an
|
||||
integer value, only last `num_logits_to_keep` logits will be calculated.
|
||||
pad_token_id (`int`, *optional*, defaults to 0):
|
||||
The id of the padding token.
|
||||
bos_token_id (`int`, *optional*, defaults to 1):
|
||||
The id of the "beginning-of-sequence" token.
|
||||
eos_token_id (`int`, *optional*, defaults to 2):
|
||||
The id of the "end-of-sequence" token.
|
||||
sliding_window (`int`, *optional*, defaults to None):
|
||||
Sliding window attention window size.
|
||||
max_position_embeddings (`int`, *optional*, defaults to 4096):
|
||||
The maximum sequence length that this model might ever be used with.
|
||||
attention_dropout (`float`, *optional*, defaults to 0.0):
|
||||
The dropout ratio for the attention probabilities.
|
||||
hidden_dropout (`float`, *optional*, defaults to 0.0):
|
||||
The dropout ratio for the hidden states.
|
||||
use_mamba_kernels (`bool`, *optional*, defaults to `True`):
|
||||
Flag indicating whether or not to use the fast mamba kernels. These are available only if `mamba-ssm` and
|
||||
`causal-conv1d` are installed, and the mamba modules are running on a CUDA device.
|
||||
ssm_state_size (`int`, *optional*, defaults to 128):
|
||||
The dimension of the mamba state space latents.
|
||||
mamba_num_heads (`int`, *optional*, defaults to 128):
|
||||
Number of heads in Mamba layers.
|
||||
mamba_n_groups (`int`, *optional*, defaults to 8):
|
||||
Number of groups in Mamba layers.
|
||||
mamba_head_dim (`int`, *optional*, defaults to 64):
|
||||
Dimension of each Mamba head.
|
||||
mamba_d_conv (`int`, *optional*, defaults to 4):
|
||||
The size of the mamba convolution kernel.
|
||||
mamba_expand (`int`, *optional*, defaults to 2):
|
||||
Expanding factor used to determine the mamba intermediate size.
|
||||
mamba_hidden_act (`str`, *optional*, defaults to "silu"):
|
||||
The non-linear activation function in the Mamba layers.
|
||||
mamba_dt_min (`float`, *optional*, defaults to 0.001):
|
||||
Minimum value for the time step in Mamba.
|
||||
mamba_dt_max (`float`, *optional*, defaults to 0.1):
|
||||
Maximum value for the time step in Mamba.
|
||||
mamba_dt_limit (`tuple`, *optional*, defaults to (0.0, float("inf"))):
|
||||
Limits for the time step in Mamba.
|
||||
mamba_dt_init_floor (`float`, *optional*, defaults to 1e-4):
|
||||
Floor value for time step initialization in Mamba.
|
||||
mamba_conv_bias (`bool`, *optional*, defaults to `True`):
|
||||
Whether to use bias in the convolution layer of the mamba mixer block.
|
||||
mamba_proj_bias (`bool`, *optional*, defaults to `False`):
|
||||
Whether to use bias in the input and output projections of the mamba mixer block.
|
||||
mamba_chunk_size (`int`, *optional*, defaults to 256):
|
||||
Size of chunks for Mamba processing.
|
||||
rescale_prenorm_residual (`bool`, *optional*, defaults to `True`):
|
||||
Whether to rescale the pre-normalization residual connections.
|
||||
"""
|
||||
|
||||
model_type = "nemotron_h"
|
||||
keys_to_ignore_at_inference = ["past_key_values"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vocab_size=131072,
|
||||
tie_word_embeddings=False,
|
||||
hidden_size=4096,
|
||||
intermediate_size=21504,
|
||||
num_hidden_layers=52,
|
||||
hybrid_override_pattern="M-M-M-M*-M-M-M-M-M*-M-M-M-M-M*-M-M-M-M-M*-M-M-M-M-M-",
|
||||
num_attention_heads=32,
|
||||
attention_head_dim=128,
|
||||
num_key_value_heads=8, # nemo: num_query_groups
|
||||
mlp_hidden_act="relu2",
|
||||
attention_bias=False,
|
||||
mlp_bias=False,
|
||||
use_bias=False,
|
||||
initializer_range=0.02, # nemo: init_method_std
|
||||
layer_norm_epsilon=1e-5, # nemo: layernorm_epsilon
|
||||
residual_in_fp32=False, # Megatron Core default value
|
||||
use_cache=True,
|
||||
num_logits_to_keep=1,
|
||||
pad_token_id=0,
|
||||
bos_token_id=1,
|
||||
eos_token_id=2,
|
||||
sliding_window=None,
|
||||
max_position_embeddings=4096,
|
||||
attention_dropout=0.0,
|
||||
hidden_dropout=0.0, # * ADDED
|
||||
use_mamba_kernels=True,
|
||||
ssm_state_size=128, # mamba_state_size
|
||||
mamba_num_heads=128,
|
||||
mamba_n_groups=8, # nemo: mamba_ssm_ngroups = num_heads
|
||||
mamba_head_dim=64,
|
||||
mamba_d_conv=4,
|
||||
mamba_expand=2,
|
||||
mamba_hidden_act="silu",
|
||||
mamba_dt_min=0.001,
|
||||
mamba_dt_max=0.1,
|
||||
mamba_dt_limit=(0.0, float("inf")),
|
||||
mamba_dt_init_floor=1e-4,
|
||||
mamba_conv_bias=True,
|
||||
mamba_proj_bias=False,
|
||||
mamba_chunk_size=256,
|
||||
rescale_prenorm_residual=True,
|
||||
**kwargs,
|
||||
):
|
||||
self.vocab_size = vocab_size
|
||||
self.tie_word_embeddings = tie_word_embeddings
|
||||
self.hidden_size = hidden_size
|
||||
self.intermediate_size = intermediate_size
|
||||
self.num_hidden_layers = num_hidden_layers
|
||||
self.hybrid_override_pattern = hybrid_override_pattern
|
||||
self.num_attention_heads = num_attention_heads
|
||||
self.attention_head_dim = attention_head_dim
|
||||
self.sliding_window = sliding_window
|
||||
self.max_position_embeddings = max_position_embeddings
|
||||
self.attention_dropout = attention_dropout
|
||||
self.hidden_dropout = hidden_dropout
|
||||
|
||||
# Validate hybrid_override_pattern
|
||||
# M: Mamba2, *: Attention, -: MLP
|
||||
assert len(self.hybrid_override_pattern) == self.num_hidden_layers, "hybrid_override_pattern must have the same length as num_hidden_layers"
|
||||
assert re.match(r"^[*-M]+$", self.hybrid_override_pattern), "hybrid_override_pattern must only contain characters 'M', '*', or '-'"
|
||||
|
||||
# for backward compatibility
|
||||
if num_key_value_heads is None:
|
||||
num_key_value_heads = num_attention_heads
|
||||
|
||||
self.num_key_value_heads = num_key_value_heads
|
||||
self.mlp_hidden_act = mlp_hidden_act
|
||||
self.attention_bias = attention_bias
|
||||
self.mlp_bias = mlp_bias
|
||||
self.use_bias = use_bias
|
||||
self.initializer_range = initializer_range
|
||||
self.layer_norm_epsilon = layer_norm_epsilon
|
||||
self.residual_in_fp32 = residual_in_fp32
|
||||
|
||||
self.use_cache = use_cache
|
||||
self.num_logits_to_keep = num_logits_to_keep
|
||||
|
||||
self.use_mamba_kernels = use_mamba_kernels
|
||||
self.n_groups = mamba_n_groups
|
||||
self.mamba_head_dim = mamba_head_dim
|
||||
self.ssm_state_size = ssm_state_size
|
||||
self.mamba_num_heads = mamba_num_heads
|
||||
self.conv_kernel = mamba_d_conv
|
||||
self.expand = mamba_expand
|
||||
self.mamba_hidden_act = mamba_hidden_act
|
||||
self.time_step_min = mamba_dt_min
|
||||
self.time_step_max = mamba_dt_max
|
||||
self.time_step_limit = mamba_dt_limit
|
||||
self.time_step_floor = mamba_dt_init_floor
|
||||
self.use_conv_bias = mamba_conv_bias
|
||||
self.mamba_proj_bias = mamba_proj_bias
|
||||
self.chunk_size = mamba_chunk_size
|
||||
self.rescale_prenorm_residual = rescale_prenorm_residual
|
||||
|
||||
super().__init__(
|
||||
pad_token_id=pad_token_id,
|
||||
bos_token_id=bos_token_id,
|
||||
eos_token_id=eos_token_id,
|
||||
tie_word_embeddings=tie_word_embeddings,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
@property
|
||||
def layers_block_type(self):
|
||||
return [
|
||||
"mamba" if self.hybrid_override_pattern[i] == "M" else
|
||||
"attention" if self.hybrid_override_pattern[i] == "*" else "mlp"
|
||||
for i in range(self.num_hidden_layers)]
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"adapter_dir": "/checkpoints/figment_sft_v11/figment-sft-v11-lora",
|
||||
"base_model_id": "nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16",
|
||||
"dtype": "bfloat16",
|
||||
"files": [
|
||||
"chat_template.jinja",
|
||||
"config.json",
|
||||
"configuration_nemotron_h.py",
|
||||
"generation_config.json",
|
||||
"model-00001-of-00002.safetensors",
|
||||
"model-00002-of-00002.safetensors",
|
||||
"model.safetensors.index.json",
|
||||
"modeling_nemotron_h.py",
|
||||
"special_tokens_map.json",
|
||||
"tokenizer.json",
|
||||
"tokenizer_config.json"
|
||||
],
|
||||
"merge_method": "peft.merge_and_unload(safe_merge=True)",
|
||||
"output_dir": "/checkpoints/figment_sft_v11/figment-sft-v11-lora-merged-bf16"
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"_from_model_config": true,
|
||||
"bos_token_id": 1,
|
||||
"eos_token_id": 2,
|
||||
"pad_token_id": 0,
|
||||
"temperature": null,
|
||||
"top_p": null,
|
||||
"transformers_version": "4.57.6"
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:d414ea3c840cd8d1d1425f3d01a698890c560a852b7d0acbd826488c6260d74b
|
||||
size 3973314912
|
||||
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:68a67041dfebf534e2661621f7bce862e1a7f1d4d0d6e3d63138c8243b8a0f60
|
||||
size 3973827728
|
||||
@@ -0,0 +1,271 @@
|
||||
{
|
||||
"metadata": {
|
||||
"total_parameters": 3973556832,
|
||||
"total_size": 7947113664
|
||||
},
|
||||
"weight_map": {
|
||||
"backbone.embeddings.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.1.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.1.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.1.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.10.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.10.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.10.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.12.mixer.k_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.12.mixer.o_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.12.mixer.q_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.12.mixer.v_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.12.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.13.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.13.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.13.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.15.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.15.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.15.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.17.mixer.k_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.17.mixer.o_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.17.mixer.q_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.17.mixer.v_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.17.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.18.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.18.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.18.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.20.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.20.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.20.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.21.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.21.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.21.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.21.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.21.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.21.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.21.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.21.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.21.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.22.mixer.down_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.22.mixer.up_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.22.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.24.mixer.k_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.24.mixer.o_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.24.mixer.q_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.24.mixer.v_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.24.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.25.mixer.down_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.25.mixer.up_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.25.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.27.mixer.down_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.27.mixer.up_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.27.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.29.mixer.down_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.29.mixer.up_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.29.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.3.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.3.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.3.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.30.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.30.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.30.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.30.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.30.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.30.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.30.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.30.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.30.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.32.mixer.k_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.32.mixer.o_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.32.mixer.q_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.32.mixer.v_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.32.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.33.mixer.down_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.33.mixer.up_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.33.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.37.mixer.down_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.37.mixer.up_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.37.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.39.mixer.down_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.39.mixer.up_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.39.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.4.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.4.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.4.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.4.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.4.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.4.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.4.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.4.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.4.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.40.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.40.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.40.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.40.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.40.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.40.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.40.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.40.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.40.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.41.mixer.down_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.41.mixer.up_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.41.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.5.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.5.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.5.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.8.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.8.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.8.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.norm_f.weight": "model-00002-of-00002.safetensors",
|
||||
"lm_head.weight": "model-00002-of-00002.safetensors"
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"bos_token": {
|
||||
"content": "<s>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"eos_token": {
|
||||
"content": "<|im_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"pad_token": {
|
||||
"content": "<|im_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"unk_token": {
|
||||
"content": "<unk>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:623c34567aebb18582765289fbe23d901c62704d6518d71866e0e58db892b5b7
|
||||
size 17077484
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:164ebf943919b4c27a54dbce3380bc156bbad3c6e893f1d185d35801eac015b7
|
||||
size 7957646688
|
||||
@@ -0,0 +1,204 @@
|
||||
{% macro render_extra_keys(json_dict, handled_keys) %}
|
||||
{%- if json_dict is mapping %}
|
||||
{%- for json_key in json_dict if json_key not in handled_keys %}
|
||||
{%- if json_dict[json_key] is mapping or (json_dict[json_key] is sequence and json_dict[json_key] is not string) %}
|
||||
{{- '\n<' ~ json_key ~ '>' ~ (json_dict[json_key] | tojson | safe) ~ '</' ~ json_key ~ '>' }}
|
||||
{%- else %}
|
||||
{{-'\n<' ~ json_key ~ '>' ~ (json_dict[json_key] | string) ~ '</' ~ json_key ~ '>' }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{% endmacro %}
|
||||
{%- set enable_thinking = enable_thinking if enable_thinking is defined else True %}
|
||||
{%- set truncate_history_thinking = truncate_history_thinking if truncate_history_thinking is defined else True %}
|
||||
|
||||
{%- set ns = namespace(last_user_idx = -1) %}
|
||||
{%- set loop_messages = messages %}
|
||||
{%- for m in loop_messages %}
|
||||
{%- if m["role"] == "user" %}
|
||||
{%- set ns.last_user_idx = loop.index0 %}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
|
||||
{%- if messages[0]["role"] == "system" %}
|
||||
{%- set system_message = messages[0]["content"] %}
|
||||
{%- set loop_messages = messages[1:] %}
|
||||
{%- else %}
|
||||
{%- set system_message = "" %}
|
||||
{%- set loop_messages = messages %}
|
||||
{%- endif %}
|
||||
{%- if not tools is defined %}
|
||||
{%- set tools = [] %}
|
||||
{%- endif %}
|
||||
{# Recompute last_user_idx relative to loop_messages after handling system #}
|
||||
{%- set ns = namespace(last_user_idx = -1) %}
|
||||
{%- for m in loop_messages %}
|
||||
{%- if m["role"] == "user" %}
|
||||
{%- set ns.last_user_idx = loop.index0 %}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- if system_message is defined %}
|
||||
{{- "<|im_start|>system\n" + system_message }}
|
||||
{%- else %}
|
||||
{%- if tools is iterable and tools | length > 0 %}
|
||||
{{- "<|im_start|>system\n" }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- if tools is iterable and tools | length > 0 %}
|
||||
{%- if system_message is defined and system_message | length > 0 %}
|
||||
{{- "\n\n" }}
|
||||
{%- endif %}
|
||||
{{- "# Tools\n\nYou have access to the following functions:\n\n" }}
|
||||
{{- "<tools>" }}
|
||||
{%- for tool in tools %}
|
||||
{%- if tool.function is defined %}
|
||||
{%- set tool = tool.function %}
|
||||
{%- endif %}
|
||||
{{- "\n<function>\n<name>" ~ tool.name ~ "</name>" }}
|
||||
{%- if tool.description is defined %}
|
||||
{{- '\n<description>' ~ (tool.description | trim) ~ '</description>' }}
|
||||
{%- endif %}
|
||||
{{- '\n<parameters>' }}
|
||||
{%- if tool.parameters is defined and tool.parameters is mapping and tool.parameters.properties is defined and tool.parameters.properties is mapping %}
|
||||
{%- for param_name, param_fields in tool.parameters.properties|items %}
|
||||
{{- '\n<parameter>' }}
|
||||
{{- '\n<name>' ~ param_name ~ '</name>' }}
|
||||
{%- if param_fields.type is defined %}
|
||||
{{- '\n<type>' ~ (param_fields.type | string) ~ '</type>' }}
|
||||
{%- endif %}
|
||||
{%- if param_fields.description is defined %}
|
||||
{{- '\n<description>' ~ (param_fields.description | trim) ~ '</description>' }}
|
||||
{%- endif %}
|
||||
{%- if param_fields.enum is defined %}
|
||||
{{- '\n<enum>' ~ (param_fields.enum | tojson | safe) ~ '</enum>' }}
|
||||
{%- endif %}
|
||||
{%- set handled_keys = ['name', 'type', 'description', 'enum'] %}
|
||||
{{- render_extra_keys(param_fields, handled_keys) }}
|
||||
{{- '\n</parameter>' }}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{% set handled_keys = ['type', 'properties', 'required'] %}
|
||||
{{- render_extra_keys(tool.parameters, handled_keys) }}
|
||||
{%- if tool.parameters is defined and tool.parameters.required is defined %}
|
||||
{{- '\n<required>' ~ (tool.parameters.required | tojson | safe) ~ '</required>' }}
|
||||
{%- endif %}
|
||||
{{- '\n</parameters>' }}
|
||||
{%- set handled_keys = ['type', 'name', 'description', 'parameters'] %}
|
||||
{{- render_extra_keys(tool, handled_keys) }}
|
||||
{{- '\n</function>' }}
|
||||
{%- endfor %}
|
||||
{{- "\n</tools>" }}
|
||||
|
||||
{{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n</IMPORTANT>' }}
|
||||
{%- endif %}
|
||||
|
||||
|
||||
{%- if system_message is defined %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- else %}
|
||||
{%- if tools is iterable and tools | length > 0 %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
|
||||
{%- for message in loop_messages %}
|
||||
{%- if message.role == "assistant" %}
|
||||
{# Add reasoning content in to content field for unified processing below. #}
|
||||
{%- if message.reasoning_content is defined and message.reasoning_content is string and message.reasoning_content | trim | length > 0 %}
|
||||
{%- set content = "<think>\n" ~ message.reasoning_content ~ "\n</think>\n" ~ (message.content | default('', true)) %}
|
||||
{%- else %}
|
||||
{%- set content = message.content | default('', true) %}
|
||||
{%- if content is string -%}
|
||||
{# Allow downstream logic to to take care of broken thought, only handle coherent reasoning here. #}
|
||||
{%- if '<think>' not in content and '</think>' not in content -%}
|
||||
{%- set content = "<think></think>" ~ content -%}
|
||||
{%- endif -%}
|
||||
{%- else -%}
|
||||
{%- set content = content -%}
|
||||
{%- endif -%}
|
||||
{%- endif %}
|
||||
{%- if message.tool_calls is defined and message.tool_calls is iterable and message.tool_calls | length > 0 %}
|
||||
{# Assistant message has tool calls. #}
|
||||
{{- '<|im_start|>assistant\n' }}
|
||||
{%- set include_content = not (truncate_history_thinking and loop.index0 < ns.last_user_idx) %}
|
||||
{%- if content is string and content | trim | length > 0 %}
|
||||
{%- if include_content %}
|
||||
{{- (content | trim) ~ '\n' -}}
|
||||
{%- else %}
|
||||
{%- set c = (content | string) %}
|
||||
{%- if '</think>' in c %}
|
||||
{# Keep only content after the last closing think. Also generation prompt causes this. #}
|
||||
{%- set c = c.split('</think>')[-1] %}
|
||||
{%- elif '<think>' in c %}
|
||||
{# If <think> was opened but never closed, drop the trailing think segment #}
|
||||
{%- set c = c.split('<think>')[0] %}
|
||||
{%- endif %}
|
||||
{%- set c = "<think></think>" ~ c | trim %}
|
||||
{%- if c | length > 0 %}
|
||||
{{- c ~ '\n' -}}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- else %}
|
||||
{{- "<think></think>" -}}
|
||||
{%- endif %}
|
||||
{%- for tool_call in message.tool_calls %}
|
||||
{%- if tool_call.function is defined %}
|
||||
{%- set tool_call = tool_call.function %}
|
||||
{%- endif %}
|
||||
{{- '<tool_call>\n<function=' ~ tool_call.name ~ '>\n' -}}
|
||||
{%- if tool_call.arguments is defined %}
|
||||
{%- for args_name, args_value in tool_call.arguments|items %}
|
||||
{{- '<parameter=' ~ args_name ~ '>\n' -}}
|
||||
{%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}
|
||||
{{- args_value ~ '\n</parameter>\n' -}}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{{- '</function>\n</tool_call>\n' -}}
|
||||
{%- endfor %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- else %}
|
||||
{# Assistant message doesn't have tool calls. #}
|
||||
{%- if not (truncate_history_thinking and loop.index0 < ns.last_user_idx) %}
|
||||
{{- '<|im_start|>assistant\n' ~ (content | default('', true) | string | trim) ~ '<|im_end|>\n' }}
|
||||
{%- else %}
|
||||
{%- set c = (content | default('', true) | string) %}
|
||||
{%- if '<think>' in c and '</think>' in c %}
|
||||
{%- set c = "<think></think>" ~ c.split('</think>')[-1] %}
|
||||
{%- endif %}
|
||||
{%- set c = c | trim %}
|
||||
{%- if c | length > 0 %}
|
||||
{{- '<|im_start|>assistant\n' ~ c ~ '<|im_end|>\n' }}
|
||||
{%- else %}
|
||||
{{- '<|im_start|>assistant\n<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- elif message.role == "user" or message.role == "system" %}
|
||||
{{- '<|im_start|>' + message.role + '\n' }}
|
||||
{%- set content = message.content | string %}
|
||||
{{- content }}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- elif message.role == "tool" %}
|
||||
{%- if loop.previtem and loop.previtem.role != "tool" %}
|
||||
{{- '<|im_start|>user\n' }}
|
||||
{%- endif %}
|
||||
{{- '<tool_response>\n' }}
|
||||
{{- message.content }}
|
||||
{{- '\n</tool_response>\n' }}
|
||||
{%- if not loop.last and loop.nextitem.role != "tool" %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- elif loop.last %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- else %}
|
||||
{{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
|
||||
{%- if add_generation_prompt %}
|
||||
{%- if enable_thinking %}
|
||||
{{- '<|im_start|>assistant\n<think>\n' }}
|
||||
{%- else %}
|
||||
{{- '<|im_start|>assistant\n<think></think>' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
59
figment_sft_v12/figment-sft-v12-lora-merged-bf16/config.json
Normal file
59
figment_sft_v12/figment-sft-v12-lora-merged-bf16/config.json
Normal file
@@ -0,0 +1,59 @@
|
||||
{
|
||||
"architectures": [
|
||||
"NemotronHForCausalLM"
|
||||
],
|
||||
"attention_bias": false,
|
||||
"attention_dropout": 0.0,
|
||||
"attention_head_dim": 128,
|
||||
"auto_map": {
|
||||
"AutoConfig": "configuration_nemotron_h.NemotronHConfig",
|
||||
"AutoModelForCausalLM": "modeling_nemotron_h.NemotronHForCausalLM"
|
||||
},
|
||||
"bos_token_id": 1,
|
||||
"chunk_size": 256,
|
||||
"conv_kernel": 4,
|
||||
"dtype": "bfloat16",
|
||||
"eos_token_id": 2,
|
||||
"expand": 2,
|
||||
"head_dim": 128,
|
||||
"hidden_dropout": 0.0,
|
||||
"hidden_size": 3136,
|
||||
"hybrid_override_pattern": "M-M-M-MM-M-M*-M-M*-M-M-M*-M-M-MM*-MMM-M-M-",
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 12544,
|
||||
"layer_norm_epsilon": 1e-05,
|
||||
"mamba_head_dim": 80,
|
||||
"mamba_hidden_act": "silu",
|
||||
"mamba_num_heads": 96,
|
||||
"mamba_proj_bias": false,
|
||||
"max_position_embeddings": 262144,
|
||||
"mlp_bias": false,
|
||||
"mlp_hidden_act": "relu2",
|
||||
"model_type": "nemotron_h",
|
||||
"n_groups": 8,
|
||||
"num_attention_heads": 40,
|
||||
"num_hidden_layers": 42,
|
||||
"num_key_value_heads": 8,
|
||||
"num_logits_to_keep": 1,
|
||||
"pad_token_id": 0,
|
||||
"rescale_prenorm_residual": true,
|
||||
"residual_in_fp32": false,
|
||||
"rms_norm_eps": 1e-05,
|
||||
"sliding_window": null,
|
||||
"ssm_state_size": 128,
|
||||
"tie_word_embeddings": false,
|
||||
"time_step_floor": 0.0001,
|
||||
"time_step_limit": [
|
||||
0.0,
|
||||
Infinity
|
||||
],
|
||||
"time_step_max": 0.1,
|
||||
"time_step_min": 0.001,
|
||||
"time_step_rank": 256,
|
||||
"transformers_version": "4.57.6",
|
||||
"use_bias": false,
|
||||
"use_cache": true,
|
||||
"use_conv_bias": true,
|
||||
"use_mamba_kernels": true,
|
||||
"vocab_size": 131072
|
||||
}
|
||||
@@ -0,0 +1,243 @@
|
||||
# coding=utf-8
|
||||
# Copyright 2024 AI21 Labs Ltd. and the HuggingFace Inc. team. All rights reserved.
|
||||
# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""NemotronH model configuration"""
|
||||
|
||||
import re
|
||||
|
||||
from transformers.configuration_utils import PretrainedConfig
|
||||
from transformers.utils import logging
|
||||
|
||||
|
||||
logger = logging.get_logger(__name__)
|
||||
|
||||
|
||||
class NemotronHConfig(PretrainedConfig):
|
||||
r"""
|
||||
This is the configuration class to store the configuration of a [`NemotronHModel`]. It is used to instantiate a
|
||||
NemotronH model according to the specified arguments, defining the model architecture. Instantiating a configuration
|
||||
with the defaults will yield a similar configuration to that of the NemotronH-v0.1 model.
|
||||
|
||||
[todo](todo)
|
||||
|
||||
Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
|
||||
documentation from [`PretrainedConfig`] for more information.
|
||||
|
||||
|
||||
Args:
|
||||
vocab_size (`int`, *optional*, defaults to 131072):
|
||||
Vocabulary size of the NemotronH model. Defines the number of different tokens that can be represented by the
|
||||
`inputs_ids` passed when calling [`NemotronHModel`]
|
||||
tie_word_embeddings (`bool`, *optional*, defaults to `False`):
|
||||
Whether the model's input and output word embeddings should be tied. Note that this is only relevant if the
|
||||
model has a output word embedding layer.
|
||||
hidden_size (`int`, *optional*, defaults to 4096):
|
||||
Dimension of the hidden representations.
|
||||
intermediate_size (`int`, *optional*, defaults to 21504):
|
||||
Dimension of the MLP representations.
|
||||
num_hidden_layers (`int`, *optional*, defaults to 52):
|
||||
Number of hidden layers in the Transformer encoder.
|
||||
hybrid_override_pattern (`str`, *optional*, defaults to `"M-M-M-M*-M-M-M-M-M*-M-M-M-M-M*-M-M-M-M-M*-M-M-M-M-M-"`):
|
||||
The pattern of the hybrid model. The pattern is a string of characters where each character represents M: Mamba2, *: Attention, -: MLP
|
||||
num_attention_heads (`int`, *optional*, defaults to 32):
|
||||
Number of attention heads for each attention layer in the Transformer encoder.
|
||||
attention_head_dim (`int`, *optional*, defaults to 128):
|
||||
Dimension of each attention head.
|
||||
num_key_value_heads (`int`, *optional*, defaults to 8):
|
||||
This is the number of key_value heads that should be used to implement Grouped Query Attention. If
|
||||
`num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
|
||||
`num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used.
|
||||
mlp_hidden_act (`str`, *optional*, defaults to "relu2"):
|
||||
The non-linear activation function in the MLP layers.
|
||||
attention_bias (`bool`, *optional*, defaults to `False`):
|
||||
Whether to use bias in attention layers.
|
||||
mlp_bias (`bool`, *optional*, defaults to `False`):
|
||||
Whether to use bias in MLP layers.
|
||||
use_bias (`bool`, *optional*, defaults to `False`):
|
||||
Whether to use bias in the model.
|
||||
initializer_range (`float`, *optional*, defaults to 0.02):
|
||||
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
|
||||
layer_norm_epsilon (`float`, *optional*, defaults to 1e-5):
|
||||
The epsilon used by the layer normalization layers.
|
||||
residual_in_fp32 (`bool`, *optional*, defaults to `False`):
|
||||
Whether or not residuals should be in `float32`. If set to `False` residuals will keep the same `dtype` as the rest of the model.
|
||||
use_cache (`bool`, *optional*, defaults to `True`):
|
||||
Whether or not the model should return the last key/values attentions (not used by all models). Only
|
||||
relevant if `config.is_decoder=True`.
|
||||
num_logits_to_keep (`int` or `None`, *optional*, defaults to 1):
|
||||
Number of prompt logits to calculate during generation. If `None`, all logits will be calculated. If an
|
||||
integer value, only last `num_logits_to_keep` logits will be calculated.
|
||||
pad_token_id (`int`, *optional*, defaults to 0):
|
||||
The id of the padding token.
|
||||
bos_token_id (`int`, *optional*, defaults to 1):
|
||||
The id of the "beginning-of-sequence" token.
|
||||
eos_token_id (`int`, *optional*, defaults to 2):
|
||||
The id of the "end-of-sequence" token.
|
||||
sliding_window (`int`, *optional*, defaults to None):
|
||||
Sliding window attention window size.
|
||||
max_position_embeddings (`int`, *optional*, defaults to 4096):
|
||||
The maximum sequence length that this model might ever be used with.
|
||||
attention_dropout (`float`, *optional*, defaults to 0.0):
|
||||
The dropout ratio for the attention probabilities.
|
||||
hidden_dropout (`float`, *optional*, defaults to 0.0):
|
||||
The dropout ratio for the hidden states.
|
||||
use_mamba_kernels (`bool`, *optional*, defaults to `True`):
|
||||
Flag indicating whether or not to use the fast mamba kernels. These are available only if `mamba-ssm` and
|
||||
`causal-conv1d` are installed, and the mamba modules are running on a CUDA device.
|
||||
ssm_state_size (`int`, *optional*, defaults to 128):
|
||||
The dimension of the mamba state space latents.
|
||||
mamba_num_heads (`int`, *optional*, defaults to 128):
|
||||
Number of heads in Mamba layers.
|
||||
mamba_n_groups (`int`, *optional*, defaults to 8):
|
||||
Number of groups in Mamba layers.
|
||||
mamba_head_dim (`int`, *optional*, defaults to 64):
|
||||
Dimension of each Mamba head.
|
||||
mamba_d_conv (`int`, *optional*, defaults to 4):
|
||||
The size of the mamba convolution kernel.
|
||||
mamba_expand (`int`, *optional*, defaults to 2):
|
||||
Expanding factor used to determine the mamba intermediate size.
|
||||
mamba_hidden_act (`str`, *optional*, defaults to "silu"):
|
||||
The non-linear activation function in the Mamba layers.
|
||||
mamba_dt_min (`float`, *optional*, defaults to 0.001):
|
||||
Minimum value for the time step in Mamba.
|
||||
mamba_dt_max (`float`, *optional*, defaults to 0.1):
|
||||
Maximum value for the time step in Mamba.
|
||||
mamba_dt_limit (`tuple`, *optional*, defaults to (0.0, float("inf"))):
|
||||
Limits for the time step in Mamba.
|
||||
mamba_dt_init_floor (`float`, *optional*, defaults to 1e-4):
|
||||
Floor value for time step initialization in Mamba.
|
||||
mamba_conv_bias (`bool`, *optional*, defaults to `True`):
|
||||
Whether to use bias in the convolution layer of the mamba mixer block.
|
||||
mamba_proj_bias (`bool`, *optional*, defaults to `False`):
|
||||
Whether to use bias in the input and output projections of the mamba mixer block.
|
||||
mamba_chunk_size (`int`, *optional*, defaults to 256):
|
||||
Size of chunks for Mamba processing.
|
||||
rescale_prenorm_residual (`bool`, *optional*, defaults to `True`):
|
||||
Whether to rescale the pre-normalization residual connections.
|
||||
"""
|
||||
|
||||
model_type = "nemotron_h"
|
||||
keys_to_ignore_at_inference = ["past_key_values"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vocab_size=131072,
|
||||
tie_word_embeddings=False,
|
||||
hidden_size=4096,
|
||||
intermediate_size=21504,
|
||||
num_hidden_layers=52,
|
||||
hybrid_override_pattern="M-M-M-M*-M-M-M-M-M*-M-M-M-M-M*-M-M-M-M-M*-M-M-M-M-M-",
|
||||
num_attention_heads=32,
|
||||
attention_head_dim=128,
|
||||
num_key_value_heads=8, # nemo: num_query_groups
|
||||
mlp_hidden_act="relu2",
|
||||
attention_bias=False,
|
||||
mlp_bias=False,
|
||||
use_bias=False,
|
||||
initializer_range=0.02, # nemo: init_method_std
|
||||
layer_norm_epsilon=1e-5, # nemo: layernorm_epsilon
|
||||
residual_in_fp32=False, # Megatron Core default value
|
||||
use_cache=True,
|
||||
num_logits_to_keep=1,
|
||||
pad_token_id=0,
|
||||
bos_token_id=1,
|
||||
eos_token_id=2,
|
||||
sliding_window=None,
|
||||
max_position_embeddings=4096,
|
||||
attention_dropout=0.0,
|
||||
hidden_dropout=0.0, # * ADDED
|
||||
use_mamba_kernels=True,
|
||||
ssm_state_size=128, # mamba_state_size
|
||||
mamba_num_heads=128,
|
||||
mamba_n_groups=8, # nemo: mamba_ssm_ngroups = num_heads
|
||||
mamba_head_dim=64,
|
||||
mamba_d_conv=4,
|
||||
mamba_expand=2,
|
||||
mamba_hidden_act="silu",
|
||||
mamba_dt_min=0.001,
|
||||
mamba_dt_max=0.1,
|
||||
mamba_dt_limit=(0.0, float("inf")),
|
||||
mamba_dt_init_floor=1e-4,
|
||||
mamba_conv_bias=True,
|
||||
mamba_proj_bias=False,
|
||||
mamba_chunk_size=256,
|
||||
rescale_prenorm_residual=True,
|
||||
**kwargs,
|
||||
):
|
||||
self.vocab_size = vocab_size
|
||||
self.tie_word_embeddings = tie_word_embeddings
|
||||
self.hidden_size = hidden_size
|
||||
self.intermediate_size = intermediate_size
|
||||
self.num_hidden_layers = num_hidden_layers
|
||||
self.hybrid_override_pattern = hybrid_override_pattern
|
||||
self.num_attention_heads = num_attention_heads
|
||||
self.attention_head_dim = attention_head_dim
|
||||
self.sliding_window = sliding_window
|
||||
self.max_position_embeddings = max_position_embeddings
|
||||
self.attention_dropout = attention_dropout
|
||||
self.hidden_dropout = hidden_dropout
|
||||
|
||||
# Validate hybrid_override_pattern
|
||||
# M: Mamba2, *: Attention, -: MLP
|
||||
assert len(self.hybrid_override_pattern) == self.num_hidden_layers, "hybrid_override_pattern must have the same length as num_hidden_layers"
|
||||
assert re.match(r"^[*-M]+$", self.hybrid_override_pattern), "hybrid_override_pattern must only contain characters 'M', '*', or '-'"
|
||||
|
||||
# for backward compatibility
|
||||
if num_key_value_heads is None:
|
||||
num_key_value_heads = num_attention_heads
|
||||
|
||||
self.num_key_value_heads = num_key_value_heads
|
||||
self.mlp_hidden_act = mlp_hidden_act
|
||||
self.attention_bias = attention_bias
|
||||
self.mlp_bias = mlp_bias
|
||||
self.use_bias = use_bias
|
||||
self.initializer_range = initializer_range
|
||||
self.layer_norm_epsilon = layer_norm_epsilon
|
||||
self.residual_in_fp32 = residual_in_fp32
|
||||
|
||||
self.use_cache = use_cache
|
||||
self.num_logits_to_keep = num_logits_to_keep
|
||||
|
||||
self.use_mamba_kernels = use_mamba_kernels
|
||||
self.n_groups = mamba_n_groups
|
||||
self.mamba_head_dim = mamba_head_dim
|
||||
self.ssm_state_size = ssm_state_size
|
||||
self.mamba_num_heads = mamba_num_heads
|
||||
self.conv_kernel = mamba_d_conv
|
||||
self.expand = mamba_expand
|
||||
self.mamba_hidden_act = mamba_hidden_act
|
||||
self.time_step_min = mamba_dt_min
|
||||
self.time_step_max = mamba_dt_max
|
||||
self.time_step_limit = mamba_dt_limit
|
||||
self.time_step_floor = mamba_dt_init_floor
|
||||
self.use_conv_bias = mamba_conv_bias
|
||||
self.mamba_proj_bias = mamba_proj_bias
|
||||
self.chunk_size = mamba_chunk_size
|
||||
self.rescale_prenorm_residual = rescale_prenorm_residual
|
||||
|
||||
super().__init__(
|
||||
pad_token_id=pad_token_id,
|
||||
bos_token_id=bos_token_id,
|
||||
eos_token_id=eos_token_id,
|
||||
tie_word_embeddings=tie_word_embeddings,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
@property
|
||||
def layers_block_type(self):
|
||||
return [
|
||||
"mamba" if self.hybrid_override_pattern[i] == "M" else
|
||||
"attention" if self.hybrid_override_pattern[i] == "*" else "mlp"
|
||||
for i in range(self.num_hidden_layers)]
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"adapter_dir": "/checkpoints/figment_sft_v12/figment-sft-v12-lora",
|
||||
"base_model_id": "nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16",
|
||||
"dtype": "bfloat16",
|
||||
"files": [
|
||||
"chat_template.jinja",
|
||||
"config.json",
|
||||
"configuration_nemotron_h.py",
|
||||
"generation_config.json",
|
||||
"model-00001-of-00002.safetensors",
|
||||
"model-00002-of-00002.safetensors",
|
||||
"model.safetensors.index.json",
|
||||
"modeling_nemotron_h.py",
|
||||
"special_tokens_map.json",
|
||||
"tokenizer.json",
|
||||
"tokenizer_config.json"
|
||||
],
|
||||
"merge_method": "peft.merge_and_unload(safe_merge=True)",
|
||||
"output_dir": "/checkpoints/figment_sft_v12/figment-sft-v12-lora-merged-bf16"
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"_from_model_config": true,
|
||||
"bos_token_id": 1,
|
||||
"eos_token_id": 2,
|
||||
"pad_token_id": 0,
|
||||
"temperature": null,
|
||||
"top_p": null,
|
||||
"transformers_version": "4.57.6"
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:fb3f334d4cb1518dbecbd8000ffeec88dd3d640158f2a34ee0f7573c01a9be62
|
||||
size 3973314912
|
||||
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:fd31cc8df047f2cf285598344738f083d87699168c55d0de9a2fb7256b179105
|
||||
size 3973827728
|
||||
@@ -0,0 +1,271 @@
|
||||
{
|
||||
"metadata": {
|
||||
"total_parameters": 3973556832,
|
||||
"total_size": 7947113664
|
||||
},
|
||||
"weight_map": {
|
||||
"backbone.embeddings.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.1.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.1.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.1.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.10.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.10.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.10.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.12.mixer.k_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.12.mixer.o_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.12.mixer.q_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.12.mixer.v_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.12.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.13.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.13.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.13.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.15.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.15.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.15.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.17.mixer.k_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.17.mixer.o_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.17.mixer.q_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.17.mixer.v_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.17.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.18.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.18.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.18.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.20.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.20.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.20.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.21.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.21.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.21.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.21.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.21.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.21.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.21.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.21.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.21.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.22.mixer.down_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.22.mixer.up_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.22.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.24.mixer.k_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.24.mixer.o_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.24.mixer.q_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.24.mixer.v_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.24.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.25.mixer.down_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.25.mixer.up_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.25.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.27.mixer.down_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.27.mixer.up_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.27.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.29.mixer.down_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.29.mixer.up_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.29.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.3.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.3.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.3.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.30.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.30.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.30.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.30.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.30.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.30.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.30.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.30.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.30.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.32.mixer.k_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.32.mixer.o_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.32.mixer.q_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.32.mixer.v_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.32.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.33.mixer.down_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.33.mixer.up_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.33.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.37.mixer.down_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.37.mixer.up_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.37.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.39.mixer.down_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.39.mixer.up_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.39.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.4.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.4.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.4.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.4.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.4.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.4.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.4.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.4.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.4.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.40.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.40.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.40.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.40.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.40.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.40.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.40.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.40.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.40.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.41.mixer.down_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.41.mixer.up_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.41.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.5.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.5.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.5.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.8.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.8.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.8.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.norm_f.weight": "model-00002-of-00002.safetensors",
|
||||
"lm_head.weight": "model-00002-of-00002.safetensors"
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"bos_token": {
|
||||
"content": "<s>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"eos_token": {
|
||||
"content": "<|im_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"pad_token": {
|
||||
"content": "<|im_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"unk_token": {
|
||||
"content": "<unk>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:623c34567aebb18582765289fbe23d901c62704d6518d71866e0e58db892b5b7
|
||||
size 17077484
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:1cedcc48d2edf82f31ebd20d8885bdd7b72d07b8d551b19838394ba57a1f2e1e
|
||||
size 7957646688
|
||||
@@ -0,0 +1,204 @@
|
||||
{% macro render_extra_keys(json_dict, handled_keys) %}
|
||||
{%- if json_dict is mapping %}
|
||||
{%- for json_key in json_dict if json_key not in handled_keys %}
|
||||
{%- if json_dict[json_key] is mapping or (json_dict[json_key] is sequence and json_dict[json_key] is not string) %}
|
||||
{{- '\n<' ~ json_key ~ '>' ~ (json_dict[json_key] | tojson | safe) ~ '</' ~ json_key ~ '>' }}
|
||||
{%- else %}
|
||||
{{-'\n<' ~ json_key ~ '>' ~ (json_dict[json_key] | string) ~ '</' ~ json_key ~ '>' }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{% endmacro %}
|
||||
{%- set enable_thinking = enable_thinking if enable_thinking is defined else True %}
|
||||
{%- set truncate_history_thinking = truncate_history_thinking if truncate_history_thinking is defined else True %}
|
||||
|
||||
{%- set ns = namespace(last_user_idx = -1) %}
|
||||
{%- set loop_messages = messages %}
|
||||
{%- for m in loop_messages %}
|
||||
{%- if m["role"] == "user" %}
|
||||
{%- set ns.last_user_idx = loop.index0 %}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
|
||||
{%- if messages[0]["role"] == "system" %}
|
||||
{%- set system_message = messages[0]["content"] %}
|
||||
{%- set loop_messages = messages[1:] %}
|
||||
{%- else %}
|
||||
{%- set system_message = "" %}
|
||||
{%- set loop_messages = messages %}
|
||||
{%- endif %}
|
||||
{%- if not tools is defined %}
|
||||
{%- set tools = [] %}
|
||||
{%- endif %}
|
||||
{# Recompute last_user_idx relative to loop_messages after handling system #}
|
||||
{%- set ns = namespace(last_user_idx = -1) %}
|
||||
{%- for m in loop_messages %}
|
||||
{%- if m["role"] == "user" %}
|
||||
{%- set ns.last_user_idx = loop.index0 %}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- if system_message is defined %}
|
||||
{{- "<|im_start|>system\n" + system_message }}
|
||||
{%- else %}
|
||||
{%- if tools is iterable and tools | length > 0 %}
|
||||
{{- "<|im_start|>system\n" }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- if tools is iterable and tools | length > 0 %}
|
||||
{%- if system_message is defined and system_message | length > 0 %}
|
||||
{{- "\n\n" }}
|
||||
{%- endif %}
|
||||
{{- "# Tools\n\nYou have access to the following functions:\n\n" }}
|
||||
{{- "<tools>" }}
|
||||
{%- for tool in tools %}
|
||||
{%- if tool.function is defined %}
|
||||
{%- set tool = tool.function %}
|
||||
{%- endif %}
|
||||
{{- "\n<function>\n<name>" ~ tool.name ~ "</name>" }}
|
||||
{%- if tool.description is defined %}
|
||||
{{- '\n<description>' ~ (tool.description | trim) ~ '</description>' }}
|
||||
{%- endif %}
|
||||
{{- '\n<parameters>' }}
|
||||
{%- if tool.parameters is defined and tool.parameters is mapping and tool.parameters.properties is defined and tool.parameters.properties is mapping %}
|
||||
{%- for param_name, param_fields in tool.parameters.properties|items %}
|
||||
{{- '\n<parameter>' }}
|
||||
{{- '\n<name>' ~ param_name ~ '</name>' }}
|
||||
{%- if param_fields.type is defined %}
|
||||
{{- '\n<type>' ~ (param_fields.type | string) ~ '</type>' }}
|
||||
{%- endif %}
|
||||
{%- if param_fields.description is defined %}
|
||||
{{- '\n<description>' ~ (param_fields.description | trim) ~ '</description>' }}
|
||||
{%- endif %}
|
||||
{%- if param_fields.enum is defined %}
|
||||
{{- '\n<enum>' ~ (param_fields.enum | tojson | safe) ~ '</enum>' }}
|
||||
{%- endif %}
|
||||
{%- set handled_keys = ['name', 'type', 'description', 'enum'] %}
|
||||
{{- render_extra_keys(param_fields, handled_keys) }}
|
||||
{{- '\n</parameter>' }}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{% set handled_keys = ['type', 'properties', 'required'] %}
|
||||
{{- render_extra_keys(tool.parameters, handled_keys) }}
|
||||
{%- if tool.parameters is defined and tool.parameters.required is defined %}
|
||||
{{- '\n<required>' ~ (tool.parameters.required | tojson | safe) ~ '</required>' }}
|
||||
{%- endif %}
|
||||
{{- '\n</parameters>' }}
|
||||
{%- set handled_keys = ['type', 'name', 'description', 'parameters'] %}
|
||||
{{- render_extra_keys(tool, handled_keys) }}
|
||||
{{- '\n</function>' }}
|
||||
{%- endfor %}
|
||||
{{- "\n</tools>" }}
|
||||
|
||||
{{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n</IMPORTANT>' }}
|
||||
{%- endif %}
|
||||
|
||||
|
||||
{%- if system_message is defined %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- else %}
|
||||
{%- if tools is iterable and tools | length > 0 %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
|
||||
{%- for message in loop_messages %}
|
||||
{%- if message.role == "assistant" %}
|
||||
{# Add reasoning content in to content field for unified processing below. #}
|
||||
{%- if message.reasoning_content is defined and message.reasoning_content is string and message.reasoning_content | trim | length > 0 %}
|
||||
{%- set content = "<think>\n" ~ message.reasoning_content ~ "\n</think>\n" ~ (message.content | default('', true)) %}
|
||||
{%- else %}
|
||||
{%- set content = message.content | default('', true) %}
|
||||
{%- if content is string -%}
|
||||
{# Allow downstream logic to to take care of broken thought, only handle coherent reasoning here. #}
|
||||
{%- if '<think>' not in content and '</think>' not in content -%}
|
||||
{%- set content = "<think></think>" ~ content -%}
|
||||
{%- endif -%}
|
||||
{%- else -%}
|
||||
{%- set content = content -%}
|
||||
{%- endif -%}
|
||||
{%- endif %}
|
||||
{%- if message.tool_calls is defined and message.tool_calls is iterable and message.tool_calls | length > 0 %}
|
||||
{# Assistant message has tool calls. #}
|
||||
{{- '<|im_start|>assistant\n' }}
|
||||
{%- set include_content = not (truncate_history_thinking and loop.index0 < ns.last_user_idx) %}
|
||||
{%- if content is string and content | trim | length > 0 %}
|
||||
{%- if include_content %}
|
||||
{{- (content | trim) ~ '\n' -}}
|
||||
{%- else %}
|
||||
{%- set c = (content | string) %}
|
||||
{%- if '</think>' in c %}
|
||||
{# Keep only content after the last closing think. Also generation prompt causes this. #}
|
||||
{%- set c = c.split('</think>')[-1] %}
|
||||
{%- elif '<think>' in c %}
|
||||
{# If <think> was opened but never closed, drop the trailing think segment #}
|
||||
{%- set c = c.split('<think>')[0] %}
|
||||
{%- endif %}
|
||||
{%- set c = "<think></think>" ~ c | trim %}
|
||||
{%- if c | length > 0 %}
|
||||
{{- c ~ '\n' -}}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- else %}
|
||||
{{- "<think></think>" -}}
|
||||
{%- endif %}
|
||||
{%- for tool_call in message.tool_calls %}
|
||||
{%- if tool_call.function is defined %}
|
||||
{%- set tool_call = tool_call.function %}
|
||||
{%- endif %}
|
||||
{{- '<tool_call>\n<function=' ~ tool_call.name ~ '>\n' -}}
|
||||
{%- if tool_call.arguments is defined %}
|
||||
{%- for args_name, args_value in tool_call.arguments|items %}
|
||||
{{- '<parameter=' ~ args_name ~ '>\n' -}}
|
||||
{%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}
|
||||
{{- args_value ~ '\n</parameter>\n' -}}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{{- '</function>\n</tool_call>\n' -}}
|
||||
{%- endfor %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- else %}
|
||||
{# Assistant message doesn't have tool calls. #}
|
||||
{%- if not (truncate_history_thinking and loop.index0 < ns.last_user_idx) %}
|
||||
{{- '<|im_start|>assistant\n' ~ (content | default('', true) | string | trim) ~ '<|im_end|>\n' }}
|
||||
{%- else %}
|
||||
{%- set c = (content | default('', true) | string) %}
|
||||
{%- if '<think>' in c and '</think>' in c %}
|
||||
{%- set c = "<think></think>" ~ c.split('</think>')[-1] %}
|
||||
{%- endif %}
|
||||
{%- set c = c | trim %}
|
||||
{%- if c | length > 0 %}
|
||||
{{- '<|im_start|>assistant\n' ~ c ~ '<|im_end|>\n' }}
|
||||
{%- else %}
|
||||
{{- '<|im_start|>assistant\n<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- elif message.role == "user" or message.role == "system" %}
|
||||
{{- '<|im_start|>' + message.role + '\n' }}
|
||||
{%- set content = message.content | string %}
|
||||
{{- content }}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- elif message.role == "tool" %}
|
||||
{%- if loop.previtem and loop.previtem.role != "tool" %}
|
||||
{{- '<|im_start|>user\n' }}
|
||||
{%- endif %}
|
||||
{{- '<tool_response>\n' }}
|
||||
{{- message.content }}
|
||||
{{- '\n</tool_response>\n' }}
|
||||
{%- if not loop.last and loop.nextitem.role != "tool" %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- elif loop.last %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- else %}
|
||||
{{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
|
||||
{%- if add_generation_prompt %}
|
||||
{%- if enable_thinking %}
|
||||
{{- '<|im_start|>assistant\n<think>\n' }}
|
||||
{%- else %}
|
||||
{{- '<|im_start|>assistant\n<think></think>' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
59
figment_sft_v13/figment-sft-v13-lora-merged-bf16/config.json
Normal file
59
figment_sft_v13/figment-sft-v13-lora-merged-bf16/config.json
Normal file
@@ -0,0 +1,59 @@
|
||||
{
|
||||
"architectures": [
|
||||
"NemotronHForCausalLM"
|
||||
],
|
||||
"attention_bias": false,
|
||||
"attention_dropout": 0.0,
|
||||
"attention_head_dim": 128,
|
||||
"auto_map": {
|
||||
"AutoConfig": "configuration_nemotron_h.NemotronHConfig",
|
||||
"AutoModelForCausalLM": "modeling_nemotron_h.NemotronHForCausalLM"
|
||||
},
|
||||
"bos_token_id": 1,
|
||||
"chunk_size": 256,
|
||||
"conv_kernel": 4,
|
||||
"dtype": "bfloat16",
|
||||
"eos_token_id": 2,
|
||||
"expand": 2,
|
||||
"head_dim": 128,
|
||||
"hidden_dropout": 0.0,
|
||||
"hidden_size": 3136,
|
||||
"hybrid_override_pattern": "M-M-M-MM-M-M*-M-M*-M-M-M*-M-M-MM*-MMM-M-M-",
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 12544,
|
||||
"layer_norm_epsilon": 1e-05,
|
||||
"mamba_head_dim": 80,
|
||||
"mamba_hidden_act": "silu",
|
||||
"mamba_num_heads": 96,
|
||||
"mamba_proj_bias": false,
|
||||
"max_position_embeddings": 262144,
|
||||
"mlp_bias": false,
|
||||
"mlp_hidden_act": "relu2",
|
||||
"model_type": "nemotron_h",
|
||||
"n_groups": 8,
|
||||
"num_attention_heads": 40,
|
||||
"num_hidden_layers": 42,
|
||||
"num_key_value_heads": 8,
|
||||
"num_logits_to_keep": 1,
|
||||
"pad_token_id": 0,
|
||||
"rescale_prenorm_residual": true,
|
||||
"residual_in_fp32": false,
|
||||
"rms_norm_eps": 1e-05,
|
||||
"sliding_window": null,
|
||||
"ssm_state_size": 128,
|
||||
"tie_word_embeddings": false,
|
||||
"time_step_floor": 0.0001,
|
||||
"time_step_limit": [
|
||||
0.0,
|
||||
Infinity
|
||||
],
|
||||
"time_step_max": 0.1,
|
||||
"time_step_min": 0.001,
|
||||
"time_step_rank": 256,
|
||||
"transformers_version": "4.57.6",
|
||||
"use_bias": false,
|
||||
"use_cache": true,
|
||||
"use_conv_bias": true,
|
||||
"use_mamba_kernels": true,
|
||||
"vocab_size": 131072
|
||||
}
|
||||
@@ -0,0 +1,243 @@
|
||||
# coding=utf-8
|
||||
# Copyright 2024 AI21 Labs Ltd. and the HuggingFace Inc. team. All rights reserved.
|
||||
# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""NemotronH model configuration"""
|
||||
|
||||
import re
|
||||
|
||||
from transformers.configuration_utils import PretrainedConfig
|
||||
from transformers.utils import logging
|
||||
|
||||
|
||||
logger = logging.get_logger(__name__)
|
||||
|
||||
|
||||
class NemotronHConfig(PretrainedConfig):
|
||||
r"""
|
||||
This is the configuration class to store the configuration of a [`NemotronHModel`]. It is used to instantiate a
|
||||
NemotronH model according to the specified arguments, defining the model architecture. Instantiating a configuration
|
||||
with the defaults will yield a similar configuration to that of the NemotronH-v0.1 model.
|
||||
|
||||
[todo](todo)
|
||||
|
||||
Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
|
||||
documentation from [`PretrainedConfig`] for more information.
|
||||
|
||||
|
||||
Args:
|
||||
vocab_size (`int`, *optional*, defaults to 131072):
|
||||
Vocabulary size of the NemotronH model. Defines the number of different tokens that can be represented by the
|
||||
`inputs_ids` passed when calling [`NemotronHModel`]
|
||||
tie_word_embeddings (`bool`, *optional*, defaults to `False`):
|
||||
Whether the model's input and output word embeddings should be tied. Note that this is only relevant if the
|
||||
model has a output word embedding layer.
|
||||
hidden_size (`int`, *optional*, defaults to 4096):
|
||||
Dimension of the hidden representations.
|
||||
intermediate_size (`int`, *optional*, defaults to 21504):
|
||||
Dimension of the MLP representations.
|
||||
num_hidden_layers (`int`, *optional*, defaults to 52):
|
||||
Number of hidden layers in the Transformer encoder.
|
||||
hybrid_override_pattern (`str`, *optional*, defaults to `"M-M-M-M*-M-M-M-M-M*-M-M-M-M-M*-M-M-M-M-M*-M-M-M-M-M-"`):
|
||||
The pattern of the hybrid model. The pattern is a string of characters where each character represents M: Mamba2, *: Attention, -: MLP
|
||||
num_attention_heads (`int`, *optional*, defaults to 32):
|
||||
Number of attention heads for each attention layer in the Transformer encoder.
|
||||
attention_head_dim (`int`, *optional*, defaults to 128):
|
||||
Dimension of each attention head.
|
||||
num_key_value_heads (`int`, *optional*, defaults to 8):
|
||||
This is the number of key_value heads that should be used to implement Grouped Query Attention. If
|
||||
`num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
|
||||
`num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used.
|
||||
mlp_hidden_act (`str`, *optional*, defaults to "relu2"):
|
||||
The non-linear activation function in the MLP layers.
|
||||
attention_bias (`bool`, *optional*, defaults to `False`):
|
||||
Whether to use bias in attention layers.
|
||||
mlp_bias (`bool`, *optional*, defaults to `False`):
|
||||
Whether to use bias in MLP layers.
|
||||
use_bias (`bool`, *optional*, defaults to `False`):
|
||||
Whether to use bias in the model.
|
||||
initializer_range (`float`, *optional*, defaults to 0.02):
|
||||
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
|
||||
layer_norm_epsilon (`float`, *optional*, defaults to 1e-5):
|
||||
The epsilon used by the layer normalization layers.
|
||||
residual_in_fp32 (`bool`, *optional*, defaults to `False`):
|
||||
Whether or not residuals should be in `float32`. If set to `False` residuals will keep the same `dtype` as the rest of the model.
|
||||
use_cache (`bool`, *optional*, defaults to `True`):
|
||||
Whether or not the model should return the last key/values attentions (not used by all models). Only
|
||||
relevant if `config.is_decoder=True`.
|
||||
num_logits_to_keep (`int` or `None`, *optional*, defaults to 1):
|
||||
Number of prompt logits to calculate during generation. If `None`, all logits will be calculated. If an
|
||||
integer value, only last `num_logits_to_keep` logits will be calculated.
|
||||
pad_token_id (`int`, *optional*, defaults to 0):
|
||||
The id of the padding token.
|
||||
bos_token_id (`int`, *optional*, defaults to 1):
|
||||
The id of the "beginning-of-sequence" token.
|
||||
eos_token_id (`int`, *optional*, defaults to 2):
|
||||
The id of the "end-of-sequence" token.
|
||||
sliding_window (`int`, *optional*, defaults to None):
|
||||
Sliding window attention window size.
|
||||
max_position_embeddings (`int`, *optional*, defaults to 4096):
|
||||
The maximum sequence length that this model might ever be used with.
|
||||
attention_dropout (`float`, *optional*, defaults to 0.0):
|
||||
The dropout ratio for the attention probabilities.
|
||||
hidden_dropout (`float`, *optional*, defaults to 0.0):
|
||||
The dropout ratio for the hidden states.
|
||||
use_mamba_kernels (`bool`, *optional*, defaults to `True`):
|
||||
Flag indicating whether or not to use the fast mamba kernels. These are available only if `mamba-ssm` and
|
||||
`causal-conv1d` are installed, and the mamba modules are running on a CUDA device.
|
||||
ssm_state_size (`int`, *optional*, defaults to 128):
|
||||
The dimension of the mamba state space latents.
|
||||
mamba_num_heads (`int`, *optional*, defaults to 128):
|
||||
Number of heads in Mamba layers.
|
||||
mamba_n_groups (`int`, *optional*, defaults to 8):
|
||||
Number of groups in Mamba layers.
|
||||
mamba_head_dim (`int`, *optional*, defaults to 64):
|
||||
Dimension of each Mamba head.
|
||||
mamba_d_conv (`int`, *optional*, defaults to 4):
|
||||
The size of the mamba convolution kernel.
|
||||
mamba_expand (`int`, *optional*, defaults to 2):
|
||||
Expanding factor used to determine the mamba intermediate size.
|
||||
mamba_hidden_act (`str`, *optional*, defaults to "silu"):
|
||||
The non-linear activation function in the Mamba layers.
|
||||
mamba_dt_min (`float`, *optional*, defaults to 0.001):
|
||||
Minimum value for the time step in Mamba.
|
||||
mamba_dt_max (`float`, *optional*, defaults to 0.1):
|
||||
Maximum value for the time step in Mamba.
|
||||
mamba_dt_limit (`tuple`, *optional*, defaults to (0.0, float("inf"))):
|
||||
Limits for the time step in Mamba.
|
||||
mamba_dt_init_floor (`float`, *optional*, defaults to 1e-4):
|
||||
Floor value for time step initialization in Mamba.
|
||||
mamba_conv_bias (`bool`, *optional*, defaults to `True`):
|
||||
Whether to use bias in the convolution layer of the mamba mixer block.
|
||||
mamba_proj_bias (`bool`, *optional*, defaults to `False`):
|
||||
Whether to use bias in the input and output projections of the mamba mixer block.
|
||||
mamba_chunk_size (`int`, *optional*, defaults to 256):
|
||||
Size of chunks for Mamba processing.
|
||||
rescale_prenorm_residual (`bool`, *optional*, defaults to `True`):
|
||||
Whether to rescale the pre-normalization residual connections.
|
||||
"""
|
||||
|
||||
model_type = "nemotron_h"
|
||||
keys_to_ignore_at_inference = ["past_key_values"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vocab_size=131072,
|
||||
tie_word_embeddings=False,
|
||||
hidden_size=4096,
|
||||
intermediate_size=21504,
|
||||
num_hidden_layers=52,
|
||||
hybrid_override_pattern="M-M-M-M*-M-M-M-M-M*-M-M-M-M-M*-M-M-M-M-M*-M-M-M-M-M-",
|
||||
num_attention_heads=32,
|
||||
attention_head_dim=128,
|
||||
num_key_value_heads=8, # nemo: num_query_groups
|
||||
mlp_hidden_act="relu2",
|
||||
attention_bias=False,
|
||||
mlp_bias=False,
|
||||
use_bias=False,
|
||||
initializer_range=0.02, # nemo: init_method_std
|
||||
layer_norm_epsilon=1e-5, # nemo: layernorm_epsilon
|
||||
residual_in_fp32=False, # Megatron Core default value
|
||||
use_cache=True,
|
||||
num_logits_to_keep=1,
|
||||
pad_token_id=0,
|
||||
bos_token_id=1,
|
||||
eos_token_id=2,
|
||||
sliding_window=None,
|
||||
max_position_embeddings=4096,
|
||||
attention_dropout=0.0,
|
||||
hidden_dropout=0.0, # * ADDED
|
||||
use_mamba_kernels=True,
|
||||
ssm_state_size=128, # mamba_state_size
|
||||
mamba_num_heads=128,
|
||||
mamba_n_groups=8, # nemo: mamba_ssm_ngroups = num_heads
|
||||
mamba_head_dim=64,
|
||||
mamba_d_conv=4,
|
||||
mamba_expand=2,
|
||||
mamba_hidden_act="silu",
|
||||
mamba_dt_min=0.001,
|
||||
mamba_dt_max=0.1,
|
||||
mamba_dt_limit=(0.0, float("inf")),
|
||||
mamba_dt_init_floor=1e-4,
|
||||
mamba_conv_bias=True,
|
||||
mamba_proj_bias=False,
|
||||
mamba_chunk_size=256,
|
||||
rescale_prenorm_residual=True,
|
||||
**kwargs,
|
||||
):
|
||||
self.vocab_size = vocab_size
|
||||
self.tie_word_embeddings = tie_word_embeddings
|
||||
self.hidden_size = hidden_size
|
||||
self.intermediate_size = intermediate_size
|
||||
self.num_hidden_layers = num_hidden_layers
|
||||
self.hybrid_override_pattern = hybrid_override_pattern
|
||||
self.num_attention_heads = num_attention_heads
|
||||
self.attention_head_dim = attention_head_dim
|
||||
self.sliding_window = sliding_window
|
||||
self.max_position_embeddings = max_position_embeddings
|
||||
self.attention_dropout = attention_dropout
|
||||
self.hidden_dropout = hidden_dropout
|
||||
|
||||
# Validate hybrid_override_pattern
|
||||
# M: Mamba2, *: Attention, -: MLP
|
||||
assert len(self.hybrid_override_pattern) == self.num_hidden_layers, "hybrid_override_pattern must have the same length as num_hidden_layers"
|
||||
assert re.match(r"^[*-M]+$", self.hybrid_override_pattern), "hybrid_override_pattern must only contain characters 'M', '*', or '-'"
|
||||
|
||||
# for backward compatibility
|
||||
if num_key_value_heads is None:
|
||||
num_key_value_heads = num_attention_heads
|
||||
|
||||
self.num_key_value_heads = num_key_value_heads
|
||||
self.mlp_hidden_act = mlp_hidden_act
|
||||
self.attention_bias = attention_bias
|
||||
self.mlp_bias = mlp_bias
|
||||
self.use_bias = use_bias
|
||||
self.initializer_range = initializer_range
|
||||
self.layer_norm_epsilon = layer_norm_epsilon
|
||||
self.residual_in_fp32 = residual_in_fp32
|
||||
|
||||
self.use_cache = use_cache
|
||||
self.num_logits_to_keep = num_logits_to_keep
|
||||
|
||||
self.use_mamba_kernels = use_mamba_kernels
|
||||
self.n_groups = mamba_n_groups
|
||||
self.mamba_head_dim = mamba_head_dim
|
||||
self.ssm_state_size = ssm_state_size
|
||||
self.mamba_num_heads = mamba_num_heads
|
||||
self.conv_kernel = mamba_d_conv
|
||||
self.expand = mamba_expand
|
||||
self.mamba_hidden_act = mamba_hidden_act
|
||||
self.time_step_min = mamba_dt_min
|
||||
self.time_step_max = mamba_dt_max
|
||||
self.time_step_limit = mamba_dt_limit
|
||||
self.time_step_floor = mamba_dt_init_floor
|
||||
self.use_conv_bias = mamba_conv_bias
|
||||
self.mamba_proj_bias = mamba_proj_bias
|
||||
self.chunk_size = mamba_chunk_size
|
||||
self.rescale_prenorm_residual = rescale_prenorm_residual
|
||||
|
||||
super().__init__(
|
||||
pad_token_id=pad_token_id,
|
||||
bos_token_id=bos_token_id,
|
||||
eos_token_id=eos_token_id,
|
||||
tie_word_embeddings=tie_word_embeddings,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
@property
|
||||
def layers_block_type(self):
|
||||
return [
|
||||
"mamba" if self.hybrid_override_pattern[i] == "M" else
|
||||
"attention" if self.hybrid_override_pattern[i] == "*" else "mlp"
|
||||
for i in range(self.num_hidden_layers)]
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"adapter_dir": "/checkpoints/figment_sft_v13/figment-sft-v13-lora",
|
||||
"base_model_id": "nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16",
|
||||
"dtype": "bfloat16",
|
||||
"files": [
|
||||
"chat_template.jinja",
|
||||
"config.json",
|
||||
"configuration_nemotron_h.py",
|
||||
"generation_config.json",
|
||||
"model-00001-of-00002.safetensors",
|
||||
"model-00002-of-00002.safetensors",
|
||||
"model.safetensors.index.json",
|
||||
"modeling_nemotron_h.py",
|
||||
"special_tokens_map.json",
|
||||
"tokenizer.json",
|
||||
"tokenizer_config.json"
|
||||
],
|
||||
"merge_method": "peft.merge_and_unload(safe_merge=True)",
|
||||
"output_dir": "/checkpoints/figment_sft_v13/figment-sft-v13-lora-merged-bf16"
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"_from_model_config": true,
|
||||
"bos_token_id": 1,
|
||||
"eos_token_id": 2,
|
||||
"pad_token_id": 0,
|
||||
"temperature": null,
|
||||
"top_p": null,
|
||||
"transformers_version": "4.57.6"
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:b694671de819a985bb9d46e0a15c4b8c0a15037483bfa2cbd24923c641cc8703
|
||||
size 3973314912
|
||||
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:14f9aa398b97733037172f00d2e8a4af2e0ad9d200cd93f85736f0b627aa14b2
|
||||
size 3973827728
|
||||
@@ -0,0 +1,271 @@
|
||||
{
|
||||
"metadata": {
|
||||
"total_parameters": 3973556832,
|
||||
"total_size": 7947113664
|
||||
},
|
||||
"weight_map": {
|
||||
"backbone.embeddings.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.1.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.1.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.1.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.10.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.10.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.10.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.12.mixer.k_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.12.mixer.o_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.12.mixer.q_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.12.mixer.v_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.12.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.13.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.13.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.13.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.15.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.15.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.15.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.17.mixer.k_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.17.mixer.o_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.17.mixer.q_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.17.mixer.v_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.17.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.18.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.18.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.18.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.20.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.20.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.20.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.21.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.21.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.21.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.21.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.21.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.21.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.21.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.21.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.21.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.22.mixer.down_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.22.mixer.up_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.22.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.24.mixer.k_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.24.mixer.o_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.24.mixer.q_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.24.mixer.v_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.24.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.25.mixer.down_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.25.mixer.up_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.25.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.27.mixer.down_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.27.mixer.up_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.27.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.29.mixer.down_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.29.mixer.up_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.29.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.3.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.3.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.3.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.30.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.30.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.30.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.30.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.30.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.30.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.30.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.30.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.30.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.32.mixer.k_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.32.mixer.o_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.32.mixer.q_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.32.mixer.v_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.32.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.33.mixer.down_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.33.mixer.up_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.33.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.37.mixer.down_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.37.mixer.up_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.37.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.39.mixer.down_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.39.mixer.up_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.39.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.4.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.4.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.4.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.4.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.4.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.4.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.4.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.4.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.4.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.40.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.40.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.40.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.40.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.40.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.40.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.40.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.40.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.40.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.41.mixer.down_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.41.mixer.up_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.41.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.5.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.5.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.5.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.8.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.8.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.8.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.norm_f.weight": "model-00002-of-00002.safetensors",
|
||||
"lm_head.weight": "model-00002-of-00002.safetensors"
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"bos_token": {
|
||||
"content": "<s>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"eos_token": {
|
||||
"content": "<|im_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"pad_token": {
|
||||
"content": "<|im_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"unk_token": {
|
||||
"content": "<unk>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:623c34567aebb18582765289fbe23d901c62704d6518d71866e0e58db892b5b7
|
||||
size 17077484
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:53de48e5f7a7fa22af7a682686adcf6c0be7c5c1fe72f72ea39d80bd68333f72
|
||||
size 7957646720
|
||||
@@ -0,0 +1,204 @@
|
||||
{% macro render_extra_keys(json_dict, handled_keys) %}
|
||||
{%- if json_dict is mapping %}
|
||||
{%- for json_key in json_dict if json_key not in handled_keys %}
|
||||
{%- if json_dict[json_key] is mapping or (json_dict[json_key] is sequence and json_dict[json_key] is not string) %}
|
||||
{{- '\n<' ~ json_key ~ '>' ~ (json_dict[json_key] | tojson | safe) ~ '</' ~ json_key ~ '>' }}
|
||||
{%- else %}
|
||||
{{-'\n<' ~ json_key ~ '>' ~ (json_dict[json_key] | string) ~ '</' ~ json_key ~ '>' }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{% endmacro %}
|
||||
{%- set enable_thinking = enable_thinking if enable_thinking is defined else True %}
|
||||
{%- set truncate_history_thinking = truncate_history_thinking if truncate_history_thinking is defined else True %}
|
||||
|
||||
{%- set ns = namespace(last_user_idx = -1) %}
|
||||
{%- set loop_messages = messages %}
|
||||
{%- for m in loop_messages %}
|
||||
{%- if m["role"] == "user" %}
|
||||
{%- set ns.last_user_idx = loop.index0 %}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
|
||||
{%- if messages[0]["role"] == "system" %}
|
||||
{%- set system_message = messages[0]["content"] %}
|
||||
{%- set loop_messages = messages[1:] %}
|
||||
{%- else %}
|
||||
{%- set system_message = "" %}
|
||||
{%- set loop_messages = messages %}
|
||||
{%- endif %}
|
||||
{%- if not tools is defined %}
|
||||
{%- set tools = [] %}
|
||||
{%- endif %}
|
||||
{# Recompute last_user_idx relative to loop_messages after handling system #}
|
||||
{%- set ns = namespace(last_user_idx = -1) %}
|
||||
{%- for m in loop_messages %}
|
||||
{%- if m["role"] == "user" %}
|
||||
{%- set ns.last_user_idx = loop.index0 %}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- if system_message is defined %}
|
||||
{{- "<|im_start|>system\n" + system_message }}
|
||||
{%- else %}
|
||||
{%- if tools is iterable and tools | length > 0 %}
|
||||
{{- "<|im_start|>system\n" }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- if tools is iterable and tools | length > 0 %}
|
||||
{%- if system_message is defined and system_message | length > 0 %}
|
||||
{{- "\n\n" }}
|
||||
{%- endif %}
|
||||
{{- "# Tools\n\nYou have access to the following functions:\n\n" }}
|
||||
{{- "<tools>" }}
|
||||
{%- for tool in tools %}
|
||||
{%- if tool.function is defined %}
|
||||
{%- set tool = tool.function %}
|
||||
{%- endif %}
|
||||
{{- "\n<function>\n<name>" ~ tool.name ~ "</name>" }}
|
||||
{%- if tool.description is defined %}
|
||||
{{- '\n<description>' ~ (tool.description | trim) ~ '</description>' }}
|
||||
{%- endif %}
|
||||
{{- '\n<parameters>' }}
|
||||
{%- if tool.parameters is defined and tool.parameters is mapping and tool.parameters.properties is defined and tool.parameters.properties is mapping %}
|
||||
{%- for param_name, param_fields in tool.parameters.properties|items %}
|
||||
{{- '\n<parameter>' }}
|
||||
{{- '\n<name>' ~ param_name ~ '</name>' }}
|
||||
{%- if param_fields.type is defined %}
|
||||
{{- '\n<type>' ~ (param_fields.type | string) ~ '</type>' }}
|
||||
{%- endif %}
|
||||
{%- if param_fields.description is defined %}
|
||||
{{- '\n<description>' ~ (param_fields.description | trim) ~ '</description>' }}
|
||||
{%- endif %}
|
||||
{%- if param_fields.enum is defined %}
|
||||
{{- '\n<enum>' ~ (param_fields.enum | tojson | safe) ~ '</enum>' }}
|
||||
{%- endif %}
|
||||
{%- set handled_keys = ['name', 'type', 'description', 'enum'] %}
|
||||
{{- render_extra_keys(param_fields, handled_keys) }}
|
||||
{{- '\n</parameter>' }}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{% set handled_keys = ['type', 'properties', 'required'] %}
|
||||
{{- render_extra_keys(tool.parameters, handled_keys) }}
|
||||
{%- if tool.parameters is defined and tool.parameters.required is defined %}
|
||||
{{- '\n<required>' ~ (tool.parameters.required | tojson | safe) ~ '</required>' }}
|
||||
{%- endif %}
|
||||
{{- '\n</parameters>' }}
|
||||
{%- set handled_keys = ['type', 'name', 'description', 'parameters'] %}
|
||||
{{- render_extra_keys(tool, handled_keys) }}
|
||||
{{- '\n</function>' }}
|
||||
{%- endfor %}
|
||||
{{- "\n</tools>" }}
|
||||
|
||||
{{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n</IMPORTANT>' }}
|
||||
{%- endif %}
|
||||
|
||||
|
||||
{%- if system_message is defined %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- else %}
|
||||
{%- if tools is iterable and tools | length > 0 %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
|
||||
{%- for message in loop_messages %}
|
||||
{%- if message.role == "assistant" %}
|
||||
{# Add reasoning content in to content field for unified processing below. #}
|
||||
{%- if message.reasoning_content is defined and message.reasoning_content is string and message.reasoning_content | trim | length > 0 %}
|
||||
{%- set content = "<think>\n" ~ message.reasoning_content ~ "\n</think>\n" ~ (message.content | default('', true)) %}
|
||||
{%- else %}
|
||||
{%- set content = message.content | default('', true) %}
|
||||
{%- if content is string -%}
|
||||
{# Allow downstream logic to to take care of broken thought, only handle coherent reasoning here. #}
|
||||
{%- if '<think>' not in content and '</think>' not in content -%}
|
||||
{%- set content = "<think></think>" ~ content -%}
|
||||
{%- endif -%}
|
||||
{%- else -%}
|
||||
{%- set content = content -%}
|
||||
{%- endif -%}
|
||||
{%- endif %}
|
||||
{%- if message.tool_calls is defined and message.tool_calls is iterable and message.tool_calls | length > 0 %}
|
||||
{# Assistant message has tool calls. #}
|
||||
{{- '<|im_start|>assistant\n' }}
|
||||
{%- set include_content = not (truncate_history_thinking and loop.index0 < ns.last_user_idx) %}
|
||||
{%- if content is string and content | trim | length > 0 %}
|
||||
{%- if include_content %}
|
||||
{{- (content | trim) ~ '\n' -}}
|
||||
{%- else %}
|
||||
{%- set c = (content | string) %}
|
||||
{%- if '</think>' in c %}
|
||||
{# Keep only content after the last closing think. Also generation prompt causes this. #}
|
||||
{%- set c = c.split('</think>')[-1] %}
|
||||
{%- elif '<think>' in c %}
|
||||
{# If <think> was opened but never closed, drop the trailing think segment #}
|
||||
{%- set c = c.split('<think>')[0] %}
|
||||
{%- endif %}
|
||||
{%- set c = "<think></think>" ~ c | trim %}
|
||||
{%- if c | length > 0 %}
|
||||
{{- c ~ '\n' -}}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- else %}
|
||||
{{- "<think></think>" -}}
|
||||
{%- endif %}
|
||||
{%- for tool_call in message.tool_calls %}
|
||||
{%- if tool_call.function is defined %}
|
||||
{%- set tool_call = tool_call.function %}
|
||||
{%- endif %}
|
||||
{{- '<tool_call>\n<function=' ~ tool_call.name ~ '>\n' -}}
|
||||
{%- if tool_call.arguments is defined %}
|
||||
{%- for args_name, args_value in tool_call.arguments|items %}
|
||||
{{- '<parameter=' ~ args_name ~ '>\n' -}}
|
||||
{%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}
|
||||
{{- args_value ~ '\n</parameter>\n' -}}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{{- '</function>\n</tool_call>\n' -}}
|
||||
{%- endfor %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- else %}
|
||||
{# Assistant message doesn't have tool calls. #}
|
||||
{%- if not (truncate_history_thinking and loop.index0 < ns.last_user_idx) %}
|
||||
{{- '<|im_start|>assistant\n' ~ (content | default('', true) | string | trim) ~ '<|im_end|>\n' }}
|
||||
{%- else %}
|
||||
{%- set c = (content | default('', true) | string) %}
|
||||
{%- if '<think>' in c and '</think>' in c %}
|
||||
{%- set c = "<think></think>" ~ c.split('</think>')[-1] %}
|
||||
{%- endif %}
|
||||
{%- set c = c | trim %}
|
||||
{%- if c | length > 0 %}
|
||||
{{- '<|im_start|>assistant\n' ~ c ~ '<|im_end|>\n' }}
|
||||
{%- else %}
|
||||
{{- '<|im_start|>assistant\n<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- elif message.role == "user" or message.role == "system" %}
|
||||
{{- '<|im_start|>' + message.role + '\n' }}
|
||||
{%- set content = message.content | string %}
|
||||
{{- content }}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- elif message.role == "tool" %}
|
||||
{%- if loop.previtem and loop.previtem.role != "tool" %}
|
||||
{{- '<|im_start|>user\n' }}
|
||||
{%- endif %}
|
||||
{{- '<tool_response>\n' }}
|
||||
{{- message.content }}
|
||||
{{- '\n</tool_response>\n' }}
|
||||
{%- if not loop.last and loop.nextitem.role != "tool" %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- elif loop.last %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- else %}
|
||||
{{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
|
||||
{%- if add_generation_prompt %}
|
||||
{%- if enable_thinking %}
|
||||
{{- '<|im_start|>assistant\n<think>\n' }}
|
||||
{%- else %}
|
||||
{{- '<|im_start|>assistant\n<think></think>' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
@@ -0,0 +1,59 @@
|
||||
{
|
||||
"architectures": [
|
||||
"NemotronHForCausalLM"
|
||||
],
|
||||
"attention_bias": false,
|
||||
"attention_dropout": 0.0,
|
||||
"attention_head_dim": 128,
|
||||
"auto_map": {
|
||||
"AutoConfig": "configuration_nemotron_h.NemotronHConfig",
|
||||
"AutoModelForCausalLM": "modeling_nemotron_h.NemotronHForCausalLM"
|
||||
},
|
||||
"bos_token_id": 1,
|
||||
"chunk_size": 256,
|
||||
"conv_kernel": 4,
|
||||
"dtype": "bfloat16",
|
||||
"eos_token_id": 2,
|
||||
"expand": 2,
|
||||
"head_dim": 128,
|
||||
"hidden_dropout": 0.0,
|
||||
"hidden_size": 3136,
|
||||
"hybrid_override_pattern": "M-M-M-MM-M-M*-M-M*-M-M-M*-M-M-MM*-MMM-M-M-",
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 12544,
|
||||
"layer_norm_epsilon": 1e-05,
|
||||
"mamba_head_dim": 80,
|
||||
"mamba_hidden_act": "silu",
|
||||
"mamba_num_heads": 96,
|
||||
"mamba_proj_bias": false,
|
||||
"max_position_embeddings": 262144,
|
||||
"mlp_bias": false,
|
||||
"mlp_hidden_act": "relu2",
|
||||
"model_type": "nemotron_h",
|
||||
"n_groups": 8,
|
||||
"num_attention_heads": 40,
|
||||
"num_hidden_layers": 42,
|
||||
"num_key_value_heads": 8,
|
||||
"num_logits_to_keep": 1,
|
||||
"pad_token_id": 0,
|
||||
"rescale_prenorm_residual": true,
|
||||
"residual_in_fp32": false,
|
||||
"rms_norm_eps": 1e-05,
|
||||
"sliding_window": null,
|
||||
"ssm_state_size": 128,
|
||||
"tie_word_embeddings": false,
|
||||
"time_step_floor": 0.0001,
|
||||
"time_step_limit": [
|
||||
0.0,
|
||||
Infinity
|
||||
],
|
||||
"time_step_max": 0.1,
|
||||
"time_step_min": 0.001,
|
||||
"time_step_rank": 256,
|
||||
"transformers_version": "4.57.6",
|
||||
"use_bias": false,
|
||||
"use_cache": true,
|
||||
"use_conv_bias": true,
|
||||
"use_mamba_kernels": true,
|
||||
"vocab_size": 131072
|
||||
}
|
||||
@@ -0,0 +1,243 @@
|
||||
# coding=utf-8
|
||||
# Copyright 2024 AI21 Labs Ltd. and the HuggingFace Inc. team. All rights reserved.
|
||||
# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""NemotronH model configuration"""
|
||||
|
||||
import re
|
||||
|
||||
from transformers.configuration_utils import PretrainedConfig
|
||||
from transformers.utils import logging
|
||||
|
||||
|
||||
logger = logging.get_logger(__name__)
|
||||
|
||||
|
||||
class NemotronHConfig(PretrainedConfig):
|
||||
r"""
|
||||
This is the configuration class to store the configuration of a [`NemotronHModel`]. It is used to instantiate a
|
||||
NemotronH model according to the specified arguments, defining the model architecture. Instantiating a configuration
|
||||
with the defaults will yield a similar configuration to that of the NemotronH-v0.1 model.
|
||||
|
||||
[todo](todo)
|
||||
|
||||
Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
|
||||
documentation from [`PretrainedConfig`] for more information.
|
||||
|
||||
|
||||
Args:
|
||||
vocab_size (`int`, *optional*, defaults to 131072):
|
||||
Vocabulary size of the NemotronH model. Defines the number of different tokens that can be represented by the
|
||||
`inputs_ids` passed when calling [`NemotronHModel`]
|
||||
tie_word_embeddings (`bool`, *optional*, defaults to `False`):
|
||||
Whether the model's input and output word embeddings should be tied. Note that this is only relevant if the
|
||||
model has a output word embedding layer.
|
||||
hidden_size (`int`, *optional*, defaults to 4096):
|
||||
Dimension of the hidden representations.
|
||||
intermediate_size (`int`, *optional*, defaults to 21504):
|
||||
Dimension of the MLP representations.
|
||||
num_hidden_layers (`int`, *optional*, defaults to 52):
|
||||
Number of hidden layers in the Transformer encoder.
|
||||
hybrid_override_pattern (`str`, *optional*, defaults to `"M-M-M-M*-M-M-M-M-M*-M-M-M-M-M*-M-M-M-M-M*-M-M-M-M-M-"`):
|
||||
The pattern of the hybrid model. The pattern is a string of characters where each character represents M: Mamba2, *: Attention, -: MLP
|
||||
num_attention_heads (`int`, *optional*, defaults to 32):
|
||||
Number of attention heads for each attention layer in the Transformer encoder.
|
||||
attention_head_dim (`int`, *optional*, defaults to 128):
|
||||
Dimension of each attention head.
|
||||
num_key_value_heads (`int`, *optional*, defaults to 8):
|
||||
This is the number of key_value heads that should be used to implement Grouped Query Attention. If
|
||||
`num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
|
||||
`num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used.
|
||||
mlp_hidden_act (`str`, *optional*, defaults to "relu2"):
|
||||
The non-linear activation function in the MLP layers.
|
||||
attention_bias (`bool`, *optional*, defaults to `False`):
|
||||
Whether to use bias in attention layers.
|
||||
mlp_bias (`bool`, *optional*, defaults to `False`):
|
||||
Whether to use bias in MLP layers.
|
||||
use_bias (`bool`, *optional*, defaults to `False`):
|
||||
Whether to use bias in the model.
|
||||
initializer_range (`float`, *optional*, defaults to 0.02):
|
||||
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
|
||||
layer_norm_epsilon (`float`, *optional*, defaults to 1e-5):
|
||||
The epsilon used by the layer normalization layers.
|
||||
residual_in_fp32 (`bool`, *optional*, defaults to `False`):
|
||||
Whether or not residuals should be in `float32`. If set to `False` residuals will keep the same `dtype` as the rest of the model.
|
||||
use_cache (`bool`, *optional*, defaults to `True`):
|
||||
Whether or not the model should return the last key/values attentions (not used by all models). Only
|
||||
relevant if `config.is_decoder=True`.
|
||||
num_logits_to_keep (`int` or `None`, *optional*, defaults to 1):
|
||||
Number of prompt logits to calculate during generation. If `None`, all logits will be calculated. If an
|
||||
integer value, only last `num_logits_to_keep` logits will be calculated.
|
||||
pad_token_id (`int`, *optional*, defaults to 0):
|
||||
The id of the padding token.
|
||||
bos_token_id (`int`, *optional*, defaults to 1):
|
||||
The id of the "beginning-of-sequence" token.
|
||||
eos_token_id (`int`, *optional*, defaults to 2):
|
||||
The id of the "end-of-sequence" token.
|
||||
sliding_window (`int`, *optional*, defaults to None):
|
||||
Sliding window attention window size.
|
||||
max_position_embeddings (`int`, *optional*, defaults to 4096):
|
||||
The maximum sequence length that this model might ever be used with.
|
||||
attention_dropout (`float`, *optional*, defaults to 0.0):
|
||||
The dropout ratio for the attention probabilities.
|
||||
hidden_dropout (`float`, *optional*, defaults to 0.0):
|
||||
The dropout ratio for the hidden states.
|
||||
use_mamba_kernels (`bool`, *optional*, defaults to `True`):
|
||||
Flag indicating whether or not to use the fast mamba kernels. These are available only if `mamba-ssm` and
|
||||
`causal-conv1d` are installed, and the mamba modules are running on a CUDA device.
|
||||
ssm_state_size (`int`, *optional*, defaults to 128):
|
||||
The dimension of the mamba state space latents.
|
||||
mamba_num_heads (`int`, *optional*, defaults to 128):
|
||||
Number of heads in Mamba layers.
|
||||
mamba_n_groups (`int`, *optional*, defaults to 8):
|
||||
Number of groups in Mamba layers.
|
||||
mamba_head_dim (`int`, *optional*, defaults to 64):
|
||||
Dimension of each Mamba head.
|
||||
mamba_d_conv (`int`, *optional*, defaults to 4):
|
||||
The size of the mamba convolution kernel.
|
||||
mamba_expand (`int`, *optional*, defaults to 2):
|
||||
Expanding factor used to determine the mamba intermediate size.
|
||||
mamba_hidden_act (`str`, *optional*, defaults to "silu"):
|
||||
The non-linear activation function in the Mamba layers.
|
||||
mamba_dt_min (`float`, *optional*, defaults to 0.001):
|
||||
Minimum value for the time step in Mamba.
|
||||
mamba_dt_max (`float`, *optional*, defaults to 0.1):
|
||||
Maximum value for the time step in Mamba.
|
||||
mamba_dt_limit (`tuple`, *optional*, defaults to (0.0, float("inf"))):
|
||||
Limits for the time step in Mamba.
|
||||
mamba_dt_init_floor (`float`, *optional*, defaults to 1e-4):
|
||||
Floor value for time step initialization in Mamba.
|
||||
mamba_conv_bias (`bool`, *optional*, defaults to `True`):
|
||||
Whether to use bias in the convolution layer of the mamba mixer block.
|
||||
mamba_proj_bias (`bool`, *optional*, defaults to `False`):
|
||||
Whether to use bias in the input and output projections of the mamba mixer block.
|
||||
mamba_chunk_size (`int`, *optional*, defaults to 256):
|
||||
Size of chunks for Mamba processing.
|
||||
rescale_prenorm_residual (`bool`, *optional*, defaults to `True`):
|
||||
Whether to rescale the pre-normalization residual connections.
|
||||
"""
|
||||
|
||||
model_type = "nemotron_h"
|
||||
keys_to_ignore_at_inference = ["past_key_values"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vocab_size=131072,
|
||||
tie_word_embeddings=False,
|
||||
hidden_size=4096,
|
||||
intermediate_size=21504,
|
||||
num_hidden_layers=52,
|
||||
hybrid_override_pattern="M-M-M-M*-M-M-M-M-M*-M-M-M-M-M*-M-M-M-M-M*-M-M-M-M-M-",
|
||||
num_attention_heads=32,
|
||||
attention_head_dim=128,
|
||||
num_key_value_heads=8, # nemo: num_query_groups
|
||||
mlp_hidden_act="relu2",
|
||||
attention_bias=False,
|
||||
mlp_bias=False,
|
||||
use_bias=False,
|
||||
initializer_range=0.02, # nemo: init_method_std
|
||||
layer_norm_epsilon=1e-5, # nemo: layernorm_epsilon
|
||||
residual_in_fp32=False, # Megatron Core default value
|
||||
use_cache=True,
|
||||
num_logits_to_keep=1,
|
||||
pad_token_id=0,
|
||||
bos_token_id=1,
|
||||
eos_token_id=2,
|
||||
sliding_window=None,
|
||||
max_position_embeddings=4096,
|
||||
attention_dropout=0.0,
|
||||
hidden_dropout=0.0, # * ADDED
|
||||
use_mamba_kernels=True,
|
||||
ssm_state_size=128, # mamba_state_size
|
||||
mamba_num_heads=128,
|
||||
mamba_n_groups=8, # nemo: mamba_ssm_ngroups = num_heads
|
||||
mamba_head_dim=64,
|
||||
mamba_d_conv=4,
|
||||
mamba_expand=2,
|
||||
mamba_hidden_act="silu",
|
||||
mamba_dt_min=0.001,
|
||||
mamba_dt_max=0.1,
|
||||
mamba_dt_limit=(0.0, float("inf")),
|
||||
mamba_dt_init_floor=1e-4,
|
||||
mamba_conv_bias=True,
|
||||
mamba_proj_bias=False,
|
||||
mamba_chunk_size=256,
|
||||
rescale_prenorm_residual=True,
|
||||
**kwargs,
|
||||
):
|
||||
self.vocab_size = vocab_size
|
||||
self.tie_word_embeddings = tie_word_embeddings
|
||||
self.hidden_size = hidden_size
|
||||
self.intermediate_size = intermediate_size
|
||||
self.num_hidden_layers = num_hidden_layers
|
||||
self.hybrid_override_pattern = hybrid_override_pattern
|
||||
self.num_attention_heads = num_attention_heads
|
||||
self.attention_head_dim = attention_head_dim
|
||||
self.sliding_window = sliding_window
|
||||
self.max_position_embeddings = max_position_embeddings
|
||||
self.attention_dropout = attention_dropout
|
||||
self.hidden_dropout = hidden_dropout
|
||||
|
||||
# Validate hybrid_override_pattern
|
||||
# M: Mamba2, *: Attention, -: MLP
|
||||
assert len(self.hybrid_override_pattern) == self.num_hidden_layers, "hybrid_override_pattern must have the same length as num_hidden_layers"
|
||||
assert re.match(r"^[*-M]+$", self.hybrid_override_pattern), "hybrid_override_pattern must only contain characters 'M', '*', or '-'"
|
||||
|
||||
# for backward compatibility
|
||||
if num_key_value_heads is None:
|
||||
num_key_value_heads = num_attention_heads
|
||||
|
||||
self.num_key_value_heads = num_key_value_heads
|
||||
self.mlp_hidden_act = mlp_hidden_act
|
||||
self.attention_bias = attention_bias
|
||||
self.mlp_bias = mlp_bias
|
||||
self.use_bias = use_bias
|
||||
self.initializer_range = initializer_range
|
||||
self.layer_norm_epsilon = layer_norm_epsilon
|
||||
self.residual_in_fp32 = residual_in_fp32
|
||||
|
||||
self.use_cache = use_cache
|
||||
self.num_logits_to_keep = num_logits_to_keep
|
||||
|
||||
self.use_mamba_kernels = use_mamba_kernels
|
||||
self.n_groups = mamba_n_groups
|
||||
self.mamba_head_dim = mamba_head_dim
|
||||
self.ssm_state_size = ssm_state_size
|
||||
self.mamba_num_heads = mamba_num_heads
|
||||
self.conv_kernel = mamba_d_conv
|
||||
self.expand = mamba_expand
|
||||
self.mamba_hidden_act = mamba_hidden_act
|
||||
self.time_step_min = mamba_dt_min
|
||||
self.time_step_max = mamba_dt_max
|
||||
self.time_step_limit = mamba_dt_limit
|
||||
self.time_step_floor = mamba_dt_init_floor
|
||||
self.use_conv_bias = mamba_conv_bias
|
||||
self.mamba_proj_bias = mamba_proj_bias
|
||||
self.chunk_size = mamba_chunk_size
|
||||
self.rescale_prenorm_residual = rescale_prenorm_residual
|
||||
|
||||
super().__init__(
|
||||
pad_token_id=pad_token_id,
|
||||
bos_token_id=bos_token_id,
|
||||
eos_token_id=eos_token_id,
|
||||
tie_word_embeddings=tie_word_embeddings,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
@property
|
||||
def layers_block_type(self):
|
||||
return [
|
||||
"mamba" if self.hybrid_override_pattern[i] == "M" else
|
||||
"attention" if self.hybrid_override_pattern[i] == "*" else "mlp"
|
||||
for i in range(self.num_hidden_layers)]
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"adapter_dir": "/checkpoints/figment_sft_v14p/figment-sft-v14p-lora",
|
||||
"base_model_id": "nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16",
|
||||
"dtype": "bfloat16",
|
||||
"files": [
|
||||
"chat_template.jinja",
|
||||
"config.json",
|
||||
"configuration_nemotron_h.py",
|
||||
"generation_config.json",
|
||||
"model-00001-of-00002.safetensors",
|
||||
"model-00002-of-00002.safetensors",
|
||||
"model.safetensors.index.json",
|
||||
"modeling_nemotron_h.py",
|
||||
"special_tokens_map.json",
|
||||
"tokenizer.json",
|
||||
"tokenizer_config.json"
|
||||
],
|
||||
"merge_method": "peft.merge_and_unload(safe_merge=True)",
|
||||
"output_dir": "/checkpoints/figment_sft_v14p/figment-sft-v14p-lora-merged-bf16"
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"_from_model_config": true,
|
||||
"bos_token_id": 1,
|
||||
"eos_token_id": 2,
|
||||
"pad_token_id": 0,
|
||||
"temperature": null,
|
||||
"top_p": null,
|
||||
"transformers_version": "4.57.6"
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:e2d5f0b2b0b6ad09de79903f30b90b6197265d67a1ed79304e04d0dbda2cebc1
|
||||
size 3973314912
|
||||
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:c7360eb7062380752d6e8ad59def991742cfa599bea8533a443fa7056aed2f78
|
||||
size 3973827728
|
||||
@@ -0,0 +1,271 @@
|
||||
{
|
||||
"metadata": {
|
||||
"total_parameters": 3973556832,
|
||||
"total_size": 7947113664
|
||||
},
|
||||
"weight_map": {
|
||||
"backbone.embeddings.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.1.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.1.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.1.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.10.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.10.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.10.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.12.mixer.k_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.12.mixer.o_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.12.mixer.q_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.12.mixer.v_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.12.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.13.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.13.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.13.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.15.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.15.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.15.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.17.mixer.k_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.17.mixer.o_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.17.mixer.q_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.17.mixer.v_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.17.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.18.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.18.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.18.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.20.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.20.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.20.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.21.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.21.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.21.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.21.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.21.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.21.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.21.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.21.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.21.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.22.mixer.down_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.22.mixer.up_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.22.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.24.mixer.k_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.24.mixer.o_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.24.mixer.q_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.24.mixer.v_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.24.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.25.mixer.down_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.25.mixer.up_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.25.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.27.mixer.down_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.27.mixer.up_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.27.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.29.mixer.down_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.29.mixer.up_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.29.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.3.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.3.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.3.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.30.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.30.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.30.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.30.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.30.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.30.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.30.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.30.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.30.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.32.mixer.k_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.32.mixer.o_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.32.mixer.q_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.32.mixer.v_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.32.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.33.mixer.down_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.33.mixer.up_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.33.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.37.mixer.down_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.37.mixer.up_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.37.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.39.mixer.down_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.39.mixer.up_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.39.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.4.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.4.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.4.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.4.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.4.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.4.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.4.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.4.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.4.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.40.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.40.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.40.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.40.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.40.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.40.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.40.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.40.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.40.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.41.mixer.down_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.41.mixer.up_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.41.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.5.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.5.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.5.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.8.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.8.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.8.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.norm_f.weight": "model-00002-of-00002.safetensors",
|
||||
"lm_head.weight": "model-00002-of-00002.safetensors"
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"bos_token": {
|
||||
"content": "<s>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"eos_token": {
|
||||
"content": "<|im_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"pad_token": {
|
||||
"content": "<|im_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"unk_token": {
|
||||
"content": "<unk>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:623c34567aebb18582765289fbe23d901c62704d6518d71866e0e58db892b5b7
|
||||
size 17077484
|
||||
File diff suppressed because it is too large
Load Diff
3
figment_sft_v5/figment-sft-v5-lora-merged-bf16.bf16.gguf
Normal file
3
figment_sft_v5/figment-sft-v5-lora-merged-bf16.bf16.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:c7f9b38d267c2ab2b791b613e0227ce3d057e61b57b568b16ca501f2e516379c
|
||||
size 7957646688
|
||||
@@ -0,0 +1,204 @@
|
||||
{% macro render_extra_keys(json_dict, handled_keys) %}
|
||||
{%- if json_dict is mapping %}
|
||||
{%- for json_key in json_dict if json_key not in handled_keys %}
|
||||
{%- if json_dict[json_key] is mapping or (json_dict[json_key] is sequence and json_dict[json_key] is not string) %}
|
||||
{{- '\n<' ~ json_key ~ '>' ~ (json_dict[json_key] | tojson | safe) ~ '</' ~ json_key ~ '>' }}
|
||||
{%- else %}
|
||||
{{-'\n<' ~ json_key ~ '>' ~ (json_dict[json_key] | string) ~ '</' ~ json_key ~ '>' }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{% endmacro %}
|
||||
{%- set enable_thinking = enable_thinking if enable_thinking is defined else True %}
|
||||
{%- set truncate_history_thinking = truncate_history_thinking if truncate_history_thinking is defined else True %}
|
||||
|
||||
{%- set ns = namespace(last_user_idx = -1) %}
|
||||
{%- set loop_messages = messages %}
|
||||
{%- for m in loop_messages %}
|
||||
{%- if m["role"] == "user" %}
|
||||
{%- set ns.last_user_idx = loop.index0 %}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
|
||||
{%- if messages[0]["role"] == "system" %}
|
||||
{%- set system_message = messages[0]["content"] %}
|
||||
{%- set loop_messages = messages[1:] %}
|
||||
{%- else %}
|
||||
{%- set system_message = "" %}
|
||||
{%- set loop_messages = messages %}
|
||||
{%- endif %}
|
||||
{%- if not tools is defined %}
|
||||
{%- set tools = [] %}
|
||||
{%- endif %}
|
||||
{# Recompute last_user_idx relative to loop_messages after handling system #}
|
||||
{%- set ns = namespace(last_user_idx = -1) %}
|
||||
{%- for m in loop_messages %}
|
||||
{%- if m["role"] == "user" %}
|
||||
{%- set ns.last_user_idx = loop.index0 %}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- if system_message is defined %}
|
||||
{{- "<|im_start|>system\n" + system_message }}
|
||||
{%- else %}
|
||||
{%- if tools is iterable and tools | length > 0 %}
|
||||
{{- "<|im_start|>system\n" }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- if tools is iterable and tools | length > 0 %}
|
||||
{%- if system_message is defined and system_message | length > 0 %}
|
||||
{{- "\n\n" }}
|
||||
{%- endif %}
|
||||
{{- "# Tools\n\nYou have access to the following functions:\n\n" }}
|
||||
{{- "<tools>" }}
|
||||
{%- for tool in tools %}
|
||||
{%- if tool.function is defined %}
|
||||
{%- set tool = tool.function %}
|
||||
{%- endif %}
|
||||
{{- "\n<function>\n<name>" ~ tool.name ~ "</name>" }}
|
||||
{%- if tool.description is defined %}
|
||||
{{- '\n<description>' ~ (tool.description | trim) ~ '</description>' }}
|
||||
{%- endif %}
|
||||
{{- '\n<parameters>' }}
|
||||
{%- if tool.parameters is defined and tool.parameters is mapping and tool.parameters.properties is defined and tool.parameters.properties is mapping %}
|
||||
{%- for param_name, param_fields in tool.parameters.properties|items %}
|
||||
{{- '\n<parameter>' }}
|
||||
{{- '\n<name>' ~ param_name ~ '</name>' }}
|
||||
{%- if param_fields.type is defined %}
|
||||
{{- '\n<type>' ~ (param_fields.type | string) ~ '</type>' }}
|
||||
{%- endif %}
|
||||
{%- if param_fields.description is defined %}
|
||||
{{- '\n<description>' ~ (param_fields.description | trim) ~ '</description>' }}
|
||||
{%- endif %}
|
||||
{%- if param_fields.enum is defined %}
|
||||
{{- '\n<enum>' ~ (param_fields.enum | tojson | safe) ~ '</enum>' }}
|
||||
{%- endif %}
|
||||
{%- set handled_keys = ['name', 'type', 'description', 'enum'] %}
|
||||
{{- render_extra_keys(param_fields, handled_keys) }}
|
||||
{{- '\n</parameter>' }}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{% set handled_keys = ['type', 'properties', 'required'] %}
|
||||
{{- render_extra_keys(tool.parameters, handled_keys) }}
|
||||
{%- if tool.parameters is defined and tool.parameters.required is defined %}
|
||||
{{- '\n<required>' ~ (tool.parameters.required | tojson | safe) ~ '</required>' }}
|
||||
{%- endif %}
|
||||
{{- '\n</parameters>' }}
|
||||
{%- set handled_keys = ['type', 'name', 'description', 'parameters'] %}
|
||||
{{- render_extra_keys(tool, handled_keys) }}
|
||||
{{- '\n</function>' }}
|
||||
{%- endfor %}
|
||||
{{- "\n</tools>" }}
|
||||
|
||||
{{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n</IMPORTANT>' }}
|
||||
{%- endif %}
|
||||
|
||||
|
||||
{%- if system_message is defined %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- else %}
|
||||
{%- if tools is iterable and tools | length > 0 %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
|
||||
{%- for message in loop_messages %}
|
||||
{%- if message.role == "assistant" %}
|
||||
{# Add reasoning content in to content field for unified processing below. #}
|
||||
{%- if message.reasoning_content is defined and message.reasoning_content is string and message.reasoning_content | trim | length > 0 %}
|
||||
{%- set content = "<think>\n" ~ message.reasoning_content ~ "\n</think>\n" ~ (message.content | default('', true)) %}
|
||||
{%- else %}
|
||||
{%- set content = message.content | default('', true) %}
|
||||
{%- if content is string -%}
|
||||
{# Allow downstream logic to to take care of broken thought, only handle coherent reasoning here. #}
|
||||
{%- if '<think>' not in content and '</think>' not in content -%}
|
||||
{%- set content = "<think></think>" ~ content -%}
|
||||
{%- endif -%}
|
||||
{%- else -%}
|
||||
{%- set content = content -%}
|
||||
{%- endif -%}
|
||||
{%- endif %}
|
||||
{%- if message.tool_calls is defined and message.tool_calls is iterable and message.tool_calls | length > 0 %}
|
||||
{# Assistant message has tool calls. #}
|
||||
{{- '<|im_start|>assistant\n' }}
|
||||
{%- set include_content = not (truncate_history_thinking and loop.index0 < ns.last_user_idx) %}
|
||||
{%- if content is string and content | trim | length > 0 %}
|
||||
{%- if include_content %}
|
||||
{{- (content | trim) ~ '\n' -}}
|
||||
{%- else %}
|
||||
{%- set c = (content | string) %}
|
||||
{%- if '</think>' in c %}
|
||||
{# Keep only content after the last closing think. Also generation prompt causes this. #}
|
||||
{%- set c = c.split('</think>')[-1] %}
|
||||
{%- elif '<think>' in c %}
|
||||
{# If <think> was opened but never closed, drop the trailing think segment #}
|
||||
{%- set c = c.split('<think>')[0] %}
|
||||
{%- endif %}
|
||||
{%- set c = "<think></think>" ~ c | trim %}
|
||||
{%- if c | length > 0 %}
|
||||
{{- c ~ '\n' -}}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- else %}
|
||||
{{- "<think></think>" -}}
|
||||
{%- endif %}
|
||||
{%- for tool_call in message.tool_calls %}
|
||||
{%- if tool_call.function is defined %}
|
||||
{%- set tool_call = tool_call.function %}
|
||||
{%- endif %}
|
||||
{{- '<tool_call>\n<function=' ~ tool_call.name ~ '>\n' -}}
|
||||
{%- if tool_call.arguments is defined %}
|
||||
{%- for args_name, args_value in tool_call.arguments|items %}
|
||||
{{- '<parameter=' ~ args_name ~ '>\n' -}}
|
||||
{%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}
|
||||
{{- args_value ~ '\n</parameter>\n' -}}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{{- '</function>\n</tool_call>\n' -}}
|
||||
{%- endfor %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- else %}
|
||||
{# Assistant message doesn't have tool calls. #}
|
||||
{%- if not (truncate_history_thinking and loop.index0 < ns.last_user_idx) %}
|
||||
{{- '<|im_start|>assistant\n' ~ (content | default('', true) | string | trim) ~ '<|im_end|>\n' }}
|
||||
{%- else %}
|
||||
{%- set c = (content | default('', true) | string) %}
|
||||
{%- if '<think>' in c and '</think>' in c %}
|
||||
{%- set c = "<think></think>" ~ c.split('</think>')[-1] %}
|
||||
{%- endif %}
|
||||
{%- set c = c | trim %}
|
||||
{%- if c | length > 0 %}
|
||||
{{- '<|im_start|>assistant\n' ~ c ~ '<|im_end|>\n' }}
|
||||
{%- else %}
|
||||
{{- '<|im_start|>assistant\n<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- elif message.role == "user" or message.role == "system" %}
|
||||
{{- '<|im_start|>' + message.role + '\n' }}
|
||||
{%- set content = message.content | string %}
|
||||
{{- content }}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- elif message.role == "tool" %}
|
||||
{%- if loop.previtem and loop.previtem.role != "tool" %}
|
||||
{{- '<|im_start|>user\n' }}
|
||||
{%- endif %}
|
||||
{{- '<tool_response>\n' }}
|
||||
{{- message.content }}
|
||||
{{- '\n</tool_response>\n' }}
|
||||
{%- if not loop.last and loop.nextitem.role != "tool" %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- elif loop.last %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- else %}
|
||||
{{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
|
||||
{%- if add_generation_prompt %}
|
||||
{%- if enable_thinking %}
|
||||
{{- '<|im_start|>assistant\n<think>\n' }}
|
||||
{%- else %}
|
||||
{{- '<|im_start|>assistant\n<think></think>' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
59
figment_sft_v5/figment-sft-v5-lora-merged-bf16/config.json
Normal file
59
figment_sft_v5/figment-sft-v5-lora-merged-bf16/config.json
Normal file
@@ -0,0 +1,59 @@
|
||||
{
|
||||
"architectures": [
|
||||
"NemotronHForCausalLM"
|
||||
],
|
||||
"attention_bias": false,
|
||||
"attention_dropout": 0.0,
|
||||
"attention_head_dim": 128,
|
||||
"auto_map": {
|
||||
"AutoConfig": "configuration_nemotron_h.NemotronHConfig",
|
||||
"AutoModelForCausalLM": "modeling_nemotron_h.NemotronHForCausalLM"
|
||||
},
|
||||
"bos_token_id": 1,
|
||||
"chunk_size": 256,
|
||||
"conv_kernel": 4,
|
||||
"dtype": "bfloat16",
|
||||
"eos_token_id": 2,
|
||||
"expand": 2,
|
||||
"head_dim": 128,
|
||||
"hidden_dropout": 0.0,
|
||||
"hidden_size": 3136,
|
||||
"hybrid_override_pattern": "M-M-M-MM-M-M*-M-M*-M-M-M*-M-M-MM*-MMM-M-M-",
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 12544,
|
||||
"layer_norm_epsilon": 1e-05,
|
||||
"mamba_head_dim": 80,
|
||||
"mamba_hidden_act": "silu",
|
||||
"mamba_num_heads": 96,
|
||||
"mamba_proj_bias": false,
|
||||
"max_position_embeddings": 262144,
|
||||
"mlp_bias": false,
|
||||
"mlp_hidden_act": "relu2",
|
||||
"model_type": "nemotron_h",
|
||||
"n_groups": 8,
|
||||
"num_attention_heads": 40,
|
||||
"num_hidden_layers": 42,
|
||||
"num_key_value_heads": 8,
|
||||
"num_logits_to_keep": 1,
|
||||
"pad_token_id": 0,
|
||||
"rescale_prenorm_residual": true,
|
||||
"residual_in_fp32": false,
|
||||
"rms_norm_eps": 1e-05,
|
||||
"sliding_window": null,
|
||||
"ssm_state_size": 128,
|
||||
"tie_word_embeddings": false,
|
||||
"time_step_floor": 0.0001,
|
||||
"time_step_limit": [
|
||||
0.0,
|
||||
Infinity
|
||||
],
|
||||
"time_step_max": 0.1,
|
||||
"time_step_min": 0.001,
|
||||
"time_step_rank": 256,
|
||||
"transformers_version": "4.57.6",
|
||||
"use_bias": false,
|
||||
"use_cache": true,
|
||||
"use_conv_bias": true,
|
||||
"use_mamba_kernels": true,
|
||||
"vocab_size": 131072
|
||||
}
|
||||
@@ -0,0 +1,243 @@
|
||||
# coding=utf-8
|
||||
# Copyright 2024 AI21 Labs Ltd. and the HuggingFace Inc. team. All rights reserved.
|
||||
# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""NemotronH model configuration"""
|
||||
|
||||
import re
|
||||
|
||||
from transformers.configuration_utils import PretrainedConfig
|
||||
from transformers.utils import logging
|
||||
|
||||
|
||||
logger = logging.get_logger(__name__)
|
||||
|
||||
|
||||
class NemotronHConfig(PretrainedConfig):
|
||||
r"""
|
||||
This is the configuration class to store the configuration of a [`NemotronHModel`]. It is used to instantiate a
|
||||
NemotronH model according to the specified arguments, defining the model architecture. Instantiating a configuration
|
||||
with the defaults will yield a similar configuration to that of the NemotronH-v0.1 model.
|
||||
|
||||
[todo](todo)
|
||||
|
||||
Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
|
||||
documentation from [`PretrainedConfig`] for more information.
|
||||
|
||||
|
||||
Args:
|
||||
vocab_size (`int`, *optional*, defaults to 131072):
|
||||
Vocabulary size of the NemotronH model. Defines the number of different tokens that can be represented by the
|
||||
`inputs_ids` passed when calling [`NemotronHModel`]
|
||||
tie_word_embeddings (`bool`, *optional*, defaults to `False`):
|
||||
Whether the model's input and output word embeddings should be tied. Note that this is only relevant if the
|
||||
model has a output word embedding layer.
|
||||
hidden_size (`int`, *optional*, defaults to 4096):
|
||||
Dimension of the hidden representations.
|
||||
intermediate_size (`int`, *optional*, defaults to 21504):
|
||||
Dimension of the MLP representations.
|
||||
num_hidden_layers (`int`, *optional*, defaults to 52):
|
||||
Number of hidden layers in the Transformer encoder.
|
||||
hybrid_override_pattern (`str`, *optional*, defaults to `"M-M-M-M*-M-M-M-M-M*-M-M-M-M-M*-M-M-M-M-M*-M-M-M-M-M-"`):
|
||||
The pattern of the hybrid model. The pattern is a string of characters where each character represents M: Mamba2, *: Attention, -: MLP
|
||||
num_attention_heads (`int`, *optional*, defaults to 32):
|
||||
Number of attention heads for each attention layer in the Transformer encoder.
|
||||
attention_head_dim (`int`, *optional*, defaults to 128):
|
||||
Dimension of each attention head.
|
||||
num_key_value_heads (`int`, *optional*, defaults to 8):
|
||||
This is the number of key_value heads that should be used to implement Grouped Query Attention. If
|
||||
`num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
|
||||
`num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used.
|
||||
mlp_hidden_act (`str`, *optional*, defaults to "relu2"):
|
||||
The non-linear activation function in the MLP layers.
|
||||
attention_bias (`bool`, *optional*, defaults to `False`):
|
||||
Whether to use bias in attention layers.
|
||||
mlp_bias (`bool`, *optional*, defaults to `False`):
|
||||
Whether to use bias in MLP layers.
|
||||
use_bias (`bool`, *optional*, defaults to `False`):
|
||||
Whether to use bias in the model.
|
||||
initializer_range (`float`, *optional*, defaults to 0.02):
|
||||
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
|
||||
layer_norm_epsilon (`float`, *optional*, defaults to 1e-5):
|
||||
The epsilon used by the layer normalization layers.
|
||||
residual_in_fp32 (`bool`, *optional*, defaults to `False`):
|
||||
Whether or not residuals should be in `float32`. If set to `False` residuals will keep the same `dtype` as the rest of the model.
|
||||
use_cache (`bool`, *optional*, defaults to `True`):
|
||||
Whether or not the model should return the last key/values attentions (not used by all models). Only
|
||||
relevant if `config.is_decoder=True`.
|
||||
num_logits_to_keep (`int` or `None`, *optional*, defaults to 1):
|
||||
Number of prompt logits to calculate during generation. If `None`, all logits will be calculated. If an
|
||||
integer value, only last `num_logits_to_keep` logits will be calculated.
|
||||
pad_token_id (`int`, *optional*, defaults to 0):
|
||||
The id of the padding token.
|
||||
bos_token_id (`int`, *optional*, defaults to 1):
|
||||
The id of the "beginning-of-sequence" token.
|
||||
eos_token_id (`int`, *optional*, defaults to 2):
|
||||
The id of the "end-of-sequence" token.
|
||||
sliding_window (`int`, *optional*, defaults to None):
|
||||
Sliding window attention window size.
|
||||
max_position_embeddings (`int`, *optional*, defaults to 4096):
|
||||
The maximum sequence length that this model might ever be used with.
|
||||
attention_dropout (`float`, *optional*, defaults to 0.0):
|
||||
The dropout ratio for the attention probabilities.
|
||||
hidden_dropout (`float`, *optional*, defaults to 0.0):
|
||||
The dropout ratio for the hidden states.
|
||||
use_mamba_kernels (`bool`, *optional*, defaults to `True`):
|
||||
Flag indicating whether or not to use the fast mamba kernels. These are available only if `mamba-ssm` and
|
||||
`causal-conv1d` are installed, and the mamba modules are running on a CUDA device.
|
||||
ssm_state_size (`int`, *optional*, defaults to 128):
|
||||
The dimension of the mamba state space latents.
|
||||
mamba_num_heads (`int`, *optional*, defaults to 128):
|
||||
Number of heads in Mamba layers.
|
||||
mamba_n_groups (`int`, *optional*, defaults to 8):
|
||||
Number of groups in Mamba layers.
|
||||
mamba_head_dim (`int`, *optional*, defaults to 64):
|
||||
Dimension of each Mamba head.
|
||||
mamba_d_conv (`int`, *optional*, defaults to 4):
|
||||
The size of the mamba convolution kernel.
|
||||
mamba_expand (`int`, *optional*, defaults to 2):
|
||||
Expanding factor used to determine the mamba intermediate size.
|
||||
mamba_hidden_act (`str`, *optional*, defaults to "silu"):
|
||||
The non-linear activation function in the Mamba layers.
|
||||
mamba_dt_min (`float`, *optional*, defaults to 0.001):
|
||||
Minimum value for the time step in Mamba.
|
||||
mamba_dt_max (`float`, *optional*, defaults to 0.1):
|
||||
Maximum value for the time step in Mamba.
|
||||
mamba_dt_limit (`tuple`, *optional*, defaults to (0.0, float("inf"))):
|
||||
Limits for the time step in Mamba.
|
||||
mamba_dt_init_floor (`float`, *optional*, defaults to 1e-4):
|
||||
Floor value for time step initialization in Mamba.
|
||||
mamba_conv_bias (`bool`, *optional*, defaults to `True`):
|
||||
Whether to use bias in the convolution layer of the mamba mixer block.
|
||||
mamba_proj_bias (`bool`, *optional*, defaults to `False`):
|
||||
Whether to use bias in the input and output projections of the mamba mixer block.
|
||||
mamba_chunk_size (`int`, *optional*, defaults to 256):
|
||||
Size of chunks for Mamba processing.
|
||||
rescale_prenorm_residual (`bool`, *optional*, defaults to `True`):
|
||||
Whether to rescale the pre-normalization residual connections.
|
||||
"""
|
||||
|
||||
model_type = "nemotron_h"
|
||||
keys_to_ignore_at_inference = ["past_key_values"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vocab_size=131072,
|
||||
tie_word_embeddings=False,
|
||||
hidden_size=4096,
|
||||
intermediate_size=21504,
|
||||
num_hidden_layers=52,
|
||||
hybrid_override_pattern="M-M-M-M*-M-M-M-M-M*-M-M-M-M-M*-M-M-M-M-M*-M-M-M-M-M-",
|
||||
num_attention_heads=32,
|
||||
attention_head_dim=128,
|
||||
num_key_value_heads=8, # nemo: num_query_groups
|
||||
mlp_hidden_act="relu2",
|
||||
attention_bias=False,
|
||||
mlp_bias=False,
|
||||
use_bias=False,
|
||||
initializer_range=0.02, # nemo: init_method_std
|
||||
layer_norm_epsilon=1e-5, # nemo: layernorm_epsilon
|
||||
residual_in_fp32=False, # Megatron Core default value
|
||||
use_cache=True,
|
||||
num_logits_to_keep=1,
|
||||
pad_token_id=0,
|
||||
bos_token_id=1,
|
||||
eos_token_id=2,
|
||||
sliding_window=None,
|
||||
max_position_embeddings=4096,
|
||||
attention_dropout=0.0,
|
||||
hidden_dropout=0.0, # * ADDED
|
||||
use_mamba_kernels=True,
|
||||
ssm_state_size=128, # mamba_state_size
|
||||
mamba_num_heads=128,
|
||||
mamba_n_groups=8, # nemo: mamba_ssm_ngroups = num_heads
|
||||
mamba_head_dim=64,
|
||||
mamba_d_conv=4,
|
||||
mamba_expand=2,
|
||||
mamba_hidden_act="silu",
|
||||
mamba_dt_min=0.001,
|
||||
mamba_dt_max=0.1,
|
||||
mamba_dt_limit=(0.0, float("inf")),
|
||||
mamba_dt_init_floor=1e-4,
|
||||
mamba_conv_bias=True,
|
||||
mamba_proj_bias=False,
|
||||
mamba_chunk_size=256,
|
||||
rescale_prenorm_residual=True,
|
||||
**kwargs,
|
||||
):
|
||||
self.vocab_size = vocab_size
|
||||
self.tie_word_embeddings = tie_word_embeddings
|
||||
self.hidden_size = hidden_size
|
||||
self.intermediate_size = intermediate_size
|
||||
self.num_hidden_layers = num_hidden_layers
|
||||
self.hybrid_override_pattern = hybrid_override_pattern
|
||||
self.num_attention_heads = num_attention_heads
|
||||
self.attention_head_dim = attention_head_dim
|
||||
self.sliding_window = sliding_window
|
||||
self.max_position_embeddings = max_position_embeddings
|
||||
self.attention_dropout = attention_dropout
|
||||
self.hidden_dropout = hidden_dropout
|
||||
|
||||
# Validate hybrid_override_pattern
|
||||
# M: Mamba2, *: Attention, -: MLP
|
||||
assert len(self.hybrid_override_pattern) == self.num_hidden_layers, "hybrid_override_pattern must have the same length as num_hidden_layers"
|
||||
assert re.match(r"^[*-M]+$", self.hybrid_override_pattern), "hybrid_override_pattern must only contain characters 'M', '*', or '-'"
|
||||
|
||||
# for backward compatibility
|
||||
if num_key_value_heads is None:
|
||||
num_key_value_heads = num_attention_heads
|
||||
|
||||
self.num_key_value_heads = num_key_value_heads
|
||||
self.mlp_hidden_act = mlp_hidden_act
|
||||
self.attention_bias = attention_bias
|
||||
self.mlp_bias = mlp_bias
|
||||
self.use_bias = use_bias
|
||||
self.initializer_range = initializer_range
|
||||
self.layer_norm_epsilon = layer_norm_epsilon
|
||||
self.residual_in_fp32 = residual_in_fp32
|
||||
|
||||
self.use_cache = use_cache
|
||||
self.num_logits_to_keep = num_logits_to_keep
|
||||
|
||||
self.use_mamba_kernels = use_mamba_kernels
|
||||
self.n_groups = mamba_n_groups
|
||||
self.mamba_head_dim = mamba_head_dim
|
||||
self.ssm_state_size = ssm_state_size
|
||||
self.mamba_num_heads = mamba_num_heads
|
||||
self.conv_kernel = mamba_d_conv
|
||||
self.expand = mamba_expand
|
||||
self.mamba_hidden_act = mamba_hidden_act
|
||||
self.time_step_min = mamba_dt_min
|
||||
self.time_step_max = mamba_dt_max
|
||||
self.time_step_limit = mamba_dt_limit
|
||||
self.time_step_floor = mamba_dt_init_floor
|
||||
self.use_conv_bias = mamba_conv_bias
|
||||
self.mamba_proj_bias = mamba_proj_bias
|
||||
self.chunk_size = mamba_chunk_size
|
||||
self.rescale_prenorm_residual = rescale_prenorm_residual
|
||||
|
||||
super().__init__(
|
||||
pad_token_id=pad_token_id,
|
||||
bos_token_id=bos_token_id,
|
||||
eos_token_id=eos_token_id,
|
||||
tie_word_embeddings=tie_word_embeddings,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
@property
|
||||
def layers_block_type(self):
|
||||
return [
|
||||
"mamba" if self.hybrid_override_pattern[i] == "M" else
|
||||
"attention" if self.hybrid_override_pattern[i] == "*" else "mlp"
|
||||
for i in range(self.num_hidden_layers)]
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"adapter_dir": "/checkpoints/figment_sft_v5/figment-sft-v5-lora",
|
||||
"base_model_id": "nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16",
|
||||
"dtype": "bfloat16",
|
||||
"files": [
|
||||
"chat_template.jinja",
|
||||
"config.json",
|
||||
"configuration_nemotron_h.py",
|
||||
"generation_config.json",
|
||||
"model-00001-of-00002.safetensors",
|
||||
"model-00002-of-00002.safetensors",
|
||||
"model.safetensors.index.json",
|
||||
"modeling_nemotron_h.py",
|
||||
"special_tokens_map.json",
|
||||
"tokenizer.json",
|
||||
"tokenizer_config.json"
|
||||
],
|
||||
"merge_method": "peft.merge_and_unload(safe_merge=True)",
|
||||
"output_dir": "/checkpoints/figment_sft_v5/figment-sft-v5-lora-merged-bf16"
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"_from_model_config": true,
|
||||
"bos_token_id": 1,
|
||||
"eos_token_id": 2,
|
||||
"pad_token_id": 0,
|
||||
"temperature": null,
|
||||
"top_p": null,
|
||||
"transformers_version": "4.57.6"
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:c52fa4ab8c9bd51ae8485127e5c578aaed686ac32ea57f913953e31919533a87
|
||||
size 3973314912
|
||||
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:f0a8ed1e11d6fd855bc0b41c93cc3d9b4eb09908f58dfda35f0f74ead5d3058c
|
||||
size 3973827728
|
||||
@@ -0,0 +1,271 @@
|
||||
{
|
||||
"metadata": {
|
||||
"total_parameters": 3973556832,
|
||||
"total_size": 7947113664
|
||||
},
|
||||
"weight_map": {
|
||||
"backbone.embeddings.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.1.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.1.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.1.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.10.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.10.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.10.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.12.mixer.k_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.12.mixer.o_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.12.mixer.q_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.12.mixer.v_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.12.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.13.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.13.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.13.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.15.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.15.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.15.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.17.mixer.k_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.17.mixer.o_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.17.mixer.q_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.17.mixer.v_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.17.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.18.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.18.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.18.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.20.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.20.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.20.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.21.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.21.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.21.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.21.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.21.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.21.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.21.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.21.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.21.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.22.mixer.down_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.22.mixer.up_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.22.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.24.mixer.k_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.24.mixer.o_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.24.mixer.q_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.24.mixer.v_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.24.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.25.mixer.down_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.25.mixer.up_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.25.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.27.mixer.down_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.27.mixer.up_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.27.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.29.mixer.down_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.29.mixer.up_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.29.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.3.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.3.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.3.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.30.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.30.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.30.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.30.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.30.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.30.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.30.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.30.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.30.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.32.mixer.k_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.32.mixer.o_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.32.mixer.q_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.32.mixer.v_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.32.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.33.mixer.down_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.33.mixer.up_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.33.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.37.mixer.down_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.37.mixer.up_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.37.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.39.mixer.down_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.39.mixer.up_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.39.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.4.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.4.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.4.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.4.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.4.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.4.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.4.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.4.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.4.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.40.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.40.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.40.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.40.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.40.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.40.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.40.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.40.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.40.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.41.mixer.down_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.41.mixer.up_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.41.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.5.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.5.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.5.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.8.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.8.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.8.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.norm_f.weight": "model-00002-of-00002.safetensors",
|
||||
"lm_head.weight": "model-00002-of-00002.safetensors"
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"bos_token": {
|
||||
"content": "<s>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"eos_token": {
|
||||
"content": "<|im_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"pad_token": {
|
||||
"content": "<|im_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"unk_token": {
|
||||
"content": "<unk>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:623c34567aebb18582765289fbe23d901c62704d6518d71866e0e58db892b5b7
|
||||
size 17077484
|
||||
8019
figment_sft_v5/figment-sft-v5-lora-merged-bf16/tokenizer_config.json
Normal file
8019
figment_sft_v5/figment-sft-v5-lora-merged-bf16/tokenizer_config.json
Normal file
File diff suppressed because it is too large
Load Diff
3
figment_sft_v6/figment-sft-v6-lora-merged-bf16.bf16.gguf
Normal file
3
figment_sft_v6/figment-sft-v6-lora-merged-bf16.bf16.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:92fb2bb4a8686230f050c1696e6df749fe49ec4d41221ab9100785afa7e34009
|
||||
size 7957646688
|
||||
@@ -0,0 +1,204 @@
|
||||
{% macro render_extra_keys(json_dict, handled_keys) %}
|
||||
{%- if json_dict is mapping %}
|
||||
{%- for json_key in json_dict if json_key not in handled_keys %}
|
||||
{%- if json_dict[json_key] is mapping or (json_dict[json_key] is sequence and json_dict[json_key] is not string) %}
|
||||
{{- '\n<' ~ json_key ~ '>' ~ (json_dict[json_key] | tojson | safe) ~ '</' ~ json_key ~ '>' }}
|
||||
{%- else %}
|
||||
{{-'\n<' ~ json_key ~ '>' ~ (json_dict[json_key] | string) ~ '</' ~ json_key ~ '>' }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{% endmacro %}
|
||||
{%- set enable_thinking = enable_thinking if enable_thinking is defined else True %}
|
||||
{%- set truncate_history_thinking = truncate_history_thinking if truncate_history_thinking is defined else True %}
|
||||
|
||||
{%- set ns = namespace(last_user_idx = -1) %}
|
||||
{%- set loop_messages = messages %}
|
||||
{%- for m in loop_messages %}
|
||||
{%- if m["role"] == "user" %}
|
||||
{%- set ns.last_user_idx = loop.index0 %}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
|
||||
{%- if messages[0]["role"] == "system" %}
|
||||
{%- set system_message = messages[0]["content"] %}
|
||||
{%- set loop_messages = messages[1:] %}
|
||||
{%- else %}
|
||||
{%- set system_message = "" %}
|
||||
{%- set loop_messages = messages %}
|
||||
{%- endif %}
|
||||
{%- if not tools is defined %}
|
||||
{%- set tools = [] %}
|
||||
{%- endif %}
|
||||
{# Recompute last_user_idx relative to loop_messages after handling system #}
|
||||
{%- set ns = namespace(last_user_idx = -1) %}
|
||||
{%- for m in loop_messages %}
|
||||
{%- if m["role"] == "user" %}
|
||||
{%- set ns.last_user_idx = loop.index0 %}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- if system_message is defined %}
|
||||
{{- "<|im_start|>system\n" + system_message }}
|
||||
{%- else %}
|
||||
{%- if tools is iterable and tools | length > 0 %}
|
||||
{{- "<|im_start|>system\n" }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- if tools is iterable and tools | length > 0 %}
|
||||
{%- if system_message is defined and system_message | length > 0 %}
|
||||
{{- "\n\n" }}
|
||||
{%- endif %}
|
||||
{{- "# Tools\n\nYou have access to the following functions:\n\n" }}
|
||||
{{- "<tools>" }}
|
||||
{%- for tool in tools %}
|
||||
{%- if tool.function is defined %}
|
||||
{%- set tool = tool.function %}
|
||||
{%- endif %}
|
||||
{{- "\n<function>\n<name>" ~ tool.name ~ "</name>" }}
|
||||
{%- if tool.description is defined %}
|
||||
{{- '\n<description>' ~ (tool.description | trim) ~ '</description>' }}
|
||||
{%- endif %}
|
||||
{{- '\n<parameters>' }}
|
||||
{%- if tool.parameters is defined and tool.parameters is mapping and tool.parameters.properties is defined and tool.parameters.properties is mapping %}
|
||||
{%- for param_name, param_fields in tool.parameters.properties|items %}
|
||||
{{- '\n<parameter>' }}
|
||||
{{- '\n<name>' ~ param_name ~ '</name>' }}
|
||||
{%- if param_fields.type is defined %}
|
||||
{{- '\n<type>' ~ (param_fields.type | string) ~ '</type>' }}
|
||||
{%- endif %}
|
||||
{%- if param_fields.description is defined %}
|
||||
{{- '\n<description>' ~ (param_fields.description | trim) ~ '</description>' }}
|
||||
{%- endif %}
|
||||
{%- if param_fields.enum is defined %}
|
||||
{{- '\n<enum>' ~ (param_fields.enum | tojson | safe) ~ '</enum>' }}
|
||||
{%- endif %}
|
||||
{%- set handled_keys = ['name', 'type', 'description', 'enum'] %}
|
||||
{{- render_extra_keys(param_fields, handled_keys) }}
|
||||
{{- '\n</parameter>' }}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{% set handled_keys = ['type', 'properties', 'required'] %}
|
||||
{{- render_extra_keys(tool.parameters, handled_keys) }}
|
||||
{%- if tool.parameters is defined and tool.parameters.required is defined %}
|
||||
{{- '\n<required>' ~ (tool.parameters.required | tojson | safe) ~ '</required>' }}
|
||||
{%- endif %}
|
||||
{{- '\n</parameters>' }}
|
||||
{%- set handled_keys = ['type', 'name', 'description', 'parameters'] %}
|
||||
{{- render_extra_keys(tool, handled_keys) }}
|
||||
{{- '\n</function>' }}
|
||||
{%- endfor %}
|
||||
{{- "\n</tools>" }}
|
||||
|
||||
{{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n</IMPORTANT>' }}
|
||||
{%- endif %}
|
||||
|
||||
|
||||
{%- if system_message is defined %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- else %}
|
||||
{%- if tools is iterable and tools | length > 0 %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
|
||||
{%- for message in loop_messages %}
|
||||
{%- if message.role == "assistant" %}
|
||||
{# Add reasoning content in to content field for unified processing below. #}
|
||||
{%- if message.reasoning_content is defined and message.reasoning_content is string and message.reasoning_content | trim | length > 0 %}
|
||||
{%- set content = "<think>\n" ~ message.reasoning_content ~ "\n</think>\n" ~ (message.content | default('', true)) %}
|
||||
{%- else %}
|
||||
{%- set content = message.content | default('', true) %}
|
||||
{%- if content is string -%}
|
||||
{# Allow downstream logic to to take care of broken thought, only handle coherent reasoning here. #}
|
||||
{%- if '<think>' not in content and '</think>' not in content -%}
|
||||
{%- set content = "<think></think>" ~ content -%}
|
||||
{%- endif -%}
|
||||
{%- else -%}
|
||||
{%- set content = content -%}
|
||||
{%- endif -%}
|
||||
{%- endif %}
|
||||
{%- if message.tool_calls is defined and message.tool_calls is iterable and message.tool_calls | length > 0 %}
|
||||
{# Assistant message has tool calls. #}
|
||||
{{- '<|im_start|>assistant\n' }}
|
||||
{%- set include_content = not (truncate_history_thinking and loop.index0 < ns.last_user_idx) %}
|
||||
{%- if content is string and content | trim | length > 0 %}
|
||||
{%- if include_content %}
|
||||
{{- (content | trim) ~ '\n' -}}
|
||||
{%- else %}
|
||||
{%- set c = (content | string) %}
|
||||
{%- if '</think>' in c %}
|
||||
{# Keep only content after the last closing think. Also generation prompt causes this. #}
|
||||
{%- set c = c.split('</think>')[-1] %}
|
||||
{%- elif '<think>' in c %}
|
||||
{# If <think> was opened but never closed, drop the trailing think segment #}
|
||||
{%- set c = c.split('<think>')[0] %}
|
||||
{%- endif %}
|
||||
{%- set c = "<think></think>" ~ c | trim %}
|
||||
{%- if c | length > 0 %}
|
||||
{{- c ~ '\n' -}}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- else %}
|
||||
{{- "<think></think>" -}}
|
||||
{%- endif %}
|
||||
{%- for tool_call in message.tool_calls %}
|
||||
{%- if tool_call.function is defined %}
|
||||
{%- set tool_call = tool_call.function %}
|
||||
{%- endif %}
|
||||
{{- '<tool_call>\n<function=' ~ tool_call.name ~ '>\n' -}}
|
||||
{%- if tool_call.arguments is defined %}
|
||||
{%- for args_name, args_value in tool_call.arguments|items %}
|
||||
{{- '<parameter=' ~ args_name ~ '>\n' -}}
|
||||
{%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}
|
||||
{{- args_value ~ '\n</parameter>\n' -}}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{{- '</function>\n</tool_call>\n' -}}
|
||||
{%- endfor %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- else %}
|
||||
{# Assistant message doesn't have tool calls. #}
|
||||
{%- if not (truncate_history_thinking and loop.index0 < ns.last_user_idx) %}
|
||||
{{- '<|im_start|>assistant\n' ~ (content | default('', true) | string | trim) ~ '<|im_end|>\n' }}
|
||||
{%- else %}
|
||||
{%- set c = (content | default('', true) | string) %}
|
||||
{%- if '<think>' in c and '</think>' in c %}
|
||||
{%- set c = "<think></think>" ~ c.split('</think>')[-1] %}
|
||||
{%- endif %}
|
||||
{%- set c = c | trim %}
|
||||
{%- if c | length > 0 %}
|
||||
{{- '<|im_start|>assistant\n' ~ c ~ '<|im_end|>\n' }}
|
||||
{%- else %}
|
||||
{{- '<|im_start|>assistant\n<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- elif message.role == "user" or message.role == "system" %}
|
||||
{{- '<|im_start|>' + message.role + '\n' }}
|
||||
{%- set content = message.content | string %}
|
||||
{{- content }}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- elif message.role == "tool" %}
|
||||
{%- if loop.previtem and loop.previtem.role != "tool" %}
|
||||
{{- '<|im_start|>user\n' }}
|
||||
{%- endif %}
|
||||
{{- '<tool_response>\n' }}
|
||||
{{- message.content }}
|
||||
{{- '\n</tool_response>\n' }}
|
||||
{%- if not loop.last and loop.nextitem.role != "tool" %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- elif loop.last %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- else %}
|
||||
{{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
|
||||
{%- if add_generation_prompt %}
|
||||
{%- if enable_thinking %}
|
||||
{{- '<|im_start|>assistant\n<think>\n' }}
|
||||
{%- else %}
|
||||
{{- '<|im_start|>assistant\n<think></think>' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
59
figment_sft_v6/figment-sft-v6-lora-merged-bf16/config.json
Normal file
59
figment_sft_v6/figment-sft-v6-lora-merged-bf16/config.json
Normal file
@@ -0,0 +1,59 @@
|
||||
{
|
||||
"architectures": [
|
||||
"NemotronHForCausalLM"
|
||||
],
|
||||
"attention_bias": false,
|
||||
"attention_dropout": 0.0,
|
||||
"attention_head_dim": 128,
|
||||
"auto_map": {
|
||||
"AutoConfig": "configuration_nemotron_h.NemotronHConfig",
|
||||
"AutoModelForCausalLM": "modeling_nemotron_h.NemotronHForCausalLM"
|
||||
},
|
||||
"bos_token_id": 1,
|
||||
"chunk_size": 256,
|
||||
"conv_kernel": 4,
|
||||
"dtype": "bfloat16",
|
||||
"eos_token_id": 2,
|
||||
"expand": 2,
|
||||
"head_dim": 128,
|
||||
"hidden_dropout": 0.0,
|
||||
"hidden_size": 3136,
|
||||
"hybrid_override_pattern": "M-M-M-MM-M-M*-M-M*-M-M-M*-M-M-MM*-MMM-M-M-",
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 12544,
|
||||
"layer_norm_epsilon": 1e-05,
|
||||
"mamba_head_dim": 80,
|
||||
"mamba_hidden_act": "silu",
|
||||
"mamba_num_heads": 96,
|
||||
"mamba_proj_bias": false,
|
||||
"max_position_embeddings": 262144,
|
||||
"mlp_bias": false,
|
||||
"mlp_hidden_act": "relu2",
|
||||
"model_type": "nemotron_h",
|
||||
"n_groups": 8,
|
||||
"num_attention_heads": 40,
|
||||
"num_hidden_layers": 42,
|
||||
"num_key_value_heads": 8,
|
||||
"num_logits_to_keep": 1,
|
||||
"pad_token_id": 0,
|
||||
"rescale_prenorm_residual": true,
|
||||
"residual_in_fp32": false,
|
||||
"rms_norm_eps": 1e-05,
|
||||
"sliding_window": null,
|
||||
"ssm_state_size": 128,
|
||||
"tie_word_embeddings": false,
|
||||
"time_step_floor": 0.0001,
|
||||
"time_step_limit": [
|
||||
0.0,
|
||||
Infinity
|
||||
],
|
||||
"time_step_max": 0.1,
|
||||
"time_step_min": 0.001,
|
||||
"time_step_rank": 256,
|
||||
"transformers_version": "4.57.6",
|
||||
"use_bias": false,
|
||||
"use_cache": true,
|
||||
"use_conv_bias": true,
|
||||
"use_mamba_kernels": true,
|
||||
"vocab_size": 131072
|
||||
}
|
||||
@@ -0,0 +1,243 @@
|
||||
# coding=utf-8
|
||||
# Copyright 2024 AI21 Labs Ltd. and the HuggingFace Inc. team. All rights reserved.
|
||||
# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""NemotronH model configuration"""
|
||||
|
||||
import re
|
||||
|
||||
from transformers.configuration_utils import PretrainedConfig
|
||||
from transformers.utils import logging
|
||||
|
||||
|
||||
logger = logging.get_logger(__name__)
|
||||
|
||||
|
||||
class NemotronHConfig(PretrainedConfig):
|
||||
r"""
|
||||
This is the configuration class to store the configuration of a [`NemotronHModel`]. It is used to instantiate a
|
||||
NemotronH model according to the specified arguments, defining the model architecture. Instantiating a configuration
|
||||
with the defaults will yield a similar configuration to that of the NemotronH-v0.1 model.
|
||||
|
||||
[todo](todo)
|
||||
|
||||
Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
|
||||
documentation from [`PretrainedConfig`] for more information.
|
||||
|
||||
|
||||
Args:
|
||||
vocab_size (`int`, *optional*, defaults to 131072):
|
||||
Vocabulary size of the NemotronH model. Defines the number of different tokens that can be represented by the
|
||||
`inputs_ids` passed when calling [`NemotronHModel`]
|
||||
tie_word_embeddings (`bool`, *optional*, defaults to `False`):
|
||||
Whether the model's input and output word embeddings should be tied. Note that this is only relevant if the
|
||||
model has a output word embedding layer.
|
||||
hidden_size (`int`, *optional*, defaults to 4096):
|
||||
Dimension of the hidden representations.
|
||||
intermediate_size (`int`, *optional*, defaults to 21504):
|
||||
Dimension of the MLP representations.
|
||||
num_hidden_layers (`int`, *optional*, defaults to 52):
|
||||
Number of hidden layers in the Transformer encoder.
|
||||
hybrid_override_pattern (`str`, *optional*, defaults to `"M-M-M-M*-M-M-M-M-M*-M-M-M-M-M*-M-M-M-M-M*-M-M-M-M-M-"`):
|
||||
The pattern of the hybrid model. The pattern is a string of characters where each character represents M: Mamba2, *: Attention, -: MLP
|
||||
num_attention_heads (`int`, *optional*, defaults to 32):
|
||||
Number of attention heads for each attention layer in the Transformer encoder.
|
||||
attention_head_dim (`int`, *optional*, defaults to 128):
|
||||
Dimension of each attention head.
|
||||
num_key_value_heads (`int`, *optional*, defaults to 8):
|
||||
This is the number of key_value heads that should be used to implement Grouped Query Attention. If
|
||||
`num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
|
||||
`num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used.
|
||||
mlp_hidden_act (`str`, *optional*, defaults to "relu2"):
|
||||
The non-linear activation function in the MLP layers.
|
||||
attention_bias (`bool`, *optional*, defaults to `False`):
|
||||
Whether to use bias in attention layers.
|
||||
mlp_bias (`bool`, *optional*, defaults to `False`):
|
||||
Whether to use bias in MLP layers.
|
||||
use_bias (`bool`, *optional*, defaults to `False`):
|
||||
Whether to use bias in the model.
|
||||
initializer_range (`float`, *optional*, defaults to 0.02):
|
||||
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
|
||||
layer_norm_epsilon (`float`, *optional*, defaults to 1e-5):
|
||||
The epsilon used by the layer normalization layers.
|
||||
residual_in_fp32 (`bool`, *optional*, defaults to `False`):
|
||||
Whether or not residuals should be in `float32`. If set to `False` residuals will keep the same `dtype` as the rest of the model.
|
||||
use_cache (`bool`, *optional*, defaults to `True`):
|
||||
Whether or not the model should return the last key/values attentions (not used by all models). Only
|
||||
relevant if `config.is_decoder=True`.
|
||||
num_logits_to_keep (`int` or `None`, *optional*, defaults to 1):
|
||||
Number of prompt logits to calculate during generation. If `None`, all logits will be calculated. If an
|
||||
integer value, only last `num_logits_to_keep` logits will be calculated.
|
||||
pad_token_id (`int`, *optional*, defaults to 0):
|
||||
The id of the padding token.
|
||||
bos_token_id (`int`, *optional*, defaults to 1):
|
||||
The id of the "beginning-of-sequence" token.
|
||||
eos_token_id (`int`, *optional*, defaults to 2):
|
||||
The id of the "end-of-sequence" token.
|
||||
sliding_window (`int`, *optional*, defaults to None):
|
||||
Sliding window attention window size.
|
||||
max_position_embeddings (`int`, *optional*, defaults to 4096):
|
||||
The maximum sequence length that this model might ever be used with.
|
||||
attention_dropout (`float`, *optional*, defaults to 0.0):
|
||||
The dropout ratio for the attention probabilities.
|
||||
hidden_dropout (`float`, *optional*, defaults to 0.0):
|
||||
The dropout ratio for the hidden states.
|
||||
use_mamba_kernels (`bool`, *optional*, defaults to `True`):
|
||||
Flag indicating whether or not to use the fast mamba kernels. These are available only if `mamba-ssm` and
|
||||
`causal-conv1d` are installed, and the mamba modules are running on a CUDA device.
|
||||
ssm_state_size (`int`, *optional*, defaults to 128):
|
||||
The dimension of the mamba state space latents.
|
||||
mamba_num_heads (`int`, *optional*, defaults to 128):
|
||||
Number of heads in Mamba layers.
|
||||
mamba_n_groups (`int`, *optional*, defaults to 8):
|
||||
Number of groups in Mamba layers.
|
||||
mamba_head_dim (`int`, *optional*, defaults to 64):
|
||||
Dimension of each Mamba head.
|
||||
mamba_d_conv (`int`, *optional*, defaults to 4):
|
||||
The size of the mamba convolution kernel.
|
||||
mamba_expand (`int`, *optional*, defaults to 2):
|
||||
Expanding factor used to determine the mamba intermediate size.
|
||||
mamba_hidden_act (`str`, *optional*, defaults to "silu"):
|
||||
The non-linear activation function in the Mamba layers.
|
||||
mamba_dt_min (`float`, *optional*, defaults to 0.001):
|
||||
Minimum value for the time step in Mamba.
|
||||
mamba_dt_max (`float`, *optional*, defaults to 0.1):
|
||||
Maximum value for the time step in Mamba.
|
||||
mamba_dt_limit (`tuple`, *optional*, defaults to (0.0, float("inf"))):
|
||||
Limits for the time step in Mamba.
|
||||
mamba_dt_init_floor (`float`, *optional*, defaults to 1e-4):
|
||||
Floor value for time step initialization in Mamba.
|
||||
mamba_conv_bias (`bool`, *optional*, defaults to `True`):
|
||||
Whether to use bias in the convolution layer of the mamba mixer block.
|
||||
mamba_proj_bias (`bool`, *optional*, defaults to `False`):
|
||||
Whether to use bias in the input and output projections of the mamba mixer block.
|
||||
mamba_chunk_size (`int`, *optional*, defaults to 256):
|
||||
Size of chunks for Mamba processing.
|
||||
rescale_prenorm_residual (`bool`, *optional*, defaults to `True`):
|
||||
Whether to rescale the pre-normalization residual connections.
|
||||
"""
|
||||
|
||||
model_type = "nemotron_h"
|
||||
keys_to_ignore_at_inference = ["past_key_values"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vocab_size=131072,
|
||||
tie_word_embeddings=False,
|
||||
hidden_size=4096,
|
||||
intermediate_size=21504,
|
||||
num_hidden_layers=52,
|
||||
hybrid_override_pattern="M-M-M-M*-M-M-M-M-M*-M-M-M-M-M*-M-M-M-M-M*-M-M-M-M-M-",
|
||||
num_attention_heads=32,
|
||||
attention_head_dim=128,
|
||||
num_key_value_heads=8, # nemo: num_query_groups
|
||||
mlp_hidden_act="relu2",
|
||||
attention_bias=False,
|
||||
mlp_bias=False,
|
||||
use_bias=False,
|
||||
initializer_range=0.02, # nemo: init_method_std
|
||||
layer_norm_epsilon=1e-5, # nemo: layernorm_epsilon
|
||||
residual_in_fp32=False, # Megatron Core default value
|
||||
use_cache=True,
|
||||
num_logits_to_keep=1,
|
||||
pad_token_id=0,
|
||||
bos_token_id=1,
|
||||
eos_token_id=2,
|
||||
sliding_window=None,
|
||||
max_position_embeddings=4096,
|
||||
attention_dropout=0.0,
|
||||
hidden_dropout=0.0, # * ADDED
|
||||
use_mamba_kernels=True,
|
||||
ssm_state_size=128, # mamba_state_size
|
||||
mamba_num_heads=128,
|
||||
mamba_n_groups=8, # nemo: mamba_ssm_ngroups = num_heads
|
||||
mamba_head_dim=64,
|
||||
mamba_d_conv=4,
|
||||
mamba_expand=2,
|
||||
mamba_hidden_act="silu",
|
||||
mamba_dt_min=0.001,
|
||||
mamba_dt_max=0.1,
|
||||
mamba_dt_limit=(0.0, float("inf")),
|
||||
mamba_dt_init_floor=1e-4,
|
||||
mamba_conv_bias=True,
|
||||
mamba_proj_bias=False,
|
||||
mamba_chunk_size=256,
|
||||
rescale_prenorm_residual=True,
|
||||
**kwargs,
|
||||
):
|
||||
self.vocab_size = vocab_size
|
||||
self.tie_word_embeddings = tie_word_embeddings
|
||||
self.hidden_size = hidden_size
|
||||
self.intermediate_size = intermediate_size
|
||||
self.num_hidden_layers = num_hidden_layers
|
||||
self.hybrid_override_pattern = hybrid_override_pattern
|
||||
self.num_attention_heads = num_attention_heads
|
||||
self.attention_head_dim = attention_head_dim
|
||||
self.sliding_window = sliding_window
|
||||
self.max_position_embeddings = max_position_embeddings
|
||||
self.attention_dropout = attention_dropout
|
||||
self.hidden_dropout = hidden_dropout
|
||||
|
||||
# Validate hybrid_override_pattern
|
||||
# M: Mamba2, *: Attention, -: MLP
|
||||
assert len(self.hybrid_override_pattern) == self.num_hidden_layers, "hybrid_override_pattern must have the same length as num_hidden_layers"
|
||||
assert re.match(r"^[*-M]+$", self.hybrid_override_pattern), "hybrid_override_pattern must only contain characters 'M', '*', or '-'"
|
||||
|
||||
# for backward compatibility
|
||||
if num_key_value_heads is None:
|
||||
num_key_value_heads = num_attention_heads
|
||||
|
||||
self.num_key_value_heads = num_key_value_heads
|
||||
self.mlp_hidden_act = mlp_hidden_act
|
||||
self.attention_bias = attention_bias
|
||||
self.mlp_bias = mlp_bias
|
||||
self.use_bias = use_bias
|
||||
self.initializer_range = initializer_range
|
||||
self.layer_norm_epsilon = layer_norm_epsilon
|
||||
self.residual_in_fp32 = residual_in_fp32
|
||||
|
||||
self.use_cache = use_cache
|
||||
self.num_logits_to_keep = num_logits_to_keep
|
||||
|
||||
self.use_mamba_kernels = use_mamba_kernels
|
||||
self.n_groups = mamba_n_groups
|
||||
self.mamba_head_dim = mamba_head_dim
|
||||
self.ssm_state_size = ssm_state_size
|
||||
self.mamba_num_heads = mamba_num_heads
|
||||
self.conv_kernel = mamba_d_conv
|
||||
self.expand = mamba_expand
|
||||
self.mamba_hidden_act = mamba_hidden_act
|
||||
self.time_step_min = mamba_dt_min
|
||||
self.time_step_max = mamba_dt_max
|
||||
self.time_step_limit = mamba_dt_limit
|
||||
self.time_step_floor = mamba_dt_init_floor
|
||||
self.use_conv_bias = mamba_conv_bias
|
||||
self.mamba_proj_bias = mamba_proj_bias
|
||||
self.chunk_size = mamba_chunk_size
|
||||
self.rescale_prenorm_residual = rescale_prenorm_residual
|
||||
|
||||
super().__init__(
|
||||
pad_token_id=pad_token_id,
|
||||
bos_token_id=bos_token_id,
|
||||
eos_token_id=eos_token_id,
|
||||
tie_word_embeddings=tie_word_embeddings,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
@property
|
||||
def layers_block_type(self):
|
||||
return [
|
||||
"mamba" if self.hybrid_override_pattern[i] == "M" else
|
||||
"attention" if self.hybrid_override_pattern[i] == "*" else "mlp"
|
||||
for i in range(self.num_hidden_layers)]
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"adapter_dir": "/checkpoints/figment_sft_v6/figment-sft-v6-lora",
|
||||
"base_model_id": "nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16",
|
||||
"dtype": "bfloat16",
|
||||
"files": [
|
||||
"chat_template.jinja",
|
||||
"config.json",
|
||||
"configuration_nemotron_h.py",
|
||||
"generation_config.json",
|
||||
"model-00001-of-00002.safetensors",
|
||||
"model-00002-of-00002.safetensors",
|
||||
"model.safetensors.index.json",
|
||||
"modeling_nemotron_h.py",
|
||||
"special_tokens_map.json",
|
||||
"tokenizer.json",
|
||||
"tokenizer_config.json"
|
||||
],
|
||||
"merge_method": "peft.merge_and_unload(safe_merge=True)",
|
||||
"output_dir": "/checkpoints/figment_sft_v6/figment-sft-v6-lora-merged-bf16"
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"_from_model_config": true,
|
||||
"bos_token_id": 1,
|
||||
"eos_token_id": 2,
|
||||
"pad_token_id": 0,
|
||||
"temperature": null,
|
||||
"top_p": null,
|
||||
"transformers_version": "4.57.6"
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:7eabefdcd7a2114c2423127b3a014b6224f536a2ab08293706c563d1c6b82ffe
|
||||
size 3973314912
|
||||
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:d4dde81a91c84ebeb38093ede809548492c43aa983867b623dac97d477068356
|
||||
size 3973827728
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user