From bfeb4429d17800676d7ed47fee335b2058d0f864 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Tue, 22 Sep 2026 00:11:13 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: CohereLabs/tiny-aya-base-32K Source: Original Platform --- .gitattributes | 48 ++++ README.md | 187 ++++++++++++++ config.json | 80 ++++++ configuration.json | 1 + generation_config.json | 7 + model.safetensors.index.json | 478 +++++++++++++++++++++++++++++++++++ special_tokens_map.json | 40 +++ tokenizer.json | 3 + tokenizer_config.json | 202 +++++++++++++++ worker-000-000.safetensors | 3 + worker-001-000.safetensors | 3 + worker-002-000.safetensors | 3 + worker-003-000.safetensors | 3 + 13 files changed, 1058 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 config.json create mode 100644 configuration.json create mode 100644 generation_config.json create mode 100644 model.safetensors.index.json create mode 100644 special_tokens_map.json create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json create mode 100644 worker-000-000.safetensors create mode 100644 worker-001-000.safetensors create mode 100644 worker-002-000.safetensors create mode 100644 worker-003-000.safetensors diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..a8e0d12 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,48 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bin.* filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zstandard filter=lfs diff=lfs merge=lfs -text +*.tfevents* filter=lfs diff=lfs merge=lfs -text +*.db* filter=lfs diff=lfs merge=lfs -text +*.ark* filter=lfs diff=lfs merge=lfs -text +**/*ckpt*data* filter=lfs diff=lfs merge=lfs -text +**/*ckpt*.meta filter=lfs diff=lfs merge=lfs -text +**/*ckpt*.index filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.gguf* filter=lfs diff=lfs merge=lfs -text +*.ggml filter=lfs diff=lfs merge=lfs -text +*.llamafile* filter=lfs diff=lfs merge=lfs -text +*.pt2 filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +"/tokenizer.json" filter=lfs diff=lfs merge=lfs -text \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..136c601 --- /dev/null +++ b/README.md @@ -0,0 +1,187 @@ +--- +language: +- multilingual +- am +- ar +- eu +- bn +- bg +- ca +- zh +- cs +- en +- fil +- fi +- fr +- de +- el +- ha +- he +- hi +- hu +- ig +- id +- ga +- it +- ja +- jv +- km +- ko +- lt +- ms +- mt +- no +- fa +- pl +- pa +- ru +- sk +- sw +- ta +- te +- th +- tr +- uk +- ur +- vi +- yo +- zu +license: cc-by-nc-4.0 +library_name: transformers +pipeline_tag: text-generation +tags: +- aya +- multilingual +- long-context +- tiny-aya +extra_gated_heading: "Acknowledge license to accept the repository" +extra_gated_button_content: "Accept license" +extra_gated_prompt: > + By submitting this form, you agree to the License Agreement and acknowledge + that the information you provide will be collected, used, and shared in + accordance with Cohere's Privacy Policy. You'll receive email updates about + Cohere Labs and Cohere research, events, products and services. You can + unsubscribe at any time. +extra_gated_fields: + Name: text + Email: text + Affiliation: text + Country: country + I agree to use this model for non-commercial use ONLY: checkbox +--- + +# Model Card for Tiny Aya Base 32K + +## Model Summary + +Cohere Labs Tiny Aya Base 32K is an open-weights research release of a pretrained 3.35 billion parameter model optimized for efficient, strong, and balanced multilingual representation across 70+ languages, including many lower-resourced ones. This variant provides a 32K context window and is designed to support downstream adaptation, instruction tuning and long-context research. + +This is a base pretrained model and has not been instruction-tuned or preference-aligned. + +This checkpoint is used as the base model for [Tiny Aya L2-Thinker](https://huggingface.co/CohereLabs/tiny-aya-l2-thinker) and [Tiny Aya En-Thinker](https://huggingface.co/CohereLabs/tiny-aya-en-thinker). + +Developed by: [Cohere](https://cohere.com/) and [Cohere Labs](https://cohere.com/research) + +- Point of Contact: [Cohere Labs](https://cohere.com/research) +- License: [CC-BY-NC](https://cohere.com/cohere-labs-cc-by-nc-license), requires also adhering to [Cohere Lab's Acceptable Use Policy](https://docs.cohere.com/docs/cohere-labs-acceptable-use-policy) +- Model: tiny-aya-base-32K +- Model Size: 3.35B +- Context Length: 32K (input + output) + +For the 8K release, see [tiny-aya-base](https://huggingface.co/CohereLabs/tiny-aya-base). For the broader family, see [tiny-aya-global](https://huggingface.co/CohereLabs/tiny-aya-global) and the [Tiny Aya collection](https://huggingface.co/collections/CohereLabs/tiny-aya). + +## Usage + +```python +from transformers import AutoModelForCausalLM, AutoTokenizer + +model_id = "CohereLabs/tiny-aya-base-32K" +tokenizer = AutoTokenizer.from_pretrained(model_id) +model = AutoModelForCausalLM.from_pretrained( + model_id, + device_map="auto", + torch_dtype="auto", +) + +prompt = "The capital of Spain is" +inputs = tokenizer(prompt, return_tensors="pt").to(model.device) + +outputs = model.generate( + **inputs, + max_new_tokens=128, + do_sample=True, + temperature=0.1, + top_p=0.9, + top_k=50, +) +print(tokenizer.decode(outputs[0], skip_special_tokens=True)) +``` + +The model can also be used with the Transformers `pipeline` abstraction: + +```python +from transformers import pipeline + +generator = pipeline( + "text-generation", + model="CohereLabs/tiny-aya-base-32K", + torch_dtype="auto", + device_map="auto", +) + +output = generator( + "The capital of Spain is", + max_new_tokens=128, + do_sample=True, + temperature=0.1, + top_p=0.9, + repetition_penalty=1.1, +) +print(output[0]["generated_text"]) +``` + +The combined prompt and generated sequence must fit within the 32K context window. + +## Model Details + +**Input:** Text only. + +**Output:** Model generates text. + +**Model Architecture:** Auto-regressive transformer in the Tiny Aya / Cohere family, using interleaved sliding-window and global-attention layers with rotary positional embeddings. + +**Languages covered:** The model was trained on 70+ languages, with a focus on English, Dutch, French, Italian, Portuguese, Romanian, Spanish, Czech, Polish, Ukrainian, Russian, Greek, German, Danish, Swedish, Norwegian, Catalan, Galician, Welsh, Irish, Basque, Croatian, Latvian, Lithuanian, Slovak, Slovenian, Estonian, Finnish, Hungarian, Serbian, Bulgarian, Arabic, Persian, Urdu, Turkish, Maltese, Hebrew, Hindi, Marathi, Bengali, Gujarati, Punjabi, Tamil, Telugu, Nepali, Tagalog, Malay, Indonesian, Vietnamese, Javanese, Khmer, Thai, Lao, Chinese, Burmese, Japanese, Korean, Amharic, Hausa, Igbo, Malagasy, Shona, Swahili, Wolof, Xhosa, Yoruba, and Zulu. + +**Context Length:** Tiny Aya Base 32K supports a context length of 32K tokens across input and output. + +## Usage and Limitations + +### Intended Usage + +Tiny Aya Base 32K is intended for continued pretraining, instruction tuning, and research on multilingual and long-context language modeling. Potential downstream applications include multilingual text generation, summarization, translation, cross-lingual tasks, and adaptation for lower-resource languages. + +Because this is a base model, prompts should use text completion rather than a chat template. Additional post-training is recommended before deploying it as a conversational assistant. + +### Strengths + +Tiny Aya provides strong open-ended generation across broad language coverage, including many lower-resource languages. Shared training signals across language families and scripts support multilingual generation, translation, summarization, and cross-lingual adaptation. This release supports substantially longer sequences than the 8K base release. + +### Limitations + +**Base-model behavior.** This checkpoint is not instruction-tuned or preference-aligned and may not reliably follow instructions or behave like a conversational assistant. + +**Reasoning tasks.** The model's strongest performance is on open-ended generation. Chain-of-thought reasoning tasks such as multilingual math are comparatively weaker. + +**Factual knowledge.** Outputs may contain incorrect or outdated statements, particularly in lower-resource languages with thinner training data coverage. + +**Uneven resource distribution.** High-resource languages benefit from richer training signals and tend to show more consistent quality. The lowest-resource languages may show greater variability, and culturally specific nuance, sarcasm, or figurative language may be less reliable. + +**Long-context use.** A 32K context window does not guarantee uniform recall or reasoning quality at every position. Long inputs also require substantially more memory and compute. + +## Model Card Contact + +For errors or additional questions about details in this model card, contact [labs@cohere.com](mailto:labs@cohere.com). + +## Terms of Use + +This model is governed by a [CC-BY-NC License](https://cohere.com/cohere-labs-cc-by-nc-license) (Non-Commercial) and also requires adhering to [Cohere Lab's Acceptable Use Policy](https://docs.cohere.com/docs/cohere-labs-acceptable-use-policy). If you are interested in commercial use, please contact Cohere’s Sales team. diff --git a/config.json b/config.json new file mode 100644 index 0000000..99630ca --- /dev/null +++ b/config.json @@ -0,0 +1,80 @@ +{ + "_sliding_window_pattern": 4, + "architectures": [ + "Cohere2ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 2, + "dtype": "bfloat16", + "eos_token_id": 3, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 11008, + "layer_norm_eps": 1e-05, + "layer_switch": 4, + "layer_types": [ + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention", + "sliding_attention", + "sliding_attention", + "sliding_attention", + "full_attention" + ], + "logit_scale": 1.0, + "max_position_embeddings": 5000000, + "model_type": "cohere2", + "norm_type": "layer_norm", + "num_attention_heads": 16, + "num_hidden_layers": 36, + "num_key_value_heads": 4, + "order_of_interleaved_layers": "local_attn_first", + "pad_token_id": 0, + "position_embedding_type": "rope_gptj", + "rms_norm_eps": null, + "rope_scaling": null, + "rope_style": "interleave", + "rope_theta": 50000, + "rotary_pct": 1.0, + "sliding_window": 4096, + "transformer_block_type": "parallel", + "transformers_version": "4.56.2.4", + "use_cache": true, + "use_embedding_sharing": true, + "use_gated_activation": true, + "use_parallel_embedding": false, + "use_qk_norm": false, + "vocab_size": 262144 +} diff --git a/configuration.json b/configuration.json new file mode 100644 index 0000000..bbeeda1 --- /dev/null +++ b/configuration.json @@ -0,0 +1 @@ +{"framework": "pytorch", "task": "text-generation", "allow_remote": true} \ No newline at end of file diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..f9ac001 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,7 @@ +{ + "_from_model_config": true, + "bos_token_id": 2, + "eos_token_id": 3, + "pad_token_id": 0, + "transformers_version": "4.56.2.4" +} diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000..e568fb0 --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,478 @@ +{ + "weight_map": { + "model.layers.10.mlp.up_proj.weight": "worker-001-000.safetensors", + "model.layers.10.mlp.gate_proj.weight": "worker-001-000.safetensors", + "model.layers.11.mlp.up_proj.weight": "worker-001-000.safetensors", + "model.layers.11.mlp.gate_proj.weight": "worker-001-000.safetensors", + "model.layers.12.mlp.up_proj.weight": "worker-001-000.safetensors", + "model.layers.12.mlp.gate_proj.weight": "worker-001-000.safetensors", + "model.layers.13.mlp.up_proj.weight": "worker-001-000.safetensors", + "model.layers.13.mlp.gate_proj.weight": "worker-001-000.safetensors", + "model.layers.14.mlp.up_proj.weight": "worker-001-000.safetensors", + "model.layers.14.mlp.gate_proj.weight": "worker-001-000.safetensors", + "model.layers.15.mlp.up_proj.weight": "worker-001-000.safetensors", + "model.layers.15.mlp.gate_proj.weight": "worker-001-000.safetensors", + "model.layers.16.mlp.up_proj.weight": "worker-001-000.safetensors", + "model.layers.16.mlp.gate_proj.weight": "worker-001-000.safetensors", + "model.layers.17.mlp.up_proj.weight": "worker-001-000.safetensors", + "model.layers.17.mlp.gate_proj.weight": "worker-001-000.safetensors", + "model.layers.18.mlp.up_proj.weight": "worker-001-000.safetensors", + "model.layers.18.mlp.gate_proj.weight": "worker-001-000.safetensors", + "model.layers.19.mlp.up_proj.weight": "worker-001-000.safetensors", + "model.layers.19.mlp.gate_proj.weight": "worker-001-000.safetensors", + "model.layers.20.mlp.up_proj.weight": "worker-001-000.safetensors", + "model.layers.20.mlp.gate_proj.weight": "worker-001-000.safetensors", + "model.layers.21.mlp.up_proj.weight": "worker-001-000.safetensors", + "model.layers.21.mlp.gate_proj.weight": "worker-001-000.safetensors", + "model.layers.22.mlp.up_proj.weight": "worker-001-000.safetensors", + "model.layers.22.mlp.gate_proj.weight": "worker-001-000.safetensors", + "model.layers.23.mlp.up_proj.weight": "worker-001-000.safetensors", + "model.layers.23.mlp.gate_proj.weight": "worker-001-000.safetensors", + "model.layers.24.mlp.up_proj.weight": "worker-001-000.safetensors", + "model.layers.24.mlp.gate_proj.weight": "worker-001-000.safetensors", + "model.layers.25.mlp.up_proj.weight": "worker-001-000.safetensors", + "model.layers.25.mlp.gate_proj.weight": "worker-001-000.safetensors", + "model.layers.26.mlp.up_proj.weight": "worker-001-000.safetensors", + "model.layers.26.mlp.gate_proj.weight": "worker-001-000.safetensors", + "model.layers.27.mlp.up_proj.weight": "worker-001-000.safetensors", + "model.layers.27.mlp.gate_proj.weight": "worker-001-000.safetensors", + "model.layers.28.mlp.up_proj.weight": "worker-001-000.safetensors", + "model.layers.28.mlp.gate_proj.weight": "worker-001-000.safetensors", + "model.layers.29.mlp.up_proj.weight": "worker-001-000.safetensors", + "model.layers.29.mlp.gate_proj.weight": "worker-001-000.safetensors", + "model.layers.30.mlp.up_proj.weight": "worker-001-000.safetensors", + "model.layers.30.mlp.gate_proj.weight": "worker-001-000.safetensors", + "model.layers.31.mlp.up_proj.weight": "worker-001-000.safetensors", + "model.layers.31.mlp.gate_proj.weight": "worker-001-000.safetensors", + "model.layers.1.self_attn.q_proj.weight": "worker-001-000.safetensors", + "model.layers.1.self_attn.k_proj.weight": "worker-001-000.safetensors", + "model.layers.1.self_attn.v_proj.weight": "worker-001-000.safetensors", + "model.layers.2.self_attn.q_proj.weight": "worker-003-000.safetensors", + "model.layers.2.self_attn.o_proj.weight": "worker-003-000.safetensors", + "model.layers.3.self_attn.q_proj.weight": "worker-003-000.safetensors", + "model.layers.3.self_attn.o_proj.weight": "worker-003-000.safetensors", + "model.layers.4.self_attn.q_proj.weight": "worker-003-000.safetensors", + "model.layers.4.self_attn.o_proj.weight": "worker-003-000.safetensors", + "model.layers.5.self_attn.q_proj.weight": "worker-003-000.safetensors", + "model.layers.5.self_attn.o_proj.weight": "worker-003-000.safetensors", + "model.layers.6.self_attn.q_proj.weight": "worker-003-000.safetensors", + "model.layers.6.self_attn.o_proj.weight": "worker-003-000.safetensors", + "model.layers.7.self_attn.q_proj.weight": "worker-003-000.safetensors", + "model.layers.7.self_attn.o_proj.weight": "worker-003-000.safetensors", + "model.layers.8.self_attn.q_proj.weight": "worker-003-000.safetensors", + "model.layers.8.self_attn.o_proj.weight": "worker-003-000.safetensors", + "model.layers.9.self_attn.q_proj.weight": "worker-003-000.safetensors", + "model.layers.9.self_attn.o_proj.weight": "worker-003-000.safetensors", + "model.layers.10.self_attn.q_proj.weight": "worker-003-000.safetensors", + "model.layers.10.self_attn.o_proj.weight": "worker-003-000.safetensors", + "model.layers.11.self_attn.q_proj.weight": "worker-003-000.safetensors", + "model.layers.11.self_attn.o_proj.weight": "worker-003-000.safetensors", + "model.layers.12.self_attn.q_proj.weight": "worker-003-000.safetensors", + "model.layers.12.self_attn.o_proj.weight": "worker-003-000.safetensors", + "model.layers.13.self_attn.q_proj.weight": "worker-003-000.safetensors", + "model.layers.13.self_attn.o_proj.weight": "worker-003-000.safetensors", + "model.layers.14.self_attn.q_proj.weight": "worker-003-000.safetensors", + "model.layers.14.self_attn.o_proj.weight": "worker-003-000.safetensors", + "model.layers.15.self_attn.q_proj.weight": "worker-003-000.safetensors", + "model.layers.15.self_attn.o_proj.weight": "worker-003-000.safetensors", + "model.layers.16.self_attn.q_proj.weight": "worker-003-000.safetensors", + "model.layers.16.self_attn.o_proj.weight": "worker-003-000.safetensors", + "model.layers.17.self_attn.q_proj.weight": "worker-003-000.safetensors", + "model.layers.17.self_attn.o_proj.weight": "worker-003-000.safetensors", + "model.layers.18.self_attn.q_proj.weight": "worker-003-000.safetensors", + "model.layers.18.self_attn.o_proj.weight": "worker-003-000.safetensors", + "model.layers.19.self_attn.q_proj.weight": "worker-003-000.safetensors", + "model.layers.19.self_attn.o_proj.weight": "worker-003-000.safetensors", + "model.layers.20.self_attn.q_proj.weight": "worker-003-000.safetensors", + "model.layers.20.self_attn.o_proj.weight": "worker-003-000.safetensors", + "model.layers.21.self_attn.q_proj.weight": "worker-003-000.safetensors", + "model.layers.21.self_attn.o_proj.weight": "worker-003-000.safetensors", + "model.layers.22.self_attn.q_proj.weight": "worker-003-000.safetensors", + "model.layers.22.self_attn.o_proj.weight": "worker-003-000.safetensors", + "model.layers.23.self_attn.q_proj.weight": "worker-003-000.safetensors", + "model.layers.23.self_attn.o_proj.weight": "worker-003-000.safetensors", + "model.layers.24.self_attn.q_proj.weight": "worker-003-000.safetensors", + "model.layers.24.self_attn.o_proj.weight": "worker-003-000.safetensors", + "model.layers.25.self_attn.q_proj.weight": "worker-003-000.safetensors", + "model.layers.25.self_attn.o_proj.weight": "worker-003-000.safetensors", + "model.layers.26.self_attn.q_proj.weight": "worker-003-000.safetensors", + "model.layers.26.self_attn.o_proj.weight": "worker-003-000.safetensors", + "model.layers.27.self_attn.q_proj.weight": "worker-003-000.safetensors", + "model.layers.27.self_attn.o_proj.weight": "worker-003-000.safetensors", + "model.layers.28.self_attn.q_proj.weight": "worker-003-000.safetensors", + "model.layers.28.self_attn.o_proj.weight": "worker-003-000.safetensors", + "model.layers.29.self_attn.q_proj.weight": "worker-003-000.safetensors", + "model.layers.29.self_attn.o_proj.weight": "worker-003-000.safetensors", + "model.layers.30.self_attn.q_proj.weight": "worker-003-000.safetensors", + "model.layers.30.self_attn.o_proj.weight": "worker-003-000.safetensors", + "model.layers.31.self_attn.q_proj.weight": "worker-003-000.safetensors", + "model.layers.31.self_attn.o_proj.weight": "worker-003-000.safetensors", + "model.layers.32.self_attn.q_proj.weight": "worker-003-000.safetensors", + "model.layers.32.self_attn.o_proj.weight": "worker-003-000.safetensors", + "model.layers.33.self_attn.q_proj.weight": "worker-003-000.safetensors", + "model.layers.33.self_attn.o_proj.weight": "worker-003-000.safetensors", + "model.layers.34.self_attn.q_proj.weight": "worker-003-000.safetensors", + "model.layers.34.self_attn.o_proj.weight": "worker-003-000.safetensors", + "model.layers.35.self_attn.q_proj.weight": "worker-003-000.safetensors", + "model.layers.35.self_attn.o_proj.weight": "worker-003-000.safetensors", + "model.layers.3.self_attn.k_proj.weight": "worker-003-000.safetensors", + "model.layers.3.self_attn.v_proj.weight": "worker-003-000.safetensors", + "model.layers.4.self_attn.k_proj.weight": "worker-003-000.safetensors", + "model.layers.4.self_attn.v_proj.weight": "worker-003-000.safetensors", + "model.layers.5.self_attn.k_proj.weight": "worker-003-000.safetensors", + "model.layers.5.self_attn.v_proj.weight": "worker-003-000.safetensors", + "model.layers.6.self_attn.k_proj.weight": "worker-003-000.safetensors", + "model.layers.6.self_attn.v_proj.weight": "worker-003-000.safetensors", + "model.layers.7.self_attn.k_proj.weight": "worker-003-000.safetensors", + "model.layers.7.self_attn.v_proj.weight": "worker-003-000.safetensors", + "model.layers.8.self_attn.k_proj.weight": "worker-003-000.safetensors", + "model.layers.8.self_attn.v_proj.weight": "worker-003-000.safetensors", + "model.layers.9.self_attn.k_proj.weight": "worker-003-000.safetensors", + "model.layers.9.self_attn.v_proj.weight": "worker-003-000.safetensors", + "model.layers.10.self_attn.k_proj.weight": "worker-003-000.safetensors", + "model.layers.10.self_attn.v_proj.weight": "worker-003-000.safetensors", + "model.layers.11.self_attn.k_proj.weight": "worker-003-000.safetensors", + "model.layers.11.self_attn.v_proj.weight": "worker-003-000.safetensors", + "model.layers.12.self_attn.k_proj.weight": "worker-003-000.safetensors", + "model.layers.12.self_attn.v_proj.weight": "worker-003-000.safetensors", + "model.layers.13.self_attn.k_proj.weight": "worker-003-000.safetensors", + "model.layers.13.self_attn.v_proj.weight": "worker-003-000.safetensors", + "model.layers.14.self_attn.k_proj.weight": "worker-003-000.safetensors", + "model.layers.14.self_attn.v_proj.weight": "worker-003-000.safetensors", + "model.layers.15.self_attn.k_proj.weight": "worker-003-000.safetensors", + "model.layers.15.self_attn.v_proj.weight": "worker-003-000.safetensors", + "model.layers.16.self_attn.k_proj.weight": "worker-003-000.safetensors", + "model.layers.16.self_attn.v_proj.weight": "worker-003-000.safetensors", + "model.layers.17.self_attn.k_proj.weight": "worker-003-000.safetensors", + "model.layers.17.self_attn.v_proj.weight": "worker-003-000.safetensors", + "model.layers.18.self_attn.k_proj.weight": "worker-003-000.safetensors", + "model.layers.18.self_attn.v_proj.weight": "worker-003-000.safetensors", + "model.layers.19.self_attn.k_proj.weight": "worker-003-000.safetensors", + "model.layers.19.self_attn.v_proj.weight": "worker-003-000.safetensors", + "model.layers.20.self_attn.k_proj.weight": "worker-003-000.safetensors", + "model.layers.20.self_attn.v_proj.weight": "worker-003-000.safetensors", + "model.layers.21.self_attn.k_proj.weight": "worker-003-000.safetensors", + "model.layers.21.self_attn.v_proj.weight": "worker-003-000.safetensors", + "model.layers.22.self_attn.k_proj.weight": "worker-003-000.safetensors", + "model.layers.22.self_attn.v_proj.weight": "worker-003-000.safetensors", + "model.layers.23.self_attn.k_proj.weight": "worker-003-000.safetensors", + "model.layers.23.self_attn.v_proj.weight": "worker-003-000.safetensors", + "model.layers.24.self_attn.k_proj.weight": "worker-003-000.safetensors", + "model.layers.24.self_attn.v_proj.weight": "worker-003-000.safetensors", + "model.layers.25.self_attn.k_proj.weight": "worker-003-000.safetensors", + "model.layers.25.self_attn.v_proj.weight": "worker-003-000.safetensors", + "model.layers.26.self_attn.k_proj.weight": "worker-003-000.safetensors", + "model.layers.26.self_attn.v_proj.weight": "worker-003-000.safetensors", + "model.layers.27.self_attn.k_proj.weight": "worker-003-000.safetensors", + "model.layers.27.self_attn.v_proj.weight": "worker-003-000.safetensors", + "model.layers.28.self_attn.k_proj.weight": "worker-003-000.safetensors", + "model.layers.28.self_attn.v_proj.weight": "worker-003-000.safetensors", + "model.layers.29.self_attn.k_proj.weight": "worker-003-000.safetensors", + "model.layers.29.self_attn.v_proj.weight": "worker-003-000.safetensors", + "model.layers.30.self_attn.k_proj.weight": "worker-003-000.safetensors", + "model.layers.30.self_attn.v_proj.weight": "worker-003-000.safetensors", + "model.layers.31.self_attn.k_proj.weight": "worker-003-000.safetensors", + "model.layers.31.self_attn.v_proj.weight": "worker-003-000.safetensors", + "model.layers.32.self_attn.k_proj.weight": "worker-003-000.safetensors", + "model.layers.32.self_attn.v_proj.weight": "worker-003-000.safetensors", + "model.layers.33.self_attn.k_proj.weight": "worker-003-000.safetensors", + "model.layers.33.self_attn.v_proj.weight": "worker-003-000.safetensors", + "model.layers.34.self_attn.k_proj.weight": "worker-003-000.safetensors", + "model.layers.34.self_attn.v_proj.weight": "worker-003-000.safetensors", + "model.layers.35.self_attn.k_proj.weight": "worker-003-000.safetensors", + "model.layers.35.self_attn.v_proj.weight": "worker-003-000.safetensors", + "model.embed_tokens.weight": "worker-000-000.safetensors", + "model.layers.0.mlp.up_proj.weight": "worker-000-000.safetensors", + "model.layers.0.mlp.gate_proj.weight": "worker-000-000.safetensors", + "model.layers.1.mlp.up_proj.weight": "worker-000-000.safetensors", + "model.layers.1.mlp.gate_proj.weight": "worker-000-000.safetensors", + "model.layers.2.mlp.up_proj.weight": "worker-000-000.safetensors", + "model.layers.2.mlp.gate_proj.weight": "worker-000-000.safetensors", + "model.layers.3.mlp.up_proj.weight": "worker-000-000.safetensors", + "model.layers.3.mlp.gate_proj.weight": "worker-000-000.safetensors", + "model.layers.4.mlp.up_proj.weight": "worker-000-000.safetensors", + "model.layers.4.mlp.gate_proj.weight": "worker-000-000.safetensors", + "model.layers.5.mlp.up_proj.weight": "worker-000-000.safetensors", + "model.layers.5.mlp.gate_proj.weight": "worker-000-000.safetensors", + "model.layers.6.mlp.up_proj.weight": "worker-000-000.safetensors", + "model.layers.6.mlp.gate_proj.weight": "worker-000-000.safetensors", + "model.layers.7.mlp.up_proj.weight": "worker-000-000.safetensors", + "model.layers.7.mlp.gate_proj.weight": "worker-000-000.safetensors", + "model.layers.8.mlp.up_proj.weight": "worker-000-000.safetensors", + "model.layers.8.mlp.gate_proj.weight": "worker-000-000.safetensors", + "model.layers.9.mlp.up_proj.weight": "worker-000-000.safetensors", + "model.layers.9.mlp.gate_proj.weight": "worker-000-000.safetensors", + "model.layers.0.self_attn.q_proj.weight": "worker-000-000.safetensors", + "model.layers.0.self_attn.o_proj.weight": "worker-000-000.safetensors", + "model.layers.0.self_attn.k_proj.weight": "worker-000-000.safetensors", + "model.layers.0.self_attn.v_proj.weight": "worker-000-000.safetensors", + "model.layers.0.mlp.up_proj.bias": "worker-000-000.safetensors", + "model.layers.0.mlp.gate_proj.bias": "worker-000-000.safetensors", + "model.layers.1.mlp.up_proj.bias": "worker-000-000.safetensors", + "model.layers.1.mlp.gate_proj.bias": "worker-000-000.safetensors", + "model.layers.2.mlp.up_proj.bias": "worker-000-000.safetensors", + "model.layers.2.mlp.gate_proj.bias": "worker-000-000.safetensors", + "model.layers.3.mlp.up_proj.bias": "worker-000-000.safetensors", + "model.layers.3.mlp.gate_proj.bias": "worker-000-000.safetensors", + "model.layers.4.mlp.up_proj.bias": "worker-000-000.safetensors", + "model.layers.4.mlp.gate_proj.bias": "worker-000-000.safetensors", + "model.layers.5.mlp.up_proj.bias": "worker-000-000.safetensors", + "model.layers.5.mlp.gate_proj.bias": "worker-000-000.safetensors", + "model.layers.6.mlp.up_proj.bias": "worker-000-000.safetensors", + "model.layers.6.mlp.gate_proj.bias": "worker-000-000.safetensors", + "model.layers.7.mlp.up_proj.bias": "worker-000-000.safetensors", + "model.layers.7.mlp.gate_proj.bias": "worker-000-000.safetensors", + "model.layers.8.mlp.up_proj.bias": "worker-000-000.safetensors", + "model.layers.8.mlp.gate_proj.bias": "worker-000-000.safetensors", + "model.layers.9.mlp.up_proj.bias": "worker-000-000.safetensors", + "model.layers.9.mlp.gate_proj.bias": "worker-000-000.safetensors", + "model.layers.10.mlp.up_proj.bias": "worker-000-000.safetensors", + "model.layers.10.mlp.gate_proj.bias": "worker-000-000.safetensors", + "model.layers.11.mlp.up_proj.bias": "worker-000-000.safetensors", + "model.layers.11.mlp.gate_proj.bias": "worker-000-000.safetensors", + "model.layers.12.mlp.up_proj.bias": "worker-000-000.safetensors", + "model.layers.12.mlp.gate_proj.bias": "worker-000-000.safetensors", + "model.layers.13.mlp.up_proj.bias": "worker-000-000.safetensors", + "model.layers.13.mlp.gate_proj.bias": "worker-000-000.safetensors", + "model.layers.14.mlp.up_proj.bias": "worker-000-000.safetensors", + "model.layers.14.mlp.gate_proj.bias": "worker-000-000.safetensors", + "model.layers.15.mlp.up_proj.bias": "worker-000-000.safetensors", + "model.layers.15.mlp.gate_proj.bias": "worker-000-000.safetensors", + "model.layers.16.mlp.up_proj.bias": "worker-000-000.safetensors", + "model.layers.16.mlp.gate_proj.bias": "worker-000-000.safetensors", + "model.layers.17.mlp.up_proj.bias": "worker-000-000.safetensors", + "model.layers.17.mlp.gate_proj.bias": "worker-000-000.safetensors", + "model.layers.18.mlp.up_proj.bias": "worker-000-000.safetensors", + "model.layers.18.mlp.gate_proj.bias": "worker-000-000.safetensors", + "model.layers.19.mlp.up_proj.bias": "worker-000-000.safetensors", + "model.layers.19.mlp.gate_proj.bias": "worker-000-000.safetensors", + "model.layers.20.mlp.up_proj.bias": "worker-000-000.safetensors", + "model.layers.20.mlp.gate_proj.bias": "worker-000-000.safetensors", + "model.layers.21.mlp.up_proj.bias": "worker-000-000.safetensors", + "model.layers.21.mlp.gate_proj.bias": "worker-000-000.safetensors", + "model.layers.22.mlp.up_proj.bias": "worker-000-000.safetensors", + "model.layers.22.mlp.gate_proj.bias": "worker-000-000.safetensors", + "model.layers.23.mlp.up_proj.bias": "worker-000-000.safetensors", + "model.layers.23.mlp.gate_proj.bias": "worker-000-000.safetensors", + "model.layers.24.mlp.up_proj.bias": "worker-000-000.safetensors", + "model.layers.24.mlp.gate_proj.bias": "worker-000-000.safetensors", + "model.layers.25.mlp.up_proj.bias": "worker-000-000.safetensors", + "model.layers.25.mlp.gate_proj.bias": "worker-000-000.safetensors", + "model.layers.26.mlp.up_proj.bias": "worker-000-000.safetensors", + "model.layers.26.mlp.gate_proj.bias": "worker-000-000.safetensors", + "model.layers.27.mlp.up_proj.bias": "worker-000-000.safetensors", + "model.layers.27.mlp.gate_proj.bias": "worker-000-000.safetensors", + "model.layers.28.mlp.up_proj.bias": "worker-000-000.safetensors", + "model.layers.28.mlp.gate_proj.bias": "worker-000-000.safetensors", + "model.layers.29.mlp.up_proj.bias": "worker-000-000.safetensors", + "model.layers.29.mlp.gate_proj.bias": "worker-000-000.safetensors", + "model.layers.30.mlp.up_proj.bias": "worker-000-000.safetensors", + "model.layers.30.mlp.gate_proj.bias": "worker-000-000.safetensors", + "model.layers.31.mlp.up_proj.bias": "worker-000-000.safetensors", + "model.layers.31.mlp.gate_proj.bias": "worker-000-000.safetensors", + "model.layers.32.mlp.up_proj.bias": "worker-000-000.safetensors", + "model.layers.32.mlp.gate_proj.bias": "worker-000-000.safetensors", + "model.layers.33.mlp.up_proj.bias": "worker-000-000.safetensors", + "model.layers.33.mlp.gate_proj.bias": "worker-000-000.safetensors", + "model.layers.34.mlp.up_proj.bias": "worker-000-000.safetensors", + "model.layers.34.mlp.gate_proj.bias": "worker-000-000.safetensors", + "model.layers.35.mlp.up_proj.bias": "worker-000-000.safetensors", + "model.layers.35.mlp.gate_proj.bias": "worker-000-000.safetensors", + "model.norm.weight": "worker-000-000.safetensors", + "model.norm.bias": "worker-000-000.safetensors", + "model.layers.0.input_layernorm.weight": "worker-000-000.safetensors", + "model.layers.0.input_layernorm.bias": "worker-000-000.safetensors", + "model.layers.0.self_attn.o_proj.bias": "worker-000-000.safetensors", + "model.layers.0.mlp.down_proj.bias": "worker-000-000.safetensors", + "model.layers.1.input_layernorm.weight": "worker-000-000.safetensors", + "model.layers.1.input_layernorm.bias": "worker-000-000.safetensors", + "model.layers.1.self_attn.o_proj.bias": "worker-000-000.safetensors", + "model.layers.1.mlp.down_proj.bias": "worker-000-000.safetensors", + "model.layers.2.input_layernorm.weight": "worker-000-000.safetensors", + "model.layers.2.input_layernorm.bias": "worker-000-000.safetensors", + "model.layers.2.self_attn.o_proj.bias": "worker-000-000.safetensors", + "model.layers.2.mlp.down_proj.bias": "worker-000-000.safetensors", + "model.layers.3.input_layernorm.weight": "worker-000-000.safetensors", + "model.layers.3.input_layernorm.bias": "worker-000-000.safetensors", + "model.layers.3.self_attn.o_proj.bias": "worker-000-000.safetensors", + "model.layers.3.mlp.down_proj.bias": "worker-000-000.safetensors", + "model.layers.4.input_layernorm.weight": "worker-000-000.safetensors", + "model.layers.4.input_layernorm.bias": "worker-000-000.safetensors", + "model.layers.4.self_attn.o_proj.bias": "worker-000-000.safetensors", + "model.layers.4.mlp.down_proj.bias": "worker-000-000.safetensors", + "model.layers.5.input_layernorm.weight": "worker-000-000.safetensors", + "model.layers.5.input_layernorm.bias": "worker-000-000.safetensors", + "model.layers.5.self_attn.o_proj.bias": "worker-000-000.safetensors", + "model.layers.5.mlp.down_proj.bias": "worker-000-000.safetensors", + "model.layers.6.input_layernorm.weight": "worker-000-000.safetensors", + "model.layers.6.input_layernorm.bias": "worker-000-000.safetensors", + "model.layers.6.self_attn.o_proj.bias": "worker-000-000.safetensors", + "model.layers.6.mlp.down_proj.bias": "worker-000-000.safetensors", + "model.layers.7.input_layernorm.weight": "worker-000-000.safetensors", + "model.layers.7.input_layernorm.bias": "worker-000-000.safetensors", + "model.layers.7.self_attn.o_proj.bias": "worker-000-000.safetensors", + "model.layers.7.mlp.down_proj.bias": "worker-000-000.safetensors", + "model.layers.8.input_layernorm.weight": "worker-000-000.safetensors", + "model.layers.8.input_layernorm.bias": "worker-000-000.safetensors", + "model.layers.8.self_attn.o_proj.bias": "worker-000-000.safetensors", + "model.layers.8.mlp.down_proj.bias": "worker-000-000.safetensors", + "model.layers.9.input_layernorm.weight": "worker-000-000.safetensors", + "model.layers.9.input_layernorm.bias": "worker-000-000.safetensors", + "model.layers.9.self_attn.o_proj.bias": "worker-000-000.safetensors", + "model.layers.9.mlp.down_proj.bias": "worker-000-000.safetensors", + "model.layers.10.input_layernorm.weight": "worker-000-000.safetensors", + "model.layers.10.input_layernorm.bias": "worker-000-000.safetensors", + "model.layers.10.self_attn.o_proj.bias": "worker-000-000.safetensors", + "model.layers.10.mlp.down_proj.bias": "worker-000-000.safetensors", + "model.layers.11.input_layernorm.weight": "worker-000-000.safetensors", + "model.layers.11.input_layernorm.bias": "worker-000-000.safetensors", + "model.layers.11.self_attn.o_proj.bias": "worker-000-000.safetensors", + "model.layers.11.mlp.down_proj.bias": "worker-000-000.safetensors", + "model.layers.12.input_layernorm.weight": "worker-000-000.safetensors", + "model.layers.12.input_layernorm.bias": "worker-000-000.safetensors", + "model.layers.12.self_attn.o_proj.bias": "worker-000-000.safetensors", + "model.layers.12.mlp.down_proj.bias": "worker-000-000.safetensors", + "model.layers.13.input_layernorm.weight": "worker-000-000.safetensors", + "model.layers.13.input_layernorm.bias": "worker-000-000.safetensors", + "model.layers.13.self_attn.o_proj.bias": "worker-000-000.safetensors", + "model.layers.13.mlp.down_proj.bias": "worker-000-000.safetensors", + "model.layers.14.input_layernorm.weight": "worker-000-000.safetensors", + "model.layers.14.input_layernorm.bias": "worker-000-000.safetensors", + "model.layers.14.self_attn.o_proj.bias": "worker-000-000.safetensors", + "model.layers.14.mlp.down_proj.bias": "worker-000-000.safetensors", + "model.layers.15.input_layernorm.weight": "worker-000-000.safetensors", + "model.layers.15.input_layernorm.bias": "worker-000-000.safetensors", + "model.layers.15.self_attn.o_proj.bias": "worker-000-000.safetensors", + "model.layers.15.mlp.down_proj.bias": "worker-000-000.safetensors", + "model.layers.16.input_layernorm.weight": "worker-000-000.safetensors", + "model.layers.16.input_layernorm.bias": "worker-000-000.safetensors", + "model.layers.16.self_attn.o_proj.bias": "worker-000-000.safetensors", + "model.layers.16.mlp.down_proj.bias": "worker-000-000.safetensors", + "model.layers.17.input_layernorm.weight": "worker-000-000.safetensors", + "model.layers.17.input_layernorm.bias": "worker-000-000.safetensors", + "model.layers.17.self_attn.o_proj.bias": "worker-000-000.safetensors", + "model.layers.17.mlp.down_proj.bias": "worker-000-000.safetensors", + "model.layers.18.input_layernorm.weight": "worker-000-000.safetensors", + "model.layers.18.input_layernorm.bias": "worker-000-000.safetensors", + "model.layers.18.self_attn.o_proj.bias": "worker-000-000.safetensors", + "model.layers.18.mlp.down_proj.bias": "worker-000-000.safetensors", + "model.layers.19.input_layernorm.weight": "worker-000-000.safetensors", + "model.layers.19.input_layernorm.bias": "worker-000-000.safetensors", + "model.layers.19.self_attn.o_proj.bias": "worker-000-000.safetensors", + "model.layers.19.mlp.down_proj.bias": "worker-000-000.safetensors", + "model.layers.20.input_layernorm.weight": "worker-000-000.safetensors", + "model.layers.20.input_layernorm.bias": "worker-000-000.safetensors", + "model.layers.20.self_attn.o_proj.bias": "worker-000-000.safetensors", + "model.layers.20.mlp.down_proj.bias": "worker-000-000.safetensors", + "model.layers.21.input_layernorm.weight": "worker-000-000.safetensors", + "model.layers.21.input_layernorm.bias": "worker-000-000.safetensors", + "model.layers.21.self_attn.o_proj.bias": "worker-000-000.safetensors", + "model.layers.21.mlp.down_proj.bias": "worker-000-000.safetensors", + "model.layers.22.input_layernorm.weight": "worker-000-000.safetensors", + "model.layers.22.input_layernorm.bias": "worker-000-000.safetensors", + "model.layers.22.self_attn.o_proj.bias": "worker-000-000.safetensors", + "model.layers.22.mlp.down_proj.bias": "worker-000-000.safetensors", + "model.layers.23.input_layernorm.weight": "worker-000-000.safetensors", + "model.layers.23.input_layernorm.bias": "worker-000-000.safetensors", + "model.layers.23.self_attn.o_proj.bias": "worker-000-000.safetensors", + "model.layers.23.mlp.down_proj.bias": "worker-000-000.safetensors", + "model.layers.24.input_layernorm.weight": "worker-000-000.safetensors", + "model.layers.24.input_layernorm.bias": "worker-000-000.safetensors", + "model.layers.24.self_attn.o_proj.bias": "worker-000-000.safetensors", + "model.layers.24.mlp.down_proj.bias": "worker-000-000.safetensors", + "model.layers.25.input_layernorm.weight": "worker-000-000.safetensors", + "model.layers.25.input_layernorm.bias": "worker-000-000.safetensors", + "model.layers.25.self_attn.o_proj.bias": "worker-000-000.safetensors", + "model.layers.25.mlp.down_proj.bias": "worker-000-000.safetensors", + "model.layers.26.input_layernorm.weight": "worker-000-000.safetensors", + "model.layers.26.input_layernorm.bias": "worker-000-000.safetensors", + "model.layers.26.self_attn.o_proj.bias": "worker-000-000.safetensors", + "model.layers.26.mlp.down_proj.bias": "worker-000-000.safetensors", + "model.layers.27.input_layernorm.weight": "worker-000-000.safetensors", + "model.layers.27.input_layernorm.bias": "worker-000-000.safetensors", + "model.layers.27.self_attn.o_proj.bias": "worker-000-000.safetensors", + "model.layers.27.mlp.down_proj.bias": "worker-000-000.safetensors", + "model.layers.28.input_layernorm.weight": "worker-000-000.safetensors", + "model.layers.28.input_layernorm.bias": "worker-000-000.safetensors", + "model.layers.28.self_attn.o_proj.bias": "worker-000-000.safetensors", + "model.layers.28.mlp.down_proj.bias": "worker-000-000.safetensors", + "model.layers.29.input_layernorm.weight": "worker-000-000.safetensors", + "model.layers.29.input_layernorm.bias": "worker-000-000.safetensors", + "model.layers.29.self_attn.o_proj.bias": "worker-000-000.safetensors", + "model.layers.29.mlp.down_proj.bias": "worker-000-000.safetensors", + "model.layers.30.input_layernorm.weight": "worker-000-000.safetensors", + "model.layers.30.input_layernorm.bias": "worker-000-000.safetensors", + "model.layers.30.self_attn.o_proj.bias": "worker-000-000.safetensors", + "model.layers.30.mlp.down_proj.bias": "worker-000-000.safetensors", + "model.layers.31.input_layernorm.weight": "worker-000-000.safetensors", + "model.layers.31.input_layernorm.bias": "worker-000-000.safetensors", + "model.layers.31.self_attn.o_proj.bias": "worker-000-000.safetensors", + "model.layers.31.mlp.down_proj.bias": "worker-000-000.safetensors", + "model.layers.32.input_layernorm.weight": "worker-000-000.safetensors", + "model.layers.32.input_layernorm.bias": "worker-000-000.safetensors", + "model.layers.32.self_attn.o_proj.bias": "worker-000-000.safetensors", + "model.layers.32.mlp.down_proj.bias": "worker-000-000.safetensors", + "model.layers.33.input_layernorm.weight": "worker-000-000.safetensors", + "model.layers.33.input_layernorm.bias": "worker-000-000.safetensors", + "model.layers.33.self_attn.o_proj.bias": "worker-000-000.safetensors", + "model.layers.33.mlp.down_proj.bias": "worker-000-000.safetensors", + "model.layers.34.input_layernorm.weight": "worker-000-000.safetensors", + "model.layers.34.input_layernorm.bias": "worker-000-000.safetensors", + "model.layers.34.self_attn.o_proj.bias": "worker-000-000.safetensors", + "model.layers.34.mlp.down_proj.bias": "worker-000-000.safetensors", + "model.layers.35.input_layernorm.weight": "worker-000-000.safetensors", + "model.layers.35.input_layernorm.bias": "worker-000-000.safetensors", + "model.layers.35.self_attn.o_proj.bias": "worker-000-000.safetensors", + "model.layers.35.mlp.down_proj.bias": "worker-000-000.safetensors", + "model.layers.32.mlp.up_proj.weight": "worker-002-000.safetensors", + "model.layers.32.mlp.gate_proj.weight": "worker-002-000.safetensors", + "model.layers.33.mlp.up_proj.weight": "worker-002-000.safetensors", + "model.layers.33.mlp.gate_proj.weight": "worker-002-000.safetensors", + "model.layers.34.mlp.up_proj.weight": "worker-002-000.safetensors", + "model.layers.34.mlp.gate_proj.weight": "worker-002-000.safetensors", + "model.layers.35.mlp.up_proj.weight": "worker-002-000.safetensors", + "model.layers.35.mlp.gate_proj.weight": "worker-002-000.safetensors", + "model.layers.0.mlp.down_proj.weight": "worker-002-000.safetensors", + "model.layers.1.mlp.down_proj.weight": "worker-002-000.safetensors", + "model.layers.2.mlp.down_proj.weight": "worker-002-000.safetensors", + "model.layers.3.mlp.down_proj.weight": "worker-002-000.safetensors", + "model.layers.4.mlp.down_proj.weight": "worker-002-000.safetensors", + "model.layers.5.mlp.down_proj.weight": "worker-002-000.safetensors", + "model.layers.6.mlp.down_proj.weight": "worker-002-000.safetensors", + "model.layers.7.mlp.down_proj.weight": "worker-002-000.safetensors", + "model.layers.8.mlp.down_proj.weight": "worker-002-000.safetensors", + "model.layers.9.mlp.down_proj.weight": "worker-002-000.safetensors", + "model.layers.10.mlp.down_proj.weight": "worker-002-000.safetensors", + "model.layers.11.mlp.down_proj.weight": "worker-002-000.safetensors", + "model.layers.12.mlp.down_proj.weight": "worker-002-000.safetensors", + "model.layers.13.mlp.down_proj.weight": "worker-002-000.safetensors", + "model.layers.14.mlp.down_proj.weight": "worker-002-000.safetensors", + "model.layers.15.mlp.down_proj.weight": "worker-002-000.safetensors", + "model.layers.16.mlp.down_proj.weight": "worker-002-000.safetensors", + "model.layers.17.mlp.down_proj.weight": "worker-002-000.safetensors", + "model.layers.18.mlp.down_proj.weight": "worker-002-000.safetensors", + "model.layers.19.mlp.down_proj.weight": "worker-002-000.safetensors", + "model.layers.20.mlp.down_proj.weight": "worker-002-000.safetensors", + "model.layers.21.mlp.down_proj.weight": "worker-002-000.safetensors", + "model.layers.22.mlp.down_proj.weight": "worker-002-000.safetensors", + "model.layers.23.mlp.down_proj.weight": "worker-002-000.safetensors", + "model.layers.24.mlp.down_proj.weight": "worker-002-000.safetensors", + "model.layers.25.mlp.down_proj.weight": "worker-002-000.safetensors", + "model.layers.26.mlp.down_proj.weight": "worker-002-000.safetensors", + "model.layers.27.mlp.down_proj.weight": "worker-002-000.safetensors", + "model.layers.28.mlp.down_proj.weight": "worker-002-000.safetensors", + "model.layers.29.mlp.down_proj.weight": "worker-002-000.safetensors", + "model.layers.30.mlp.down_proj.weight": "worker-002-000.safetensors", + "model.layers.31.mlp.down_proj.weight": "worker-002-000.safetensors", + "model.layers.32.mlp.down_proj.weight": "worker-002-000.safetensors", + "model.layers.33.mlp.down_proj.weight": "worker-002-000.safetensors", + "model.layers.34.mlp.down_proj.weight": "worker-002-000.safetensors", + "model.layers.35.mlp.down_proj.weight": "worker-002-000.safetensors", + "model.layers.1.self_attn.o_proj.weight": "worker-002-000.safetensors", + "model.layers.2.self_attn.k_proj.weight": "worker-002-000.safetensors", + "model.layers.2.self_attn.v_proj.weight": "worker-002-000.safetensors" + }, + "metadata": { + "total_size": 6700486656 + } +} \ No newline at end of file diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..553cc1f --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,40 @@ +{ + "bos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "unk_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "additional_special_tokens": [ + "<|START_RESPONSE|>", + "<|END_RESPONSE|>", + "<|START_ACTION|>", + "<|END_ACTION|>", + "<|START_TOOL_RESULT|>", + "<|END_TOOL_RESULT|>", + "<|START_THINKING|>", + "<|END_THINKING|>" + ] +} \ No newline at end of file diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..9505840 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55388a861271fb6b9eb631670e5eee0562bcddf204dc63494d98ee67f5551839 +size 30245131 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..e9080bb --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,202 @@ +{ + "add_bos_token": true, + "add_eos_token": false, + "add_prefix_space": false, + "clean_up_tokenization_spaces": false, + "legacy": true, + "spaces_between_special_tokens": false, + "use_default_system_prompt": false, + "bos_token": "", + "eos_token": "", + "pad_token": "", + "unk_token": "", + "tokenizer_class": "CohereTokenizerFast", + "model_max_length": 32768, + "sp_model_kwargs": {}, + "added_tokens_decoder": { + "0": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "1": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "2": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "3": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "4": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "5": { + "content": "<|START_OF_TURN_TOKEN|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "6": { + "content": "<|END_OF_TURN_TOKEN|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "7": { + "content": "<|USER_TOKEN|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "8": { + "content": "<|CHATBOT_TOKEN|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "9": { + "content": "<|SYSTEM_TOKEN|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "10": { + "content": "<|NEW_FILE|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "11": { + "content": "<|BEGINNING_OF_PREFIX_FIM_TOKEN|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "12": { + "content": "<|BEGINNING_OF_MIDDLE_FIM_TOKEN|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "13": { + "content": "<|BEGINNING_OF_SUFFIX_FIM_TOKEN|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "14": { + "content": "<|END_OF_MIDDLE_FIM_TOKEN|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "261000": { + "content": "<|START_RESPONSE|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "261001": { + "content": "<|END_RESPONSE|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "261002": { + "content": "<|START_ACTION|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "261003": { + "content": "<|END_ACTION|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "261004": { + "content": "<|START_TOOL_RESULT|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "261005": { + "content": "<|END_TOOL_RESULT|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "261006": { + "content": "<|START_THINKING|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "261007": { + "content": "<|END_THINKING|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + } +} \ No newline at end of file diff --git a/worker-000-000.safetensors b/worker-000-000.safetensors new file mode 100644 index 0000000..c8bba94 --- /dev/null +++ b/worker-000-000.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b8ed41b3ee2726ca0d701bbac53f7739e5f4f0dab235abd10aa75c94d64a1ad +size 1998698496 diff --git a/worker-001-000.safetensors b/worker-001-000.safetensors new file mode 100644 index 0000000..d7e83bf --- /dev/null +++ b/worker-001-000.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd9d7dab86ecdfdbfc120f0d0e9a17ff36da700a85c0d8943c14c4d702a9833a +size 1996494056 diff --git a/worker-002-000.safetensors b/worker-002-000.safetensors new file mode 100644 index 0000000..38af6a0 --- /dev/null +++ b/worker-002-000.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77efd96abd6c1c274e33a64c079d6c3ccbc31f7c3a34484d7af5a9f2a796fc90 +size 1996494088 diff --git a/worker-003-000.safetensors b/worker-003-000.safetensors new file mode 100644 index 0000000..64f940d --- /dev/null +++ b/worker-003-000.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92c66c9a943579db14b8a5451f3014a68155994857a2904df9f482a19c2397ec +size 708852792