commit 988a0f9abc9a621a043d6eeccc34da9c5c8cb395 Author: ModelHub XC Date: Mon Jul 20 05:19:09 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: flowxai/sentinel-gate Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..c36de7d --- /dev/null +++ b/.gitattributes @@ -0,0 +1,38 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text +gguf/sentinel-gate-4b-Q4_K_M.gguf filter=lfs diff=lfs merge=lfs -text +gguf/sentinel-gate-4b-Q8_0.gguf filter=lfs diff=lfs merge=lfs -text diff --git a/EVAL_RESULTS.md b/EVAL_RESULTS.md new file mode 100644 index 0000000..cbfc7dd --- /dev/null +++ b/EVAL_RESULTS.md @@ -0,0 +1,59 @@ +# FlowX Sentinel Gate (4B) — Evaluation Results + +Held-out evaluation of the released model (`flowx-sentinel-gate-4b-v2`). Numbers are on a +**held-out set of 71 cases (46 ESCALATE / 17 DECIDE)** across banking, insurance, logistics, +and labor. Decoding: greedy (temperature 0), `enable_thinking=False`. The safety-critical +metric is the **false-negative rate** (gold=ESCALATE but the model says DECIDE, i.e. +auto-deciding a case that should have gone to a human). + +## Headline (held-out, n=71) + +| Metric | Result | Note | +| --- | --- | --- | +| False-negative rate (missed escalations) | **0.000** (0/46) | the safety metric | +| Action accuracy (ESCALATE vs DECIDE) | **1.000** | | +| False-positive rate (over-escalation) | **0.000** (0/17) | | +| JSON validity (raw) | 0.89 | deploy with the deterministic repair step | +| Category accuracy (on true-escalate) | 0.61 | the human-routing hint | + +## Decision confusion (the gate's actual job) + +| gold \\ predicted | ESCALATE | DECIDE | +| --- | --- | --- | +| ESCALATE (n=46) | 46 | 0 | +| DECIDE (n=17) | 0 | 17 | + +The ESCALATE/DECIDE decision is correct on every held-out case: no missed escalations, no +over-escalation. That decision is what gates automation and is the field to trust. + +## Baseline (before the data rebalance/retrain) + +| Metric | baseline (n=24, 2 DECIDE) | this release (n=71, 17 DECIDE) | +| --- | --- | --- | +| False-negative rate | 0.000 | 0.000 | +| Action accuracy | 0.944 | 1.000 | +| False-positive rate | 0.50 (noisy, n=2) | 0.000 (trustworthy, n=17) | +| JSON validity | 0.75 | 0.89 | +| Category accuracy | 0.875 (n=16) | 0.61 (n=46) | + +Rebalancing DECIDE from ~15% to ~35% of the corpus and retraining made the false-positive +rate trustworthy (17 DECIDE cases vs 2) and lifted JSON validity 0.75 → 0.89 while holding the +safety metric at 0.000. The category-accuracy "drop" is the honest number emerging on a larger +held-out (the old 0.875 was small-sample noise on 16 cases). + +## Honest reading / caveats + +- **Category is a routing hint, not a gate.** The model gets the ESCALATE/DECIDE decision right + every time here, but the `escalation_category` label is ~0.61 (categories legitimately + overlap for some cases). Route on the decision; treat the category as a suggestion. +- **JSON validity 0.89 raw.** Deploy with the deterministic JSON repair step the pipeline pairs + with the model. +- **Home-field note.** Scenarios are realistic-synthetic, grounded in real regulatory + citations. Validate on your own case distribution before production. + +## Reproduction + +Held-out: `mlx_data/escalation_v2/valid.jsonl`. Scorer: +`eval_sentinel_4b.py mlx_data/escalation_v2/valid.jsonl`. + +_Author: Bogdan Răduță, Head of Research, FlowX.AI._ diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000..ac66afb --- /dev/null +++ b/NOTICE @@ -0,0 +1,34 @@ +FlowX.AI Regulatory Ontology Models +Copyright 2026 FlowX.AI + +Licensed under the Apache License, Version 2.0 (the "License"); you may not use +these model artifacts except in compliance with the License. You may obtain a +copy of the License at http://www.apache.org/licenses/LICENSE-2.0. + +This release includes two LoRA-fine-tuned models developed by FlowX.AI: + - FlowX Semantic Mapper (regulatory-text -> structured ontology JSON) + - FlowX Sentinel Gate (case -> ESCALATE / DECIDE with category + audit trail) + +Base model + Qwen3-4B, Copyright Alibaba Cloud, licensed under Apache-2.0 + (https://huggingface.co/Qwen/Qwen3-4B). These models are LoRA adaptations of + that base and inherit its Apache-2.0 terms. + +Training data provenance + - Semantic Mapper: real, verbatim regulatory / legislative text from official + public sources (US eCFR and state codes; EU EUR-Lex; France Legifrance; + Germany Gesetze im Internet; Romania legislatie.just.ro; UNECE/ADR), used for + ontology annotation. Official legislation is reproduced with source + acknowledgement; each training record carries its source URL. + - Sentinel Gate: realistic synthetic escalation scenarios authored by FlowX.AI, + grounded in real regulatory citations. + Concept labels use a controlled 252-concept taxonomy authored by FlowX.AI. + +Responsible use + These models are decision-support tools for regulatory/compliance workflows. + They are NOT legal advice and must not be the sole basis for a legal or + compliance decision. Outputs should be reviewed by a qualified professional. + +Attribution + If you use or redistribute these models, please retain this NOTICE and credit + "FlowX.AI". diff --git a/README.md b/README.md new file mode 100644 index 0000000..a001668 --- /dev/null +++ b/README.md @@ -0,0 +1,385 @@ +--- +license: apache-2.0 +language: + - en +base_model: Qwen/Qwen3-4B +library_name: transformers +pipeline_tag: text-generation +tags: + - lora + - regulatory + - compliance + - escalation + - decision-gate + - mlx + - gguf + - flowx +model-index: + - name: sentinel-gate + results: + - task: + type: text-generation + name: Escalation gate decision (ESCALATE vs DECIDE) on regulated-decision cases + dataset: + name: FlowX Sentinel held-out cases + type: flowxai/sentinel-gate + metrics: + - type: false_negative_rate + name: false-negative rate (missed escalations, the safety metric) + value: 0.000 + - type: accuracy + name: action accuracy (ESCALATE vs DECIDE) + value: 1.000 + - type: false_positive_rate + name: false-positive rate (over-escalation) + value: 0.000 + - type: accuracy + name: raw JSON validity (deploy with deterministic repair) + value: 0.89 + - type: accuracy + name: category accuracy on true-escalate (routing hint) + value: 0.61 +--- + +## Inference contract + +Running this model correctly requires its **frozen inference contract** - the exact +system prompt, output JSON schema, user-turn format, and decode spec it was trained +against. See [`inference_contract/`](./inference_contract): + +- [`INFERENCE.md`](./inference_contract/INFERENCE.md) - wiring guide: system prompt, user turn `Case:\n\n\nDecide: ESCALATE or DECIDE?`, decode settings (`enable_thinking=False`, temp 0, `max_new_tokens` ~1200), the six categories, and the **deterministic JSON repair step** (raw JSON validity 0.89) the deployed pipeline pairs with the model. +- [`prompt_sentinel_sys_v1.txt`](./inference_contract/prompt_sentinel_sys_v1.txt) - the system prompt, verbatim. +- [`schema_sentinel_v1.json`](./inference_contract/schema_sentinel_v1.json) - output JSON Schema for the oracle decision. + +Prompt version `sentinel_sys_v1`. Do not edit the prompt/schema; the weights are trained against them. + +# FlowX Sentinel Gate (4B) - the escalation decision gate + +**An on-device escalation gate for regulated decisions: given a complete case, it decides ESCALATE (route to a human) vs DECIDE (safe to automate), with a category, rationale, calibrated confidence, and an audit trail.** + +Sentinel Gate is a **LoRA fine-tune of Qwen3-4B** (Apache-2.0), built by FlowX.AI. It reads +a complete regulated-decision case - the domain facts plus the applicable `policy_schema` +(a `PDP...` policy id) - and returns a single strict JSON decision. It sits **after +[`flowxai/semantic-mapper`](https://huggingface.co/flowxai/semantic-mapper)** in a compliance +pipeline: the Mapper tags knowledge-base chunks, a policy layer assembles a case, and the +Sentinel Gate decides what is safe to automate versus what must be routed to a human. + +The product insight that shapes everything: **the gate's job is the ESCALATE/DECIDE +decision, and that decision is safety-critical, so it is the metric we optimize and the field +you gate on.** A compliance automation gate that misses an escalation is dangerous in a way +that a wrong routing label is not. So we lead with the false-negative rate (missed +escalations), hold it at **0.000**, and treat the escalation *category* as a secondary +routing hint (~0.61) rather than a second gate. The model never over-escalates either +(false-positive rate 0.000), which is what makes the automation worth having. + +Part of the **FlowX on-device model family**: +[`flowxai/caveat`](https://huggingface.co/flowxai/caveat), +[`flowxai/scam-guard-qwen06b`](https://huggingface.co/flowxai/scam-guard-qwen06b), +[`flowxai/scam-guard-qwen17b`](https://huggingface.co/flowxai/scam-guard-qwen17b), +[`flowxai/semantic-mapper`](https://huggingface.co/flowxai/semantic-mapper). + +> **Not legal advice - decision-support only.** Sentinel Gate is a compliance automation +> gate, not a legal opinion and not a substitute for a compliance officer. It decides +> *whether a case is safe to automate or must go to a human*; a human owns every escalated +> case. Deploy with the confidence threshold your risk posture requires. + +English at v1. Domains span **banking, insurance, logistics, and labor** (cases are EN). +On-device formats: **fp16 safetensors** (transformers/CUDA/vLLM), **MLX-quantized** (int4 + +int8, Apple Silicon), and **GGUF** (llama.cpp/CPU/Ollama, `Q8_0` + `Q4_K_M`). + +--- + +## How do I use it? + +Three copy-pasteable ways to turn a case into a decision. The system prompt is the exact +two-liner from [`prompt_sentinel_sys_v1.txt`](./inference_contract/prompt_sentinel_sys_v1.txt). +The user turn is `Case:\n\n\nDecide: ESCALATE or DECIDE?`. + +Real example input (an OFAC-screening wire transfer - a `POLICY_VIOLATION` escalation): + +``` +Case: +{ + "transaction_facts": { + "product": "international wire transfer", + "originator": "Crestwood Imports Inc", + "beneficiary_name": "Volna Trading LLC", + "beneficiary_country": "Cyprus", + "amount_usd": 118000, + "purpose": "machinery purchase", + "screening_result": "potential match - beneficiary owner on SDN-adjacent watchlist (50% rule concern)" + }, + "documents_provided": ["wire request form", "commercial invoice", "originator KYC on file"], + "documents_missing": [], + "policy_schema": "PDP.lending.payments.ofac_sanctions_block" +} + +Decide: ESCALATE or DECIDE? +``` + +### (a) transformers / CUDA (fp16 safetensors) + +The root of the repo is fp16 safetensors for transformers / vLLM. + +```python +from transformers import AutoModelForCausalLM, AutoTokenizer + +SYSTEM = ("You are an escalation gate for regulated decisions.\n" + "Determine: ESCALATE or DECIDE? Output ONLY JSON.") +tok = AutoTokenizer.from_pretrained("flowxai/sentinel-gate") +model = AutoModelForCausalLM.from_pretrained("flowxai/sentinel-gate", torch_dtype="float16", device_map="cuda") + +case = open("wire_case.json").read() # the case JSON above +user = f"Case:\n{case}\n\nDecide: ESCALATE or DECIDE?" + +prompt = tok.apply_chat_template( + [{"role": "system", "content": SYSTEM}, + {"role": "user", "content": user}], + add_generation_prompt=True, enable_thinking=False, return_tensors="pt", +).to("cuda") +out = model.generate(prompt, max_new_tokens=1200, do_sample=False, temperature=0.0) +raw = tok.decode(out[0][prompt.shape[-1]:], skip_special_tokens=True) +# then: deterministic JSON repair -> validate against schema_sentinel_v1.json +``` + +### (b) MLX (Apple Silicon) + +Off the MLX-quantized weights (int4/int8): + +```bash +mlx_lm.generate --model flowxai/sentinel-gate-mlx-int4 --temp 0 --max-tokens 1200 \ + --system-prompt "$(cat inference_contract/prompt_sentinel_sys_v1.txt)" \ + --prompt "$(printf 'Case:\n%s\n\nDecide: ESCALATE or DECIDE?' "$(cat wire_case.json)")" +``` + +```python +from mlx_lm import load, generate + +SYSTEM = open("inference_contract/prompt_sentinel_sys_v1.txt").read() +model, tok = load("flowxai/sentinel-gate-mlx-int4") +case = open("wire_case.json").read() +user = f"Case:\n{case}\n\nDecide: ESCALATE or DECIDE?" +prompt = tok.apply_chat_template( + [{"role": "system", "content": SYSTEM}, + {"role": "user", "content": user}], + add_generation_prompt=True, enable_thinking=False, +) +raw = generate(model, tok, prompt=prompt, max_tokens=1200, verbose=False) +``` + +### (c) Short note - decode discipline + +Two things are non-negotiable: **`enable_thinking=False`** (the adapter was trained on pure +JSON, so the default thinking template yields empty/degraded output) and **temp 0** (the gate +must be reproducible for audit). Give it **~1200 new tokens** - the oracle JSON can run long +and truncation is the main cause of invalid JSON. Always run the output through the +deterministic JSON repair step (see [`INFERENCE.md`](./inference_contract/INFERENCE.md)). + +All three backends return the **same strict JSON** for the wire-transfer case above: + +```json +{ + "action": "ESCALATE", + "escalation_category": "POLICY_VIOLATION", + "policy_violations": { + "ofac_potential_match": { + "triggered": true, + "policy": "PDP.lending.payments.ofac_sanctions_block", + "regulation": "31 CFR 501 / OFAC sanctions regulations; OFAC 50 Percent Rule; 31 CFR 1010 (BSA recordkeeping for funds transfers)", + "restriction": "Beneficiary ownership screening returned a potential SDN-adjacent match invoking the OFAC 50% Rule; payment is automatically held pending sanctions determination.", + "consequence": "Releasing the wire without OFAC clearance risks a sanctions violation, blocking/rejection obligations, and civil penalties; the transaction must be blocked from auto-release." + } + }, + "confidence_score": 0.33, + "confidence_reasoning": "Transaction documentation is complete, but a potential OFAC match on beneficiary ownership is a hard stop; only a sanctions analyst can clear, reject, or block the wire.", + "human_action_required": "OFAC/sanctions analyst to investigate the potential match, apply the 50% Rule analysis, and clear, reject, or block the wire per OFAC procedures.", + "audit_trail": [ + "Ran beneficiary screening; returned potential SDN-adjacent ownership match", + "Triggered OFAC 50 Percent Rule review condition", + "Matched POLICY_VIOLATION on sanctions block policy", + "Held wire and routed to sanctions analyst; auto-release prohibited" + ] +} +``` + +For a case with no blocking condition and complete documentation, the gate returns +`{"action": "DECIDE", "escalation_category": null, "decision": "...", ...}` with a high +`confidence_score` and `human_action_required: "NONE"`. + +--- + +## How it works + +``` + +----------------------------------+ + case JSON (facts + | Sentinel Gate 4B | + policy_schema: "PDP...") ---> | Qwen3-4B (LoRA), thinking off | ---> strict JSON + Decide: ESCALATE or DECIDE? | greedy decode -> JSON repair | + +----------------------------------+ + | + + action (ESCALATE | DECIDE) <- the gate + + escalation_category <- routing hint + + category-specific block + + confidence_score / reasoning + + human_action_required + + audit_trail +``` + +In the pipeline: `semantic-mapper` tags KB chunks -> a policy layer assembles a case -> +**Sentinel Gate decides automate vs route-to-human**. Under the hood, one turn is: +`apply_chat_template(enable_thinking=False) -> Qwen3-4B (LoRA fine-tuned) -> greedy decode -> +deterministic JSON repair -> validate against schema_sentinel_v1.json`. + +**The `action` field is the safety-critical output.** You branch your automation off it and +nothing else. The escalation_category, block, and confidence enrich the human hand-off but +never decide it. + +--- + +## Output schema + +A single strict JSON object. Full schema: +[`schema_sentinel_v1.json`](./inference_contract/schema_sentinel_v1.json). + +- `action` - `ESCALATE` | `DECIDE`. **The gate.** `ESCALATE` routes the case to a human; + `DECIDE` marks it safe to automate. +- `escalation_category` - one of the six ids when `action=ESCALATE`, else `null`. A + **routing hint** (~0.61), not a second gate. +- **category-specific block** - one object whose key depends on the category: + - `policy_violations` (POLICY_VIOLATION) - keyed by violation; each entry names `policy`, + `regulation`, `restriction`, `consequence`. + - `missing_preconditions` (MISSING_REQUIRED_DOCUMENTATION) - each entry names `required_by`, + `regulation`, `severity`, `reason`. + - `boundary_analysis` (BOUNDARY_CONDITION) - `policy_threshold`, the case value, + `distance_from_threshold`, `assessment`. + - `confidence_factors` (INSUFFICIENT_CONFIDENCE) - `ambiguous_signals[]`, `why_uncertain`. + - `conflicting_signals` (CONFLICTING_SIGNALS) - the competing `{source, value}` items. + - `external_dependency` (EXTERNAL_DEPENDENCY) - `awaiting`, `blocking_gate`. +- `confidence_score` - number 0.0–1.0. Calibrated; apply your own threshold. Advisory. +- `confidence_reasoning` - one to three sentences explaining the score and the decision. +- `human_action_required` - the concrete next step a human must take, or `"NONE"` for DECIDE. +- `audit_trail` - ordered, append-only list of reasoning steps and policy gates, for review. +- (DECIDE cases also carry `decision` / `selected_route` / `policy_gates_passed`.) + +**The six escalation categories:** `MISSING_REQUIRED_DOCUMENTATION`, `POLICY_VIOLATION`, +`BOUNDARY_CONDITION`, `INSUFFICIENT_CONFIDENCE`, `CONFLICTING_SIGNALS`, `EXTERNAL_DEPENDENCY`. + +--- + +## Evaluation + +Held-out set: **n=71 cases (46 escalate / 17 decide)**. Metrics in order of what they tell you. + +### The safety metric first: missed escalations + +The dangerous failure for a compliance gate is **waving a case through that should have gone +to a human** (a missed escalation, i.e. a false negative). That is the number to read first. + +| Metric | Result | What it means | +| --- | --- | --- | +| **False-negative rate (missed escalations)** | **0.000** (0/46) | **the safety metric** - every case that should escalate did | +| Action accuracy (ESCALATE vs DECIDE) | **1.000** | the gate decision is correct on every held-out case | +| False-positive rate (over-escalation) | **0.000** (0/17) | no safe-to-automate case was needlessly routed to a human | +| Raw JSON validity (no repair) | 0.89 | deploy with the deterministic repair step (see below) | +| Category accuracy (on true-escalate) | 0.61 | the human-routing label; categories legitimately overlap | + +### Decision confusion (ESCALATE vs DECIDE), n=71 + +Rows = gold, cols = predicted: + +| gold \ pred | ESCALATE | DECIDE | recall | +| --- | --- | --- | --- | +| ESCALATE | **46** | 0 | 1.000 (n=46) | +| DECIDE | 0 | **17** | 1.000 (n=17) | + +The diagonal is complete: **no missed escalations, no over-escalation, action decided +correctly every time** on this held-out set. This is the gate's job, and on this set it is +perfect. + +### Honest reading + +- **The decision is the product, and it is perfect on this set.** Gate on `action`. +- **The category is a secondary routing hint at ~0.61.** The six categories legitimately + overlap for some cases (a boundary case that is also a policy edge; a missing-doc case that + is also insufficient-confidence), so the label is genuinely ambiguous for a slice of + escalations. Use it to pick a specialist queue, not to make a correctness-critical branch, + and let a human re-label at intake. +- **Raw JSON validity is 0.89, so ship the deterministic repair step.** The deployed pipeline + pairs the model with a deterministic JSON repair/retry pass and schema validation; 0.89 is + the *raw* number before that pass, not the deployed one. +- **Home-field caveat.** These 71 cases are **realistic synthetic** - grounded in real + regulatory citations (31 CFR 1010, Solvency II, 49 CFR 172, Directive 2003/88/EC, Codul + muncii) but authored for the benchmark, not drawn from live traffic. The scenarios share the + distribution the model trained on. **Validate on your own case distribution before + production**; a perfect held-out gate is a necessary signal, not a promise for your traffic. + +--- + +## Formats + +| Path | Format | Runs on | +| --- | --- | --- | +| `/` (root) | fp16 safetensors | transformers / CUDA / vLLM | +| `mlx-int4/` | MLX int4 | Apple Silicon (smallest footprint) | +| `mlx-int8/` | MLX int8 | Apple Silicon (higher fidelity) | +| `gguf/*.gguf` | GGUF `Q8_0` / `Q4_K_M` | llama.cpp / CPU / Ollama | + +`Q8_0` is the recommended GGUF quant; MLX int4 is the fastest quality-holding path on Apple +Silicon. All formats use the same frozen inference contract (`enable_thinking=False`, temp 0, +`max_new_tokens` ~1200). + +--- + +## Intended use & limitations + +**Intended use.** A **compliance automation gate**: it decides which regulated-decision cases +are safe to automate and which must be routed to a human, with a structured rationale and an +audit trail for the hand-off. It sits after `semantic-mapper` and before a human queue. + +**Out of scope & limitations.** + +- **Not legal advice.** A verdict is an automation-routing signal, not a legal opinion. A + human owns every escalated case, and the recommended action always routes to a human. +- **Category label is ~61% accurate.** Gate on `action` (ESCALATE/DECIDE); treat + `escalation_category` as a routing suggestion, not ground truth. +- **JSON validity is 0.89 raw.** Deploy with the deterministic repair/retry step; do not rely + on raw output being parseable. +- **Evaluated on 71 realistic-synthetic cases.** No real client data was used in training or + eval. Validate on your own case distribution before production. +- **English, four domains (banking / insurance / logistics / labor) at v1.** Cases outside + this distribution - other domains, other languages, malformed `policy_schema` - are out of + scope and should default to escalation, not automation. +- **Depends on the input case being complete and correct.** The gate reasons over the facts it + is given; a case assembled with wrong or missing facts can produce a wrong decision. The + upstream policy layer and `semantic-mapper` are part of the trust boundary. + +--- + +## Training data + +**471 realistic-synthetic escalation cases** (400 train / 71 held-out), balanced **~35% DECIDE +/ 65% ESCALATE** across the six categories and four regulated domains (banking, insurance, +logistics, labor). Each case is **grounded in a real regulatory citation** (for example 31 CFR +1010, Solvency II, 49 CFR 172, Directive 2003/88/EC, Codul muncii). The scenarios are +**realistic synthetic** - authored to reflect real regulatory conditions - with **no real +client data**. The assistant turn in each record is the exact oracle JSON the model must emit, +with thinking disabled. + +## Fine-tuning + +LoRA (rank 32 / scale 16 / dropout 0.05), `num_layers -1` (all layers), from `Qwen/Qwen3-4B`. +Trained with **MLX-LM on Apple Silicon**: 400 iterations (~4 epochs), **cosine LR 5e-5 → +5e-6** with a 40-step warmup, `max_seq_length 2048`, seed 42, thinking disabled. + +> **Stability note.** An initial `1e-4` run **diverged**; `5e-5` with the 40-step warmup is +> the stable recipe. + +--- + +## License + +Apache-2.0 (weights and code). Copyright 2026 FlowX.AI. `NOTICE` present. Base model +`Qwen/Qwen3-4B` is Apache-2.0. Escalation scenarios are realistic synthetic, grounded in real +regulatory citations (no real client data). + +_Author: Bogdan Răduță, Head of Research, FlowX.AI._ diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..699ff8d --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..df14f86 --- /dev/null +++ b/config.json @@ -0,0 +1,30 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 151643, + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2560, + "initializer_range": 0.02, + "intermediate_size": 9728, + "max_position_embeddings": 40960, + "max_window_layers": 36, + "model_type": "qwen3", + "num_attention_heads": 32, + "num_hidden_layers": 36, + "num_key_value_heads": 8, + "rms_norm_eps": 1e-06, + "rope_scaling": null, + "rope_theta": 1000000, + "sliding_window": null, + "tie_word_embeddings": true, + "torch_dtype": "bfloat16", + "transformers_version": "4.51.0", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 151936 +} \ No newline at end of file diff --git a/gguf/sentinel-gate-4b-Q4_K_M.gguf b/gguf/sentinel-gate-4b-Q4_K_M.gguf new file mode 100644 index 0000000..1683182 --- /dev/null +++ b/gguf/sentinel-gate-4b-Q4_K_M.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b347edd82aaee5edbe44a36598b9eac4b8c6e9be16baaa36c3e929dbc163bf9 +size 2497280704 diff --git a/gguf/sentinel-gate-4b-Q8_0.gguf b/gguf/sentinel-gate-4b-Q8_0.gguf new file mode 100644 index 0000000..f5834b9 --- /dev/null +++ b/gguf/sentinel-gate-4b-Q8_0.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e4499f3f1d0dbf1ce6483619dc8326571c7c2151f17cad3f2828e1731f8091a +size 4280405184 diff --git a/inference_contract/INFERENCE.md b/inference_contract/INFERENCE.md new file mode 100644 index 0000000..bc0fa80 --- /dev/null +++ b/inference_contract/INFERENCE.md @@ -0,0 +1,105 @@ +# Inference contract - FlowX Sentinel Gate + +This is the **frozen inference contract** for `flowxai/sentinel-gate`: the exact system +prompt, user-turn format, decode settings, and output schema the weights were trained +against. Do not edit the prompt or schema; the LoRA was trained on them verbatim. + +Prompt version: `sentinel_sys_v1`. + +Files in this directory: + +- [`prompt_sentinel_sys_v1.txt`](./prompt_sentinel_sys_v1.txt) - the system prompt, verbatim. +- [`schema_sentinel_v1.json`](./schema_sentinel_v1.json) - JSON Schema for the oracle output. + +--- + +## System prompt (verbatim) + +The exact two-line system prompt is in +[`prompt_sentinel_sys_v1.txt`](./prompt_sentinel_sys_v1.txt): + +``` +You are an escalation gate for regulated decisions. +Determine: ESCALATE or DECIDE? Output ONLY JSON. +``` + +## User-turn format + +One case per turn. The case JSON carries the domain facts plus the applicable +`policy_schema` (a `PDP...` policy id), then a fixed trailing question: + +``` +Case: + + +Decide: ESCALATE or DECIDE? +``` + +The `Case:\n` prefix and the trailing `\n\nDecide: ESCALATE or DECIDE?` line are part of +the contract - keep them exactly. The model was trained with these delimiters framing the +case object. + +## Decode settings + +| Setting | Value | Why | +|---|---|---| +| `enable_thinking` | **`False`** | Qwen3-4B is a thinking model, but the adapter was trained on pure JSON with no thinking block. The default template yields empty/degraded output. Set this at `apply_chat_template`. | +| `temperature` | **`0`** (greedy) | Deterministic decisions; the gate must be reproducible for audit. | +| `max_new_tokens` | **~1200** | The oracle JSON (category block + reasoning + audit trail) can run long, especially for BOUNDARY_CONDITION and EXTERNAL_DEPENDENCY. Truncation is the main cause of invalid JSON. | + +## The six escalation categories + +Present as `escalation_category` when `action` is `ESCALATE` (it is `null` for `DECIDE`): + +1. `MISSING_REQUIRED_DOCUMENTATION` - a hard precondition document/evidence is absent. +2. `POLICY_VIOLATION` - a policy/regulation rule is triggered and blocks auto-release. +3. `BOUNDARY_CONDITION` - the case sits near a policy threshold; the edge needs a human read. +4. `INSUFFICIENT_CONFIDENCE` - the facts do not resolve the decision to an actionable degree. +5. `CONFLICTING_SIGNALS` - two or more trusted sources disagree materially. +6. `EXTERNAL_DEPENDENCY` - the decision is blocked awaiting an outside result (screening, ruling). + +Each category emits a category-specific block under a distinct key +(`policy_violations`, `missing_preconditions`, `boundary_analysis`, `confidence_factors`, +`conflicting_signals`, `external_dependency`). See `schema_sentinel_v1.json`. + +## Deterministic JSON repair (deploy with it) + +Raw JSON validity from the model is **0.89** on the held-out set. The deployed pipeline +pairs the model with a **deterministic JSON repair step**: parse the raw output; if it +fails, apply structural fixes (close unterminated strings/brackets, strip any trailing +prose after the final `}`, drop a leading thinking artifact if one leaks) and re-parse, then +validate against `schema_sentinel_v1.json`. On a repair failure, retry the decode once. Do +not rely on raw output being parseable; treat the repair step as part of the contract. + +## What to gate on + +- **Gate on the `action` field (ESCALATE vs DECIDE).** This is the decision the model is + for, and it is **perfect on the held-out set** (n=71): zero missed escalations + (false-negative rate 0.000) and zero over-escalation (false-positive rate 0.000). Wire + your automate-vs-route branch off `action` alone. +- **Treat `escalation_category` as a routing hint, not ground truth.** Category accuracy on + true-escalate is **~0.61**; the categories legitimately overlap for some cases (e.g. a + boundary case that is also a policy edge). Use it to pick a specialist queue, but do not + make correctness-critical branches depend on it, and let a human re-label at intake. +- `confidence_score` is calibrated per the training oracle; apply the threshold your risk + posture requires. It is advisory, not a second gate. + +## Minimal wiring (MLX) + +```python +from mlx_lm import load, generate + +SYSTEM = open("prompt_sentinel_sys_v1.txt").read() +model, tok = load("flowxai/sentinel-gate-mlx-int4") + +case_json = "" +user = f"Case:\n{case_json}\n\nDecide: ESCALATE or DECIDE?" + +prompt = tok.apply_chat_template( + [{"role": "system", "content": SYSTEM}, + {"role": "user", "content": user}], + add_generation_prompt=True, enable_thinking=False, +) +raw = generate(model, tok, prompt=prompt, max_tokens=1200, verbose=False) +# then: deterministic JSON repair -> validate against schema_sentinel_v1.json +``` diff --git a/inference_contract/prompt_sentinel_sys_v1.txt b/inference_contract/prompt_sentinel_sys_v1.txt new file mode 100644 index 0000000..79f083e --- /dev/null +++ b/inference_contract/prompt_sentinel_sys_v1.txt @@ -0,0 +1,2 @@ +You are an escalation gate for regulated decisions. +Determine: ESCALATE or DECIDE? Output ONLY JSON. \ No newline at end of file diff --git a/inference_contract/schema_sentinel_v1.json b/inference_contract/schema_sentinel_v1.json new file mode 100644 index 0000000..e3e99d0 --- /dev/null +++ b/inference_contract/schema_sentinel_v1.json @@ -0,0 +1,129 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://huggingface.co/flowxai/sentinel-gate/inference_contract/schema_sentinel_v1.json", + "title": "FlowX Sentinel Gate output (schema_sentinel_v1)", + "description": "The single JSON object the Sentinel Gate emits for one regulated-decision case. The gate decides ESCALATE (route to a human) vs DECIDE (safe to automate). When action=ESCALATE, escalation_category is one of six ids and a category-specific block is present; when action=DECIDE, escalation_category is null. Prompt version sentinel_sys_v1. Note: additionalProperties is intentionally true because the category-specific block key varies by category (policy_violations, missing_preconditions, boundary_analysis, confidence_factors, conflicting_signals, external_dependency) and DECIDE cases carry decision/rationale keys.", + "type": "object", + "additionalProperties": true, + "required": [ + "action", + "escalation_category", + "confidence_score", + "audit_trail" + ], + "properties": { + "action": { + "type": "string", + "description": "The gate decision. ESCALATE routes the case to a human; DECIDE marks it safe to automate. This is the field to gate on (perfect on the held-out set).", + "enum": ["ESCALATE", "DECIDE"] + }, + "escalation_category": { + "description": "The human-routing label for an escalation, or null when action=DECIDE. A secondary routing hint (~0.61 accuracy on true-escalate), not the gate.", + "type": ["string", "null"], + "enum": [ + "MISSING_REQUIRED_DOCUMENTATION", + "POLICY_VIOLATION", + "BOUNDARY_CONDITION", + "INSUFFICIENT_CONFIDENCE", + "CONFLICTING_SIGNALS", + "EXTERNAL_DEPENDENCY", + null + ] + }, + "confidence_score": { + "type": "number", + "description": "Calibrated confidence in the decision, 0.0-1.0. For ESCALATE this is typically the confidence that the case is safe to automate (low), so a low score supports escalation; for DECIDE it is the confidence in the auto-decision (high).", + "minimum": 0.0, + "maximum": 1.0 + }, + "confidence_reasoning": { + "type": "string", + "description": "One to three sentences explaining the confidence_score and why the case was escalated or auto-decided." + }, + "human_action_required": { + "type": "string", + "description": "For ESCALATE: the concrete next step a human owner must take (who does what). For DECIDE: the string \"NONE\"." + }, + "audit_trail": { + "type": "array", + "description": "Ordered, append-only log of the reasoning steps and policy gates evaluated, for compliance review.", + "items": { "type": "string" }, + "minItems": 1 + }, + "policy_violations": { + "type": "object", + "description": "Category-specific block for POLICY_VIOLATION. Keyed by violation id; each entry names the policy, regulation, restriction, and consequence.", + "additionalProperties": true + }, + "missing_preconditions": { + "type": "object", + "description": "Category-specific block for MISSING_REQUIRED_DOCUMENTATION. Keyed by the missing precondition; each entry names required_by, regulation, severity, and reason.", + "additionalProperties": true + }, + "boundary_analysis": { + "type": "object", + "description": "Category-specific block for BOUNDARY_CONDITION. Names the policy_threshold, the shipment/case value, distance_from_threshold, and an assessment of the edge case.", + "additionalProperties": true + }, + "confidence_factors": { + "type": "object", + "description": "Category-specific block for INSUFFICIENT_CONFIDENCE. Lists ambiguous_signals and why_uncertain.", + "additionalProperties": true + }, + "conflicting_signals": { + "type": "array", + "description": "Category-specific block for CONFLICTING_SIGNALS. The competing sources/values that disagree.", + "items": { "type": "object", "additionalProperties": true } + }, + "external_dependency": { + "type": "object", + "description": "Category-specific block for EXTERNAL_DEPENDENCY. Names what the decision is awaiting and the blocking_gate.", + "additionalProperties": true + }, + "escalation_path": { + "type": "string", + "description": "Optional routing hint naming the specialist queue or workflow that should own the escalation." + }, + "policy_gates_passed": { + "type": "array", + "description": "Optional list of policy gates that were checked and passed before the decision (present on some ESCALATE edge cases and on DECIDE cases).", + "items": { "type": "string" } + }, + "decision": { + "type": "string", + "description": "For DECIDE cases: the automated outcome selected (e.g. ROUTE_APPROVED)." + }, + "selected_route": { + "type": "string", + "description": "For DECIDE cases where a route/option is chosen: the selected option." + }, + "rationale": { + "type": "string", + "description": "For DECIDE cases: the plain rationale for auto-deciding (some records use confidence_reasoning for this)." + } + }, + "allOf": [ + { + "if": { "properties": { "action": { "const": "DECIDE" } } }, + "then": { "properties": { "escalation_category": { "type": "null" } } } + }, + { + "if": { "properties": { "action": { "const": "ESCALATE" } } }, + "then": { + "properties": { + "escalation_category": { + "type": "string", + "enum": [ + "MISSING_REQUIRED_DOCUMENTATION", + "POLICY_VIOLATION", + "BOUNDARY_CONDITION", + "INSUFFICIENT_CONFIDENCE", + "CONFLICTING_SIGNALS", + "EXTERNAL_DEPENDENCY" + ] + } + } + } + } + ] +} diff --git a/mlx-int4/README.md b/mlx-int4/README.md new file mode 100644 index 0000000..200f8d0 --- /dev/null +++ b/mlx-int4/README.md @@ -0,0 +1,7 @@ +--- +language: en +pipeline_tag: text-generation +library_name: mlx +tags: +- mlx +--- diff --git a/mlx-int4/chat_template.jinja b/mlx-int4/chat_template.jinja new file mode 100644 index 0000000..699ff8d --- /dev/null +++ b/mlx-int4/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/mlx-int4/config.json b/mlx-int4/config.json new file mode 100644 index 0000000..30b673c --- /dev/null +++ b/mlx-int4/config.json @@ -0,0 +1,40 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 151643, + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2560, + "initializer_range": 0.02, + "intermediate_size": 9728, + "max_position_embeddings": 40960, + "max_window_layers": 36, + "model_type": "qwen3", + "num_attention_heads": 32, + "num_hidden_layers": 36, + "num_key_value_heads": 8, + "quantization": { + "group_size": 64, + "bits": 4, + "mode": "affine" + }, + "quantization_config": { + "group_size": 64, + "bits": 4, + "mode": "affine" + }, + "rms_norm_eps": 1e-06, + "rope_scaling": null, + "rope_theta": 1000000, + "sliding_window": null, + "tie_word_embeddings": true, + "torch_dtype": "bfloat16", + "transformers_version": "4.51.0", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 151936 +} \ No newline at end of file diff --git a/mlx-int4/model.safetensors b/mlx-int4/model.safetensors new file mode 100644 index 0000000..d24fb97 --- /dev/null +++ b/mlx-int4/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c73bf73db975fc1872ceca7fc4493ebf41a71ec9a436e2288d646cc05aa1dd9f +size 2263022417 diff --git a/mlx-int4/model.safetensors.index.json b/mlx-int4/model.safetensors.index.json new file mode 100644 index 0000000..4741a21 --- /dev/null +++ b/mlx-int4/model.safetensors.index.json @@ -0,0 +1,912 @@ +{ + "metadata": { + "total_size": 2262920192, + "total_parameters": 4022468096 + }, + "weight_map": { + "model.embed_tokens.biases": "model.safetensors", + "model.embed_tokens.scales": "model.safetensors", + "model.embed_tokens.weight": "model.safetensors", + "model.layers.0.input_layernorm.weight": "model.safetensors", + "model.layers.0.mlp.down_proj.biases": "model.safetensors", + "model.layers.0.mlp.down_proj.scales": "model.safetensors", + "model.layers.0.mlp.down_proj.weight": "model.safetensors", + "model.layers.0.mlp.gate_proj.biases": "model.safetensors", + "model.layers.0.mlp.gate_proj.scales": "model.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model.safetensors", + "model.layers.0.mlp.up_proj.biases": "model.safetensors", + "model.layers.0.mlp.up_proj.scales": "model.safetensors", + "model.layers.0.mlp.up_proj.weight": "model.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model.safetensors", + "model.layers.0.self_attn.k_norm.weight": "model.safetensors", + "model.layers.0.self_attn.k_proj.biases": "model.safetensors", + "model.layers.0.self_attn.k_proj.scales": "model.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model.safetensors", + "model.layers.0.self_attn.o_proj.biases": "model.safetensors", + "model.layers.0.self_attn.o_proj.scales": "model.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model.safetensors", + "model.layers.0.self_attn.q_norm.weight": "model.safetensors", + "model.layers.0.self_attn.q_proj.biases": "model.safetensors", + "model.layers.0.self_attn.q_proj.scales": "model.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model.safetensors", + "model.layers.0.self_attn.v_proj.biases": "model.safetensors", + "model.layers.0.self_attn.v_proj.scales": "model.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model.safetensors", + "model.layers.1.input_layernorm.weight": "model.safetensors", + "model.layers.1.mlp.down_proj.biases": "model.safetensors", + "model.layers.1.mlp.down_proj.scales": "model.safetensors", + "model.layers.1.mlp.down_proj.weight": "model.safetensors", + "model.layers.1.mlp.gate_proj.biases": "model.safetensors", + "model.layers.1.mlp.gate_proj.scales": "model.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model.safetensors", + "model.layers.1.mlp.up_proj.biases": "model.safetensors", + "model.layers.1.mlp.up_proj.scales": "model.safetensors", + "model.layers.1.mlp.up_proj.weight": "model.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model.safetensors", + "model.layers.1.self_attn.k_norm.weight": "model.safetensors", + "model.layers.1.self_attn.k_proj.biases": "model.safetensors", + "model.layers.1.self_attn.k_proj.scales": "model.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model.safetensors", + "model.layers.1.self_attn.o_proj.biases": "model.safetensors", + "model.layers.1.self_attn.o_proj.scales": "model.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model.safetensors", + "model.layers.1.self_attn.q_norm.weight": "model.safetensors", + "model.layers.1.self_attn.q_proj.biases": "model.safetensors", + "model.layers.1.self_attn.q_proj.scales": "model.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model.safetensors", + "model.layers.1.self_attn.v_proj.biases": "model.safetensors", + "model.layers.1.self_attn.v_proj.scales": "model.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model.safetensors", + "model.layers.10.input_layernorm.weight": "model.safetensors", + "model.layers.10.mlp.down_proj.biases": "model.safetensors", + "model.layers.10.mlp.down_proj.scales": "model.safetensors", + "model.layers.10.mlp.down_proj.weight": "model.safetensors", + "model.layers.10.mlp.gate_proj.biases": "model.safetensors", + "model.layers.10.mlp.gate_proj.scales": "model.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model.safetensors", + "model.layers.10.mlp.up_proj.biases": "model.safetensors", + "model.layers.10.mlp.up_proj.scales": "model.safetensors", + "model.layers.10.mlp.up_proj.weight": "model.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model.safetensors", + "model.layers.10.self_attn.k_norm.weight": "model.safetensors", + "model.layers.10.self_attn.k_proj.biases": "model.safetensors", + "model.layers.10.self_attn.k_proj.scales": "model.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model.safetensors", + "model.layers.10.self_attn.o_proj.biases": "model.safetensors", + "model.layers.10.self_attn.o_proj.scales": "model.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model.safetensors", + "model.layers.10.self_attn.q_norm.weight": "model.safetensors", + "model.layers.10.self_attn.q_proj.biases": "model.safetensors", + "model.layers.10.self_attn.q_proj.scales": "model.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model.safetensors", + "model.layers.10.self_attn.v_proj.biases": "model.safetensors", + "model.layers.10.self_attn.v_proj.scales": "model.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model.safetensors", + "model.layers.11.input_layernorm.weight": "model.safetensors", + "model.layers.11.mlp.down_proj.biases": "model.safetensors", + "model.layers.11.mlp.down_proj.scales": "model.safetensors", + "model.layers.11.mlp.down_proj.weight": "model.safetensors", + "model.layers.11.mlp.gate_proj.biases": "model.safetensors", + "model.layers.11.mlp.gate_proj.scales": "model.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model.safetensors", + "model.layers.11.mlp.up_proj.biases": "model.safetensors", + "model.layers.11.mlp.up_proj.scales": "model.safetensors", + "model.layers.11.mlp.up_proj.weight": "model.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model.safetensors", + "model.layers.11.self_attn.k_norm.weight": "model.safetensors", + "model.layers.11.self_attn.k_proj.biases": "model.safetensors", + "model.layers.11.self_attn.k_proj.scales": "model.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model.safetensors", + "model.layers.11.self_attn.o_proj.biases": "model.safetensors", + "model.layers.11.self_attn.o_proj.scales": "model.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model.safetensors", + "model.layers.11.self_attn.q_norm.weight": "model.safetensors", + "model.layers.11.self_attn.q_proj.biases": "model.safetensors", + "model.layers.11.self_attn.q_proj.scales": "model.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model.safetensors", + "model.layers.11.self_attn.v_proj.biases": "model.safetensors", + "model.layers.11.self_attn.v_proj.scales": "model.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model.safetensors", + "model.layers.12.input_layernorm.weight": "model.safetensors", + "model.layers.12.mlp.down_proj.biases": "model.safetensors", + "model.layers.12.mlp.down_proj.scales": "model.safetensors", + "model.layers.12.mlp.down_proj.weight": "model.safetensors", + "model.layers.12.mlp.gate_proj.biases": "model.safetensors", + "model.layers.12.mlp.gate_proj.scales": "model.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model.safetensors", + "model.layers.12.mlp.up_proj.biases": "model.safetensors", + "model.layers.12.mlp.up_proj.scales": "model.safetensors", + "model.layers.12.mlp.up_proj.weight": "model.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model.safetensors", + "model.layers.12.self_attn.k_norm.weight": "model.safetensors", + "model.layers.12.self_attn.k_proj.biases": "model.safetensors", + "model.layers.12.self_attn.k_proj.scales": "model.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model.safetensors", + "model.layers.12.self_attn.o_proj.biases": "model.safetensors", + "model.layers.12.self_attn.o_proj.scales": "model.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model.safetensors", + "model.layers.12.self_attn.q_norm.weight": "model.safetensors", + "model.layers.12.self_attn.q_proj.biases": "model.safetensors", + "model.layers.12.self_attn.q_proj.scales": "model.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model.safetensors", + "model.layers.12.self_attn.v_proj.biases": "model.safetensors", + "model.layers.12.self_attn.v_proj.scales": "model.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model.safetensors", + "model.layers.13.input_layernorm.weight": "model.safetensors", + "model.layers.13.mlp.down_proj.biases": "model.safetensors", + "model.layers.13.mlp.down_proj.scales": "model.safetensors", + "model.layers.13.mlp.down_proj.weight": "model.safetensors", + "model.layers.13.mlp.gate_proj.biases": "model.safetensors", + "model.layers.13.mlp.gate_proj.scales": "model.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model.safetensors", + "model.layers.13.mlp.up_proj.biases": "model.safetensors", + "model.layers.13.mlp.up_proj.scales": "model.safetensors", + "model.layers.13.mlp.up_proj.weight": "model.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model.safetensors", + "model.layers.13.self_attn.k_norm.weight": "model.safetensors", + "model.layers.13.self_attn.k_proj.biases": "model.safetensors", + "model.layers.13.self_attn.k_proj.scales": "model.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model.safetensors", + "model.layers.13.self_attn.o_proj.biases": "model.safetensors", + "model.layers.13.self_attn.o_proj.scales": "model.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model.safetensors", + "model.layers.13.self_attn.q_norm.weight": "model.safetensors", + "model.layers.13.self_attn.q_proj.biases": "model.safetensors", + "model.layers.13.self_attn.q_proj.scales": "model.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model.safetensors", + "model.layers.13.self_attn.v_proj.biases": "model.safetensors", + "model.layers.13.self_attn.v_proj.scales": "model.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model.safetensors", + "model.layers.14.input_layernorm.weight": "model.safetensors", + "model.layers.14.mlp.down_proj.biases": "model.safetensors", + "model.layers.14.mlp.down_proj.scales": "model.safetensors", + "model.layers.14.mlp.down_proj.weight": "model.safetensors", + "model.layers.14.mlp.gate_proj.biases": "model.safetensors", + "model.layers.14.mlp.gate_proj.scales": "model.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model.safetensors", + "model.layers.14.mlp.up_proj.biases": "model.safetensors", + "model.layers.14.mlp.up_proj.scales": "model.safetensors", + "model.layers.14.mlp.up_proj.weight": "model.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model.safetensors", + "model.layers.14.self_attn.k_norm.weight": "model.safetensors", + "model.layers.14.self_attn.k_proj.biases": "model.safetensors", + "model.layers.14.self_attn.k_proj.scales": "model.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model.safetensors", + "model.layers.14.self_attn.o_proj.biases": "model.safetensors", + "model.layers.14.self_attn.o_proj.scales": "model.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model.safetensors", + "model.layers.14.self_attn.q_norm.weight": "model.safetensors", + "model.layers.14.self_attn.q_proj.biases": "model.safetensors", + "model.layers.14.self_attn.q_proj.scales": "model.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model.safetensors", + "model.layers.14.self_attn.v_proj.biases": "model.safetensors", + "model.layers.14.self_attn.v_proj.scales": "model.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model.safetensors", + "model.layers.15.input_layernorm.weight": "model.safetensors", + "model.layers.15.mlp.down_proj.biases": "model.safetensors", + "model.layers.15.mlp.down_proj.scales": "model.safetensors", + "model.layers.15.mlp.down_proj.weight": "model.safetensors", + "model.layers.15.mlp.gate_proj.biases": "model.safetensors", + "model.layers.15.mlp.gate_proj.scales": "model.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model.safetensors", + "model.layers.15.mlp.up_proj.biases": "model.safetensors", + "model.layers.15.mlp.up_proj.scales": "model.safetensors", + "model.layers.15.mlp.up_proj.weight": "model.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model.safetensors", + "model.layers.15.self_attn.k_norm.weight": "model.safetensors", + "model.layers.15.self_attn.k_proj.biases": "model.safetensors", + "model.layers.15.self_attn.k_proj.scales": "model.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model.safetensors", + "model.layers.15.self_attn.o_proj.biases": "model.safetensors", + "model.layers.15.self_attn.o_proj.scales": "model.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model.safetensors", + "model.layers.15.self_attn.q_norm.weight": "model.safetensors", + "model.layers.15.self_attn.q_proj.biases": "model.safetensors", + "model.layers.15.self_attn.q_proj.scales": "model.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model.safetensors", + "model.layers.15.self_attn.v_proj.biases": "model.safetensors", + "model.layers.15.self_attn.v_proj.scales": "model.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model.safetensors", + "model.layers.16.input_layernorm.weight": "model.safetensors", + "model.layers.16.mlp.down_proj.biases": "model.safetensors", + "model.layers.16.mlp.down_proj.scales": "model.safetensors", + "model.layers.16.mlp.down_proj.weight": "model.safetensors", + "model.layers.16.mlp.gate_proj.biases": "model.safetensors", + "model.layers.16.mlp.gate_proj.scales": "model.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model.safetensors", + "model.layers.16.mlp.up_proj.biases": "model.safetensors", + "model.layers.16.mlp.up_proj.scales": "model.safetensors", + "model.layers.16.mlp.up_proj.weight": "model.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model.safetensors", + "model.layers.16.self_attn.k_norm.weight": "model.safetensors", + "model.layers.16.self_attn.k_proj.biases": "model.safetensors", + "model.layers.16.self_attn.k_proj.scales": "model.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model.safetensors", + "model.layers.16.self_attn.o_proj.biases": "model.safetensors", + "model.layers.16.self_attn.o_proj.scales": "model.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model.safetensors", + "model.layers.16.self_attn.q_norm.weight": "model.safetensors", + "model.layers.16.self_attn.q_proj.biases": "model.safetensors", + "model.layers.16.self_attn.q_proj.scales": "model.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model.safetensors", + "model.layers.16.self_attn.v_proj.biases": "model.safetensors", + "model.layers.16.self_attn.v_proj.scales": "model.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model.safetensors", + "model.layers.17.input_layernorm.weight": "model.safetensors", + "model.layers.17.mlp.down_proj.biases": "model.safetensors", + "model.layers.17.mlp.down_proj.scales": "model.safetensors", + "model.layers.17.mlp.down_proj.weight": "model.safetensors", + "model.layers.17.mlp.gate_proj.biases": "model.safetensors", + "model.layers.17.mlp.gate_proj.scales": "model.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model.safetensors", + "model.layers.17.mlp.up_proj.biases": "model.safetensors", + "model.layers.17.mlp.up_proj.scales": "model.safetensors", + "model.layers.17.mlp.up_proj.weight": "model.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model.safetensors", + "model.layers.17.self_attn.k_norm.weight": "model.safetensors", + "model.layers.17.self_attn.k_proj.biases": "model.safetensors", + "model.layers.17.self_attn.k_proj.scales": "model.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model.safetensors", + "model.layers.17.self_attn.o_proj.biases": "model.safetensors", + "model.layers.17.self_attn.o_proj.scales": "model.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model.safetensors", + "model.layers.17.self_attn.q_norm.weight": "model.safetensors", + "model.layers.17.self_attn.q_proj.biases": "model.safetensors", + "model.layers.17.self_attn.q_proj.scales": "model.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model.safetensors", + "model.layers.17.self_attn.v_proj.biases": "model.safetensors", + "model.layers.17.self_attn.v_proj.scales": "model.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model.safetensors", + "model.layers.18.input_layernorm.weight": "model.safetensors", + "model.layers.18.mlp.down_proj.biases": "model.safetensors", + "model.layers.18.mlp.down_proj.scales": "model.safetensors", + "model.layers.18.mlp.down_proj.weight": "model.safetensors", + "model.layers.18.mlp.gate_proj.biases": "model.safetensors", + "model.layers.18.mlp.gate_proj.scales": "model.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model.safetensors", + "model.layers.18.mlp.up_proj.biases": "model.safetensors", + "model.layers.18.mlp.up_proj.scales": "model.safetensors", + "model.layers.18.mlp.up_proj.weight": "model.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model.safetensors", + "model.layers.18.self_attn.k_norm.weight": "model.safetensors", + "model.layers.18.self_attn.k_proj.biases": "model.safetensors", + "model.layers.18.self_attn.k_proj.scales": "model.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model.safetensors", + "model.layers.18.self_attn.o_proj.biases": "model.safetensors", + "model.layers.18.self_attn.o_proj.scales": "model.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model.safetensors", + "model.layers.18.self_attn.q_norm.weight": "model.safetensors", + "model.layers.18.self_attn.q_proj.biases": "model.safetensors", + "model.layers.18.self_attn.q_proj.scales": "model.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model.safetensors", + "model.layers.18.self_attn.v_proj.biases": "model.safetensors", + "model.layers.18.self_attn.v_proj.scales": "model.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model.safetensors", + "model.layers.19.input_layernorm.weight": "model.safetensors", + "model.layers.19.mlp.down_proj.biases": "model.safetensors", + "model.layers.19.mlp.down_proj.scales": "model.safetensors", + "model.layers.19.mlp.down_proj.weight": "model.safetensors", + "model.layers.19.mlp.gate_proj.biases": "model.safetensors", + "model.layers.19.mlp.gate_proj.scales": "model.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model.safetensors", + "model.layers.19.mlp.up_proj.biases": "model.safetensors", + "model.layers.19.mlp.up_proj.scales": "model.safetensors", + "model.layers.19.mlp.up_proj.weight": "model.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model.safetensors", + "model.layers.19.self_attn.k_norm.weight": "model.safetensors", + "model.layers.19.self_attn.k_proj.biases": "model.safetensors", + "model.layers.19.self_attn.k_proj.scales": "model.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model.safetensors", + "model.layers.19.self_attn.o_proj.biases": "model.safetensors", + "model.layers.19.self_attn.o_proj.scales": "model.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model.safetensors", + "model.layers.19.self_attn.q_norm.weight": "model.safetensors", + "model.layers.19.self_attn.q_proj.biases": "model.safetensors", + "model.layers.19.self_attn.q_proj.scales": "model.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model.safetensors", + "model.layers.19.self_attn.v_proj.biases": "model.safetensors", + "model.layers.19.self_attn.v_proj.scales": "model.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model.safetensors", + "model.layers.2.input_layernorm.weight": "model.safetensors", + "model.layers.2.mlp.down_proj.biases": "model.safetensors", + "model.layers.2.mlp.down_proj.scales": "model.safetensors", + "model.layers.2.mlp.down_proj.weight": "model.safetensors", + "model.layers.2.mlp.gate_proj.biases": "model.safetensors", + "model.layers.2.mlp.gate_proj.scales": "model.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model.safetensors", + "model.layers.2.mlp.up_proj.biases": "model.safetensors", + "model.layers.2.mlp.up_proj.scales": "model.safetensors", + "model.layers.2.mlp.up_proj.weight": "model.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model.safetensors", + "model.layers.2.self_attn.k_norm.weight": "model.safetensors", + "model.layers.2.self_attn.k_proj.biases": "model.safetensors", + "model.layers.2.self_attn.k_proj.scales": "model.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model.safetensors", + "model.layers.2.self_attn.o_proj.biases": "model.safetensors", + "model.layers.2.self_attn.o_proj.scales": "model.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model.safetensors", + "model.layers.2.self_attn.q_norm.weight": "model.safetensors", + "model.layers.2.self_attn.q_proj.biases": "model.safetensors", + "model.layers.2.self_attn.q_proj.scales": "model.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model.safetensors", + "model.layers.2.self_attn.v_proj.biases": "model.safetensors", + "model.layers.2.self_attn.v_proj.scales": "model.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model.safetensors", + "model.layers.20.input_layernorm.weight": "model.safetensors", + "model.layers.20.mlp.down_proj.biases": "model.safetensors", + "model.layers.20.mlp.down_proj.scales": "model.safetensors", + "model.layers.20.mlp.down_proj.weight": "model.safetensors", + "model.layers.20.mlp.gate_proj.biases": "model.safetensors", + "model.layers.20.mlp.gate_proj.scales": "model.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model.safetensors", + "model.layers.20.mlp.up_proj.biases": "model.safetensors", + "model.layers.20.mlp.up_proj.scales": "model.safetensors", + "model.layers.20.mlp.up_proj.weight": "model.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model.safetensors", + "model.layers.20.self_attn.k_norm.weight": "model.safetensors", + "model.layers.20.self_attn.k_proj.biases": "model.safetensors", + "model.layers.20.self_attn.k_proj.scales": "model.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model.safetensors", + "model.layers.20.self_attn.o_proj.biases": "model.safetensors", + "model.layers.20.self_attn.o_proj.scales": "model.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model.safetensors", + "model.layers.20.self_attn.q_norm.weight": "model.safetensors", + "model.layers.20.self_attn.q_proj.biases": "model.safetensors", + "model.layers.20.self_attn.q_proj.scales": "model.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model.safetensors", + "model.layers.20.self_attn.v_proj.biases": "model.safetensors", + "model.layers.20.self_attn.v_proj.scales": "model.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model.safetensors", + "model.layers.21.input_layernorm.weight": "model.safetensors", + "model.layers.21.mlp.down_proj.biases": "model.safetensors", + "model.layers.21.mlp.down_proj.scales": "model.safetensors", + "model.layers.21.mlp.down_proj.weight": "model.safetensors", + "model.layers.21.mlp.gate_proj.biases": "model.safetensors", + "model.layers.21.mlp.gate_proj.scales": "model.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model.safetensors", + "model.layers.21.mlp.up_proj.biases": "model.safetensors", + "model.layers.21.mlp.up_proj.scales": "model.safetensors", + "model.layers.21.mlp.up_proj.weight": "model.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model.safetensors", + "model.layers.21.self_attn.k_norm.weight": "model.safetensors", + "model.layers.21.self_attn.k_proj.biases": "model.safetensors", + "model.layers.21.self_attn.k_proj.scales": "model.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model.safetensors", + "model.layers.21.self_attn.o_proj.biases": "model.safetensors", + "model.layers.21.self_attn.o_proj.scales": "model.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model.safetensors", + "model.layers.21.self_attn.q_norm.weight": "model.safetensors", + "model.layers.21.self_attn.q_proj.biases": "model.safetensors", + "model.layers.21.self_attn.q_proj.scales": "model.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model.safetensors", + "model.layers.21.self_attn.v_proj.biases": "model.safetensors", + "model.layers.21.self_attn.v_proj.scales": "model.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model.safetensors", + "model.layers.22.input_layernorm.weight": "model.safetensors", + "model.layers.22.mlp.down_proj.biases": "model.safetensors", + "model.layers.22.mlp.down_proj.scales": "model.safetensors", + "model.layers.22.mlp.down_proj.weight": "model.safetensors", + "model.layers.22.mlp.gate_proj.biases": "model.safetensors", + "model.layers.22.mlp.gate_proj.scales": "model.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model.safetensors", + "model.layers.22.mlp.up_proj.biases": "model.safetensors", + "model.layers.22.mlp.up_proj.scales": "model.safetensors", + "model.layers.22.mlp.up_proj.weight": "model.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model.safetensors", + "model.layers.22.self_attn.k_norm.weight": "model.safetensors", + "model.layers.22.self_attn.k_proj.biases": "model.safetensors", + "model.layers.22.self_attn.k_proj.scales": "model.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model.safetensors", + "model.layers.22.self_attn.o_proj.biases": "model.safetensors", + "model.layers.22.self_attn.o_proj.scales": "model.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model.safetensors", + "model.layers.22.self_attn.q_norm.weight": "model.safetensors", + "model.layers.22.self_attn.q_proj.biases": "model.safetensors", + "model.layers.22.self_attn.q_proj.scales": "model.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model.safetensors", + "model.layers.22.self_attn.v_proj.biases": "model.safetensors", + "model.layers.22.self_attn.v_proj.scales": "model.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model.safetensors", + "model.layers.23.input_layernorm.weight": "model.safetensors", + "model.layers.23.mlp.down_proj.biases": "model.safetensors", + "model.layers.23.mlp.down_proj.scales": "model.safetensors", + "model.layers.23.mlp.down_proj.weight": "model.safetensors", + "model.layers.23.mlp.gate_proj.biases": "model.safetensors", + "model.layers.23.mlp.gate_proj.scales": "model.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model.safetensors", + "model.layers.23.mlp.up_proj.biases": "model.safetensors", + "model.layers.23.mlp.up_proj.scales": "model.safetensors", + "model.layers.23.mlp.up_proj.weight": "model.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model.safetensors", + "model.layers.23.self_attn.k_norm.weight": "model.safetensors", + "model.layers.23.self_attn.k_proj.biases": "model.safetensors", + "model.layers.23.self_attn.k_proj.scales": "model.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model.safetensors", + "model.layers.23.self_attn.o_proj.biases": "model.safetensors", + "model.layers.23.self_attn.o_proj.scales": "model.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model.safetensors", + "model.layers.23.self_attn.q_norm.weight": "model.safetensors", + "model.layers.23.self_attn.q_proj.biases": "model.safetensors", + "model.layers.23.self_attn.q_proj.scales": "model.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model.safetensors", + "model.layers.23.self_attn.v_proj.biases": "model.safetensors", + "model.layers.23.self_attn.v_proj.scales": "model.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model.safetensors", + "model.layers.24.input_layernorm.weight": "model.safetensors", + "model.layers.24.mlp.down_proj.biases": "model.safetensors", + "model.layers.24.mlp.down_proj.scales": "model.safetensors", + "model.layers.24.mlp.down_proj.weight": "model.safetensors", + "model.layers.24.mlp.gate_proj.biases": "model.safetensors", + "model.layers.24.mlp.gate_proj.scales": "model.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model.safetensors", + "model.layers.24.mlp.up_proj.biases": "model.safetensors", + "model.layers.24.mlp.up_proj.scales": "model.safetensors", + "model.layers.24.mlp.up_proj.weight": "model.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model.safetensors", + "model.layers.24.self_attn.k_norm.weight": "model.safetensors", + "model.layers.24.self_attn.k_proj.biases": "model.safetensors", + "model.layers.24.self_attn.k_proj.scales": "model.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model.safetensors", + "model.layers.24.self_attn.o_proj.biases": "model.safetensors", + "model.layers.24.self_attn.o_proj.scales": "model.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model.safetensors", + "model.layers.24.self_attn.q_norm.weight": "model.safetensors", + "model.layers.24.self_attn.q_proj.biases": "model.safetensors", + "model.layers.24.self_attn.q_proj.scales": "model.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model.safetensors", + "model.layers.24.self_attn.v_proj.biases": "model.safetensors", + "model.layers.24.self_attn.v_proj.scales": "model.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model.safetensors", + "model.layers.25.input_layernorm.weight": "model.safetensors", + "model.layers.25.mlp.down_proj.biases": "model.safetensors", + "model.layers.25.mlp.down_proj.scales": "model.safetensors", + "model.layers.25.mlp.down_proj.weight": "model.safetensors", + "model.layers.25.mlp.gate_proj.biases": "model.safetensors", + "model.layers.25.mlp.gate_proj.scales": "model.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model.safetensors", + "model.layers.25.mlp.up_proj.biases": "model.safetensors", + "model.layers.25.mlp.up_proj.scales": "model.safetensors", + "model.layers.25.mlp.up_proj.weight": "model.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model.safetensors", + "model.layers.25.self_attn.k_norm.weight": "model.safetensors", + "model.layers.25.self_attn.k_proj.biases": "model.safetensors", + "model.layers.25.self_attn.k_proj.scales": "model.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model.safetensors", + "model.layers.25.self_attn.o_proj.biases": "model.safetensors", + "model.layers.25.self_attn.o_proj.scales": "model.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model.safetensors", + "model.layers.25.self_attn.q_norm.weight": "model.safetensors", + "model.layers.25.self_attn.q_proj.biases": "model.safetensors", + "model.layers.25.self_attn.q_proj.scales": "model.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model.safetensors", + "model.layers.25.self_attn.v_proj.biases": "model.safetensors", + "model.layers.25.self_attn.v_proj.scales": "model.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model.safetensors", + "model.layers.26.input_layernorm.weight": "model.safetensors", + "model.layers.26.mlp.down_proj.biases": "model.safetensors", + "model.layers.26.mlp.down_proj.scales": "model.safetensors", + "model.layers.26.mlp.down_proj.weight": "model.safetensors", + "model.layers.26.mlp.gate_proj.biases": "model.safetensors", + "model.layers.26.mlp.gate_proj.scales": "model.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model.safetensors", + "model.layers.26.mlp.up_proj.biases": "model.safetensors", + "model.layers.26.mlp.up_proj.scales": "model.safetensors", + "model.layers.26.mlp.up_proj.weight": "model.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model.safetensors", + "model.layers.26.self_attn.k_norm.weight": "model.safetensors", + "model.layers.26.self_attn.k_proj.biases": "model.safetensors", + "model.layers.26.self_attn.k_proj.scales": "model.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model.safetensors", + "model.layers.26.self_attn.o_proj.biases": "model.safetensors", + "model.layers.26.self_attn.o_proj.scales": "model.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model.safetensors", + "model.layers.26.self_attn.q_norm.weight": "model.safetensors", + "model.layers.26.self_attn.q_proj.biases": "model.safetensors", + "model.layers.26.self_attn.q_proj.scales": "model.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model.safetensors", + "model.layers.26.self_attn.v_proj.biases": "model.safetensors", + "model.layers.26.self_attn.v_proj.scales": "model.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model.safetensors", + "model.layers.27.input_layernorm.weight": "model.safetensors", + "model.layers.27.mlp.down_proj.biases": "model.safetensors", + "model.layers.27.mlp.down_proj.scales": "model.safetensors", + "model.layers.27.mlp.down_proj.weight": "model.safetensors", + "model.layers.27.mlp.gate_proj.biases": "model.safetensors", + "model.layers.27.mlp.gate_proj.scales": "model.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model.safetensors", + "model.layers.27.mlp.up_proj.biases": "model.safetensors", + "model.layers.27.mlp.up_proj.scales": "model.safetensors", + "model.layers.27.mlp.up_proj.weight": "model.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model.safetensors", + "model.layers.27.self_attn.k_norm.weight": "model.safetensors", + "model.layers.27.self_attn.k_proj.biases": "model.safetensors", + "model.layers.27.self_attn.k_proj.scales": "model.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model.safetensors", + "model.layers.27.self_attn.o_proj.biases": "model.safetensors", + "model.layers.27.self_attn.o_proj.scales": "model.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model.safetensors", + "model.layers.27.self_attn.q_norm.weight": "model.safetensors", + "model.layers.27.self_attn.q_proj.biases": "model.safetensors", + "model.layers.27.self_attn.q_proj.scales": "model.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model.safetensors", + "model.layers.27.self_attn.v_proj.biases": "model.safetensors", + "model.layers.27.self_attn.v_proj.scales": "model.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model.safetensors", + "model.layers.28.input_layernorm.weight": "model.safetensors", + "model.layers.28.mlp.down_proj.biases": "model.safetensors", + "model.layers.28.mlp.down_proj.scales": "model.safetensors", + "model.layers.28.mlp.down_proj.weight": "model.safetensors", + "model.layers.28.mlp.gate_proj.biases": "model.safetensors", + "model.layers.28.mlp.gate_proj.scales": "model.safetensors", + "model.layers.28.mlp.gate_proj.weight": "model.safetensors", + "model.layers.28.mlp.up_proj.biases": "model.safetensors", + "model.layers.28.mlp.up_proj.scales": "model.safetensors", + "model.layers.28.mlp.up_proj.weight": "model.safetensors", + "model.layers.28.post_attention_layernorm.weight": "model.safetensors", + "model.layers.28.self_attn.k_norm.weight": "model.safetensors", + "model.layers.28.self_attn.k_proj.biases": "model.safetensors", + "model.layers.28.self_attn.k_proj.scales": "model.safetensors", + "model.layers.28.self_attn.k_proj.weight": "model.safetensors", + "model.layers.28.self_attn.o_proj.biases": "model.safetensors", + "model.layers.28.self_attn.o_proj.scales": "model.safetensors", + "model.layers.28.self_attn.o_proj.weight": "model.safetensors", + "model.layers.28.self_attn.q_norm.weight": "model.safetensors", + "model.layers.28.self_attn.q_proj.biases": "model.safetensors", + "model.layers.28.self_attn.q_proj.scales": "model.safetensors", + "model.layers.28.self_attn.q_proj.weight": "model.safetensors", + "model.layers.28.self_attn.v_proj.biases": "model.safetensors", + "model.layers.28.self_attn.v_proj.scales": "model.safetensors", + "model.layers.28.self_attn.v_proj.weight": "model.safetensors", + "model.layers.29.input_layernorm.weight": "model.safetensors", + "model.layers.29.mlp.down_proj.biases": "model.safetensors", + "model.layers.29.mlp.down_proj.scales": "model.safetensors", + "model.layers.29.mlp.down_proj.weight": "model.safetensors", + "model.layers.29.mlp.gate_proj.biases": "model.safetensors", + "model.layers.29.mlp.gate_proj.scales": "model.safetensors", + "model.layers.29.mlp.gate_proj.weight": "model.safetensors", + "model.layers.29.mlp.up_proj.biases": "model.safetensors", + "model.layers.29.mlp.up_proj.scales": "model.safetensors", + "model.layers.29.mlp.up_proj.weight": "model.safetensors", + "model.layers.29.post_attention_layernorm.weight": "model.safetensors", + "model.layers.29.self_attn.k_norm.weight": "model.safetensors", + "model.layers.29.self_attn.k_proj.biases": "model.safetensors", + "model.layers.29.self_attn.k_proj.scales": "model.safetensors", + "model.layers.29.self_attn.k_proj.weight": "model.safetensors", + "model.layers.29.self_attn.o_proj.biases": "model.safetensors", + "model.layers.29.self_attn.o_proj.scales": "model.safetensors", + "model.layers.29.self_attn.o_proj.weight": "model.safetensors", + "model.layers.29.self_attn.q_norm.weight": "model.safetensors", + "model.layers.29.self_attn.q_proj.biases": "model.safetensors", + "model.layers.29.self_attn.q_proj.scales": "model.safetensors", + "model.layers.29.self_attn.q_proj.weight": "model.safetensors", + "model.layers.29.self_attn.v_proj.biases": "model.safetensors", + "model.layers.29.self_attn.v_proj.scales": "model.safetensors", + "model.layers.29.self_attn.v_proj.weight": "model.safetensors", + "model.layers.3.input_layernorm.weight": "model.safetensors", + "model.layers.3.mlp.down_proj.biases": "model.safetensors", + "model.layers.3.mlp.down_proj.scales": "model.safetensors", + "model.layers.3.mlp.down_proj.weight": "model.safetensors", + "model.layers.3.mlp.gate_proj.biases": "model.safetensors", + "model.layers.3.mlp.gate_proj.scales": "model.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model.safetensors", + "model.layers.3.mlp.up_proj.biases": "model.safetensors", + "model.layers.3.mlp.up_proj.scales": "model.safetensors", + "model.layers.3.mlp.up_proj.weight": "model.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model.safetensors", + "model.layers.3.self_attn.k_norm.weight": "model.safetensors", + "model.layers.3.self_attn.k_proj.biases": "model.safetensors", + "model.layers.3.self_attn.k_proj.scales": "model.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model.safetensors", + "model.layers.3.self_attn.o_proj.biases": "model.safetensors", + "model.layers.3.self_attn.o_proj.scales": "model.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model.safetensors", + "model.layers.3.self_attn.q_norm.weight": "model.safetensors", + "model.layers.3.self_attn.q_proj.biases": "model.safetensors", + "model.layers.3.self_attn.q_proj.scales": "model.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model.safetensors", + "model.layers.3.self_attn.v_proj.biases": "model.safetensors", + "model.layers.3.self_attn.v_proj.scales": "model.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model.safetensors", + "model.layers.30.input_layernorm.weight": "model.safetensors", + "model.layers.30.mlp.down_proj.biases": "model.safetensors", + "model.layers.30.mlp.down_proj.scales": "model.safetensors", + "model.layers.30.mlp.down_proj.weight": "model.safetensors", + "model.layers.30.mlp.gate_proj.biases": "model.safetensors", + "model.layers.30.mlp.gate_proj.scales": "model.safetensors", + "model.layers.30.mlp.gate_proj.weight": "model.safetensors", + "model.layers.30.mlp.up_proj.biases": "model.safetensors", + "model.layers.30.mlp.up_proj.scales": "model.safetensors", + "model.layers.30.mlp.up_proj.weight": "model.safetensors", + "model.layers.30.post_attention_layernorm.weight": "model.safetensors", + "model.layers.30.self_attn.k_norm.weight": "model.safetensors", + "model.layers.30.self_attn.k_proj.biases": "model.safetensors", + "model.layers.30.self_attn.k_proj.scales": "model.safetensors", + "model.layers.30.self_attn.k_proj.weight": "model.safetensors", + "model.layers.30.self_attn.o_proj.biases": "model.safetensors", + "model.layers.30.self_attn.o_proj.scales": "model.safetensors", + "model.layers.30.self_attn.o_proj.weight": "model.safetensors", + "model.layers.30.self_attn.q_norm.weight": "model.safetensors", + "model.layers.30.self_attn.q_proj.biases": "model.safetensors", + "model.layers.30.self_attn.q_proj.scales": "model.safetensors", + "model.layers.30.self_attn.q_proj.weight": "model.safetensors", + "model.layers.30.self_attn.v_proj.biases": "model.safetensors", + "model.layers.30.self_attn.v_proj.scales": "model.safetensors", + "model.layers.30.self_attn.v_proj.weight": "model.safetensors", + "model.layers.31.input_layernorm.weight": "model.safetensors", + "model.layers.31.mlp.down_proj.biases": "model.safetensors", + "model.layers.31.mlp.down_proj.scales": "model.safetensors", + "model.layers.31.mlp.down_proj.weight": "model.safetensors", + "model.layers.31.mlp.gate_proj.biases": "model.safetensors", + "model.layers.31.mlp.gate_proj.scales": "model.safetensors", + "model.layers.31.mlp.gate_proj.weight": "model.safetensors", + "model.layers.31.mlp.up_proj.biases": "model.safetensors", + "model.layers.31.mlp.up_proj.scales": "model.safetensors", + "model.layers.31.mlp.up_proj.weight": "model.safetensors", + "model.layers.31.post_attention_layernorm.weight": "model.safetensors", + "model.layers.31.self_attn.k_norm.weight": "model.safetensors", + "model.layers.31.self_attn.k_proj.biases": "model.safetensors", + "model.layers.31.self_attn.k_proj.scales": "model.safetensors", + "model.layers.31.self_attn.k_proj.weight": "model.safetensors", + "model.layers.31.self_attn.o_proj.biases": "model.safetensors", + "model.layers.31.self_attn.o_proj.scales": "model.safetensors", + "model.layers.31.self_attn.o_proj.weight": "model.safetensors", + "model.layers.31.self_attn.q_norm.weight": "model.safetensors", + "model.layers.31.self_attn.q_proj.biases": "model.safetensors", + "model.layers.31.self_attn.q_proj.scales": "model.safetensors", + "model.layers.31.self_attn.q_proj.weight": "model.safetensors", + "model.layers.31.self_attn.v_proj.biases": "model.safetensors", + "model.layers.31.self_attn.v_proj.scales": "model.safetensors", + "model.layers.31.self_attn.v_proj.weight": "model.safetensors", + "model.layers.32.input_layernorm.weight": "model.safetensors", + "model.layers.32.mlp.down_proj.biases": "model.safetensors", + "model.layers.32.mlp.down_proj.scales": "model.safetensors", + "model.layers.32.mlp.down_proj.weight": "model.safetensors", + "model.layers.32.mlp.gate_proj.biases": "model.safetensors", + "model.layers.32.mlp.gate_proj.scales": "model.safetensors", + "model.layers.32.mlp.gate_proj.weight": "model.safetensors", + "model.layers.32.mlp.up_proj.biases": "model.safetensors", + "model.layers.32.mlp.up_proj.scales": "model.safetensors", + "model.layers.32.mlp.up_proj.weight": "model.safetensors", + "model.layers.32.post_attention_layernorm.weight": "model.safetensors", + "model.layers.32.self_attn.k_norm.weight": "model.safetensors", + "model.layers.32.self_attn.k_proj.biases": "model.safetensors", + "model.layers.32.self_attn.k_proj.scales": "model.safetensors", + "model.layers.32.self_attn.k_proj.weight": "model.safetensors", + "model.layers.32.self_attn.o_proj.biases": "model.safetensors", + "model.layers.32.self_attn.o_proj.scales": "model.safetensors", + "model.layers.32.self_attn.o_proj.weight": "model.safetensors", + "model.layers.32.self_attn.q_norm.weight": "model.safetensors", + "model.layers.32.self_attn.q_proj.biases": "model.safetensors", + "model.layers.32.self_attn.q_proj.scales": "model.safetensors", + "model.layers.32.self_attn.q_proj.weight": "model.safetensors", + "model.layers.32.self_attn.v_proj.biases": "model.safetensors", + "model.layers.32.self_attn.v_proj.scales": "model.safetensors", + "model.layers.32.self_attn.v_proj.weight": "model.safetensors", + "model.layers.33.input_layernorm.weight": "model.safetensors", + "model.layers.33.mlp.down_proj.biases": "model.safetensors", + "model.layers.33.mlp.down_proj.scales": "model.safetensors", + "model.layers.33.mlp.down_proj.weight": "model.safetensors", + "model.layers.33.mlp.gate_proj.biases": "model.safetensors", + "model.layers.33.mlp.gate_proj.scales": "model.safetensors", + "model.layers.33.mlp.gate_proj.weight": "model.safetensors", + "model.layers.33.mlp.up_proj.biases": "model.safetensors", + "model.layers.33.mlp.up_proj.scales": "model.safetensors", + "model.layers.33.mlp.up_proj.weight": "model.safetensors", + "model.layers.33.post_attention_layernorm.weight": "model.safetensors", + "model.layers.33.self_attn.k_norm.weight": "model.safetensors", + "model.layers.33.self_attn.k_proj.biases": "model.safetensors", + "model.layers.33.self_attn.k_proj.scales": "model.safetensors", + "model.layers.33.self_attn.k_proj.weight": "model.safetensors", + "model.layers.33.self_attn.o_proj.biases": "model.safetensors", + "model.layers.33.self_attn.o_proj.scales": "model.safetensors", + "model.layers.33.self_attn.o_proj.weight": "model.safetensors", + "model.layers.33.self_attn.q_norm.weight": "model.safetensors", + "model.layers.33.self_attn.q_proj.biases": "model.safetensors", + "model.layers.33.self_attn.q_proj.scales": "model.safetensors", + "model.layers.33.self_attn.q_proj.weight": "model.safetensors", + "model.layers.33.self_attn.v_proj.biases": "model.safetensors", + "model.layers.33.self_attn.v_proj.scales": "model.safetensors", + "model.layers.33.self_attn.v_proj.weight": "model.safetensors", + "model.layers.34.input_layernorm.weight": "model.safetensors", + "model.layers.34.mlp.down_proj.biases": "model.safetensors", + "model.layers.34.mlp.down_proj.scales": "model.safetensors", + "model.layers.34.mlp.down_proj.weight": "model.safetensors", + "model.layers.34.mlp.gate_proj.biases": "model.safetensors", + "model.layers.34.mlp.gate_proj.scales": "model.safetensors", + "model.layers.34.mlp.gate_proj.weight": "model.safetensors", + "model.layers.34.mlp.up_proj.biases": "model.safetensors", + "model.layers.34.mlp.up_proj.scales": "model.safetensors", + "model.layers.34.mlp.up_proj.weight": "model.safetensors", + "model.layers.34.post_attention_layernorm.weight": "model.safetensors", + "model.layers.34.self_attn.k_norm.weight": "model.safetensors", + "model.layers.34.self_attn.k_proj.biases": "model.safetensors", + "model.layers.34.self_attn.k_proj.scales": "model.safetensors", + "model.layers.34.self_attn.k_proj.weight": "model.safetensors", + "model.layers.34.self_attn.o_proj.biases": "model.safetensors", + "model.layers.34.self_attn.o_proj.scales": "model.safetensors", + "model.layers.34.self_attn.o_proj.weight": "model.safetensors", + "model.layers.34.self_attn.q_norm.weight": "model.safetensors", + "model.layers.34.self_attn.q_proj.biases": "model.safetensors", + "model.layers.34.self_attn.q_proj.scales": "model.safetensors", + "model.layers.34.self_attn.q_proj.weight": "model.safetensors", + "model.layers.34.self_attn.v_proj.biases": "model.safetensors", + "model.layers.34.self_attn.v_proj.scales": "model.safetensors", + "model.layers.34.self_attn.v_proj.weight": "model.safetensors", + "model.layers.35.input_layernorm.weight": "model.safetensors", + "model.layers.35.mlp.down_proj.biases": "model.safetensors", + "model.layers.35.mlp.down_proj.scales": "model.safetensors", + "model.layers.35.mlp.down_proj.weight": "model.safetensors", + "model.layers.35.mlp.gate_proj.biases": "model.safetensors", + "model.layers.35.mlp.gate_proj.scales": "model.safetensors", + "model.layers.35.mlp.gate_proj.weight": "model.safetensors", + "model.layers.35.mlp.up_proj.biases": "model.safetensors", + "model.layers.35.mlp.up_proj.scales": "model.safetensors", + "model.layers.35.mlp.up_proj.weight": "model.safetensors", + "model.layers.35.post_attention_layernorm.weight": "model.safetensors", + "model.layers.35.self_attn.k_norm.weight": "model.safetensors", + "model.layers.35.self_attn.k_proj.biases": "model.safetensors", + "model.layers.35.self_attn.k_proj.scales": "model.safetensors", + "model.layers.35.self_attn.k_proj.weight": "model.safetensors", + "model.layers.35.self_attn.o_proj.biases": "model.safetensors", + "model.layers.35.self_attn.o_proj.scales": "model.safetensors", + "model.layers.35.self_attn.o_proj.weight": "model.safetensors", + "model.layers.35.self_attn.q_norm.weight": "model.safetensors", + "model.layers.35.self_attn.q_proj.biases": "model.safetensors", + "model.layers.35.self_attn.q_proj.scales": "model.safetensors", + "model.layers.35.self_attn.q_proj.weight": "model.safetensors", + "model.layers.35.self_attn.v_proj.biases": "model.safetensors", + "model.layers.35.self_attn.v_proj.scales": "model.safetensors", + "model.layers.35.self_attn.v_proj.weight": "model.safetensors", + "model.layers.4.input_layernorm.weight": "model.safetensors", + "model.layers.4.mlp.down_proj.biases": "model.safetensors", + "model.layers.4.mlp.down_proj.scales": "model.safetensors", + "model.layers.4.mlp.down_proj.weight": "model.safetensors", + "model.layers.4.mlp.gate_proj.biases": "model.safetensors", + "model.layers.4.mlp.gate_proj.scales": "model.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model.safetensors", + "model.layers.4.mlp.up_proj.biases": "model.safetensors", + "model.layers.4.mlp.up_proj.scales": "model.safetensors", + "model.layers.4.mlp.up_proj.weight": "model.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model.safetensors", + "model.layers.4.self_attn.k_norm.weight": "model.safetensors", + "model.layers.4.self_attn.k_proj.biases": "model.safetensors", + "model.layers.4.self_attn.k_proj.scales": "model.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model.safetensors", + "model.layers.4.self_attn.o_proj.biases": "model.safetensors", + "model.layers.4.self_attn.o_proj.scales": "model.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model.safetensors", + "model.layers.4.self_attn.q_norm.weight": "model.safetensors", + "model.layers.4.self_attn.q_proj.biases": "model.safetensors", + "model.layers.4.self_attn.q_proj.scales": "model.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model.safetensors", + "model.layers.4.self_attn.v_proj.biases": "model.safetensors", + "model.layers.4.self_attn.v_proj.scales": "model.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model.safetensors", + "model.layers.5.input_layernorm.weight": "model.safetensors", + "model.layers.5.mlp.down_proj.biases": "model.safetensors", + "model.layers.5.mlp.down_proj.scales": "model.safetensors", + "model.layers.5.mlp.down_proj.weight": "model.safetensors", + "model.layers.5.mlp.gate_proj.biases": "model.safetensors", + "model.layers.5.mlp.gate_proj.scales": "model.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model.safetensors", + "model.layers.5.mlp.up_proj.biases": "model.safetensors", + "model.layers.5.mlp.up_proj.scales": "model.safetensors", + "model.layers.5.mlp.up_proj.weight": "model.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model.safetensors", + "model.layers.5.self_attn.k_norm.weight": "model.safetensors", + "model.layers.5.self_attn.k_proj.biases": "model.safetensors", + "model.layers.5.self_attn.k_proj.scales": "model.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model.safetensors", + "model.layers.5.self_attn.o_proj.biases": "model.safetensors", + "model.layers.5.self_attn.o_proj.scales": "model.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model.safetensors", + "model.layers.5.self_attn.q_norm.weight": "model.safetensors", + "model.layers.5.self_attn.q_proj.biases": "model.safetensors", + "model.layers.5.self_attn.q_proj.scales": "model.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model.safetensors", + "model.layers.5.self_attn.v_proj.biases": "model.safetensors", + "model.layers.5.self_attn.v_proj.scales": "model.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model.safetensors", + "model.layers.6.input_layernorm.weight": "model.safetensors", + "model.layers.6.mlp.down_proj.biases": "model.safetensors", + "model.layers.6.mlp.down_proj.scales": "model.safetensors", + "model.layers.6.mlp.down_proj.weight": "model.safetensors", + "model.layers.6.mlp.gate_proj.biases": "model.safetensors", + "model.layers.6.mlp.gate_proj.scales": "model.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model.safetensors", + "model.layers.6.mlp.up_proj.biases": "model.safetensors", + "model.layers.6.mlp.up_proj.scales": "model.safetensors", + "model.layers.6.mlp.up_proj.weight": "model.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model.safetensors", + "model.layers.6.self_attn.k_norm.weight": "model.safetensors", + "model.layers.6.self_attn.k_proj.biases": "model.safetensors", + "model.layers.6.self_attn.k_proj.scales": "model.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model.safetensors", + "model.layers.6.self_attn.o_proj.biases": "model.safetensors", + "model.layers.6.self_attn.o_proj.scales": "model.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model.safetensors", + "model.layers.6.self_attn.q_norm.weight": "model.safetensors", + "model.layers.6.self_attn.q_proj.biases": "model.safetensors", + "model.layers.6.self_attn.q_proj.scales": "model.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model.safetensors", + "model.layers.6.self_attn.v_proj.biases": "model.safetensors", + "model.layers.6.self_attn.v_proj.scales": "model.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model.safetensors", + "model.layers.7.input_layernorm.weight": "model.safetensors", + "model.layers.7.mlp.down_proj.biases": "model.safetensors", + "model.layers.7.mlp.down_proj.scales": "model.safetensors", + "model.layers.7.mlp.down_proj.weight": "model.safetensors", + "model.layers.7.mlp.gate_proj.biases": "model.safetensors", + "model.layers.7.mlp.gate_proj.scales": "model.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model.safetensors", + "model.layers.7.mlp.up_proj.biases": "model.safetensors", + "model.layers.7.mlp.up_proj.scales": "model.safetensors", + "model.layers.7.mlp.up_proj.weight": "model.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model.safetensors", + "model.layers.7.self_attn.k_norm.weight": "model.safetensors", + "model.layers.7.self_attn.k_proj.biases": "model.safetensors", + "model.layers.7.self_attn.k_proj.scales": "model.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model.safetensors", + "model.layers.7.self_attn.o_proj.biases": "model.safetensors", + "model.layers.7.self_attn.o_proj.scales": "model.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model.safetensors", + "model.layers.7.self_attn.q_norm.weight": "model.safetensors", + "model.layers.7.self_attn.q_proj.biases": "model.safetensors", + "model.layers.7.self_attn.q_proj.scales": "model.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model.safetensors", + "model.layers.7.self_attn.v_proj.biases": "model.safetensors", + "model.layers.7.self_attn.v_proj.scales": "model.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model.safetensors", + "model.layers.8.input_layernorm.weight": "model.safetensors", + "model.layers.8.mlp.down_proj.biases": "model.safetensors", + "model.layers.8.mlp.down_proj.scales": "model.safetensors", + "model.layers.8.mlp.down_proj.weight": "model.safetensors", + "model.layers.8.mlp.gate_proj.biases": "model.safetensors", + "model.layers.8.mlp.gate_proj.scales": "model.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model.safetensors", + "model.layers.8.mlp.up_proj.biases": "model.safetensors", + "model.layers.8.mlp.up_proj.scales": "model.safetensors", + "model.layers.8.mlp.up_proj.weight": "model.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model.safetensors", + "model.layers.8.self_attn.k_norm.weight": "model.safetensors", + "model.layers.8.self_attn.k_proj.biases": "model.safetensors", + "model.layers.8.self_attn.k_proj.scales": "model.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model.safetensors", + "model.layers.8.self_attn.o_proj.biases": "model.safetensors", + "model.layers.8.self_attn.o_proj.scales": "model.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model.safetensors", + "model.layers.8.self_attn.q_norm.weight": "model.safetensors", + "model.layers.8.self_attn.q_proj.biases": "model.safetensors", + "model.layers.8.self_attn.q_proj.scales": "model.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model.safetensors", + "model.layers.8.self_attn.v_proj.biases": "model.safetensors", + "model.layers.8.self_attn.v_proj.scales": "model.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model.safetensors", + "model.layers.9.input_layernorm.weight": "model.safetensors", + "model.layers.9.mlp.down_proj.biases": "model.safetensors", + "model.layers.9.mlp.down_proj.scales": "model.safetensors", + "model.layers.9.mlp.down_proj.weight": "model.safetensors", + "model.layers.9.mlp.gate_proj.biases": "model.safetensors", + "model.layers.9.mlp.gate_proj.scales": "model.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model.safetensors", + "model.layers.9.mlp.up_proj.biases": "model.safetensors", + "model.layers.9.mlp.up_proj.scales": "model.safetensors", + "model.layers.9.mlp.up_proj.weight": "model.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model.safetensors", + "model.layers.9.self_attn.k_norm.weight": "model.safetensors", + "model.layers.9.self_attn.k_proj.biases": "model.safetensors", + "model.layers.9.self_attn.k_proj.scales": "model.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model.safetensors", + "model.layers.9.self_attn.o_proj.biases": "model.safetensors", + "model.layers.9.self_attn.o_proj.scales": "model.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model.safetensors", + "model.layers.9.self_attn.q_norm.weight": "model.safetensors", + "model.layers.9.self_attn.q_proj.biases": "model.safetensors", + "model.layers.9.self_attn.q_proj.scales": "model.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model.safetensors", + "model.layers.9.self_attn.v_proj.biases": "model.safetensors", + "model.layers.9.self_attn.v_proj.scales": "model.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model.safetensors", + "model.norm.weight": "model.safetensors" + } +} \ No newline at end of file diff --git a/mlx-int4/tokenizer.json b/mlx-int4/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/mlx-int4/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/mlx-int4/tokenizer_config.json b/mlx-int4/tokenizer_config.json new file mode 100644 index 0000000..d89bb6f --- /dev/null +++ b/mlx-int4/tokenizer_config.json @@ -0,0 +1,16 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "is_local": true, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "tool_parser_type": "json_tools", + "unk_token": null +} diff --git a/mlx-int8/README.md b/mlx-int8/README.md new file mode 100644 index 0000000..93ff76c --- /dev/null +++ b/mlx-int8/README.md @@ -0,0 +1,7 @@ +--- +language: en +pipeline_tag: text-generation +tags: +- mlx +library_name: mlx +--- diff --git a/mlx-int8/chat_template.jinja b/mlx-int8/chat_template.jinja new file mode 100644 index 0000000..699ff8d --- /dev/null +++ b/mlx-int8/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/mlx-int8/config.json b/mlx-int8/config.json new file mode 100644 index 0000000..2210eee --- /dev/null +++ b/mlx-int8/config.json @@ -0,0 +1,40 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 151643, + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2560, + "initializer_range": 0.02, + "intermediate_size": 9728, + "max_position_embeddings": 40960, + "max_window_layers": 36, + "model_type": "qwen3", + "num_attention_heads": 32, + "num_hidden_layers": 36, + "num_key_value_heads": 8, + "quantization": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "quantization_config": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "rms_norm_eps": 1e-06, + "rope_scaling": null, + "rope_theta": 1000000, + "sliding_window": null, + "tie_word_embeddings": true, + "torch_dtype": "bfloat16", + "transformers_version": "4.51.0", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 151936 +} \ No newline at end of file diff --git a/mlx-int8/model.safetensors b/mlx-int8/model.safetensors new file mode 100644 index 0000000..65a315e --- /dev/null +++ b/mlx-int8/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5719e311009e684e40759f2dba1e7981f24fe44d1fae03dda0aab1bb3b612161 +size 4274158989 diff --git a/mlx-int8/model.safetensors.index.json b/mlx-int8/model.safetensors.index.json new file mode 100644 index 0000000..d54e06a --- /dev/null +++ b/mlx-int8/model.safetensors.index.json @@ -0,0 +1,912 @@ +{ + "metadata": { + "total_size": 4274056192, + "total_parameters": 4022468096 + }, + "weight_map": { + "model.embed_tokens.biases": "model.safetensors", + "model.embed_tokens.scales": "model.safetensors", + "model.embed_tokens.weight": "model.safetensors", + "model.layers.0.input_layernorm.weight": "model.safetensors", + "model.layers.0.mlp.down_proj.biases": "model.safetensors", + "model.layers.0.mlp.down_proj.scales": "model.safetensors", + "model.layers.0.mlp.down_proj.weight": "model.safetensors", + "model.layers.0.mlp.gate_proj.biases": "model.safetensors", + "model.layers.0.mlp.gate_proj.scales": "model.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model.safetensors", + "model.layers.0.mlp.up_proj.biases": "model.safetensors", + "model.layers.0.mlp.up_proj.scales": "model.safetensors", + "model.layers.0.mlp.up_proj.weight": "model.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model.safetensors", + "model.layers.0.self_attn.k_norm.weight": "model.safetensors", + "model.layers.0.self_attn.k_proj.biases": "model.safetensors", + "model.layers.0.self_attn.k_proj.scales": "model.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model.safetensors", + "model.layers.0.self_attn.o_proj.biases": "model.safetensors", + "model.layers.0.self_attn.o_proj.scales": "model.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model.safetensors", + "model.layers.0.self_attn.q_norm.weight": "model.safetensors", + "model.layers.0.self_attn.q_proj.biases": "model.safetensors", + "model.layers.0.self_attn.q_proj.scales": "model.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model.safetensors", + "model.layers.0.self_attn.v_proj.biases": "model.safetensors", + "model.layers.0.self_attn.v_proj.scales": "model.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model.safetensors", + "model.layers.1.input_layernorm.weight": "model.safetensors", + "model.layers.1.mlp.down_proj.biases": "model.safetensors", + "model.layers.1.mlp.down_proj.scales": "model.safetensors", + "model.layers.1.mlp.down_proj.weight": "model.safetensors", + "model.layers.1.mlp.gate_proj.biases": "model.safetensors", + "model.layers.1.mlp.gate_proj.scales": "model.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model.safetensors", + "model.layers.1.mlp.up_proj.biases": "model.safetensors", + "model.layers.1.mlp.up_proj.scales": "model.safetensors", + "model.layers.1.mlp.up_proj.weight": "model.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model.safetensors", + "model.layers.1.self_attn.k_norm.weight": "model.safetensors", + "model.layers.1.self_attn.k_proj.biases": "model.safetensors", + "model.layers.1.self_attn.k_proj.scales": "model.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model.safetensors", + "model.layers.1.self_attn.o_proj.biases": "model.safetensors", + "model.layers.1.self_attn.o_proj.scales": "model.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model.safetensors", + "model.layers.1.self_attn.q_norm.weight": "model.safetensors", + "model.layers.1.self_attn.q_proj.biases": "model.safetensors", + "model.layers.1.self_attn.q_proj.scales": "model.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model.safetensors", + "model.layers.1.self_attn.v_proj.biases": "model.safetensors", + "model.layers.1.self_attn.v_proj.scales": "model.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model.safetensors", + "model.layers.10.input_layernorm.weight": "model.safetensors", + "model.layers.10.mlp.down_proj.biases": "model.safetensors", + "model.layers.10.mlp.down_proj.scales": "model.safetensors", + "model.layers.10.mlp.down_proj.weight": "model.safetensors", + "model.layers.10.mlp.gate_proj.biases": "model.safetensors", + "model.layers.10.mlp.gate_proj.scales": "model.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model.safetensors", + "model.layers.10.mlp.up_proj.biases": "model.safetensors", + "model.layers.10.mlp.up_proj.scales": "model.safetensors", + "model.layers.10.mlp.up_proj.weight": "model.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model.safetensors", + "model.layers.10.self_attn.k_norm.weight": "model.safetensors", + "model.layers.10.self_attn.k_proj.biases": "model.safetensors", + "model.layers.10.self_attn.k_proj.scales": "model.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model.safetensors", + "model.layers.10.self_attn.o_proj.biases": "model.safetensors", + "model.layers.10.self_attn.o_proj.scales": "model.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model.safetensors", + "model.layers.10.self_attn.q_norm.weight": "model.safetensors", + "model.layers.10.self_attn.q_proj.biases": "model.safetensors", + "model.layers.10.self_attn.q_proj.scales": "model.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model.safetensors", + "model.layers.10.self_attn.v_proj.biases": "model.safetensors", + "model.layers.10.self_attn.v_proj.scales": "model.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model.safetensors", + "model.layers.11.input_layernorm.weight": "model.safetensors", + "model.layers.11.mlp.down_proj.biases": "model.safetensors", + "model.layers.11.mlp.down_proj.scales": "model.safetensors", + "model.layers.11.mlp.down_proj.weight": "model.safetensors", + "model.layers.11.mlp.gate_proj.biases": "model.safetensors", + "model.layers.11.mlp.gate_proj.scales": "model.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model.safetensors", + "model.layers.11.mlp.up_proj.biases": "model.safetensors", + "model.layers.11.mlp.up_proj.scales": "model.safetensors", + "model.layers.11.mlp.up_proj.weight": "model.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model.safetensors", + "model.layers.11.self_attn.k_norm.weight": "model.safetensors", + "model.layers.11.self_attn.k_proj.biases": "model.safetensors", + "model.layers.11.self_attn.k_proj.scales": "model.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model.safetensors", + "model.layers.11.self_attn.o_proj.biases": "model.safetensors", + "model.layers.11.self_attn.o_proj.scales": "model.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model.safetensors", + "model.layers.11.self_attn.q_norm.weight": "model.safetensors", + "model.layers.11.self_attn.q_proj.biases": "model.safetensors", + "model.layers.11.self_attn.q_proj.scales": "model.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model.safetensors", + "model.layers.11.self_attn.v_proj.biases": "model.safetensors", + "model.layers.11.self_attn.v_proj.scales": "model.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model.safetensors", + "model.layers.12.input_layernorm.weight": "model.safetensors", + "model.layers.12.mlp.down_proj.biases": "model.safetensors", + "model.layers.12.mlp.down_proj.scales": "model.safetensors", + "model.layers.12.mlp.down_proj.weight": "model.safetensors", + "model.layers.12.mlp.gate_proj.biases": "model.safetensors", + "model.layers.12.mlp.gate_proj.scales": "model.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model.safetensors", + "model.layers.12.mlp.up_proj.biases": "model.safetensors", + "model.layers.12.mlp.up_proj.scales": "model.safetensors", + "model.layers.12.mlp.up_proj.weight": "model.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model.safetensors", + "model.layers.12.self_attn.k_norm.weight": "model.safetensors", + "model.layers.12.self_attn.k_proj.biases": "model.safetensors", + "model.layers.12.self_attn.k_proj.scales": "model.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model.safetensors", + "model.layers.12.self_attn.o_proj.biases": "model.safetensors", + "model.layers.12.self_attn.o_proj.scales": "model.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model.safetensors", + "model.layers.12.self_attn.q_norm.weight": "model.safetensors", + "model.layers.12.self_attn.q_proj.biases": "model.safetensors", + "model.layers.12.self_attn.q_proj.scales": "model.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model.safetensors", + "model.layers.12.self_attn.v_proj.biases": "model.safetensors", + "model.layers.12.self_attn.v_proj.scales": "model.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model.safetensors", + "model.layers.13.input_layernorm.weight": "model.safetensors", + "model.layers.13.mlp.down_proj.biases": "model.safetensors", + "model.layers.13.mlp.down_proj.scales": "model.safetensors", + "model.layers.13.mlp.down_proj.weight": "model.safetensors", + "model.layers.13.mlp.gate_proj.biases": "model.safetensors", + "model.layers.13.mlp.gate_proj.scales": "model.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model.safetensors", + "model.layers.13.mlp.up_proj.biases": "model.safetensors", + "model.layers.13.mlp.up_proj.scales": "model.safetensors", + "model.layers.13.mlp.up_proj.weight": "model.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model.safetensors", + "model.layers.13.self_attn.k_norm.weight": "model.safetensors", + "model.layers.13.self_attn.k_proj.biases": "model.safetensors", + "model.layers.13.self_attn.k_proj.scales": "model.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model.safetensors", + "model.layers.13.self_attn.o_proj.biases": "model.safetensors", + "model.layers.13.self_attn.o_proj.scales": "model.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model.safetensors", + "model.layers.13.self_attn.q_norm.weight": "model.safetensors", + "model.layers.13.self_attn.q_proj.biases": "model.safetensors", + "model.layers.13.self_attn.q_proj.scales": "model.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model.safetensors", + "model.layers.13.self_attn.v_proj.biases": "model.safetensors", + "model.layers.13.self_attn.v_proj.scales": "model.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model.safetensors", + "model.layers.14.input_layernorm.weight": "model.safetensors", + "model.layers.14.mlp.down_proj.biases": "model.safetensors", + "model.layers.14.mlp.down_proj.scales": "model.safetensors", + "model.layers.14.mlp.down_proj.weight": "model.safetensors", + "model.layers.14.mlp.gate_proj.biases": "model.safetensors", + "model.layers.14.mlp.gate_proj.scales": "model.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model.safetensors", + "model.layers.14.mlp.up_proj.biases": "model.safetensors", + "model.layers.14.mlp.up_proj.scales": "model.safetensors", + "model.layers.14.mlp.up_proj.weight": "model.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model.safetensors", + "model.layers.14.self_attn.k_norm.weight": "model.safetensors", + "model.layers.14.self_attn.k_proj.biases": "model.safetensors", + "model.layers.14.self_attn.k_proj.scales": "model.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model.safetensors", + "model.layers.14.self_attn.o_proj.biases": "model.safetensors", + "model.layers.14.self_attn.o_proj.scales": "model.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model.safetensors", + "model.layers.14.self_attn.q_norm.weight": "model.safetensors", + "model.layers.14.self_attn.q_proj.biases": "model.safetensors", + "model.layers.14.self_attn.q_proj.scales": "model.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model.safetensors", + "model.layers.14.self_attn.v_proj.biases": "model.safetensors", + "model.layers.14.self_attn.v_proj.scales": "model.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model.safetensors", + "model.layers.15.input_layernorm.weight": "model.safetensors", + "model.layers.15.mlp.down_proj.biases": "model.safetensors", + "model.layers.15.mlp.down_proj.scales": "model.safetensors", + "model.layers.15.mlp.down_proj.weight": "model.safetensors", + "model.layers.15.mlp.gate_proj.biases": "model.safetensors", + "model.layers.15.mlp.gate_proj.scales": "model.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model.safetensors", + "model.layers.15.mlp.up_proj.biases": "model.safetensors", + "model.layers.15.mlp.up_proj.scales": "model.safetensors", + "model.layers.15.mlp.up_proj.weight": "model.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model.safetensors", + "model.layers.15.self_attn.k_norm.weight": "model.safetensors", + "model.layers.15.self_attn.k_proj.biases": "model.safetensors", + "model.layers.15.self_attn.k_proj.scales": "model.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model.safetensors", + "model.layers.15.self_attn.o_proj.biases": "model.safetensors", + "model.layers.15.self_attn.o_proj.scales": "model.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model.safetensors", + "model.layers.15.self_attn.q_norm.weight": "model.safetensors", + "model.layers.15.self_attn.q_proj.biases": "model.safetensors", + "model.layers.15.self_attn.q_proj.scales": "model.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model.safetensors", + "model.layers.15.self_attn.v_proj.biases": "model.safetensors", + "model.layers.15.self_attn.v_proj.scales": "model.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model.safetensors", + "model.layers.16.input_layernorm.weight": "model.safetensors", + "model.layers.16.mlp.down_proj.biases": "model.safetensors", + "model.layers.16.mlp.down_proj.scales": "model.safetensors", + "model.layers.16.mlp.down_proj.weight": "model.safetensors", + "model.layers.16.mlp.gate_proj.biases": "model.safetensors", + "model.layers.16.mlp.gate_proj.scales": "model.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model.safetensors", + "model.layers.16.mlp.up_proj.biases": "model.safetensors", + "model.layers.16.mlp.up_proj.scales": "model.safetensors", + "model.layers.16.mlp.up_proj.weight": "model.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model.safetensors", + "model.layers.16.self_attn.k_norm.weight": "model.safetensors", + "model.layers.16.self_attn.k_proj.biases": "model.safetensors", + "model.layers.16.self_attn.k_proj.scales": "model.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model.safetensors", + "model.layers.16.self_attn.o_proj.biases": "model.safetensors", + "model.layers.16.self_attn.o_proj.scales": "model.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model.safetensors", + "model.layers.16.self_attn.q_norm.weight": "model.safetensors", + "model.layers.16.self_attn.q_proj.biases": "model.safetensors", + "model.layers.16.self_attn.q_proj.scales": "model.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model.safetensors", + "model.layers.16.self_attn.v_proj.biases": "model.safetensors", + "model.layers.16.self_attn.v_proj.scales": "model.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model.safetensors", + "model.layers.17.input_layernorm.weight": "model.safetensors", + "model.layers.17.mlp.down_proj.biases": "model.safetensors", + "model.layers.17.mlp.down_proj.scales": "model.safetensors", + "model.layers.17.mlp.down_proj.weight": "model.safetensors", + "model.layers.17.mlp.gate_proj.biases": "model.safetensors", + "model.layers.17.mlp.gate_proj.scales": "model.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model.safetensors", + "model.layers.17.mlp.up_proj.biases": "model.safetensors", + "model.layers.17.mlp.up_proj.scales": "model.safetensors", + "model.layers.17.mlp.up_proj.weight": "model.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model.safetensors", + "model.layers.17.self_attn.k_norm.weight": "model.safetensors", + "model.layers.17.self_attn.k_proj.biases": "model.safetensors", + "model.layers.17.self_attn.k_proj.scales": "model.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model.safetensors", + "model.layers.17.self_attn.o_proj.biases": "model.safetensors", + "model.layers.17.self_attn.o_proj.scales": "model.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model.safetensors", + "model.layers.17.self_attn.q_norm.weight": "model.safetensors", + "model.layers.17.self_attn.q_proj.biases": "model.safetensors", + "model.layers.17.self_attn.q_proj.scales": "model.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model.safetensors", + "model.layers.17.self_attn.v_proj.biases": "model.safetensors", + "model.layers.17.self_attn.v_proj.scales": "model.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model.safetensors", + "model.layers.18.input_layernorm.weight": "model.safetensors", + "model.layers.18.mlp.down_proj.biases": "model.safetensors", + "model.layers.18.mlp.down_proj.scales": "model.safetensors", + "model.layers.18.mlp.down_proj.weight": "model.safetensors", + "model.layers.18.mlp.gate_proj.biases": "model.safetensors", + "model.layers.18.mlp.gate_proj.scales": "model.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model.safetensors", + "model.layers.18.mlp.up_proj.biases": "model.safetensors", + "model.layers.18.mlp.up_proj.scales": "model.safetensors", + "model.layers.18.mlp.up_proj.weight": "model.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model.safetensors", + "model.layers.18.self_attn.k_norm.weight": "model.safetensors", + "model.layers.18.self_attn.k_proj.biases": "model.safetensors", + "model.layers.18.self_attn.k_proj.scales": "model.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model.safetensors", + "model.layers.18.self_attn.o_proj.biases": "model.safetensors", + "model.layers.18.self_attn.o_proj.scales": "model.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model.safetensors", + "model.layers.18.self_attn.q_norm.weight": "model.safetensors", + "model.layers.18.self_attn.q_proj.biases": "model.safetensors", + "model.layers.18.self_attn.q_proj.scales": "model.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model.safetensors", + "model.layers.18.self_attn.v_proj.biases": "model.safetensors", + "model.layers.18.self_attn.v_proj.scales": "model.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model.safetensors", + "model.layers.19.input_layernorm.weight": "model.safetensors", + "model.layers.19.mlp.down_proj.biases": "model.safetensors", + "model.layers.19.mlp.down_proj.scales": "model.safetensors", + "model.layers.19.mlp.down_proj.weight": "model.safetensors", + "model.layers.19.mlp.gate_proj.biases": "model.safetensors", + "model.layers.19.mlp.gate_proj.scales": "model.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model.safetensors", + "model.layers.19.mlp.up_proj.biases": "model.safetensors", + "model.layers.19.mlp.up_proj.scales": "model.safetensors", + "model.layers.19.mlp.up_proj.weight": "model.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model.safetensors", + "model.layers.19.self_attn.k_norm.weight": "model.safetensors", + "model.layers.19.self_attn.k_proj.biases": "model.safetensors", + "model.layers.19.self_attn.k_proj.scales": "model.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model.safetensors", + "model.layers.19.self_attn.o_proj.biases": "model.safetensors", + "model.layers.19.self_attn.o_proj.scales": "model.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model.safetensors", + "model.layers.19.self_attn.q_norm.weight": "model.safetensors", + "model.layers.19.self_attn.q_proj.biases": "model.safetensors", + "model.layers.19.self_attn.q_proj.scales": "model.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model.safetensors", + "model.layers.19.self_attn.v_proj.biases": "model.safetensors", + "model.layers.19.self_attn.v_proj.scales": "model.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model.safetensors", + "model.layers.2.input_layernorm.weight": "model.safetensors", + "model.layers.2.mlp.down_proj.biases": "model.safetensors", + "model.layers.2.mlp.down_proj.scales": "model.safetensors", + "model.layers.2.mlp.down_proj.weight": "model.safetensors", + "model.layers.2.mlp.gate_proj.biases": "model.safetensors", + "model.layers.2.mlp.gate_proj.scales": "model.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model.safetensors", + "model.layers.2.mlp.up_proj.biases": "model.safetensors", + "model.layers.2.mlp.up_proj.scales": "model.safetensors", + "model.layers.2.mlp.up_proj.weight": "model.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model.safetensors", + "model.layers.2.self_attn.k_norm.weight": "model.safetensors", + "model.layers.2.self_attn.k_proj.biases": "model.safetensors", + "model.layers.2.self_attn.k_proj.scales": "model.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model.safetensors", + "model.layers.2.self_attn.o_proj.biases": "model.safetensors", + "model.layers.2.self_attn.o_proj.scales": "model.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model.safetensors", + "model.layers.2.self_attn.q_norm.weight": "model.safetensors", + "model.layers.2.self_attn.q_proj.biases": "model.safetensors", + "model.layers.2.self_attn.q_proj.scales": "model.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model.safetensors", + "model.layers.2.self_attn.v_proj.biases": "model.safetensors", + "model.layers.2.self_attn.v_proj.scales": "model.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model.safetensors", + "model.layers.20.input_layernorm.weight": "model.safetensors", + "model.layers.20.mlp.down_proj.biases": "model.safetensors", + "model.layers.20.mlp.down_proj.scales": "model.safetensors", + "model.layers.20.mlp.down_proj.weight": "model.safetensors", + "model.layers.20.mlp.gate_proj.biases": "model.safetensors", + "model.layers.20.mlp.gate_proj.scales": "model.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model.safetensors", + "model.layers.20.mlp.up_proj.biases": "model.safetensors", + "model.layers.20.mlp.up_proj.scales": "model.safetensors", + "model.layers.20.mlp.up_proj.weight": "model.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model.safetensors", + "model.layers.20.self_attn.k_norm.weight": "model.safetensors", + "model.layers.20.self_attn.k_proj.biases": "model.safetensors", + "model.layers.20.self_attn.k_proj.scales": "model.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model.safetensors", + "model.layers.20.self_attn.o_proj.biases": "model.safetensors", + "model.layers.20.self_attn.o_proj.scales": "model.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model.safetensors", + "model.layers.20.self_attn.q_norm.weight": "model.safetensors", + "model.layers.20.self_attn.q_proj.biases": "model.safetensors", + "model.layers.20.self_attn.q_proj.scales": "model.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model.safetensors", + "model.layers.20.self_attn.v_proj.biases": "model.safetensors", + "model.layers.20.self_attn.v_proj.scales": "model.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model.safetensors", + "model.layers.21.input_layernorm.weight": "model.safetensors", + "model.layers.21.mlp.down_proj.biases": "model.safetensors", + "model.layers.21.mlp.down_proj.scales": "model.safetensors", + "model.layers.21.mlp.down_proj.weight": "model.safetensors", + "model.layers.21.mlp.gate_proj.biases": "model.safetensors", + "model.layers.21.mlp.gate_proj.scales": "model.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model.safetensors", + "model.layers.21.mlp.up_proj.biases": "model.safetensors", + "model.layers.21.mlp.up_proj.scales": "model.safetensors", + "model.layers.21.mlp.up_proj.weight": "model.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model.safetensors", + "model.layers.21.self_attn.k_norm.weight": "model.safetensors", + "model.layers.21.self_attn.k_proj.biases": "model.safetensors", + "model.layers.21.self_attn.k_proj.scales": "model.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model.safetensors", + "model.layers.21.self_attn.o_proj.biases": "model.safetensors", + "model.layers.21.self_attn.o_proj.scales": "model.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model.safetensors", + "model.layers.21.self_attn.q_norm.weight": "model.safetensors", + "model.layers.21.self_attn.q_proj.biases": "model.safetensors", + "model.layers.21.self_attn.q_proj.scales": "model.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model.safetensors", + "model.layers.21.self_attn.v_proj.biases": "model.safetensors", + "model.layers.21.self_attn.v_proj.scales": "model.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model.safetensors", + "model.layers.22.input_layernorm.weight": "model.safetensors", + "model.layers.22.mlp.down_proj.biases": "model.safetensors", + "model.layers.22.mlp.down_proj.scales": "model.safetensors", + "model.layers.22.mlp.down_proj.weight": "model.safetensors", + "model.layers.22.mlp.gate_proj.biases": "model.safetensors", + "model.layers.22.mlp.gate_proj.scales": "model.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model.safetensors", + "model.layers.22.mlp.up_proj.biases": "model.safetensors", + "model.layers.22.mlp.up_proj.scales": "model.safetensors", + "model.layers.22.mlp.up_proj.weight": "model.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model.safetensors", + "model.layers.22.self_attn.k_norm.weight": "model.safetensors", + "model.layers.22.self_attn.k_proj.biases": "model.safetensors", + "model.layers.22.self_attn.k_proj.scales": "model.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model.safetensors", + "model.layers.22.self_attn.o_proj.biases": "model.safetensors", + "model.layers.22.self_attn.o_proj.scales": "model.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model.safetensors", + "model.layers.22.self_attn.q_norm.weight": "model.safetensors", + "model.layers.22.self_attn.q_proj.biases": "model.safetensors", + "model.layers.22.self_attn.q_proj.scales": "model.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model.safetensors", + "model.layers.22.self_attn.v_proj.biases": "model.safetensors", + "model.layers.22.self_attn.v_proj.scales": "model.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model.safetensors", + "model.layers.23.input_layernorm.weight": "model.safetensors", + "model.layers.23.mlp.down_proj.biases": "model.safetensors", + "model.layers.23.mlp.down_proj.scales": "model.safetensors", + "model.layers.23.mlp.down_proj.weight": "model.safetensors", + "model.layers.23.mlp.gate_proj.biases": "model.safetensors", + "model.layers.23.mlp.gate_proj.scales": "model.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model.safetensors", + "model.layers.23.mlp.up_proj.biases": "model.safetensors", + "model.layers.23.mlp.up_proj.scales": "model.safetensors", + "model.layers.23.mlp.up_proj.weight": "model.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model.safetensors", + "model.layers.23.self_attn.k_norm.weight": "model.safetensors", + "model.layers.23.self_attn.k_proj.biases": "model.safetensors", + "model.layers.23.self_attn.k_proj.scales": "model.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model.safetensors", + "model.layers.23.self_attn.o_proj.biases": "model.safetensors", + "model.layers.23.self_attn.o_proj.scales": "model.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model.safetensors", + "model.layers.23.self_attn.q_norm.weight": "model.safetensors", + "model.layers.23.self_attn.q_proj.biases": "model.safetensors", + "model.layers.23.self_attn.q_proj.scales": "model.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model.safetensors", + "model.layers.23.self_attn.v_proj.biases": "model.safetensors", + "model.layers.23.self_attn.v_proj.scales": "model.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model.safetensors", + "model.layers.24.input_layernorm.weight": "model.safetensors", + "model.layers.24.mlp.down_proj.biases": "model.safetensors", + "model.layers.24.mlp.down_proj.scales": "model.safetensors", + "model.layers.24.mlp.down_proj.weight": "model.safetensors", + "model.layers.24.mlp.gate_proj.biases": "model.safetensors", + "model.layers.24.mlp.gate_proj.scales": "model.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model.safetensors", + "model.layers.24.mlp.up_proj.biases": "model.safetensors", + "model.layers.24.mlp.up_proj.scales": "model.safetensors", + "model.layers.24.mlp.up_proj.weight": "model.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model.safetensors", + "model.layers.24.self_attn.k_norm.weight": "model.safetensors", + "model.layers.24.self_attn.k_proj.biases": "model.safetensors", + "model.layers.24.self_attn.k_proj.scales": "model.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model.safetensors", + "model.layers.24.self_attn.o_proj.biases": "model.safetensors", + "model.layers.24.self_attn.o_proj.scales": "model.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model.safetensors", + "model.layers.24.self_attn.q_norm.weight": "model.safetensors", + "model.layers.24.self_attn.q_proj.biases": "model.safetensors", + "model.layers.24.self_attn.q_proj.scales": "model.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model.safetensors", + "model.layers.24.self_attn.v_proj.biases": "model.safetensors", + "model.layers.24.self_attn.v_proj.scales": "model.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model.safetensors", + "model.layers.25.input_layernorm.weight": "model.safetensors", + "model.layers.25.mlp.down_proj.biases": "model.safetensors", + "model.layers.25.mlp.down_proj.scales": "model.safetensors", + "model.layers.25.mlp.down_proj.weight": "model.safetensors", + "model.layers.25.mlp.gate_proj.biases": "model.safetensors", + "model.layers.25.mlp.gate_proj.scales": "model.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model.safetensors", + "model.layers.25.mlp.up_proj.biases": "model.safetensors", + "model.layers.25.mlp.up_proj.scales": "model.safetensors", + "model.layers.25.mlp.up_proj.weight": "model.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model.safetensors", + "model.layers.25.self_attn.k_norm.weight": "model.safetensors", + "model.layers.25.self_attn.k_proj.biases": "model.safetensors", + "model.layers.25.self_attn.k_proj.scales": "model.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model.safetensors", + "model.layers.25.self_attn.o_proj.biases": "model.safetensors", + "model.layers.25.self_attn.o_proj.scales": "model.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model.safetensors", + "model.layers.25.self_attn.q_norm.weight": "model.safetensors", + "model.layers.25.self_attn.q_proj.biases": "model.safetensors", + "model.layers.25.self_attn.q_proj.scales": "model.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model.safetensors", + "model.layers.25.self_attn.v_proj.biases": "model.safetensors", + "model.layers.25.self_attn.v_proj.scales": "model.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model.safetensors", + "model.layers.26.input_layernorm.weight": "model.safetensors", + "model.layers.26.mlp.down_proj.biases": "model.safetensors", + "model.layers.26.mlp.down_proj.scales": "model.safetensors", + "model.layers.26.mlp.down_proj.weight": "model.safetensors", + "model.layers.26.mlp.gate_proj.biases": "model.safetensors", + "model.layers.26.mlp.gate_proj.scales": "model.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model.safetensors", + "model.layers.26.mlp.up_proj.biases": "model.safetensors", + "model.layers.26.mlp.up_proj.scales": "model.safetensors", + "model.layers.26.mlp.up_proj.weight": "model.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model.safetensors", + "model.layers.26.self_attn.k_norm.weight": "model.safetensors", + "model.layers.26.self_attn.k_proj.biases": "model.safetensors", + "model.layers.26.self_attn.k_proj.scales": "model.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model.safetensors", + "model.layers.26.self_attn.o_proj.biases": "model.safetensors", + "model.layers.26.self_attn.o_proj.scales": "model.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model.safetensors", + "model.layers.26.self_attn.q_norm.weight": "model.safetensors", + "model.layers.26.self_attn.q_proj.biases": "model.safetensors", + "model.layers.26.self_attn.q_proj.scales": "model.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model.safetensors", + "model.layers.26.self_attn.v_proj.biases": "model.safetensors", + "model.layers.26.self_attn.v_proj.scales": "model.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model.safetensors", + "model.layers.27.input_layernorm.weight": "model.safetensors", + "model.layers.27.mlp.down_proj.biases": "model.safetensors", + "model.layers.27.mlp.down_proj.scales": "model.safetensors", + "model.layers.27.mlp.down_proj.weight": "model.safetensors", + "model.layers.27.mlp.gate_proj.biases": "model.safetensors", + "model.layers.27.mlp.gate_proj.scales": "model.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model.safetensors", + "model.layers.27.mlp.up_proj.biases": "model.safetensors", + "model.layers.27.mlp.up_proj.scales": "model.safetensors", + "model.layers.27.mlp.up_proj.weight": "model.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model.safetensors", + "model.layers.27.self_attn.k_norm.weight": "model.safetensors", + "model.layers.27.self_attn.k_proj.biases": "model.safetensors", + "model.layers.27.self_attn.k_proj.scales": "model.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model.safetensors", + "model.layers.27.self_attn.o_proj.biases": "model.safetensors", + "model.layers.27.self_attn.o_proj.scales": "model.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model.safetensors", + "model.layers.27.self_attn.q_norm.weight": "model.safetensors", + "model.layers.27.self_attn.q_proj.biases": "model.safetensors", + "model.layers.27.self_attn.q_proj.scales": "model.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model.safetensors", + "model.layers.27.self_attn.v_proj.biases": "model.safetensors", + "model.layers.27.self_attn.v_proj.scales": "model.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model.safetensors", + "model.layers.28.input_layernorm.weight": "model.safetensors", + "model.layers.28.mlp.down_proj.biases": "model.safetensors", + "model.layers.28.mlp.down_proj.scales": "model.safetensors", + "model.layers.28.mlp.down_proj.weight": "model.safetensors", + "model.layers.28.mlp.gate_proj.biases": "model.safetensors", + "model.layers.28.mlp.gate_proj.scales": "model.safetensors", + "model.layers.28.mlp.gate_proj.weight": "model.safetensors", + "model.layers.28.mlp.up_proj.biases": "model.safetensors", + "model.layers.28.mlp.up_proj.scales": "model.safetensors", + "model.layers.28.mlp.up_proj.weight": "model.safetensors", + "model.layers.28.post_attention_layernorm.weight": "model.safetensors", + "model.layers.28.self_attn.k_norm.weight": "model.safetensors", + "model.layers.28.self_attn.k_proj.biases": "model.safetensors", + "model.layers.28.self_attn.k_proj.scales": "model.safetensors", + "model.layers.28.self_attn.k_proj.weight": "model.safetensors", + "model.layers.28.self_attn.o_proj.biases": "model.safetensors", + "model.layers.28.self_attn.o_proj.scales": "model.safetensors", + "model.layers.28.self_attn.o_proj.weight": "model.safetensors", + "model.layers.28.self_attn.q_norm.weight": "model.safetensors", + "model.layers.28.self_attn.q_proj.biases": "model.safetensors", + "model.layers.28.self_attn.q_proj.scales": "model.safetensors", + "model.layers.28.self_attn.q_proj.weight": "model.safetensors", + "model.layers.28.self_attn.v_proj.biases": "model.safetensors", + "model.layers.28.self_attn.v_proj.scales": "model.safetensors", + "model.layers.28.self_attn.v_proj.weight": "model.safetensors", + "model.layers.29.input_layernorm.weight": "model.safetensors", + "model.layers.29.mlp.down_proj.biases": "model.safetensors", + "model.layers.29.mlp.down_proj.scales": "model.safetensors", + "model.layers.29.mlp.down_proj.weight": "model.safetensors", + "model.layers.29.mlp.gate_proj.biases": "model.safetensors", + "model.layers.29.mlp.gate_proj.scales": "model.safetensors", + "model.layers.29.mlp.gate_proj.weight": "model.safetensors", + "model.layers.29.mlp.up_proj.biases": "model.safetensors", + "model.layers.29.mlp.up_proj.scales": "model.safetensors", + "model.layers.29.mlp.up_proj.weight": "model.safetensors", + "model.layers.29.post_attention_layernorm.weight": "model.safetensors", + "model.layers.29.self_attn.k_norm.weight": "model.safetensors", + "model.layers.29.self_attn.k_proj.biases": "model.safetensors", + "model.layers.29.self_attn.k_proj.scales": "model.safetensors", + "model.layers.29.self_attn.k_proj.weight": "model.safetensors", + "model.layers.29.self_attn.o_proj.biases": "model.safetensors", + "model.layers.29.self_attn.o_proj.scales": "model.safetensors", + "model.layers.29.self_attn.o_proj.weight": "model.safetensors", + "model.layers.29.self_attn.q_norm.weight": "model.safetensors", + "model.layers.29.self_attn.q_proj.biases": "model.safetensors", + "model.layers.29.self_attn.q_proj.scales": "model.safetensors", + "model.layers.29.self_attn.q_proj.weight": "model.safetensors", + "model.layers.29.self_attn.v_proj.biases": "model.safetensors", + "model.layers.29.self_attn.v_proj.scales": "model.safetensors", + "model.layers.29.self_attn.v_proj.weight": "model.safetensors", + "model.layers.3.input_layernorm.weight": "model.safetensors", + "model.layers.3.mlp.down_proj.biases": "model.safetensors", + "model.layers.3.mlp.down_proj.scales": "model.safetensors", + "model.layers.3.mlp.down_proj.weight": "model.safetensors", + "model.layers.3.mlp.gate_proj.biases": "model.safetensors", + "model.layers.3.mlp.gate_proj.scales": "model.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model.safetensors", + "model.layers.3.mlp.up_proj.biases": "model.safetensors", + "model.layers.3.mlp.up_proj.scales": "model.safetensors", + "model.layers.3.mlp.up_proj.weight": "model.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model.safetensors", + "model.layers.3.self_attn.k_norm.weight": "model.safetensors", + "model.layers.3.self_attn.k_proj.biases": "model.safetensors", + "model.layers.3.self_attn.k_proj.scales": "model.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model.safetensors", + "model.layers.3.self_attn.o_proj.biases": "model.safetensors", + "model.layers.3.self_attn.o_proj.scales": "model.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model.safetensors", + "model.layers.3.self_attn.q_norm.weight": "model.safetensors", + "model.layers.3.self_attn.q_proj.biases": "model.safetensors", + "model.layers.3.self_attn.q_proj.scales": "model.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model.safetensors", + "model.layers.3.self_attn.v_proj.biases": "model.safetensors", + "model.layers.3.self_attn.v_proj.scales": "model.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model.safetensors", + "model.layers.30.input_layernorm.weight": "model.safetensors", + "model.layers.30.mlp.down_proj.biases": "model.safetensors", + "model.layers.30.mlp.down_proj.scales": "model.safetensors", + "model.layers.30.mlp.down_proj.weight": "model.safetensors", + "model.layers.30.mlp.gate_proj.biases": "model.safetensors", + "model.layers.30.mlp.gate_proj.scales": "model.safetensors", + "model.layers.30.mlp.gate_proj.weight": "model.safetensors", + "model.layers.30.mlp.up_proj.biases": "model.safetensors", + "model.layers.30.mlp.up_proj.scales": "model.safetensors", + "model.layers.30.mlp.up_proj.weight": "model.safetensors", + "model.layers.30.post_attention_layernorm.weight": "model.safetensors", + "model.layers.30.self_attn.k_norm.weight": "model.safetensors", + "model.layers.30.self_attn.k_proj.biases": "model.safetensors", + "model.layers.30.self_attn.k_proj.scales": "model.safetensors", + "model.layers.30.self_attn.k_proj.weight": "model.safetensors", + "model.layers.30.self_attn.o_proj.biases": "model.safetensors", + "model.layers.30.self_attn.o_proj.scales": "model.safetensors", + "model.layers.30.self_attn.o_proj.weight": "model.safetensors", + "model.layers.30.self_attn.q_norm.weight": "model.safetensors", + "model.layers.30.self_attn.q_proj.biases": "model.safetensors", + "model.layers.30.self_attn.q_proj.scales": "model.safetensors", + "model.layers.30.self_attn.q_proj.weight": "model.safetensors", + "model.layers.30.self_attn.v_proj.biases": "model.safetensors", + "model.layers.30.self_attn.v_proj.scales": "model.safetensors", + "model.layers.30.self_attn.v_proj.weight": "model.safetensors", + "model.layers.31.input_layernorm.weight": "model.safetensors", + "model.layers.31.mlp.down_proj.biases": "model.safetensors", + "model.layers.31.mlp.down_proj.scales": "model.safetensors", + "model.layers.31.mlp.down_proj.weight": "model.safetensors", + "model.layers.31.mlp.gate_proj.biases": "model.safetensors", + "model.layers.31.mlp.gate_proj.scales": "model.safetensors", + "model.layers.31.mlp.gate_proj.weight": "model.safetensors", + "model.layers.31.mlp.up_proj.biases": "model.safetensors", + "model.layers.31.mlp.up_proj.scales": "model.safetensors", + "model.layers.31.mlp.up_proj.weight": "model.safetensors", + "model.layers.31.post_attention_layernorm.weight": "model.safetensors", + "model.layers.31.self_attn.k_norm.weight": "model.safetensors", + "model.layers.31.self_attn.k_proj.biases": "model.safetensors", + "model.layers.31.self_attn.k_proj.scales": "model.safetensors", + "model.layers.31.self_attn.k_proj.weight": "model.safetensors", + "model.layers.31.self_attn.o_proj.biases": "model.safetensors", + "model.layers.31.self_attn.o_proj.scales": "model.safetensors", + "model.layers.31.self_attn.o_proj.weight": "model.safetensors", + "model.layers.31.self_attn.q_norm.weight": "model.safetensors", + "model.layers.31.self_attn.q_proj.biases": "model.safetensors", + "model.layers.31.self_attn.q_proj.scales": "model.safetensors", + "model.layers.31.self_attn.q_proj.weight": "model.safetensors", + "model.layers.31.self_attn.v_proj.biases": "model.safetensors", + "model.layers.31.self_attn.v_proj.scales": "model.safetensors", + "model.layers.31.self_attn.v_proj.weight": "model.safetensors", + "model.layers.32.input_layernorm.weight": "model.safetensors", + "model.layers.32.mlp.down_proj.biases": "model.safetensors", + "model.layers.32.mlp.down_proj.scales": "model.safetensors", + "model.layers.32.mlp.down_proj.weight": "model.safetensors", + "model.layers.32.mlp.gate_proj.biases": "model.safetensors", + "model.layers.32.mlp.gate_proj.scales": "model.safetensors", + "model.layers.32.mlp.gate_proj.weight": "model.safetensors", + "model.layers.32.mlp.up_proj.biases": "model.safetensors", + "model.layers.32.mlp.up_proj.scales": "model.safetensors", + "model.layers.32.mlp.up_proj.weight": "model.safetensors", + "model.layers.32.post_attention_layernorm.weight": "model.safetensors", + "model.layers.32.self_attn.k_norm.weight": "model.safetensors", + "model.layers.32.self_attn.k_proj.biases": "model.safetensors", + "model.layers.32.self_attn.k_proj.scales": "model.safetensors", + "model.layers.32.self_attn.k_proj.weight": "model.safetensors", + "model.layers.32.self_attn.o_proj.biases": "model.safetensors", + "model.layers.32.self_attn.o_proj.scales": "model.safetensors", + "model.layers.32.self_attn.o_proj.weight": "model.safetensors", + "model.layers.32.self_attn.q_norm.weight": "model.safetensors", + "model.layers.32.self_attn.q_proj.biases": "model.safetensors", + "model.layers.32.self_attn.q_proj.scales": "model.safetensors", + "model.layers.32.self_attn.q_proj.weight": "model.safetensors", + "model.layers.32.self_attn.v_proj.biases": "model.safetensors", + "model.layers.32.self_attn.v_proj.scales": "model.safetensors", + "model.layers.32.self_attn.v_proj.weight": "model.safetensors", + "model.layers.33.input_layernorm.weight": "model.safetensors", + "model.layers.33.mlp.down_proj.biases": "model.safetensors", + "model.layers.33.mlp.down_proj.scales": "model.safetensors", + "model.layers.33.mlp.down_proj.weight": "model.safetensors", + "model.layers.33.mlp.gate_proj.biases": "model.safetensors", + "model.layers.33.mlp.gate_proj.scales": "model.safetensors", + "model.layers.33.mlp.gate_proj.weight": "model.safetensors", + "model.layers.33.mlp.up_proj.biases": "model.safetensors", + "model.layers.33.mlp.up_proj.scales": "model.safetensors", + "model.layers.33.mlp.up_proj.weight": "model.safetensors", + "model.layers.33.post_attention_layernorm.weight": "model.safetensors", + "model.layers.33.self_attn.k_norm.weight": "model.safetensors", + "model.layers.33.self_attn.k_proj.biases": "model.safetensors", + "model.layers.33.self_attn.k_proj.scales": "model.safetensors", + "model.layers.33.self_attn.k_proj.weight": "model.safetensors", + "model.layers.33.self_attn.o_proj.biases": "model.safetensors", + "model.layers.33.self_attn.o_proj.scales": "model.safetensors", + "model.layers.33.self_attn.o_proj.weight": "model.safetensors", + "model.layers.33.self_attn.q_norm.weight": "model.safetensors", + "model.layers.33.self_attn.q_proj.biases": "model.safetensors", + "model.layers.33.self_attn.q_proj.scales": "model.safetensors", + "model.layers.33.self_attn.q_proj.weight": "model.safetensors", + "model.layers.33.self_attn.v_proj.biases": "model.safetensors", + "model.layers.33.self_attn.v_proj.scales": "model.safetensors", + "model.layers.33.self_attn.v_proj.weight": "model.safetensors", + "model.layers.34.input_layernorm.weight": "model.safetensors", + "model.layers.34.mlp.down_proj.biases": "model.safetensors", + "model.layers.34.mlp.down_proj.scales": "model.safetensors", + "model.layers.34.mlp.down_proj.weight": "model.safetensors", + "model.layers.34.mlp.gate_proj.biases": "model.safetensors", + "model.layers.34.mlp.gate_proj.scales": "model.safetensors", + "model.layers.34.mlp.gate_proj.weight": "model.safetensors", + "model.layers.34.mlp.up_proj.biases": "model.safetensors", + "model.layers.34.mlp.up_proj.scales": "model.safetensors", + "model.layers.34.mlp.up_proj.weight": "model.safetensors", + "model.layers.34.post_attention_layernorm.weight": "model.safetensors", + "model.layers.34.self_attn.k_norm.weight": "model.safetensors", + "model.layers.34.self_attn.k_proj.biases": "model.safetensors", + "model.layers.34.self_attn.k_proj.scales": "model.safetensors", + "model.layers.34.self_attn.k_proj.weight": "model.safetensors", + "model.layers.34.self_attn.o_proj.biases": "model.safetensors", + "model.layers.34.self_attn.o_proj.scales": "model.safetensors", + "model.layers.34.self_attn.o_proj.weight": "model.safetensors", + "model.layers.34.self_attn.q_norm.weight": "model.safetensors", + "model.layers.34.self_attn.q_proj.biases": "model.safetensors", + "model.layers.34.self_attn.q_proj.scales": "model.safetensors", + "model.layers.34.self_attn.q_proj.weight": "model.safetensors", + "model.layers.34.self_attn.v_proj.biases": "model.safetensors", + "model.layers.34.self_attn.v_proj.scales": "model.safetensors", + "model.layers.34.self_attn.v_proj.weight": "model.safetensors", + "model.layers.35.input_layernorm.weight": "model.safetensors", + "model.layers.35.mlp.down_proj.biases": "model.safetensors", + "model.layers.35.mlp.down_proj.scales": "model.safetensors", + "model.layers.35.mlp.down_proj.weight": "model.safetensors", + "model.layers.35.mlp.gate_proj.biases": "model.safetensors", + "model.layers.35.mlp.gate_proj.scales": "model.safetensors", + "model.layers.35.mlp.gate_proj.weight": "model.safetensors", + "model.layers.35.mlp.up_proj.biases": "model.safetensors", + "model.layers.35.mlp.up_proj.scales": "model.safetensors", + "model.layers.35.mlp.up_proj.weight": "model.safetensors", + "model.layers.35.post_attention_layernorm.weight": "model.safetensors", + "model.layers.35.self_attn.k_norm.weight": "model.safetensors", + "model.layers.35.self_attn.k_proj.biases": "model.safetensors", + "model.layers.35.self_attn.k_proj.scales": "model.safetensors", + "model.layers.35.self_attn.k_proj.weight": "model.safetensors", + "model.layers.35.self_attn.o_proj.biases": "model.safetensors", + "model.layers.35.self_attn.o_proj.scales": "model.safetensors", + "model.layers.35.self_attn.o_proj.weight": "model.safetensors", + "model.layers.35.self_attn.q_norm.weight": "model.safetensors", + "model.layers.35.self_attn.q_proj.biases": "model.safetensors", + "model.layers.35.self_attn.q_proj.scales": "model.safetensors", + "model.layers.35.self_attn.q_proj.weight": "model.safetensors", + "model.layers.35.self_attn.v_proj.biases": "model.safetensors", + "model.layers.35.self_attn.v_proj.scales": "model.safetensors", + "model.layers.35.self_attn.v_proj.weight": "model.safetensors", + "model.layers.4.input_layernorm.weight": "model.safetensors", + "model.layers.4.mlp.down_proj.biases": "model.safetensors", + "model.layers.4.mlp.down_proj.scales": "model.safetensors", + "model.layers.4.mlp.down_proj.weight": "model.safetensors", + "model.layers.4.mlp.gate_proj.biases": "model.safetensors", + "model.layers.4.mlp.gate_proj.scales": "model.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model.safetensors", + "model.layers.4.mlp.up_proj.biases": "model.safetensors", + "model.layers.4.mlp.up_proj.scales": "model.safetensors", + "model.layers.4.mlp.up_proj.weight": "model.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model.safetensors", + "model.layers.4.self_attn.k_norm.weight": "model.safetensors", + "model.layers.4.self_attn.k_proj.biases": "model.safetensors", + "model.layers.4.self_attn.k_proj.scales": "model.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model.safetensors", + "model.layers.4.self_attn.o_proj.biases": "model.safetensors", + "model.layers.4.self_attn.o_proj.scales": "model.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model.safetensors", + "model.layers.4.self_attn.q_norm.weight": "model.safetensors", + "model.layers.4.self_attn.q_proj.biases": "model.safetensors", + "model.layers.4.self_attn.q_proj.scales": "model.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model.safetensors", + "model.layers.4.self_attn.v_proj.biases": "model.safetensors", + "model.layers.4.self_attn.v_proj.scales": "model.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model.safetensors", + "model.layers.5.input_layernorm.weight": "model.safetensors", + "model.layers.5.mlp.down_proj.biases": "model.safetensors", + "model.layers.5.mlp.down_proj.scales": "model.safetensors", + "model.layers.5.mlp.down_proj.weight": "model.safetensors", + "model.layers.5.mlp.gate_proj.biases": "model.safetensors", + "model.layers.5.mlp.gate_proj.scales": "model.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model.safetensors", + "model.layers.5.mlp.up_proj.biases": "model.safetensors", + "model.layers.5.mlp.up_proj.scales": "model.safetensors", + "model.layers.5.mlp.up_proj.weight": "model.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model.safetensors", + "model.layers.5.self_attn.k_norm.weight": "model.safetensors", + "model.layers.5.self_attn.k_proj.biases": "model.safetensors", + "model.layers.5.self_attn.k_proj.scales": "model.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model.safetensors", + "model.layers.5.self_attn.o_proj.biases": "model.safetensors", + "model.layers.5.self_attn.o_proj.scales": "model.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model.safetensors", + "model.layers.5.self_attn.q_norm.weight": "model.safetensors", + "model.layers.5.self_attn.q_proj.biases": "model.safetensors", + "model.layers.5.self_attn.q_proj.scales": "model.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model.safetensors", + "model.layers.5.self_attn.v_proj.biases": "model.safetensors", + "model.layers.5.self_attn.v_proj.scales": "model.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model.safetensors", + "model.layers.6.input_layernorm.weight": "model.safetensors", + "model.layers.6.mlp.down_proj.biases": "model.safetensors", + "model.layers.6.mlp.down_proj.scales": "model.safetensors", + "model.layers.6.mlp.down_proj.weight": "model.safetensors", + "model.layers.6.mlp.gate_proj.biases": "model.safetensors", + "model.layers.6.mlp.gate_proj.scales": "model.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model.safetensors", + "model.layers.6.mlp.up_proj.biases": "model.safetensors", + "model.layers.6.mlp.up_proj.scales": "model.safetensors", + "model.layers.6.mlp.up_proj.weight": "model.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model.safetensors", + "model.layers.6.self_attn.k_norm.weight": "model.safetensors", + "model.layers.6.self_attn.k_proj.biases": "model.safetensors", + "model.layers.6.self_attn.k_proj.scales": "model.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model.safetensors", + "model.layers.6.self_attn.o_proj.biases": "model.safetensors", + "model.layers.6.self_attn.o_proj.scales": "model.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model.safetensors", + "model.layers.6.self_attn.q_norm.weight": "model.safetensors", + "model.layers.6.self_attn.q_proj.biases": "model.safetensors", + "model.layers.6.self_attn.q_proj.scales": "model.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model.safetensors", + "model.layers.6.self_attn.v_proj.biases": "model.safetensors", + "model.layers.6.self_attn.v_proj.scales": "model.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model.safetensors", + "model.layers.7.input_layernorm.weight": "model.safetensors", + "model.layers.7.mlp.down_proj.biases": "model.safetensors", + "model.layers.7.mlp.down_proj.scales": "model.safetensors", + "model.layers.7.mlp.down_proj.weight": "model.safetensors", + "model.layers.7.mlp.gate_proj.biases": "model.safetensors", + "model.layers.7.mlp.gate_proj.scales": "model.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model.safetensors", + "model.layers.7.mlp.up_proj.biases": "model.safetensors", + "model.layers.7.mlp.up_proj.scales": "model.safetensors", + "model.layers.7.mlp.up_proj.weight": "model.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model.safetensors", + "model.layers.7.self_attn.k_norm.weight": "model.safetensors", + "model.layers.7.self_attn.k_proj.biases": "model.safetensors", + "model.layers.7.self_attn.k_proj.scales": "model.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model.safetensors", + "model.layers.7.self_attn.o_proj.biases": "model.safetensors", + "model.layers.7.self_attn.o_proj.scales": "model.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model.safetensors", + "model.layers.7.self_attn.q_norm.weight": "model.safetensors", + "model.layers.7.self_attn.q_proj.biases": "model.safetensors", + "model.layers.7.self_attn.q_proj.scales": "model.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model.safetensors", + "model.layers.7.self_attn.v_proj.biases": "model.safetensors", + "model.layers.7.self_attn.v_proj.scales": "model.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model.safetensors", + "model.layers.8.input_layernorm.weight": "model.safetensors", + "model.layers.8.mlp.down_proj.biases": "model.safetensors", + "model.layers.8.mlp.down_proj.scales": "model.safetensors", + "model.layers.8.mlp.down_proj.weight": "model.safetensors", + "model.layers.8.mlp.gate_proj.biases": "model.safetensors", + "model.layers.8.mlp.gate_proj.scales": "model.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model.safetensors", + "model.layers.8.mlp.up_proj.biases": "model.safetensors", + "model.layers.8.mlp.up_proj.scales": "model.safetensors", + "model.layers.8.mlp.up_proj.weight": "model.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model.safetensors", + "model.layers.8.self_attn.k_norm.weight": "model.safetensors", + "model.layers.8.self_attn.k_proj.biases": "model.safetensors", + "model.layers.8.self_attn.k_proj.scales": "model.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model.safetensors", + "model.layers.8.self_attn.o_proj.biases": "model.safetensors", + "model.layers.8.self_attn.o_proj.scales": "model.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model.safetensors", + "model.layers.8.self_attn.q_norm.weight": "model.safetensors", + "model.layers.8.self_attn.q_proj.biases": "model.safetensors", + "model.layers.8.self_attn.q_proj.scales": "model.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model.safetensors", + "model.layers.8.self_attn.v_proj.biases": "model.safetensors", + "model.layers.8.self_attn.v_proj.scales": "model.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model.safetensors", + "model.layers.9.input_layernorm.weight": "model.safetensors", + "model.layers.9.mlp.down_proj.biases": "model.safetensors", + "model.layers.9.mlp.down_proj.scales": "model.safetensors", + "model.layers.9.mlp.down_proj.weight": "model.safetensors", + "model.layers.9.mlp.gate_proj.biases": "model.safetensors", + "model.layers.9.mlp.gate_proj.scales": "model.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model.safetensors", + "model.layers.9.mlp.up_proj.biases": "model.safetensors", + "model.layers.9.mlp.up_proj.scales": "model.safetensors", + "model.layers.9.mlp.up_proj.weight": "model.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model.safetensors", + "model.layers.9.self_attn.k_norm.weight": "model.safetensors", + "model.layers.9.self_attn.k_proj.biases": "model.safetensors", + "model.layers.9.self_attn.k_proj.scales": "model.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model.safetensors", + "model.layers.9.self_attn.o_proj.biases": "model.safetensors", + "model.layers.9.self_attn.o_proj.scales": "model.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model.safetensors", + "model.layers.9.self_attn.q_norm.weight": "model.safetensors", + "model.layers.9.self_attn.q_proj.biases": "model.safetensors", + "model.layers.9.self_attn.q_proj.scales": "model.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model.safetensors", + "model.layers.9.self_attn.v_proj.biases": "model.safetensors", + "model.layers.9.self_attn.v_proj.scales": "model.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model.safetensors", + "model.norm.weight": "model.safetensors" + } +} \ No newline at end of file diff --git a/mlx-int8/tokenizer.json b/mlx-int8/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/mlx-int8/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/mlx-int8/tokenizer_config.json b/mlx-int8/tokenizer_config.json new file mode 100644 index 0000000..d89bb6f --- /dev/null +++ b/mlx-int8/tokenizer_config.json @@ -0,0 +1,16 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "is_local": true, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "tool_parser_type": "json_tools", + "unk_token": null +} diff --git a/model-00001-of-00002.safetensors b/model-00001-of-00002.safetensors new file mode 100644 index 0000000..7a80722 --- /dev/null +++ b/model-00001-of-00002.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58ab1ca3279142521e04faf5a39ab43177732e3ac056e5b2899ecf413d439e0d +size 5321133607 diff --git a/model-00002-of-00002.safetensors b/model-00002-of-00002.safetensors new file mode 100644 index 0000000..cb7d267 --- /dev/null +++ b/model-00002-of-00002.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2c0e962f4fd3ac9a51d7b5d177e79b8f84da5052ea5d2b5dba29e26b3f6d9dd +size 2723848286 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000..767079a --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,406 @@ +{ + "metadata": { + "total_size": 8044936192, + "total_parameters": 4022468096 + }, + "weight_map": { + "model.embed_tokens.weight": "model-00001-of-00002.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.0.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.1.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.10.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.11.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.12.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.13.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.14.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.15.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.16.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.17.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.18.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.19.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.2.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.20.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.21.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.21.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.21.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.22.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.22.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.22.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.23.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.24.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.25.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.26.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.27.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.28.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.28.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.28.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.28.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.28.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.28.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.layers.28.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.28.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.28.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.layers.28.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.28.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.29.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.29.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.29.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.29.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.29.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.29.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.layers.29.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.29.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.29.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.layers.29.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.29.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.3.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.30.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.30.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.30.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.30.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.30.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.30.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.layers.30.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.30.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.30.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.layers.30.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.30.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.31.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.31.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.31.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.31.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.31.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.31.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.layers.31.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.31.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.31.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.layers.31.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.31.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.32.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.32.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.32.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.32.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.32.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.32.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.layers.32.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.32.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.32.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.layers.32.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.32.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.33.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.33.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.33.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.33.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.33.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.33.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.layers.33.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.33.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.33.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.layers.33.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.33.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.34.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.34.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.34.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.34.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.34.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.34.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.layers.34.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.34.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.34.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.layers.34.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.34.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.35.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.35.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.35.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.35.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.35.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.35.self_attn.k_norm.weight": "model-00002-of-00002.safetensors", + "model.layers.35.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.35.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.35.self_attn.q_norm.weight": "model-00002-of-00002.safetensors", + "model.layers.35.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.35.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.4.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.5.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.6.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.7.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.8.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.9.self_attn.k_norm.weight": "model-00001-of-00002.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.self_attn.q_norm.weight": "model-00001-of-00002.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.norm.weight": "model-00002-of-00002.safetensors" + } +} \ No newline at end of file diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..d89bb6f --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,16 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "is_local": true, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "tool_parser_type": "json_tools", + "unk_token": null +}