初始化项目，由ModelHub XC社区提供模型
Model: Ademola265/Qwen3-4B-Thinking-2507-GLM-4.7-Distilled-GGUF Source: Original Platform
This commit is contained in:
ModelHub XC
49
.gitattributes
vendored
Normal file
49
.gitattributes
vendored
Normal file
@@ -0,0 +1,49 @@
|
||||
*.7z filter=lfs diff=lfs merge=lfs -text
|
||||
*.arrow filter=lfs diff=lfs merge=lfs -text
|
||||
*.bin filter=lfs diff=lfs merge=lfs -text
|
||||
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
||||
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
||||
*.ftz filter=lfs diff=lfs merge=lfs -text
|
||||
*.gz filter=lfs diff=lfs merge=lfs -text
|
||||
*.h5 filter=lfs diff=lfs merge=lfs -text
|
||||
*.joblib filter=lfs diff=lfs merge=lfs -text
|
||||
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
||||
*.model filter=lfs diff=lfs merge=lfs -text
|
||||
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
||||
*.npy filter=lfs diff=lfs merge=lfs -text
|
||||
*.npz filter=lfs diff=lfs merge=lfs -text
|
||||
*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||
*.ot filter=lfs diff=lfs merge=lfs -text
|
||||
*.parquet filter=lfs diff=lfs merge=lfs -text
|
||||
*.pb filter=lfs diff=lfs merge=lfs -text
|
||||
*.pickle filter=lfs diff=lfs merge=lfs -text
|
||||
*.pkl filter=lfs diff=lfs merge=lfs -text
|
||||
*.pt filter=lfs diff=lfs merge=lfs -text
|
||||
*.pth filter=lfs diff=lfs merge=lfs -text
|
||||
*.rar filter=lfs diff=lfs merge=lfs -text
|
||||
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar filter=lfs diff=lfs merge=lfs -text
|
||||
*.tflite filter=lfs diff=lfs merge=lfs -text
|
||||
*.tgz filter=lfs diff=lfs merge=lfs -text
|
||||
*.wasm filter=lfs diff=lfs merge=lfs -text
|
||||
*.xz filter=lfs diff=lfs merge=lfs -text
|
||||
*.zip filter=lfs diff=lfs merge=lfs -text
|
||||
*.zst filter=lfs diff=lfs merge=lfs -text
|
||||
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||
DASD-4B-Thinking-2507-GRPO-v2.Q8_0.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
DASD-4B-Thinking-2507-GRPO-v2.Q4_K_M.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
Qwen3-4B-Thinking-2507-GLM-4.7-Distilled.Q4_K_M.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
Qwen3-4B-Thinking-2507-GLM-4.7-Distilled.Q8_0.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
Qwen3-4B-Thinking-2507-GLM-4.7-Distilled-Q2_K.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
Qwen3-4B-Thinking-2507-GLM-4.7-Distilled-Q3_K_S.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
Qwen3-4B-Thinking-2507-GLM-4.7-Distilled-Q3_K_M.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
Qwen3-4B-Thinking-2507-GLM-4.7-Distilled-Q3_K_L.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
Qwen3-4B-Thinking-2507-GLM-4.7-Distilled-IQ4_XS.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
Qwen3-4B-Thinking-2507-GLM-4.7-Distilled-Q4_K_S.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
Qwen3-4B-Thinking-2507-GLM-4.7-Distilled-Q5_K_S.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
Qwen3-4B-Thinking-2507-GLM-4.7-Distilled-Q5_K_M.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
Qwen3-4B-Thinking-2507-GLM-4.7-Distilled-Q6_K.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
Qwen3-4B-Thinking-2507-GLM-4.7-Distilled-BF16.gguf filter=lfs diff=lfs merge=lfs -text
|
||||
3
Qwen3-4B-Thinking-2507-GLM-4.7-Distilled-BF16.gguf
Normal file
3
Qwen3-4B-Thinking-2507-GLM-4.7-Distilled-BF16.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:2383fa8d1f2f1313fd079be550230373ce0d2821b483865de4d8324d3fbd09e5
|
||||
size 8051285280
|
||||
3
Qwen3-4B-Thinking-2507-GLM-4.7-Distilled-IQ4_XS.gguf
Normal file
3
Qwen3-4B-Thinking-2507-GLM-4.7-Distilled-IQ4_XS.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:039599ae7137e0a7d40c5118f659ed939d5e09a8c035ee2556e5e5aa54ae8019
|
||||
size 2286316320
|
||||
3
Qwen3-4B-Thinking-2507-GLM-4.7-Distilled-Q2_K.gguf
Normal file
3
Qwen3-4B-Thinking-2507-GLM-4.7-Distilled-Q2_K.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:f7e187f7c9c81076f62ad0340d399034cec8c543053ecf0414ca5f733504aec3
|
||||
size 1669499680
|
||||
3
Qwen3-4B-Thinking-2507-GLM-4.7-Distilled-Q3_K_L.gguf
Normal file
3
Qwen3-4B-Thinking-2507-GLM-4.7-Distilled-Q3_K_L.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:1416f8f7abf0e15f541ec54fdc268f066e2c7b3939d2baa041ca36aaddf7b187
|
||||
size 2239785760
|
||||
3
Qwen3-4B-Thinking-2507-GLM-4.7-Distilled-Q3_K_M.gguf
Normal file
3
Qwen3-4B-Thinking-2507-GLM-4.7-Distilled-Q3_K_M.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:99e88aa0d33ff5e9320a7fca8a24b48ef24de1696775e1c6db64298bdec2049a
|
||||
size 2075618080
|
||||
3
Qwen3-4B-Thinking-2507-GLM-4.7-Distilled-Q3_K_S.gguf
Normal file
3
Qwen3-4B-Thinking-2507-GLM-4.7-Distilled-Q3_K_S.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:f1ed26f16e504ad2b08802fea33e6e4eaa263963dd6c219a9096c6b00027feef
|
||||
size 1886997280
|
||||
3
Qwen3-4B-Thinking-2507-GLM-4.7-Distilled-Q4_K_S.gguf
Normal file
3
Qwen3-4B-Thinking-2507-GLM-4.7-Distilled-Q4_K_S.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:3f6a51730a63ed351cf38cda61aefd3d86862d9432fc0322a38fdff6b0f1e9d2
|
||||
size 2383309600
|
||||
3
Qwen3-4B-Thinking-2507-GLM-4.7-Distilled-Q5_K_M.gguf
Normal file
3
Qwen3-4B-Thinking-2507-GLM-4.7-Distilled-Q5_K_M.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:9fc96a49500b5c5da068ed2eef2697a0ab865cd35fd9a459360c3818b43dd2b1
|
||||
size 2889513760
|
||||
3
Qwen3-4B-Thinking-2507-GLM-4.7-Distilled-Q5_K_S.gguf
Normal file
3
Qwen3-4B-Thinking-2507-GLM-4.7-Distilled-Q5_K_S.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:43a9e3030d0ef735bf4209fcebe58bad440d1dbe49df1c0685d9b747106487f6
|
||||
size 2823711520
|
||||
3
Qwen3-4B-Thinking-2507-GLM-4.7-Distilled-Q6_K.gguf
Normal file
3
Qwen3-4B-Thinking-2507-GLM-4.7-Distilled-Q6_K.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:5aed5f599cb11bc69d95c7932375d2cf277fe06afbb0179c9b7de155dfe5fa5a
|
||||
size 3306261280
|
||||
3
Qwen3-4B-Thinking-2507-GLM-4.7-Distilled.Q4_K_M.gguf
Normal file
3
Qwen3-4B-Thinking-2507-GLM-4.7-Distilled.Q4_K_M.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:91236de2657fef394ad573a1e85a5b3b7c1d1e81ebb724084e7dfa2d0cbde13c
|
||||
size 2497280192
|
||||
3
Qwen3-4B-Thinking-2507-GLM-4.7-Distilled.Q8_0.gguf
Normal file
3
Qwen3-4B-Thinking-2507-GLM-4.7-Distilled.Q8_0.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:e8e7a95c7906b521fc6b20518e58b4e248834436805fa5453fc8efdede8f22ab
|
||||
size 4280404672
|
||||
163
README.md
Normal file
163
README.md
Normal file
@@ -0,0 +1,163 @@
|
||||
---
|
||||
language:
|
||||
- en
|
||||
- zh
|
||||
license: apache-2.0
|
||||
base_model: Jackrong/DASD-4B-Thinking-2507-GRPO-v2
|
||||
tags:
|
||||
- qwen3
|
||||
- unsloth
|
||||
- text-generation
|
||||
- reasoning
|
||||
- math
|
||||
- grpo
|
||||
- sft
|
||||
- distillation
|
||||
- conversational
|
||||
- glm-4.7
|
||||
pipeline_tag: text-generation
|
||||
---
|
||||
|
||||
# Qwen3-4B-Thinking-2507-GLM-4.7-Distilled
|
||||
|
||||
**Qwen3-4B-Thinking-2507-GLM-4.7-Distilled** is a fine-tuned model built upon the GRPO-optimized `Jackrong/DASD-4B-Thinking-2507-GRPO-v2` (originally based on [`Qwen/Qwen3-4B-Thinking-2507`](https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507)). This model was developed using a Supervised Fine-Tuning (SFT) strategy heavily distilled from the GLM-4.7 model series (at a default temperature of 1.0), with a central focus on multi-turn conversational alignment and **structured Chain-of-Thought (CoT) execution**.
|
||||
|
||||
🎯 **Core Improvement:**
|
||||
The primary objective of this fine-tuning was to transform the model's reasoning pattern for everyday and lightweight tasks. Instead of the typical linear, free-associative, and highly self-correcting ("think-as-you-go") stream of consciousness, this model has learned to adopt a highly confident, **"Plan-then-Execute"** paradigm. It systematically breaks down tasks into logical outlines and executes modular, report-like responses without unnecessary self-doubt or hesitation.
|
||||
|
||||
---
|
||||
|
||||
## 🧬 Training Pipeline Overview
|
||||
|
||||
This model is the culmination of two sequential training stages targeting mathematical reasoning and conversational CoT tracking:
|
||||
|
||||
```text
|
||||
Qwen/Qwen3-4B-Thinking-2507
|
||||
│
|
||||
▼ Stage 0: GRPO (RL on Math & Reasoning)
|
||||
DASD-4B-Thinking-2507-GRPO-v2
|
||||
│
|
||||
▼ Stage 1: SFT with GLM-4.7 Series Distilled Datasets (T=1.0)
|
||||
Qwen3-4B-Thinking-2507-GLM-4.7-Distilled ← (this model)
|
||||
```
|
||||
|
||||
### 🧠 Chain of Thought (CoT) Evolution: Base vs. Distilled
|
||||
|
||||
A significant shift in the model's reasoning style is observed after distillation from the GLM-4.7 series data. The model transitions from a **spontaneous thinker** into a **structured planner**:
|
||||
|
||||
| 🎯 Feature | 🌀 Base Model (Qwen3-4B-Thinking) | ✨ Distilled Model (GLM-4.7-Distilled) |
|
||||
| :--- | :--- | :--- |
|
||||
| **Thinking Style** | 🌊 Linear, stream-of-consciousness | 🧱 Modularized, report-like |
|
||||
| **Execution** | 🏃 Thinks on the fly, writes as it thinks | 📝 "Plan-then-Execute" framework |
|
||||
| **Structure** | 🔀 Unstructured, organic self-correction mid-thought | 📑 Highly structured with headings & logical phases |
|
||||
| **Confidence** | 🤔 High self-doubt ("Wait...", "Maybe...", "Should I...") | 🚀 Highly confident, rarely hesitates |
|
||||
| **Output Tone** | 🗣️ Conversational, exploring multiple paths | 📊 Objective, direct, and systematic |
|
||||
|
||||
**🌟 Key Takeaway:**
|
||||
Through the GLM-4.7 dataset distillation, the model successfully learned the **modular thinking paradigm**. Instead of continuously questioning itself, it now *breaks down tasks, creates a clear outline, and systematically executes each step* like writing a formal report.
|
||||
|
||||
---
|
||||
|
||||
## 📚 Stage Details
|
||||
|
||||
### Stage 0 — GRPO Reinforcement Learning: `DASD-4B-Thinking-2507-GRPO-v2`
|
||||
|
||||
Starting from the base model `Qwen/Qwen3-4B-Thinking-2507`, Group Relative Policy Optimization (GRPO) was applied. This stage consisted of:
|
||||
- **Cold Start:** Fine-tuning on the [`unsloth/OpenMathReasoning-mini`](https://huggingface.co/datasets/unsloth/OpenMathReasoning-mini) dataset.
|
||||
- **Reinforcement Learning:** Applying GRPO via the [`open-r1/DAPO-Math-17k-Processed`](https://huggingface.co/datasets/open-r1/DAPO-Math-17k-Processed) dataset.
|
||||
|
||||
This stage significantly improved the model's:
|
||||
|
||||
- Correctness on math problem solving
|
||||
- Step-by-step logical reasoning
|
||||
- Reward signal alignment for verifiable tasks
|
||||
|
||||
---
|
||||
|
||||
### Stage 1 — SFT GLM-4.7 Distillation (T=1.0): `Qwen3-4B-Thinking-2507-GLM-4.7-Distilled` (this model)
|
||||
|
||||
Building on the reasoning foundation of `DASD-4B-Thinking-2507-GRPO-v2`, Stage 1 SFT was performed using a mixed dataset heavily utilizing **GLM-4.7** synthetic data generated at a **default temperature of 1.0**, along with multi-turn alignments.
|
||||
|
||||
Higher-temperature data introduces greater **lexical diversity, broader mode coverage, and more formatted/structured chain-of-thought traces**, enabling the model to generalize better across diverse conversational reasoning patterns and problem domains. It helps the model handle multi-turn conversations effectively while protecting its internal structure of `<think>...</think>` tracking.
|
||||
|
||||
---
|
||||
|
||||
## 🗂️ All Datasets Used
|
||||
|
||||
| Stage | Dataset | Purpose |
|
||||
|-------|---------|---------|
|
||||
| GRPO (Cold Start) | [`unsloth/OpenMathReasoning-mini`](https://huggingface.co/datasets/unsloth/OpenMathReasoning-mini) | Initial foundational mathematical reasoning |
|
||||
| GRPO (RL) | [`open-r1/DAPO-Math-17k-Processed`](https://huggingface.co/datasets/open-r1/DAPO-Math-17k-Processed) | Math & reasoning RL training via GRPO |
|
||||
| SFT Distillation | [`Alibaba-Apsara/Superior-Reasoning-SFT-gpt-oss-120b`](https://huggingface.co/datasets/Alibaba-Apsara/Superior-Reasoning-SFT-gpt-oss-120b) (Stage 2) | Diverse reasoning structures |
|
||||
| SFT Distillation | [`Jackrong/glm-4.7-multiturn-CoT`](https://huggingface.co/datasets/Jackrong/glm-4.7-multiturn-CoT) | Multi-turn CoT alignment |
|
||||
| SFT Distillation | [`Jackrong/glm-4.7-Superior-Reasoning-stage1`](https://huggingface.co/datasets/Jackrong/glm-4.7-Superior-Reasoning-stage1) | Enhanced fundamental reasoning |
|
||||
| SFT Distillation | [`TeichAI/glm-4.7-2000x`](https://huggingface.co/datasets/TeichAI/glm-4.7-2000x) | Generalization and lexical diversity |
|
||||
| SFT Distillation | [`Jackrong/MultiReason-ChatAlpaca`](https://huggingface.co/datasets/Jackrong/MultiReason-ChatAlpaca) | Conversational multi-turn tracking |
|
||||
|
||||
---
|
||||
|
||||
## 🏃 Quickstart
|
||||
|
||||
```python
|
||||
from transformers import AutoModelForCausalLM, AutoTokenizer
|
||||
|
||||
model_name = "Jackrong/Qwen3-4B-Thinking-2507-GLM-4.7-Distilled"
|
||||
tokenizer = AutoTokenizer.from_pretrained(model_name)
|
||||
model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", device_map="auto")
|
||||
|
||||
messages = [
|
||||
{"role": "user", "content": "Solve: find all real solutions to x^3 - 6x^2 + 11x - 6 = 0."}
|
||||
]
|
||||
|
||||
text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
|
||||
inputs = tokenizer([text], return_tensors="pt").to(model.device)
|
||||
outputs = model.generate(**inputs, max_new_tokens=4096)
|
||||
response = tokenizer.decode(outputs[0][inputs.input_ids.shape[1]:], skip_special_tokens=True)
|
||||
print(response)
|
||||
```
|
||||
|
||||
> **Tip:** This model naturally generates `<think>...</think>` reasoning traces before the final answer. You can parse these to inspect the chain-of-thought.
|
||||
|
||||
---
|
||||
|
||||
## 📋 Model Details
|
||||
|
||||
| Attribute | Value |
|
||||
|-----------|-------|
|
||||
| **Base Model** | `Jackrong/DASD-4B-Thinking-2507-GRPO-v2` |
|
||||
| **Architecture** | Qwen3 (4B Dense) |
|
||||
| **License** | Apache 2.0 |
|
||||
| **Language(s)** | English, Chinese |
|
||||
| **Training Framework** | [Unsloth](https://github.com/unslothai/unsloth) + Hugging Face TRL |
|
||||
| **RL Algorithm** | GRPO (Group Relative Policy Optimization) |
|
||||
| **Fine-tuning Method** | SFT (GLM-4.7 Distillation at T=1.0) |
|
||||
| **Developed by** | Jackrong |
|
||||
|
||||
---
|
||||
|
||||
## ⚠️ Limitations & Intended Use
|
||||
|
||||
- This model is intended for **research and educational purposes** related to reasoning and mathematical problem-solving.
|
||||
- While mathematical and logical reasoning capabilities have been enhanced, the model may still produce incorrect answers or hallucinations — always verify outputs on critical tasks.
|
||||
- The model inherits the capabilities and limitations of the underlying `Qwen3-4B-Thinking-2507` architecture.
|
||||
- Not intended for deployment in high-stakes applications without additional safety evaluation.
|
||||
|
||||
---
|
||||
|
||||
## 📎 Related Models
|
||||
|
||||
| Model | Description |
|
||||
|-------|-------------|
|
||||
| [`Qwen/Qwen3-4B-Thinking-2507`](https://huggingface.co/Qwen/Qwen3-4B-Thinking-2507) | Base model |
|
||||
| [`Jackrong/DASD-4B-Thinking-2507-GRPO-v2`](https://huggingface.co/Jackrong/DASD-4B-Thinking-2507-GRPO-v2) | After GRPO RL training |
|
||||
| [`Jackrong/Qwen3-4B-Thinking-2507-GLM-4.7-Distilled`](https://huggingface.co/Jackrong/Qwen3-4B-Thinking-2507-GLM-4.7-Distilled) | **This model** — GLM-4.7 Distilled |
|
||||
|
||||
---
|
||||
|
||||
## 🙏 Acknowledgements
|
||||
|
||||
- [Zhipu AI](https://huggingface.co/THUDM) for the GLM-4.7 model series capability
|
||||
- [Alibaba Cloud Apsara Lab](https://huggingface.co/Alibaba-Apsara) for reasoning datasets
|
||||
- [Open-R1](https://huggingface.co/open-r1) for the DAPO Math dataset
|
||||
- [Unsloth](https://github.com/unslothai/unsloth) for efficient fine-tuning infrastructure
|
||||
- [Qwen Team](https://huggingface.co/Qwen) for the excellent base model
|
||||
69
config.json
Normal file
69
config.json
Normal file
@@ -0,0 +1,69 @@
|
||||
{
|
||||
"architectures": [
|
||||
"Qwen3ForCausalLM"
|
||||
],
|
||||
"attention_bias": false,
|
||||
"attention_dropout": 0.0,
|
||||
"torch_dtype": "bfloat16",
|
||||
"eos_token_id": 151645,
|
||||
"head_dim": 128,
|
||||
"hidden_act": "silu",
|
||||
"hidden_size": 2560,
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 9728,
|
||||
"layer_types": [
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention"
|
||||
],
|
||||
"max_position_embeddings": 262144,
|
||||
"max_window_layers": 36,
|
||||
"model_type": "qwen3",
|
||||
"num_attention_heads": 32,
|
||||
"num_hidden_layers": 36,
|
||||
"num_key_value_heads": 8,
|
||||
"pad_token_id": 151654,
|
||||
"rms_norm_eps": 1e-06,
|
||||
"rope_scaling": null,
|
||||
"rope_theta": 5000000,
|
||||
"sliding_window": null,
|
||||
"tie_word_embeddings": true,
|
||||
"unsloth_fixed": true,
|
||||
"unsloth_version": "2026.2.1",
|
||||
"use_cache": true,
|
||||
"use_sliding_window": false,
|
||||
"vocab_size": 151936
|
||||
}
|
||||
Reference in New Issue
Block a user