From ae4d8eb8b1cfe7324e4398df6a6afdf72089e718 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Mon, 20 Jul 2026 15:31:09 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: DireDreadlord/Dragon-1.5-0.5B Source: Original Platform --- .gitattributes | 37 ++++++++++++++ README.md | 107 +++++++++++++++++++++++++++++++++++++++++ chat_template.jinja | 14 ++++++ config.json | 57 ++++++++++++++++++++++ dragon_logo_a.png | 3 ++ generation_config.json | 13 +++++ model.safetensors | 3 ++ requirements.txt | 14 ++++++ tokenizer.json | 3 ++ tokenizer_config.json | 19 ++++++++ training_args.bin | 3 ++ 11 files changed, 273 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 chat_template.jinja create mode 100644 config.json create mode 100644 dragon_logo_a.png create mode 100644 generation_config.json create mode 100644 model.safetensors create mode 100644 requirements.txt create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json create mode 100644 training_args.bin diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..6dcb038 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,37 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +dragon_logo_a.png filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..0d26244 --- /dev/null +++ b/README.md @@ -0,0 +1,107 @@ +--- +license: apache-2.0 +datasets: +- Jackrong/DeepSeek-V4-Distill-8000x +language: +- en +base_model: +- Qwen/Qwen2-0.5B-Instruct +pipeline_tag: text-generation +tags: +- slm +- trl +- text-generation-inference +- reasoning +- thinking +- chat +--- + +# Dragon-1.5-0.5B (qwen2-0.5b-reasoning v3.1.1) + +![Dragon Logo](./dragon_logo_a.png) + +Dragon is a lightweight general reasoning model built upon the base [Qwen2-0.5B-instruct model](https://huggingface.co/Qwen/Qwen2-0.5B-Instruct). It offers accurate and quick text generation on a variety of topics(including code related problems). +It's small size (0.5B parameters) allows it to run comfortably on most laptop/commercial grade GPUs. +This model also offers Q/A and subject matter expert capabilities on general and code related subjects. + +The Dragon-1.5 is the next generation for the [Dragon-1/1.5 series](https://huggingface.co/collections/DireDreadlord/dragon-1-15) which incorporates high-end reasoning capabilities into the standard Qwen2 architecture. + +The 0.5B variant has been SFT trained on general/code reasoning traces found [here](https://huggingface.co/datasets/Jackrong/DeepSeek-V4-Distill-8000x) with further RL training carried out via. a GRPO algorithm. This endows the model with enhanced reasoning capabilities which allows it to serve higher quality and hallucination-free generations. + +--- + + +**Estimated parameters:** ~0.5B + +**Architecture:** Qwen2 + +**Intended use:** Advanced reasoning, instruction following along with enhanced code snippet and long form code generation + +--- + + +## Training data +**Phase-1** +- Source: deepseek-v4-distill-8000x dataset (https://huggingface.co/datasets/Jackrong/DeepSeek-V4-Distill-8000x) +- Rows: ~7,716 rows templated with a custom .jinja chat format +- Training: trained for 4,000 steps on an A10 (24GB VRAM) + +**Phase-2** +- Source: deepseek-v4-reasoning-code-2500 dataset (https://huggingface.co/datasets/Banaxi-Tech/Deepseek-V4-Reasoning-Code-2500) +- Rows: ~7,716 rows templated with a custom .jinja chat format +- Training: trained via. GRPO for 350 steps on an A10 (24GB VRAM) + + +## Usage + +Install requirements: + +```bash +pip install -r requirements.txt +pip install transformers datasets accelerate safetensors +``` + + +## Usage (Hugging Face Hub) +You can load it directly from HuggingFace: + +```python +import torch +from transformers import AutoModelForCausalLM, AutoTokenizer, TextStreamer + + +device = "cuda" if torch.cuda.is_available() else "cpu" + +model_id = "DireDreadlord/Dragon-1.5-0.5B" +model = AutoModelForCausalLM.from_pretrained( + model_id, + device_map="auto", + dtype="auto" +) +model.to(device) +tokenizer = AutoTokenizer.from_pretrained(model_id) +streamer = TextStreamer(tokenizer, skip_prompt=True, skip_special_tokens=False) + +prompt = "Solve the leetcode problem 1: two sum using the hash map technique" + +input_ids = tokenizer.apply_chat_template( + [{"role": "user", "content": prompt}], + add_generation_prompt=True, + return_tensors="pt", + tokenize=True, +)["input_ids"].to(device) + +output = model.generate( + input_ids, + do_sample=True, + temperature=0.4, + top_k=50, + repetition_penalty=1.05, + max_new_tokens=2048, + streamer=streamer, +) +``` +**For optimal long-form generation(with reasoning), set `max_new_tokens=2048`** + +## Limitations +- Model for experimental use only; users should employ it as such under license. \ No newline at end of file diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..86e52aa --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,14 @@ +{%- for message in messages -%} + {%- if loop.first and messages[0]["role"] != "system" -%} + {{- "<|im_start|>system +You are a helpful assistant.<|im_end|> +" -}} + {%- endif -%} + {{- "<|im_start|>" + message["role"] + " +" + message["content"] + "<|im_end|>" + " +" -}} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{- "<|im_start|>assistant +" -}} +{%- endif -%} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..08959bc --- /dev/null +++ b/config.json @@ -0,0 +1,57 @@ +{ + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "hidden_act": "silu", + "hidden_size": 896, + "initializer_range": 0.02, + "intermediate_size": 4864, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 32768, + "max_window_layers": 24, + "model_type": "qwen2", + "num_attention_heads": 14, + "num_hidden_layers": 24, + "num_key_value_heads": 2, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000.0, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.13.1", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151646 +} diff --git a/dragon_logo_a.png b/dragon_logo_a.png new file mode 100644 index 0000000..63090c4 --- /dev/null +++ b/dragon_logo_a.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:69a8e0992b877825de99661050b2a31ec1eca703b46b7179796f4dfa7268afcb +size 2427802 diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..6e9d7f4 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,13 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "repetition_penalty": 1.1, + "temperature": 0.7, + "top_k": 20, + "top_p": 0.8, + "transformers_version": "5.13.1" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..8445991 --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ccd0085cba01584e074e47650f7e83d2e510f613d73565868d1220c88b599a7 +size 987578144 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..8f7b11c --- /dev/null +++ b/requirements.txt @@ -0,0 +1,14 @@ +torch +numpy +tiktoken +datasets +transformers +peft +bitsandbytes +tqdm +matplotlib +safetensors +huggingface_hub +accelerate +trl +diffusers \ No newline at end of file diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..408c9fb --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:48f722bc04c884e2fe1525fdcd85a1293a8499b6e620c1ac7c083c49632305fb +size 11418262 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..53ed36e --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,19 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>" + ], + "is_local": false, + "local_files_only": false, + "model_max_length": 32768, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..730f5ba --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d7eea10ef2b1ef7a745486e31fe4251613785f1f4930a892f835cde33dfe518 +size 7096