From 6c861f8a4bd845f30caca948e65e82a5e65ba0ca Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Tue, 16 Jun 2026 04:57:16 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: Xerv-AI/Ada Source: Original Platform --- .gitattributes | 36 ++++++++ README.md | 131 +++++++++++++++++++++++++++ config.json | 63 +++++++++++++ model.safetensors | 3 + tokenizer.json | 3 + tokenizer_config.json | 202 ++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 438 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 config.json create mode 100644 model.safetensors create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..73f032b --- /dev/null +++ b/README.md @@ -0,0 +1,131 @@ +--- +license: apache-2.0 +language: +- en +pipeline_tag: text-generation +tags: +- unsloth +- qwen +- qwen2.5 +- math +- reasoning +- alpaca +- pytorch +- custom-finetune +- lora-merged +base_model: unsloth/Qwen2.5-Math-1.5B +datasets: +- Xerv-AI/GRAD +- yahma/alpaca-cleaned +inference: + parameters: + repetition_penalty: 1.15 + max_new_tokens: 256 + temperature: 0.5 + examples: + - text: "### Instruction:\nProvide a step-by-step logical proof finding the eigenvalues of the matrix [[2, 1], [1, 2]].\n### Response:\n" + +widget: + - example_title: Fibonacci (Python) + messages: + - role: system + content: You are a chatbot who can help code! + - role: user + content: Write me a function to calculate the first 10 digits of the fibonacci sequence in Python and print it out to the CLI. +--- + + +## 🌌 Xerv-AI/Ada: The Multi-Modal Mathematical Generalist SLM +**Ada** is an ultra-lightweight, high-speed, and highly optimized reasoning Small Language Model (SLM) derived from the powerful **Qwen2.5-Math-1.5B** architecture. Engineered specifically to bridge the gap between hyper-specialized graduate-level mathematical proofs and standard conversational utility, Ada solves the notorious "catastrophic forgetting" problem often found in math-heavy fine-tunes. +Whether you need a step-by-step calculus breakdown, a topological proof in LaTeX, or just a simple conversational assistant for daily tasks, Ada delivers state-of-the-art performance for a 1.5 Billion parameter model. + +### 🚀 Model Overview +Standard math-specific LLMs frequently suffer from domain overfitting. When prompted with basic conversational queries, they either hallucinate lengthy pseudo-proofs or fail entirely to understand the user's intent. **Xerv-AI/Ada** was meticulously engineered to resolve this by utilizing a carefully balanced, dual-distribution training dataset, allowing it to act as both a rigorous STEM assistant and a general-purpose chat model. + +| Specification | Details | +| :--- | :--- | +| **Model Name** | Xerv-AI/Ada | +| **Base Architecture** | unsloth/Qwen2.5-Math-1.5B | +| **Parameter Count** | 1.5 Billion | +| **Primary Capabilities** | Graduate-level STEM reasoning, logical deduction, and mathematical proofs. | +| **Secondary Capabilities** | General conversational instruction-following, roleplay, and basic coding. | +| **Training Framework** | QLoRA via Unsloth (Triton kernels). | +| **Precision** | Merged 16-bit (Fine-tuned in 4-bit). | +| **License** | Apache-2.0 |
### 🔬 Core Capabilities & Strengths
* **Balanced Generalization:** Ada seamlessly transitions between casual conversation and intense analytical problem-solving without format-forced hallucinations.
* **Advanced STEM Reasoning:** Fully optimized to generate detailed, multi-step logical proofs in advanced algebra, calculus, topology, and physics.
* **Hardware Optimized for Edge Deployment:** Designed to run at maximum inference throughput on low-VRAM consumer hardware (such as a single 16GB NVIDIA T4 GPU, Mac M-series chips, or edge devices) using 4-bit quantization.
* **Impeccable Formatting:** Native understanding of structural formatting, easily outputting highly readable markdown and structured logic steps.
### 🏗 Architecture & Training Methodology
Ada was trained using Supervised Fine-Tuning (SFT) targeting the attention mechanisms of the base model. Utilizing **Unsloth** on a standard Google Colab NVIDIA T4 GPU, the training leveraged Low-Rank Adaptation (LoRA) to maximize efficiency before being merged into a standalone 16-bit Hugging Face model.
* **Target Modules:** q_proj, k_proj, v_proj, o_proj, gate_proj, up_proj, down_proj
* **LoRA Rank (r):** 16
* **LoRA Alpha:** 16
* **Optimizer:** adamw_8bit
* **Learning Rate:** 2e-4
* **Effective Batch Size:** 8 (Batch size 2 with 4 Gradient Accumulation steps)
### 📚 The Dataset: Dual-Distribution Blending
To achieve generalization and prevent catastrophic forgetting, Ada was fine-tuned on a strict 50/50 blend of two distinct datasets, batched and streamed via high-throughput Parquet files: +| Dataset | Sample Size | Description & Purpose | +| :--- | :--- | :--- | +| **Xerv-AI/GRAD** | ~1.93k rows | A proprietary synthetic dataset containing exceptionally long (average 8,000 characters) graduate and research-level mathematical proofs. This instills deep reasoning and strict formatting. | +| **yahma/alpaca-cleaned** | ~2.00k rows | A refined subset of the standard Alpaca dataset. This teaches the model conversational flow, roleplay, basic Q&A, and crucially, *when not to use complex math*. | + +### 💻 Usage & Python Inference Guide +The model is highly responsive to the standard **Alpaca Instruction/Response template**. +**Important Inference Note:** For best results, use a repetition_penalty of roughly **1.15**. This acts as a crucial guardrail to prevent the model from infinitely looping through mathematical steps on overly simple arithmetic queries. +**1. Installation Requirements** +```bash +pip install unsloth transformers accelerate torch +``` +**2. Fast Inference Script** +```python +from unsloth import FastLanguageModel +import torch +# Configuration +repo_name = "Xerv-AI/Ada" +max_seq_length = 2048 +# Load the model and tokenizer (4-bit recommended for low-VRAM) +model, tokenizer = FastLanguageModel.from_pretrained( + model_name = repo_name, + max_seq_length = max_seq_length, + dtype = None, + load_in_4bit = True, +) +# Enable optimized inference mode +FastLanguageModel.for_inference(model) +# Define the universal prompt template +universal_prompt = """### Instruction: +{} +### Response: +{}""" +# Prepare your query +query = "Provide a step-by-step logical proof finding the eigenvalues of the matrix [[2, 1], [1, 2]]." +inputs = tokenizer( + [universal_prompt.format(query, "")], + return_tensors = "pt" +).to("cuda") +print("Generating analytical response...") +# Generate the output +outputs = model.generate( + **inputs, + max_new_tokens = 1024, + max_length = None, + use_cache = True, + repetition_penalty = 1.15, # Critical: prevents generation loops + pad_token_id = tokenizer.eos_token_id +) +# Decode and print the result +response = tokenizer.batch_decode(outputs, skip_special_tokens = True)[0] +print(f"\n{'='*50}\nOutput:\n{'='*50}") +print(response.split("### Response:\n")[-1]) +``` + +### Performance Summary + +| Dataset | Accuracy | +| :--- | :--- | +| **GSM8K** | **40.00%** | +| **MATH** |**60.00%** | +| **MATH-Hard** |**50.00%** | +| **GRAD** |**40.00%** | + +### 🛡️ Safety & Alignment Guardrails +Despite being fine-tuned on raw mathematical logic and conversational instruction data, Ada successfully retains its foundational safety alignments. Because only 1% to 2% of the parameters were actively updated via LoRA (and subsequently merged), the original base Qwen2.5 weights responsible for safety remain fully intact. + * **Content Moderation:** The model actively refuses to generate explicit, illegal, or harmful content, relying on the RLHF and DPO safety guardrails instilled during Alibaba's original pre-training phase. +### ⚠️ Limitations & Known Biases +While Ada punches well above its 1.5B weight class, it is important to acknowledge the limitations inherent to Small Language Models: + * **Arithmetic Hallucinations:** Ada is exceptionally capable at symbolic logic, structural breakdowns, and mathematical theory. However, like many SLMs, it can occasionally suffer from minor arithmetic errors (e.g., basic addition/subtraction mistakes) deep within multi-page proofs. Always verify raw calculations. + * **Language Constraint:** The model is optimized exclusively for **English** text and standard mathematical notation. + * **Prompt Sensitivity:** Ada performs at its absolute peak when math queries explicitly ask for a "proof," "step-by-step breakdown," or "logical analysis" within the instruction block. + * **World Knowledge:** It lacks the broad, encyclopedic trivia knowledge found in massive 70B+ parameter models. +### 🤝 Acknowledgements + * **Alibaba Cloud:** For the phenomenal, state-of-the-art base Qwen2.5-Math architecture. + * **Unsloth AI:** For the Triton-optimized training kernels that made compiling and fine-tuning this model possible and highly efficient on consumer hardware. + * **Xerv-AI:** For the curation of the GRAD synthetic dataset powering the advanced reasoning capabilities. \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..e30f8c4 --- /dev/null +++ b/config.json @@ -0,0 +1,63 @@ +{ + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": null, + "torch_dtype": "float16", + "eos_token_id": 151643, + "hidden_act": "silu", + "hidden_size": 1536, + "initializer_range": 0.02, + "intermediate_size": 8960, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 4096, + "max_window_layers": 21, + "model_type": "qwen2", + "num_attention_heads": 12, + "num_hidden_layers": 28, + "num_key_value_heads": 2, + "pad_token_id": 151665, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 10000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "unsloth_fixed": true, + "unsloth_version": "2026.4.8", + "use_cache": false, + "use_mrope": false, + "use_sliding_window": false, + "vocab_size": 151936 +} \ No newline at end of file diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..397fc49 --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e5e98d13031bc815f88803f41e0527c902bea9fb39f95f61293aa6f75a60592e +size 3087467144 diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..5340d81 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd5948af71b4f56cf697f7580814c7ce8b80595ef985544efcacf716126a2e31 +size 11422356 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..beda3aa --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,202 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [], + "is_local": false, + "model_max_length": 4096, + "pad_token": "<|PAD_TOKEN|>", + "padding_side": "left", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null, + "added_tokens_decoder": { + "151643": { + "content": "<|endoftext|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151644": { + "content": "<|im_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151645": { + "content": "<|im_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151646": { + "content": "<|object_ref_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151647": { + "content": "<|object_ref_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151648": { + "content": "<|box_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151649": { + "content": "<|box_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151650": { + "content": "<|quad_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151651": { + "content": "<|quad_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151652": { + "content": "<|vision_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151653": { + "content": "<|vision_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151654": { + "content": "<|vision_pad|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151655": { + "content": "<|image_pad|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151656": { + "content": "<|video_pad|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + "151657": { + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151658": { + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151659": { + "content": "<|fim_prefix|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151660": { + "content": "<|fim_middle|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151661": { + "content": "<|fim_suffix|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151662": { + "content": "<|fim_pad|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151663": { + "content": "<|repo_name|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151664": { + "content": "<|file_sep|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": false + }, + "151665": { + "content": "<|PAD_TOKEN|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + } + } +} \ No newline at end of file