commit c66cd5146cc8e4250c66d02ef46409782728610f Author: ModelHub XC Date: Thu Jul 9 23:06:25 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: saadxsalman/ss-medical-moe Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..c0ba16e --- /dev/null +++ b/README.md @@ -0,0 +1,123 @@ +--- +language: + - en +license: llama3.2 +base_model: meta-llama/Llama-3.2-1B +datasets: + - Malikeh1375/medical-question-answering-datasets +tags: + - medical + - healthcare + - mixture-of-experts + - lora + - causal-lm + - llama + - llama-3 + - fine-tuned +pipeline_tag: text-generation +library_name: transformers +--- + +# 🏥 Medical MoE Llama-3.2-1B + +**Model by [saadxsalman](https://huggingface.co/saadxsalman)** + +A **Mixture of Experts (MoE)** language model fine-tuned for medical question +answering, built on Llama-3.2-1B with QLoRA adaptation and a custom +4-expert routing layer trained on 1.26M real doctor-patient conversations. + +## Architecture +Input Tokens +↓ +Llama-3.2-1B (1B params, QLoRA-merged, fp16) +↓ +MoE Layer — 4 domain experts, top-2 routing +↓ +LayerNorm + Residual +↓ +LM Head → Medical Answer + +### Expert Specializations + +| Expert | Medical Domain | +|----------|-----------------------------| +| Expert 1 | Symptoms & Diagnosis | +| Expert 2 | Drug Interactions | +| Expert 3 | Anatomy & Physiology | +| Expert 4 | Treatment & Procedures | + +## Training Details + +| Parameter | Value | +|------------------------|----------------------------------------------------| +| Base model | `meta-llama/Llama-3.2-1B` | +| Fine-tuning method | QLoRA (4-bit NF4) + MoE layer | +| LoRA rank | 16 | +| LoRA alpha | 32 | +| LoRA target modules | q_proj, v_proj, k_proj, o_proj | +| Number of experts | 4 | +| Top-K routing | 2 | +| Training dataset | Malikeh1375/medical-question-answering-datasets | +| Training samples | 15,000 | +| Total epochs | 6 (3 learning + 3 specialization) | +| Phase 1 | LR=2e-4, aux_coef=0.01 (domain learning) | +| Phase 2 | LR=5e-5, aux_coef=0.05 (expert specialization) | +| Max sequence length | 256 | +| Hardware | Kaggle T4 GPU (16GB VRAM) | + +## Dataset + +Trained on [`Malikeh1375/medical-question-answering-datasets`](https://huggingface.co/datasets/Malikeh1375/medical-question-answering-datasets) +— 1.26M medical QA pairs from 12 sources including ChatDoctor (112K real +doctor-patient conversations), iCliniq, MedQA, WikiDoc, PubMed, and CORD-19. + +## Expert Routing Results + +| Domain | Expert 1 | Expert 2 | Expert 3 | Expert 4 | +|-------------------------|----------|----------|----------|----------| +| Symptoms & Diagnosis | 0.338 | 0.212 | 0.241 | 0.210 | +| Drug Interactions | 0.340 | 0.209 | 0.246 | 0.204 | +| Anatomy & Physiology | 0.266 | 0.248 | 0.232 | 0.254 | +| Treatment & Procedures | 0.299 | 0.231 | 0.227 | 0.243 | + +## How to Use + +```python +import torch +from transformers import AutoTokenizer, AutoModelForCausalLM +from huggingface_hub import hf_hub_download + +# Load tokenizer and merged backbone +tokenizer = AutoTokenizer.from_pretrained("saadxsalman/medical-moe-llama3.2-1b") +backbone = AutoModelForCausalLM.from_pretrained( + "saadxsalman/medical-moe-llama3.2-1b", + torch_dtype=torch.float16, + device_map="auto" +) + +# Load MoE weights separately +moe_ckpt = torch.load( + hf_hub_download("saadxsalman/medical-moe-llama3.2-1b", "moe_components.pt"), + map_location="cuda" +) +# See moe_config.json in the repo for full architecture details +``` + +## Files in This Repository + +| File | Description | +|-----------------------|-----------------------------------------------| +| `model.safetensors` | Llama-3.2-1B backbone (LoRA merged, fp16) | +| `moe_components.pt` | MoE layer + LayerNorm + LM head weights | +| `moe_config.json` | Full architecture + training configuration | +| `tokenizer.json` | Llama-3 tokenizer | +| `routing_matrix.npy` | Expert routing analysis results | +| `routing_heatmap.png` | Expert specialization visualization | +| `training_curves.png` | Loss curves across all 6 epochs | +| `domain_dist.png` | Training data domain distribution | + +## ⚠️ Disclaimer + +This model is for **research purposes only**. It is not validated for +clinical use and may produce incorrect medical information. Always consult +a qualified healthcare professional for medical advice. diff --git a/config.json b/config.json new file mode 100644 index 0000000..af40ccd --- /dev/null +++ b/config.json @@ -0,0 +1,36 @@ +{ + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "dtype": "float16", + "eos_token_id": 128001, + "head_dim": 64, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 32, + "num_hidden_layers": 16, + "num_key_value_heads": 8, + "pad_token_id": null, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_parameters": { + "factor": 32.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_theta": 500000.0, + "rope_type": "llama3" + }, + "tie_word_embeddings": true, + "transformers_version": "5.0.0", + "use_cache": true, + "vocab_size": 128256 +} diff --git a/domain_dist.png b/domain_dist.png new file mode 100644 index 0000000..29fbcf1 Binary files /dev/null and b/domain_dist.png differ diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..0f6e971 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "bos_token_id": 128000, + "do_sample": true, + "eos_token_id": 128001, + "temperature": 0.6, + "top_p": 0.9, + "transformers_version": "5.0.0" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..713bfd7 --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a57e2be260b9eeb4d1fe9e809cbc041c8937633b5de24e08b1cbc2cae774b71d +size 2471645464 diff --git a/moe_components.pt b/moe_components.pt new file mode 100644 index 0000000..b42423f --- /dev/null +++ b/moe_components.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e8e775364d80fc562fe631fbceede759452c3697a30950c07ff98600a6971b31 +size 1319264285 diff --git a/moe_config.json b/moe_config.json new file mode 100644 index 0000000..533712f --- /dev/null +++ b/moe_config.json @@ -0,0 +1,35 @@ +{ + "base_model": "meta-llama/Llama-3.2-1B", + "hidden_size": 2048, + "ffn_hidden": 4096, + "lora_r": 16, + "lora_alpha": 32, + "lora_dropout": 0.05, + "lora_targets": [ + "q_proj", + "v_proj", + "k_proj", + "o_proj" + ], + "num_experts": 4, + "top_k": 2, + "dataset_name": "Malikeh1375/medical-question-answering-datasets", + "dataset_subset": "all-processed", + "num_samples": 15000, + "max_length": 256, + "batch_size": 2, + "grad_accum": 16, + "warmup_ratio": 0.1, + "val_split": 0.1, + "save_path": "/kaggle/working/llama_moe_medical", + "model_type": "llama-3.2-1b-lora-moe", + "lora_merged": true, + "hf_username": "saadxsalman", + "repo_id": "saadxsalman/ss-medical-moe", + "domain_experts": { + "expert_1": "Symptoms & Diagnosis", + "expert_2": "Drug Interactions", + "expert_3": "Anatomy & Physiology", + "expert_4": "Treatment & Procedures" + } +} \ No newline at end of file diff --git a/routing_heatmap.png b/routing_heatmap.png new file mode 100644 index 0000000..68a9d68 Binary files /dev/null and b/routing_heatmap.png differ diff --git a/routing_matrix.npy b/routing_matrix.npy new file mode 100644 index 0000000..947667b --- /dev/null +++ b/routing_matrix.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a93525f92be7641ed239717720605be6eb9a29b938a3c528ab4b1eaffbb71183 +size 256 diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..ba0eafa --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0dcdea11cb7ecb49d61d951e27fd0e5d92248771bdfa6b63992b633bf870eb82 +size 17210018 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..f11592a --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,14 @@ +{ + "backend": "tokenizers", + "bos_token": "<|begin_of_text|>", + "clean_up_tokenization_spaces": true, + "eos_token": "<|end_of_text|>", + "is_local": false, + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 131072, + "pad_token": "<|end_of_text|>", + "tokenizer_class": "TokenizersBackend" +} diff --git a/training_curves.png b/training_curves.png new file mode 100644 index 0000000..c3a036e Binary files /dev/null and b/training_curves.png differ