commit 5d88d579a39c5ceb2c500a161ca90542449a2e0d Author: ModelHub XC Date: Tue Jul 14 16:03:09 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: Rumiii/Qwen2.5-0.5B-Med-Post-Trained-92k Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..f564314 --- /dev/null +++ b/README.md @@ -0,0 +1,172 @@ +--- +library_name: transformers +language: +- en +license: apache-2.0 +base_model: Qwen/Qwen2.5-0.5B +tags: +- transformers +- medical +- biomedical +- continued-pretraining +- supervised-fine-tuning +- qwen2.5 +- pubmed +- conversational +- causal-lm +arxiv: 2506.09513 +--- + +# Qwen2.5-0.5B-Med-Post-Trained-92k + +A domain-adapted and instruction-tuned variant of [Qwen/Qwen2.5-0.5B](https://huggingface.co/Qwen/Qwen2.5-0.5B), produced through a two-stage training pipeline: full-parameter continued pre-training (CPT) on biomedical text followed by supervised fine-tuning (SFT) on a general instruction dataset. + +![image](https://cdn-uploads.huggingface.co/production/uploads/66e00ba55e4fd4bfead4a97c/VtJDMizAWLMxFIeIaZN9S.png) + +## Training Pipeline + +### Stage 1 — Continued Pre-Training (CPT) + +| Property | Value | +|---|---| +| Base model | Qwen/Qwen2.5-0.5B | +| Training type | Full-parameter CPT (no LoRA) | +| Dataset | VietAI/vi_pubmed (92k English abstracts) | +| Tokens | ~23.6 million | +| Objective | Causal Language Modeling (CLM) | +| Optimizer | AdamW 8-bit | +| Learning rate | 2e-5 cosine | +| Hardware | Kaggle Tesla T4 | +| Training time | ~3h 45m | +| Loss | 2.581 → 2.478 | + +### Stage 2 — Supervised Fine-Tuning (SFT) + +| Property | Value | +|---|---| +| Base model | Rumiii/Qwen2.5-0.5B-Med-Pre-Trained-92k | +| Training type | Full-parameter SFT (no LoRA) | +| Dataset | causal-lm/ultrachat (20k samples) | +| Format | Qwen2.5 ChatML chat template | +| Optimizer | AdamW 8-bit | +| Learning rate | 2e-5 cosine | +| Hardware | Kaggle Tesla T4 | +| Training time | ~1h 13m | +| Loss | 2.093 → 1.289 | + +## Usage + +This model uses the Qwen2.5 ChatML chat template. A system prompt is required for best results. The recommended inference setup is shown below. + +### Recommended system prompt + +```python +SYSTEM_PROMPT = ( + "You are a knowledgeable medical AI assistant named MedAssist. " + "Answer all questions clearly, directly, and informatively. " + "For medical questions provide accurate information. " + "Never generate multiple choice questions unless explicitly asked." +) +``` + +### Basic inference + +```python +from transformers import AutoTokenizer, AutoModelForCausalLM +import torch + +model_id = "Rumiii/Qwen2.5-0.5B-Med-Post-Trained-92k" + +tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True) +model = AutoModelForCausalLM.from_pretrained( + model_id, + torch_dtype=torch.float16, + device_map="auto", + trust_remote_code=True, +) +model.eval() + +SYSTEM_PROMPT = ( + "You are a knowledgeable medical AI assistant named MedAssist. " + "Answer all questions clearly, directly, and informatively. " + "For medical questions provide accurate information. " + "Never generate multiple choice questions unless explicitly asked." +) + +messages = [ + {"role": "system", "content": SYSTEM_PROMPT}, + {"role": "user", "content": "What are the symptoms of pneumonia?"}, +] + +inputs = tokenizer.apply_chat_template( + messages, + tokenize=True, + add_generation_prompt=True, + return_tensors="pt", + return_dict=True, +).to(model.device) + +im_end_id = tokenizer.convert_tokens_to_ids("<|im_end|>") +stop_ids = [tokenizer.eos_token_id] +if im_end_id and im_end_id != tokenizer.eos_token_id: + stop_ids.append(im_end_id) + +with torch.no_grad(): + outputs = model.generate( + **inputs, + max_new_tokens=512, + temperature=0.2, + top_p=0.9, + repetition_penalty=1.15, + do_sample=True, + eos_token_id=stop_ids, + pad_token_id=tokenizer.eos_token_id, + ) + +new_tokens = outputs[0][inputs["input_ids"].shape[-1]:] +response = tokenizer.decode(new_tokens, skip_special_tokens=True).strip() +print(response) +``` + +### Recommended generation parameters + +| Parameter | Value | Reason | +|---|---|---| +| temperature | 0.2 | Low temperature for factual medical responses | +| top_p | 0.9 | Stable token sampling | +| repetition_penalty | 1.15 | Prevents response loops | +| max_new_tokens | 512 | Sufficient for complete answers | +| eos_token_id | include `<\|im_end\|>` | Required to stop at turn boundary | + +### Important note on greetings + +As a 0.5B model fine-tuned on instruction data, this model may produce inconsistent responses to simple greetings such as "Hi" or "Hello." It performs best when given direct questions or requests. For production deployments, greeting inputs should be handled with a fixed response rather than passed to the model. + +## Intended Use + +- Medical question answering and clinical education +- Research into small biomedical language models +- Lightweight medical AI prototyping +- Demonstration of CPT + SFT pipeline on consumer hardware + +## Not Intended For + +- Clinical decision making in real patient care +- Diagnostic or treatment decisions +- Replacement of licensed medical professionals + +## Limitations + +- 494M parameters — reasoning depth is limited compared to larger models +- Trained on single-turn instruction pairs — multi-turn coherence is basic +- Clinical accuracy not guaranteed — all outputs require expert verification +- Simple greetings may produce inconsistent responses at this model scale +- English only + +## Author + +Rumi Iqbal Sufi +Graduate Trainee, Excelra Knowledge Solutions, Hyderabad +HuggingFace: [Rumiii](https://huggingface.co/Rumiii) +GitHub: [sufirumii](https://github.com/sufirumii) +arXiv: [2506.09513](https://arxiv.org/abs/2506.09513) \ No newline at end of file diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..28028c0 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,54 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0]['role'] == 'system' %} + {{- messages[0]['content'] }} + {%- else %} + {{- 'You are a helpful assistant.' }} + {%- endif %} + {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0]['role'] == 'system' %} + {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }} + {%- else %} + {{- '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role }} + {%- if message.content %} + {{- '\n' + message.content }} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {{- tool_call.arguments | tojson }} + {{- '}\n' }} + {%- endfor %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/config.json b/config.json new file mode 100644 index 0000000..b33d2ee --- /dev/null +++ b/config.json @@ -0,0 +1,58 @@ +{ + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "float16", + "eos_token_id": 151643, + "hidden_act": "silu", + "hidden_size": 896, + "initializer_range": 0.02, + "intermediate_size": 4864, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 32768, + "max_window_layers": 24, + "model_type": "qwen2", + "num_attention_heads": 14, + "num_hidden_layers": 24, + "num_key_value_heads": 2, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000.0, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.0.0", + "use_cache": false, + "use_mrope": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..0165322 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,9 @@ +{ + "do_sample": false, + "eos_token_id": [ + 151643 + ], + "max_new_tokens": 2048, + "pad_token_id": 151643, + "transformers_version": "5.0.0" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..9179937 --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b394faff447419258f6e888f3efa524adfd7ead86e9cf796180a7e97a6ac80c +size 988097536 diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..34510ff --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8 +size 11421892 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..1ce8d7b --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": true, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +}