From ca47e0c5e74815c24d6071b021f07b2609a41022 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Sat, 25 Jul 2026 05:09:09 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: FlameF0X/Qwen2-0.2B-it Source: Original Platform --- .eval_results/mmlu_pro.yaml | 8 + .eval_results/results.yaml | 17 ++ .gitattributes | 36 +++++ README.md | 126 +++++++++++++++ chat_template.jinja | 1 + config.json | 41 +++++ generation_config.json | 9 ++ model.safetensors | 3 + optimizer.pt | 3 + rng_state.pth | 3 + scaler.pt | 3 + scheduler.pt | 3 + tokenizer.json | 3 + tokenizer_config.json | 29 ++++ trainer_state.json | 314 ++++++++++++++++++++++++++++++++++++ training_args.bin | 3 + 16 files changed, 602 insertions(+) create mode 100644 .eval_results/mmlu_pro.yaml create mode 100644 .eval_results/results.yaml create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 chat_template.jinja create mode 100644 config.json create mode 100644 generation_config.json create mode 100644 model.safetensors create mode 100644 optimizer.pt create mode 100644 rng_state.pth create mode 100644 scaler.pt create mode 100644 scheduler.pt create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json create mode 100644 trainer_state.json create mode 100644 training_args.bin diff --git a/.eval_results/mmlu_pro.yaml b/.eval_results/mmlu_pro.yaml new file mode 100644 index 0000000..6408dfc --- /dev/null +++ b/.eval_results/mmlu_pro.yaml @@ -0,0 +1,8 @@ +- dataset: + id: TIGER-Lab/MMLU-Pro + task_id: mmlu_pro + value: 4.0 + source: + url: https://huggingface.co/FlameF0X/Qwen2-0.2B-it + name: Model Card + user: FlameF0X \ No newline at end of file diff --git a/.eval_results/results.yaml b/.eval_results/results.yaml new file mode 100644 index 0000000..7cb3212 --- /dev/null +++ b/.eval_results/results.yaml @@ -0,0 +1,17 @@ +- dataset: + id: TIGER-Lab/MMLU-Pro + task_id: mmlu_pro + value: 4.0 + source: + url: https://huggingface.co/FlameF0X/Qwen2-0.2B-it + name: Model Card + user: FlameF0X + +- dataset: + id: openai/gsm8k + task_id: gsm8k + value: 2.0 + source: + url: https://huggingface.co/FlameF0X/Qwen2-0.2B-it + name: Model Card + user: FlameF0X \ No newline at end of file diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..28c3254 --- /dev/null +++ b/README.md @@ -0,0 +1,126 @@ +--- +library_name: transformers +base_model: +- FlameF0X/Qwen2-0.2B-pt +license: apache-2.0 +datasets: +- Salesforce/wikitext +- roneneldan/TinyStories +- FlameF0X/arXiv-AI-ML +- Skylion007/openwebtext +- flytech/python-codes-25k +- bookcorpus/bookcorpus +- HuggingFaceH4/ultrachat_200k +- openai/gsm8k +- microsoft/orca-math-word-problems-200k +- laion/OIG +- microsoft/wiki_qa +metrics: +- accuracy +model-index: + - name: FlameF0X/Qwen2-0.2B-it + results: + - task: + type: text-generation + name: Text Generation + dataset: + id: openai/gsm8k + name: GSM8K + type: gsm8k + config: main + split: test + metrics: + - name: Accuracy + type: accuracy + value: 2.00 + verified: false + source: + name: Local Benchmark + url: https://huggingface.co/FlameF0X/Qwen2-0.2B-it + - task: + type: text-generation + name: Text Generation + dataset: + id: TIGER-Lab/MMLU-Pro + name: MMLU-Pro + type: TIGER-Lab/MMLU-Pro + config: default + split: test + metrics: + - name: Accuracy + type: accuracy + value: 4.00 + verified: false + source: + name: Local Benchmark + url: https://huggingface.co/FlameF0X/Qwen2-0.2B-it +--- + +## Evaluation Results + +| Benchmark | Score | +|-----------|-------| +| GSM8K (test) | 2.00% | +| MMLU-Pro (test) | 4.00% | + +> Results obtained via local evaluation. Given the model size (0.2B parameters), low benchmark scores are expected. + +## Model Usage +```python +import torch +from transformers import AutoTokenizer, AutoModelForCausalLM + +model_path = "FlameF0X/Qwen2-0.2B-it" + +tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True) +model = AutoModelForCausalLM.from_pretrained( + model_path, + torch_dtype="auto", + device_map="auto", + trust_remote_code=True +) + +messages = [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Explain how a transformer model works in one sentence."} +] + +text = tokenizer.apply_chat_template( + messages, + tokenize=False, + add_generation_prompt=True +) + +model_inputs = tokenizer([text], return_tensors="pt").to(model.device) + +generated_ids = model.generate( + **model_inputs, + max_new_tokens=128, + do_sample=True, + temperature=0.7 +) + +generated_ids = [ + output_ids[len(input_ids):] + for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids) +] + +response = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0] +print(f"--- Assistant Response ---\n{response}") +``` + +## Training Data + +This model was instruction-tuned on a mixture of: + +- `Salesforce/wikitext` — General text +- `roneneldan/TinyStories` — Short story generation +- `FlameF0X/arXiv-AI-ML` — AI/ML research papers +- `Skylion007/openwebtext` — Web text +- `flytech/python-codes-25k` — Python code +- `bookcorpus/bookcorpus` — Books +- `HuggingFaceH4/ultrachat_200k` — Instruction following +- `openai/gsm8k` — Math reasoning +- `microsoft/orca-math-word-problems-200k` — Math word problems +- `laion/OIG` — Open instruction generalist +- `microsoft/wiki_qa` — Question answering \ No newline at end of file diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..7be44a6 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1 @@ +{% for message in messages %}{{ '<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n' }}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..7ac658d --- /dev/null +++ b/config.json @@ -0,0 +1,41 @@ +{ + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "float32", + "eos_token_id": 151643, + "hidden_act": "silu", + "hidden_size": 512, + "initializer_range": 0.02, + "intermediate_size": 1376, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 2048, + "max_window_layers": 28, + "model_type": "qwen2", + "num_attention_heads": 8, + "num_hidden_layers": 8, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 10000.0, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": false, + "transformers_version": "5.2.0", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..355f282 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,9 @@ +{ + "_from_model_config": true, + "eos_token_id": 151643, + "output_attentions": false, + "output_hidden_states": false, + "pad_token_id": 151643, + "transformers_version": "5.2.0", + "use_cache": true +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..40bb105 --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f34ca4b7fea430ee68446e9d66f21dc0db8d5c674a3ba30b6b634694e8d9e501 +size 723612256 diff --git a/optimizer.pt b/optimizer.pt new file mode 100644 index 0000000..5bb369c --- /dev/null +++ b/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de8b2cf2a988ec7f7c7f403ba328be18f8874faaf5ecc21164260f960eb24b0f +size 1447288075 diff --git a/rng_state.pth b/rng_state.pth new file mode 100644 index 0000000..9cd53dd --- /dev/null +++ b/rng_state.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17cd930da9783ca70bad4b9cdeee6a06c0acea8f34645a333c93341f487f66a3 +size 14645 diff --git a/scaler.pt b/scaler.pt new file mode 100644 index 0000000..05c6428 --- /dev/null +++ b/scaler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4aa03f6e0cd07cf67ce1fbe3101d545f5771ef9148b9debf02b11cf6948da5c +size 1383 diff --git a/scheduler.pt b/scheduler.pt new file mode 100644 index 0000000..cf1f9ec --- /dev/null +++ b/scheduler.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8918e26db947366e457e93daaf6ae845d9aef0ef762ad16d34c2fab1781df115 +size 1465 diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..46b69f2 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d429fe753aea0ff87a94e86396d5508abb0d1d0e1f7a0d47c787ff72e0bf2691 +size 11422170 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..c960ecf --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..2c0654d --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,314 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 9.807384615384615, + "eval_steps": 500, + "global_step": 2000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.24615384615384617, + "grad_norm": 0.09971032291650772, + "learning_rate": 7.921600000000001e-05, + "loss": 3.8301455688476564, + "step": 50 + }, + { + "epoch": 0.49230769230769234, + "grad_norm": 0.0911579355597496, + "learning_rate": 7.8416e-05, + "loss": 3.0332159423828124, + "step": 100 + }, + { + "epoch": 0.7384615384615385, + "grad_norm": 0.10190442204475403, + "learning_rate": 7.7616e-05, + "loss": 2.7591339111328126, + "step": 150 + }, + { + "epoch": 0.9846153846153847, + "grad_norm": 0.09316737949848175, + "learning_rate": 7.6816e-05, + "loss": 2.617864074707031, + "step": 200 + }, + { + "epoch": 1.2264615384615385, + "grad_norm": 0.08559751510620117, + "learning_rate": 7.601600000000001e-05, + "loss": 2.3154002380371095, + "step": 250 + }, + { + "epoch": 1.4726153846153847, + "grad_norm": 0.0818333625793457, + "learning_rate": 7.5216e-05, + "loss": 2.227242126464844, + "step": 300 + }, + { + "epoch": 1.7187692307692308, + "grad_norm": 0.09082052856683731, + "learning_rate": 7.441600000000001e-05, + "loss": 2.1838539123535154, + "step": 350 + }, + { + "epoch": 1.964923076923077, + "grad_norm": 0.08575436472892761, + "learning_rate": 7.361600000000001e-05, + "loss": 2.1341651916503905, + "step": 400 + }, + { + "epoch": 2.206769230769231, + "grad_norm": 0.08966775238513947, + "learning_rate": 7.2816e-05, + "loss": 1.9519345092773437, + "step": 450 + }, + { + "epoch": 2.452923076923077, + "grad_norm": 0.08918383717536926, + "learning_rate": 7.201600000000001e-05, + "loss": 1.8568724060058595, + "step": 500 + }, + { + "epoch": 2.699076923076923, + "grad_norm": 0.09143362194299698, + "learning_rate": 7.1216e-05, + "loss": 1.8383006286621093, + "step": 550 + }, + { + "epoch": 2.9452307692307693, + "grad_norm": 0.0969746857881546, + "learning_rate": 7.0416e-05, + "loss": 1.8297712707519531, + "step": 600 + }, + { + "epoch": 3.187076923076923, + "grad_norm": 0.09824724495410919, + "learning_rate": 6.9616e-05, + "loss": 1.6545506286621094, + "step": 650 + }, + { + "epoch": 3.4332307692307693, + "grad_norm": 0.08493440598249435, + "learning_rate": 6.8816e-05, + "loss": 1.6011067199707032, + "step": 700 + }, + { + "epoch": 3.6793846153846155, + "grad_norm": 0.08676321804523468, + "learning_rate": 6.801600000000001e-05, + "loss": 1.5964797973632812, + "step": 750 + }, + { + "epoch": 3.9255384615384616, + "grad_norm": 0.09471355378627777, + "learning_rate": 6.7216e-05, + "loss": 1.5877139282226562, + "step": 800 + }, + { + "epoch": 4.1673846153846155, + "grad_norm": 0.08782424032688141, + "learning_rate": 6.641600000000001e-05, + "loss": 1.4405186462402344, + "step": 850 + }, + { + "epoch": 4.413538461538462, + "grad_norm": 0.09666605293750763, + "learning_rate": 6.5616e-05, + "loss": 1.383701171875, + "step": 900 + }, + { + "epoch": 4.659692307692308, + "grad_norm": 0.10380911082029343, + "learning_rate": 6.481600000000001e-05, + "loss": 1.375392608642578, + "step": 950 + }, + { + "epoch": 4.905846153846154, + "grad_norm": 0.10152668505907059, + "learning_rate": 6.4016e-05, + "loss": 1.3678237915039062, + "step": 1000 + }, + { + "epoch": 5.147692307692307, + "grad_norm": 0.102939173579216, + "learning_rate": 6.321600000000001e-05, + "loss": 1.257767333984375, + "step": 1050 + }, + { + "epoch": 5.3938461538461535, + "grad_norm": 0.09689474105834961, + "learning_rate": 6.241600000000001e-05, + "loss": 1.179140625, + "step": 1100 + }, + { + "epoch": 5.64, + "grad_norm": 0.09813547879457474, + "learning_rate": 6.1616e-05, + "loss": 1.1883023071289063, + "step": 1150 + }, + { + "epoch": 5.886153846153846, + "grad_norm": 0.10634688287973404, + "learning_rate": 6.0816e-05, + "loss": 1.196829605102539, + "step": 1200 + }, + { + "epoch": 6.128, + "grad_norm": 0.10477212816476822, + "learning_rate": 6.0016e-05, + "loss": 1.0858740234375, + "step": 1250 + }, + { + "epoch": 6.374153846153846, + "grad_norm": 0.10455155372619629, + "learning_rate": 5.9216000000000004e-05, + "loss": 0.9896931457519531, + "step": 1300 + }, + { + "epoch": 6.6203076923076924, + "grad_norm": 0.10506019741296768, + "learning_rate": 5.8416000000000004e-05, + "loss": 1.0163351440429687, + "step": 1350 + }, + { + "epoch": 6.866461538461539, + "grad_norm": 0.1090712919831276, + "learning_rate": 5.7616e-05, + "loss": 1.0064225006103515, + "step": 1400 + }, + { + "epoch": 7.108307692307692, + "grad_norm": 0.10628747940063477, + "learning_rate": 5.6816000000000006e-05, + "loss": 0.931029052734375, + "step": 1450 + }, + { + "epoch": 7.354461538461538, + "grad_norm": 0.1018829494714737, + "learning_rate": 5.601600000000001e-05, + "loss": 0.8393967437744141, + "step": 1500 + }, + { + "epoch": 7.600615384615384, + "grad_norm": 0.11559069901704788, + "learning_rate": 5.521600000000001e-05, + "loss": 0.8411316680908203, + "step": 1550 + }, + { + "epoch": 7.8467692307692305, + "grad_norm": 0.12172011286020279, + "learning_rate": 5.441600000000001e-05, + "loss": 0.8445599365234375, + "step": 1600 + }, + { + "epoch": 8.088615384615384, + "grad_norm": 0.09611531347036362, + "learning_rate": 5.3616e-05, + "loss": 0.7935149383544922, + "step": 1650 + }, + { + "epoch": 8.334769230769231, + "grad_norm": 0.11105120927095413, + "learning_rate": 5.2816000000000004e-05, + "loss": 0.6863140106201172, + "step": 1700 + }, + { + "epoch": 8.580923076923076, + "grad_norm": 0.10133285075426102, + "learning_rate": 5.2016000000000005e-05, + "loss": 0.6949832153320312, + "step": 1750 + }, + { + "epoch": 8.827076923076923, + "grad_norm": 0.11194281280040741, + "learning_rate": 5.1216000000000006e-05, + "loss": 0.7070323944091796, + "step": 1800 + }, + { + "epoch": 9.068923076923078, + "grad_norm": 0.093862384557724, + "learning_rate": 5.0416e-05, + "loss": 0.6766038513183594, + "step": 1850 + }, + { + "epoch": 9.315076923076923, + "grad_norm": 0.1024981439113617, + "learning_rate": 4.9616e-05, + "loss": 0.5633995056152343, + "step": 1900 + }, + { + "epoch": 9.56123076923077, + "grad_norm": 0.10483216494321823, + "learning_rate": 4.8816e-05, + "loss": 0.5755315399169922, + "step": 1950 + }, + { + "epoch": 9.807384615384615, + "grad_norm": 0.11396331340074539, + "learning_rate": 4.8016e-05, + "loss": 0.5865091705322265, + "step": 2000 + } + ], + "logging_steps": 50, + "max_steps": 5000, + "num_input_tokens_seen": 0, + "num_train_epochs": 25, + "save_steps": 1000, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.019225655836672e+16, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..17a0e71 --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:197af5fb66d4f0b05823de4e1b82d51e93a46f42124f1586c8e67d8a8a0dfbdc +size 5201