commit 347372d0d90f12cdc38aa9f23e8806cb9422abfd Author: ModelHub XC Date: Mon Jul 27 19:23:11 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: sascha-frank-ai-research/tsft-rag-qwen2.5-7b-instruct Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..6634c8c --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2024 Alibaba Cloud + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..bbcfa51 --- /dev/null +++ b/README.md @@ -0,0 +1,346 @@ +--- +license: apache-2.0 +library_name: transformers +pipeline_tag: text-generation +base_model: Qwen/Qwen2.5-7B-Instruct +base_model_relation: finetune +language: + - de + - en +tags: + - rag + - retrieval-augmented-generation + - task-specific-fine-tuning + - full-fine-tuning + - qwen2.5 + - text-generation + - structured-output + - abstention + - research + - tsft-rag + - model-card-v1 +--- + +# TSFT-RAG Qwen2.5-7B-Instruct + +**Task-Specific Full Fine-Tuning for Retrieval-Augmented Generation** + +TSFT-RAG Qwen2.5-7B-Instruct is a full-parameter fine-tuned derivative of +[Qwen/Qwen2.5-7B-Instruct](https://huggingface.co/Qwen/Qwen2.5-7B-Instruct). + +Part of the [TSFT-RAG model family](https://huggingface.co/sascha-frank-ai-research) for Retrieval-Augmented Generation across Gemma, Qwen and Llama architectures. + +The objective is to investigate how supervised full fine-tuning changes the behaviour of modern language models when used as Retrieval-Augmented Generation (RAG) systems. + +--- + +# TSFT-RAG Model Series + +| Model | Parameters | Status | +|--------|-----------:|--------| +| [Qwen2.5-0.5B-Instruct](https://huggingface.co/sascha-frank-ai-research/tsft-rag-qwen2.5-0.5b-instruct) | 0.5B | Released | +| [Qwen2.5-1.5B-Instruct](https://huggingface.co/sascha-frank-ai-research/tsft-rag-qwen2.5-1.5b-instruct) | 1.5B | Released | +| **Qwen2.5-7B-Instruct** | **7B** | **Released** | +| [Qwen2.5-14B-Instruct](https://huggingface.co/sascha-frank-ai-research/tsft-rag-qwen2.5-14b-instruct) | 14B | Released | + +--- + +The model was trained to improve several behaviours that are important in practical +RAG systems: + +- answering from supplied context; +- abstaining when the context does not support an answer; +- following task-specific output requirements; +- producing structured JSON output; +- generating source-aware and citation-oriented responses. + +The strongest observed gains are in hard-negative handling, structured-output validity, +and citation-related behaviour. Performance does not improve uniformly across all +metrics; grounded-QA and some analysis metrics show small declines in the reported +evaluation. + +## Model Details + +| Property | Value | +|---|---| +| Model family | TSFT-RAG | +| Base model | Qwen/Qwen2.5-7B-Instruct | +| Architecture | Qwen2ForCausalLM | +| Parameters | approximately 7B | +| Training method | full-parameter supervised fine-tuning | +| Primary language of the training corpus | German | +| Context length supported by the configuration | 32,768 tokens | +| Training sequence length | 1,024 tokens | +| Weight dtype | bfloat16 | +| Framework | Transformers | +| License | Apache-2.0 | + +## Intended Use + +This model is intended for research and experimental use in RAG-oriented workflows. + +Suitable use cases include: + +- context-grounded question answering; +- evaluation of abstention behaviour; +- generation of structured RAG responses; +- source-aware answer generation; +- comparison of base and fully fine-tuned language models; +- local or server-based RAG experiments. + +The model should be used with retrieved evidence or other explicitly supplied context. +It is not intended to serve as a standalone factual knowledge source. + +## Training Data + +The model was trained on a fixed, task-specific German-language corpus designed for +RAG-oriented instruction following. + +The training examples cover: + +- answerable context-grounded questions; +- hard-negative and unanswerable questions; +- multiple assistant roles and response styles; +- analysis tasks for main topic, key message, and main arguments; +- JSON and short-answer output formats; +- citation-aware responses. + +The complete raw training corpus is not distributed with this checkpoint. The +accompanying repository documents the data-processing, training, and evaluation +pipeline. + +## Training Procedure + +This checkpoint was produced using full-parameter supervised fine-tuning rather than +LoRA, QLoRA, or another adapter-based method. + +| Training argument | Value | +|---|---:| +| Epochs | 3 | +| Learning rate | 1e-6 | +| Per-device train batch size | 4 | +| Per-device evaluation batch size | 4 | +| Gradient accumulation steps | 4 | +| Effective batch size per process | 16 | +| Optimizer | AdamW (PyTorch) | +| Scheduler | cosine | +| Warmup ratio | 0.08 | +| Weight decay | 0.01 | +| Maximum gradient norm | 1.0 | +| Precision | bfloat16 | +| FP16 | disabled | +| Maximum sequence length | 1,024 | +| Packing | disabled | +| Seed | 42 | + +The recorded training runtime was approximately **5,656 seconds** +(about **1 h 34 min**), with a final reported training loss of +**0.8019**. + +## Evaluation + +The fine-tuned model and the unchanged base model were evaluated on the same held-out +test set of **1,940 instances**. + +| Metric | Base model | TSFT-RAG | Difference | +|---|---:|---:|---:| +| Aggregate score | 0.4016 | 0.5617 | +0.1601 | +| Hard-negative handling | 0.0791 | 0.5523 | +0.4732 | +| Grounded QA | 0.5505 | 0.5256 | -0.0249 | +| Analysis: main topic | 0.6158 | 0.6241 | +0.0083 | +| Analysis: key message | 0.6122 | 0.5726 | -0.0396 | +| Analysis: main arguments | 0.5823 | 0.5584 | -0.0239 | +| JSON validity | 0.5739 | 0.9069 | +0.3330 | +| Citation precision | 0.0000 | 0.8300 | +0.8300 | +| Citation recall | 0.0000 | 0.8300 | +0.8300 | + +The largest improvement is in hard-negative handling. The unchanged base model +correctly rejected **56 of 708** hard-negative cases, while the fine-tuned model +correctly rejected **391 of 708**. + +The aggregate score, hard-negative handling, JSON validity, citation precision, and +citation recall improve substantially. Grounded QA, key-message analysis, and +main-argument analysis show modest declines, while main-topic analysis improves +slightly. + +These results should be interpreted as task-specific benchmark outcomes rather than +general-purpose capability claims. + +Further evaluation details, scripts, and analysis are available in the accompanying +repository and paper. + +## Usage + +```python +import torch +from transformers import AutoModelForCausalLM, AutoTokenizer + +model_id = "sascha-frank-ai-research/tsft-rag-qwen2.5-7b-instruct" + +tokenizer = AutoTokenizer.from_pretrained(model_id) + +model = AutoModelForCausalLM.from_pretrained( + model_id, + torch_dtype=torch.bfloat16, + device_map="auto", +) + +messages = [ + { + "role": "system", + "content": ( + "Du bist ein sachlicher RAG-Assistent. " + "Beantworte die Frage präzise nur auf Grundlage des Kontexts. " + "Wenn der Kontext die Antwort nicht belegt, sage dies ausdrücklich." + ), + }, + { + "role": "user", + "content": ( + "Kontext:\n" + "Ein Retrieval-Modul liefert relevante Dokumentpassagen an das Sprachmodell.\n\n" + "Frage:\n" + "Welche Komponente stellt dem Sprachmodell die Dokumentpassagen bereit?" + ), + }, +] + +text = tokenizer.apply_chat_template( + messages, + tokenize=False, + add_generation_prompt=True, +) + +inputs = tokenizer(text, return_tensors="pt").to(model.device) + +with torch.inference_mode(): + output = model.generate( + **inputs, + max_new_tokens=256, + do_sample=False, + ) + +generated_tokens = output[0, inputs["input_ids"].shape[1]:] + +print( + tokenizer.decode( + generated_tokens, + skip_special_tokens=True, + ) +) +``` + +## Prompting Notes + +The model was trained with Qwen's chat template and should be used through +`tokenizer.apply_chat_template(...)`. + +For RAG use, prompts should clearly separate: + +1. the behavioural instruction; +2. the retrieved context; +3. the user question; +4. any required output format. + +For deterministic evaluation, use greedy decoding or another fixed decoding setup. + +## Limitations + +- The model may produce incorrect, incomplete, or unsupported answers. +- Fine-tuning does not guarantee faithful use of retrieved evidence. +- Retrieval quality remains a major determinant of answer quality. +- The model may fail to abstain even when evidence is insufficient. +- Citation-like output does not guarantee that a cited passage truly supports a claim. +- Performance outside the evaluated RAG task formats has not been systematically established. +- The training corpus is primarily German, so behaviour may differ in other languages. +- The model inherits limitations and biases from the Qwen2.5 base model and from the + task-specific training data. +- The observed metric trade-offs should be considered when selecting the model for deployment. + +The model should not be used for high-stakes medical, legal, financial, safety-critical, +or administrative decisions without independent validation and human oversight. + +## Ethical Considerations + +Retrieved documents may contain inaccurate, biased, confidential, or outdated +information. Such content can be reproduced or amplified by the model. + +Users are responsible for: + +- validating retrieval sources; +- protecting confidential material; +- testing model behaviour in the intended domain; +- monitoring unsupported or misleading output; +- complying with applicable law, policy, and licensing requirements. + +## Relationship to the Research Project + +This checkpoint is one model from a controlled cross-family study of task-specific +full fine-tuning for RAG. The study compares base and fine-tuned configurations from +the Gemma, Llama, and Qwen model families across parameter scales from approximately +0.5B to 14B. + +Code, training scripts, evaluation scripts, and documentation: + +https://github.com/frankmst/rag-task-specific-full-finetuning + +## Base Model and License + +This model is derived from +[Qwen/Qwen2.5-7B-Instruct](https://huggingface.co/Qwen/Qwen2.5-7B-Instruct), +which is distributed under the Apache License 2.0. + +This derivative checkpoint is also released under the Apache License 2.0. Users +should review the license file included in this repository and the licensing +information of the base model. + +## Citation + +A formal paper citation will be added after publication of the accompanying paper. + +Until then, please cite the software repository: + +```bibtex +@software{frank_tsft_rag_2026, + author = {Sascha Frank}, + title = {Task-Specific Full Fine-Tuning for Retrieval-Augmented Generation}, + year = {2026}, + url = {https://github.com/frankmst/rag-task-specific-full-finetuning} +} +``` + +## Project + +The complete TSFT-RAG research project, including training scripts, +evaluation pipeline, benchmark methodology and documentation, is available at: + +**Project repository** +https://github.com/frankmst/rag-task-specific-full-finetuning + +**TSFT-RAG model family** +https://huggingface.co/sascha-frank-ai-research + +--- + +## Author + +**Sascha Frank** + +Independent AI Researcher + +**ORCID** +https://orcid.org/0000-0002-0588-0081 + +**GitHub** +https://github.com/frankmst + +**Hugging Face** +https://huggingface.co/sascha-frank-ai-research + + +## Links + +- Project repository: https://github.com/frankmst/rag-task-specific-full-finetuning +- Base model: https://huggingface.co/Qwen/Qwen2.5-7B-Instruct +- Project website: https://sascha-frank.com +- Paper: forthcoming \ No newline at end of file diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..bdf7919 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,54 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0]['role'] == 'system' %} + {{- messages[0]['content'] }} + {%- else %} + {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }} + {%- endif %} + {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0]['role'] == 'system' %} + {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }} + {%- else %} + {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {{- '<|im_start|>' + message.role }} + {%- if message.content %} + {{- '\n' + message.content }} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {{- tool_call.arguments | tojson }} + {{- '}\n' }} + {%- endfor %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/config.json b/config.json new file mode 100644 index 0000000..e564cdd --- /dev/null +++ b/config.json @@ -0,0 +1,62 @@ +{ + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "hidden_act": "silu", + "hidden_size": 3584, + "initializer_range": 0.02, + "intermediate_size": 18944, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 32768, + "max_window_layers": 28, + "model_type": "qwen2", + "num_attention_heads": 28, + "num_hidden_layers": 28, + "num_key_value_heads": 4, + "pad_token_id": 151654, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000.0, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": false, + "transformers_version": "5.12.1", + "unsloth_fixed": true, + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 152064 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..7ece780 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,14 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "max_length": 32768, + "pad_token_id": 151654, + "repetition_penalty": 1.05, + "temperature": 0.7, + "top_k": 20, + "top_p": 0.8, + "transformers_version": "5.12.1" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..bb20a72 --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b1f5c572165efd1fcba6d7c34980108f61faf934b0b212bb45fb7ae9b43c1213 +size 15231272152 diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..34510ff --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8 +size 11421892 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..63093d1 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,16 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "is_local": false, + "local_files_only": false, + "model_max_length": 32768, + "pad_token": "<|vision_pad|>", + "padding_side": "right", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/train_metrics.json b/train_metrics.json new file mode 100644 index 0000000..70c79a8 --- /dev/null +++ b/train_metrics.json @@ -0,0 +1,8 @@ +{ + "train_runtime": 5656.4446, + "train_samples_per_second": 8.233, + "train_steps_per_second": 0.515, + "total_flos": 5.3836610453744026e+17, + "train_loss": 0.8018893963112849, + "epoch": 3.0 +} \ No newline at end of file