初始化项目，由ModelHub XC社区提供模型
Model: Atomic-Ai/GPT-Chat-German Source: Original Platform
This commit is contained in:
ModelHub XC
35
.gitattributes
vendored
Normal file
35
.gitattributes
vendored
Normal file
@@ -0,0 +1,35 @@
|
||||
*.7z filter=lfs diff=lfs merge=lfs -text
|
||||
*.arrow filter=lfs diff=lfs merge=lfs -text
|
||||
*.bin filter=lfs diff=lfs merge=lfs -text
|
||||
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
||||
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
||||
*.ftz filter=lfs diff=lfs merge=lfs -text
|
||||
*.gz filter=lfs diff=lfs merge=lfs -text
|
||||
*.h5 filter=lfs diff=lfs merge=lfs -text
|
||||
*.joblib filter=lfs diff=lfs merge=lfs -text
|
||||
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
||||
*.model filter=lfs diff=lfs merge=lfs -text
|
||||
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
||||
*.npy filter=lfs diff=lfs merge=lfs -text
|
||||
*.npz filter=lfs diff=lfs merge=lfs -text
|
||||
*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||
*.ot filter=lfs diff=lfs merge=lfs -text
|
||||
*.parquet filter=lfs diff=lfs merge=lfs -text
|
||||
*.pb filter=lfs diff=lfs merge=lfs -text
|
||||
*.pickle filter=lfs diff=lfs merge=lfs -text
|
||||
*.pkl filter=lfs diff=lfs merge=lfs -text
|
||||
*.pt filter=lfs diff=lfs merge=lfs -text
|
||||
*.pth filter=lfs diff=lfs merge=lfs -text
|
||||
*.rar filter=lfs diff=lfs merge=lfs -text
|
||||
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar filter=lfs diff=lfs merge=lfs -text
|
||||
*.tflite filter=lfs diff=lfs merge=lfs -text
|
||||
*.tgz filter=lfs diff=lfs merge=lfs -text
|
||||
*.wasm filter=lfs diff=lfs merge=lfs -text
|
||||
*.xz filter=lfs diff=lfs merge=lfs -text
|
||||
*.zip filter=lfs diff=lfs merge=lfs -text
|
||||
*.zst filter=lfs diff=lfs merge=lfs -text
|
||||
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||
158
README.md
Normal file
158
README.md
Normal file
@@ -0,0 +1,158 @@
|
||||
---
|
||||
license: mit
|
||||
language:
|
||||
- de
|
||||
base_model: kkirchheim/german-gpt2-medium
|
||||
tags:
|
||||
- german
|
||||
- gpt2
|
||||
- gpt2-medium
|
||||
- reasoning
|
||||
- chain-of-thought
|
||||
- sft
|
||||
- full-finetuning
|
||||
pipeline_tag: text-generation
|
||||
datasets:
|
||||
- Atomic-Ai/GPT-Think
|
||||
---
|
||||
|
||||
# 🧠 German GPT-2 Medium – Reasoning Edition (Full SFT)
|
||||
|
||||
Dieses Modell ist eine vollständig feinabgestimmte ("Full Fine-Tuned") Version von [kkirchheim/german-gpt2-medium](https://huggingface.co/kkirchheim/german-gpt2-medium). Es wurde auf **über 50 MB** deutschsprachigen Textdaten trainiert, die auf **logisches Denken und Chain-of-Thought-Reasoning** ausgerichtet sind.
|
||||
|
||||
> ⚠️ **Dieses Modell ist ausschließlich auf Deutsch trainiert.** Englische Eingaben werden wahrscheinlich nicht korrekt verarbeitet.
|
||||
|
||||
## 💡 Besonderheiten
|
||||
|
||||
* **Reasoning-Fähigkeit:** Das Modell generiert einen internen Denkprozess (`<think>...</think>`), bevor es antwortet – ähnlich wie Chain-of-Thought-Prompting.
|
||||
* **GPT-2 Medium Basis:** Mit ~355M Parametern deutlich leistungsfähiger als die Small-Variante (124M).
|
||||
* **Full Fine-Tuning:** Keine Adapter (LoRA etc.) – 100% der Parameter wurden auf den Reasoning-Datensatz angepasst.
|
||||
* **100% Deutsch:** Trainiert ausschließlich auf deutschen Daten. Englisch wird nicht unterstützt.
|
||||
|
||||
## 📝 Prompt-Format (Wichtig!)
|
||||
|
||||
Das Modell erwartet eine strikte Eingabestruktur. Wenn dieses Format nicht eingehalten wird, funktioniert die Reasoning-Funktion möglicherweise nicht.
|
||||
|
||||
```text
|
||||
<user> Deine Frage hier <End> <AI Assistent>
|
||||
```
|
||||
|
||||
Das Modell wird dann einen `<think>`-Block generieren, ihn mit `</think>` schließen und anschließend die endgültige Antwort ausgeben. Die Generierung stoppt automatisch beim `<End>`-Token oder wenn das Modell versucht, eine neue `<user>`-Runde zu starten.
|
||||
|
||||
## 💻 Nutzung
|
||||
|
||||
### Einfache Nutzung
|
||||
|
||||
```python
|
||||
import torch
|
||||
from transformers import AutoTokenizer, AutoModelForCausalLM
|
||||
|
||||
model_name = "Atomic-Ai/GPT-Chat-German"
|
||||
tokenizer = AutoTokenizer.from_pretrained(model_name)
|
||||
model = AutoModelForCausalLM.from_pretrained(
|
||||
model_name,
|
||||
torch_dtype=torch.float16 if torch.cuda.is_available() else torch.float32
|
||||
)
|
||||
|
||||
device = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
model = model.to(device)
|
||||
|
||||
def generate_response(user_input):
|
||||
prompt = f"<user> {user_input} <End> <AI Assistent>"
|
||||
inputs = tokenizer(prompt, return_tensors="pt").to(device)
|
||||
|
||||
with torch.no_grad():
|
||||
output_ids = model.generate(
|
||||
**inputs,
|
||||
max_new_tokens=1024,
|
||||
do_sample=True,
|
||||
temperature=0.7,
|
||||
top_p=0.9,
|
||||
repetition_penalty=1.2,
|
||||
pad_token_id=tokenizer.eos_token_id,
|
||||
eos_token_id=[
|
||||
tokenizer.eos_token_id,
|
||||
tokenizer.convert_tokens_to_ids("<user>")
|
||||
]
|
||||
)
|
||||
|
||||
full_output = tokenizer.decode(output_ids[0], skip_special_tokens=False)
|
||||
|
||||
# Antwort nach dem <AI Assistent> Tag extrahieren
|
||||
if "<AI Assistent>" in full_output:
|
||||
response = full_output.split("<AI Assistent>")[-1]
|
||||
else:
|
||||
response = full_output
|
||||
|
||||
# Bereinigung
|
||||
response = response.replace("<End>", "").replace("<user>", "").strip()
|
||||
return response
|
||||
|
||||
print(generate_response("Was ist die Hauptstadt von Deutschland?"))
|
||||
```
|
||||
|
||||
### Interaktiver Chat
|
||||
|
||||
```python
|
||||
# Nach dem Laden des Modells (siehe oben):
|
||||
while True:
|
||||
user_input = input("Du: ")
|
||||
if user_input.lower() in ["exit", "quit", "ende"]:
|
||||
print("👋 Tschüss!")
|
||||
break
|
||||
print(f"Bot: {generate_response(user_input)}\n")
|
||||
```
|
||||
|
||||
## ⚙️ Trainings-Details
|
||||
|
||||
Das Training wurde auf einer **NVIDIA T4 GPU** mit Mixed Precision (FP16) durchgeführt.
|
||||
|
||||
| Parameter | Wert |
|
||||
| :--- | :--- |
|
||||
| **Base Model** | kkirchheim/german-gpt2-medium (~355M Parameter) |
|
||||
| **Methode** | Full Fine-Tuning (100% Parameter) |
|
||||
| **Trainingsdaten** | ~50 MB kuratierter deutscher Text (Reasoning-fokussiert) |
|
||||
| **Epochen** | 1 |
|
||||
| **Batch Size** | 4 |
|
||||
| **Gradient Accumulation** | 2 (effektive Batch Size: 8) |
|
||||
| **Max Sequence Length** | 512 Token |
|
||||
| **Learning Rate** | 5e-5 |
|
||||
| **Scheduler** | Linear mit 3% Warmup |
|
||||
| **Optimizer** | AdamW (Fused) |
|
||||
| **Precision** | FP16 (Mixed Precision) |
|
||||
| **Weight Decay** | 0.01 |
|
||||
|
||||
### 📉 Trainingsverlauf (Loss)
|
||||
|
||||
| Step | Loss |
|
||||
| :--- | :--- |
|
||||
| 1450 | 1.188 |
|
||||
| 1500 | 1.063 |
|
||||
| 1550 | 1.155 |
|
||||
| 1600 | 1.124 |
|
||||
| 1650 | 1.125 |
|
||||
| 1700 | 1.109 |
|
||||
| 1750 | 1.099 |
|
||||
| 1800 | 1.063 |
|
||||
|
||||
Der Loss konvergiert stabil im Bereich von **~1.06–1.19**, was auf ein gut angepasstes Modell ohne starkes Overfitting hindeutet.
|
||||
|
||||
### Special Tokens
|
||||
|
||||
| Token | Funktion |
|
||||
| :--- | :--- |
|
||||
| `<user>` | Markiert den Beginn der Nutzereingabe |
|
||||
| `<AI Assistent>` | Markiert den Beginn der Modellantwort |
|
||||
| `<think>` / `</think>` | Umschließt den internen Denkprozess |
|
||||
| `<End>` | End-of-Sequence Token |
|
||||
|
||||
## ⚠️ Limitierungen
|
||||
|
||||
* **Nur Deutsch:** Das Modell wurde ausschließlich auf deutschen Daten trainiert. Englische oder mehrsprachige Eingaben führen zu schlechten Ergebnissen.
|
||||
* **Kontextfenster:** Begrenzt auf 1024 Token (GPT-2 Architektur).
|
||||
* **Modellgröße:** Trotz Medium-Variante (~355M) ist die Kapazität für komplexes Weltwissen begrenzt.
|
||||
* **Reasoning:** Der Chain-of-Thought-Ansatz verbessert die Antwortqualität, kann aber bei sehr einfachen Fragen unnötig ausführlich sein.
|
||||
|
||||
---
|
||||
|
||||
*Erstellt mit Liebe und viel GPU-Power. 🚀*
|
||||
8
added_tokens.json
Normal file
8
added_tokens.json
Normal file
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"</think>": 50270,
|
||||
"<AI Assistent>": 50268,
|
||||
"<End>": 50267,
|
||||
"<think>": 50269,
|
||||
"<user>": 50266,
|
||||
"<|endoftext|>": 50265
|
||||
}
|
||||
40
config.json
Normal file
40
config.json
Normal file
@@ -0,0 +1,40 @@
|
||||
{
|
||||
"activation_function": "gelu_new",
|
||||
"architectures": [
|
||||
"GPT2LMHeadModel"
|
||||
],
|
||||
"attn_pdrop": 0.1,
|
||||
"bos_token_id": 50256,
|
||||
"dtype": "float32",
|
||||
"embd_pdrop": 0.1,
|
||||
"eos_token_id": 50256,
|
||||
"initializer_range": 0.02,
|
||||
"layer_norm_epsilon": 1e-05,
|
||||
"model_type": "gpt2",
|
||||
"n_ctx": 2048,
|
||||
"n_embd": 1024,
|
||||
"n_head": 16,
|
||||
"n_inner": null,
|
||||
"n_layer": 24,
|
||||
"n_positions": 2048,
|
||||
"n_special": 0,
|
||||
"predict_special_tokens": true,
|
||||
"reorder_and_upcast_attn": false,
|
||||
"resid_pdrop": 0.1,
|
||||
"scale_attn_by_inverse_layer_idx": false,
|
||||
"scale_attn_weights": true,
|
||||
"summary_activation": null,
|
||||
"summary_first_dropout": 0.1,
|
||||
"summary_proj_to_labels": true,
|
||||
"summary_type": "cls_index",
|
||||
"summary_use_proj": true,
|
||||
"task_specific_params": {
|
||||
"text-generation": {
|
||||
"do_sample": true,
|
||||
"max_length": 50
|
||||
}
|
||||
},
|
||||
"transformers_version": "4.57.1",
|
||||
"use_cache": false,
|
||||
"vocab_size": 50271
|
||||
}
|
||||
6
generation_config.json
Normal file
6
generation_config.json
Normal file
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"_from_model_config": true,
|
||||
"bos_token_id": 50256,
|
||||
"eos_token_id": 50256,
|
||||
"transformers_version": "4.57.1"
|
||||
}
|
||||
50005
merges.txt
Normal file
50005
merges.txt
Normal file
File diff suppressed because it is too large
Load Diff
3
model.safetensors
Normal file
3
model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:5ee5226ba08c647f95f9a948bc90de30cb837448fd3b2643298ebc9266e45f25
|
||||
size 1423574528
|
||||
60
special_tokens_map.json
Normal file
60
special_tokens_map.json
Normal file
@@ -0,0 +1,60 @@
|
||||
{
|
||||
"additional_special_tokens": [
|
||||
{
|
||||
"content": "<user>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
{
|
||||
"content": "<AI Assistent>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
{
|
||||
"content": "<think>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
{
|
||||
"content": "</think>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
}
|
||||
],
|
||||
"bos_token": {
|
||||
"content": "<|endoftext|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"eos_token": {
|
||||
"content": "<End>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"pad_token": {
|
||||
"content": "<pad>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"unk_token": {
|
||||
"content": "<|endoftext|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
}
|
||||
}
|
||||
250434
tokenizer.json
Normal file
250434
tokenizer.json
Normal file
File diff suppressed because it is too large
Load Diff
114
tokenizer_config.json
Normal file
114
tokenizer_config.json
Normal file
@@ -0,0 +1,114 @@
|
||||
{
|
||||
"add_prefix_space": false,
|
||||
"added_tokens_decoder": {
|
||||
"0": {
|
||||
"content": "<s>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"1": {
|
||||
"content": "<pad>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"2": {
|
||||
"content": "</s>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"3": {
|
||||
"content": "<unk>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"4": {
|
||||
"content": "<mask>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"50265": {
|
||||
"content": "<|endoftext|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"50266": {
|
||||
"content": "<user>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"50267": {
|
||||
"content": "<End>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"50268": {
|
||||
"content": "<AI Assistent>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"50269": {
|
||||
"content": "<think>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"50270": {
|
||||
"content": "</think>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
}
|
||||
},
|
||||
"additional_special_tokens": [
|
||||
"<user>",
|
||||
"<AI Assistent>",
|
||||
"<think>",
|
||||
"</think>"
|
||||
],
|
||||
"bos_token": "<|endoftext|>",
|
||||
"clean_up_tokenization_spaces": false,
|
||||
"eos_token": "<End>",
|
||||
"extra_special_tokens": {},
|
||||
"max_length": 256,
|
||||
"model_max_length": 1000000000000000019884624838656,
|
||||
"pad_to_multiple_of": null,
|
||||
"pad_token": "<pad>",
|
||||
"pad_token_type_id": 0,
|
||||
"padding_side": "right",
|
||||
"stride": 0,
|
||||
"tokenizer_class": "GPT2Tokenizer",
|
||||
"truncation_side": "right",
|
||||
"truncation_strategy": "longest_first",
|
||||
"unk_token": "<|endoftext|>"
|
||||
}
|
||||
3
training_args.bin
Normal file
3
training_args.bin
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:f990d10d9a996cd02b29102dba3ce889623b631bdce9b82479c5d02b7e6e5868
|
||||
size 5841
|
||||
1
vocab.json
Normal file
1
vocab.json
Normal file
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user