burtenshaw/terminus-pi-trl-async-grpo
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

初始化项目，由ModelHub XC社区提供模型

Browse Source 
Model: burtenshaw/terminus-pi-trl-async-grpo
Source: Original Platform
This commit is contained in:
ModelHub XC
2026-07-20 20:49:07 +08:00
commit e6b239e143
11 changed files with 584 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
.gitattributes vendored Normal file
View File

@@ -0,0 +1,53 @@
*.7z filter=lfs diff=lfs merge=lfs -text
*.arrow filter=lfs diff=lfs merge=lfs -text
*.bz2 filter=lfs diff=lfs merge=lfs -text
*.ftz filter=lfs diff=lfs merge=lfs -text
*.gz filter=lfs diff=lfs merge=lfs -text
*.h5 filter=lfs diff=lfs merge=lfs -text
*.joblib filter=lfs diff=lfs merge=lfs -text
*.lfs.* filter=lfs diff=lfs merge=lfs -text
*.model filter=lfs diff=lfs merge=lfs -text
*.msgpack filter=lfs diff=lfs merge=lfs -text
*.onnx filter=lfs diff=lfs merge=lfs -text
*.ot filter=lfs diff=lfs merge=lfs -text
*.parquet filter=lfs diff=lfs merge=lfs -text
*.pb filter=lfs diff=lfs merge=lfs -text
*.pt filter=lfs diff=lfs merge=lfs -text
*.pth filter=lfs diff=lfs merge=lfs -text
*.rar filter=lfs diff=lfs merge=lfs -text
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
*.tar.* filter=lfs diff=lfs merge=lfs -text
*.tflite filter=lfs diff=lfs merge=lfs -text
*.tgz filter=lfs diff=lfs merge=lfs -text
*.xz filter=lfs diff=lfs merge=lfs -text
*.zip filter=lfs diff=lfs merge=lfs -text
*.zstandard filter=lfs diff=lfs merge=lfs -text
*.tfevents* filter=lfs diff=lfs merge=lfs -text
*.db* filter=lfs diff=lfs merge=lfs -text
*.ark* filter=lfs diff=lfs merge=lfs -text
**/*ckpt*data* filter=lfs diff=lfs merge=lfs -text
**/*ckpt*.meta filter=lfs diff=lfs merge=lfs -text
**/*ckpt*.index filter=lfs diff=lfs merge=lfs -text
*.ckpt filter=lfs diff=lfs merge=lfs -text
*.gguf* filter=lfs diff=lfs merge=lfs -text
*.ggml filter=lfs diff=lfs merge=lfs -text
*.llamafile* filter=lfs diff=lfs merge=lfs -text
*.pt2 filter=lfs diff=lfs merge=lfs -text
*.mlmodel filter=lfs diff=lfs merge=lfs -text
*.npy filter=lfs diff=lfs merge=lfs -text
*.npz filter=lfs diff=lfs merge=lfs -text
*.pickle filter=lfs diff=lfs merge=lfs -text
*.pkl filter=lfs diff=lfs merge=lfs -text
*.tar filter=lfs diff=lfs merge=lfs -text
*.wasm filter=lfs diff=lfs merge=lfs -text
*.zst filter=lfs diff=lfs merge=lfs -text
*tfevents* filter=lfs diff=lfs merge=lfs -text
model.safetensors filter=lfs diff=lfs merge=lfs -text
tokenizer.json filter=lfs diff=lfs merge=lfs -text
training_args.bin filter=lfs diff=lfs merge=lfs -text

69
README.md Normal file
View File

@@ -0,0 +1,69 @@
---
base_model: Qwen/Qwen3-0.6B
library_name: transformers
model_name: terminus-pi-trl-async-grpo
tags:
- generated_from_trainer
- trackio
- async-grpo
- trl
- trackio:https://huggingface.co/spaces/burtenshaw/terminus-pi-trl-static-a22f63
licence: license
---
# Model Card for terminus-pi-trl-async-grpo
This model is a fine-tuned version of [Qwen/Qwen3-0.6B](https://huggingface.co/Qwen/Qwen3-0.6B).
It has been trained using [TRL](https://github.com/huggingface/trl).
## Quick start
```python
from transformers import pipeline
question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
generator = pipeline("text-generation", model="burtenshaw/terminus-pi-trl-async-grpo", device="cuda")
output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
print(output["generated_text"])
```
## Training procedure
This model was trained with AsyncGRPO, a method introduced in [DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models](https://huggingface.co/papers/2402.03300).
### Framework versions
- TRL: 1.6.0.dev0
- Transformers: 5.10.0.dev0
- Pytorch: 2.10.0
- Datasets: 4.8.5
- Tokenizers: 0.22.2
## Citations
Cite AsyncGRPO as:
```bibtex
@article{shao2024deepseekmath,
title = {{DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models}},
author = {Zhihong Shao and Peiyi Wang and Qihao Zhu and Runxin Xu and Junxiao Song and Mingchuan Zhang and Y. K. Li and Y. Wu and Daya Guo},
year = 2024,
eprint = {arXiv:2402.03300},
}
```
Cite TRL as:
```bibtex
@software{vonwerra2020trl,
title = {{TRL: Transformers Reinforcement Learning}},
author = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin},
license = {Apache-2.0},
url = {https://github.com/huggingface/trl},
year = {2020}
}
```

89
chat_template.jinja Normal file
View File

@@ -0,0 +1,89 @@
{%- if tools %}
{{- '<|im_start|>system\n' }}
{%- if messages[0].role == 'system' %}
{{- messages[0].content + '\n\n' }}
{%- endif %}
{{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
{%- for tool in tools %}
{{- "\n" }}
{{- tool | tojson }}
{%- endfor %}
{{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
{%- else %}
{%- if messages[0].role == 'system' %}
{{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
{%- endif %}
{%- endif %}
{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
{%- for message in messages[::-1] %}
{%- set index = (messages|length - 1) - loop.index0 %}
{%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
{%- set ns.multi_step_tool = false %}
{%- set ns.last_query_index = index %}
{%- endif %}
{%- endfor %}
{%- for message in messages %}
{%- if message.content is string %}
{%- set content = message.content %}
{%- else %}
{%- set content = '' %}
{%- endif %}
{%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
{{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
{%- elif message.role == "assistant" %}
{%- set reasoning_content = '' %}
{%- if message.reasoning_content is string %}
{%- set reasoning_content = message.reasoning_content %}
{%- else %}
{%- if '</think>' in content %}
{%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
{%- set content = content.split('</think>')[-1].lstrip('\n') %}
{%- endif %}
{%- endif %}
{%- if loop.index0 > ns.last_query_index %}
{%- if loop.last or (not loop.last and reasoning_content) %}
{{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
{%- else %}
{{- '<|im_start|>' + message.role + '\n' + content }}
{%- endif %}
{%- else %}
{{- '<|im_start|>' + message.role + '\n' + content }}
{%- endif %}
{%- if message.tool_calls %}
{%- for tool_call in message.tool_calls %}
{%- if (loop.first and content) or (not loop.first) %}
{{- '\n' }}
{%- endif %}
{%- if tool_call.function %}
{%- set tool_call = tool_call.function %}
{%- endif %}
{{- '<tool_call>\n{"name": "' }}
{{- tool_call.name }}
{{- '", "arguments": ' }}
{%- if tool_call.arguments is string %}
{{- tool_call.arguments }}
{%- else %}
{{- tool_call.arguments | tojson }}
{%- endif %}
{{- '}\n</tool_call>' }}
{%- endfor %}
{%- endif %}
{{- '<|im_end|>\n' }}
{%- elif message.role == "tool" %}
{%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
{{- '<|im_start|>user' }}
{%- endif %}
{{- '\n<tool_response>\n' }}
{{- content }}
{{- '\n</tool_response>' }}
{%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
{{- '<|im_end|>\n' }}
{%- endif %}
{%- endif %}
{%- endfor %}
{%- if add_generation_prompt %}
{{- '<|im_start|>assistant\n' }}
{%- if enable_thinking is defined and enable_thinking is false %}
{{- '<think>\n\n</think>\n\n' }}
{%- endif %}
{%- endif %}

63
config.json Normal file
View File

@@ -0,0 +1,63 @@
{
"architectures": [
"Qwen3ForCausalLM"
],
"attention_bias": false,
"attention_dropout": 0.0,
"bos_token_id": null,
"dtype": "float32",
"eos_token_id": 151645,
"head_dim": 128,
"hidden_act": "silu",
"hidden_size": 1024,
"initializer_range": 0.02,
"intermediate_size": 3072,
"layer_types": [
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention",
"full_attention"
],
"max_position_embeddings": 40960,
"max_window_layers": 28,
"model_type": "qwen3",
"num_attention_heads": 16,
"num_hidden_layers": 28,
"num_key_value_heads": 8,
"pad_token_id": 151643,
"rms_norm_eps": 1e-06,
"rope_parameters": {
"rope_theta": 1000000,
"rope_type": "default"
},
"sliding_window": null,
"tie_word_embeddings": true,
"transformers_version": "5.10.0.dev0",
"use_cache": false,
"use_sliding_window": false,
"vocab_size": 151936
}

1
configuration.json Normal file
View File

@@ -0,0 +1 @@
{"framework": "pytorch", "task": "text-generation", "allow_remote": true}

12
generation_config.json Normal file
View File

@@ -0,0 +1,12 @@
{
"do_sample": true,
"eos_token_id": [
151645,
151643
],
"pad_token_id": 151643,
"temperature": 0.6,
"top_k": 20,
"top_p": 0.95,
"transformers_version": "5.10.0.dev0"
}

3
model.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:0013beb29261fb78ef2f1932eece31c07e1376794000cb11119d923330bcef1a
size 2384234968

215
summary.json Normal file
View File

@@ -0,0 +1,215 @@
{
"model": "Qwen/Qwen3-0.6B",
"trainer": "async-grpo",
"experiment": "first-command",
"env_url": "http://ip-26-0-168-52.ec2.internal:8000",
"vllm_server_url": "http://ip-26-0-168-95.ec2.internal:8000",
"hub_model_id": "burtenshaw/terminus-pi-trl-async-grpo",
"trackio_project": "terminus-pi-trl",
"report_to": "trackio",
"push_to_hub": true,
"run_name": "terminus-async-grpo-22179048",
"max_inflight_tasks": 2,
"output_dir": "/fsx/benjamin_burtenshaw/OpenEnv-codex-terminus-pi-trl-space/examples/end_to_end/tbench2_pi_trl/logs/output-22179048",
"train_result": {
"train_runtime": 189.2986,
"train_samples_per_second": 0.127,
"train_steps_per_second": 0.063,
"total_flos": 69970898190336.0,
"train_loss": 0.0,
"epoch": 1.0
},
"log_history": [
{
"loss": 0.0,
"grad_norm": 0.0,
"learning_rate": 5.833333333333334e-07,
"ratio": 0.9964810609817505,
"kl": 7.434934377670288e-05,
"entropy": 0.033603236079216,
"clip_ratio": 0.0,
"reward": 1.0,
"reward_std": 0.0,
"rewards/terminus_reward": 1.0,
"tools/call_frequency": 4.0,
"tools/failure_frequency": 0.0,
"generation_tok_per_s": 60.3897705078125,
"scoring_time_ms": 117.16629791259766,
"wait_scoring_ms": 68.45825958251953,
"buffer_qsize": 6.0,
"queue_wait_time_s": 4.076957702636719e-05,
"completions/mean_length": 55.0,
"training_tok/s": 6.806692854321114,
"forward_time_s": 0.07218480110168457,
"train_seq_len": 1104.0,
"weight_sync_time_s": 0.13250494003295898,
"epoch": 0.5,
"step": 6
},
{
"loss": 0.0,
"grad_norm": 0.0,
"learning_rate": 5e-07,
"ratio": 0.9964810609817505,
"kl": 7.434934377670288e-05,
"entropy": 0.033603236079216,
"clip_ratio": 0.0,
"reward": 1.0,
"reward_std": 0.0,
"rewards/terminus_reward": 1.0,
"tools/call_frequency": 4.0,
"tools/failure_frequency": 0.0,
"generation_tok_per_s": 63.50257110595703,
"scoring_time_ms": 112.0641860961914,
"wait_scoring_ms": 65.2970199584961,
"buffer_qsize": 8.0,
"queue_wait_time_s": 4.649162292480469e-05,
"completions/mean_length": 55.0,
"training_tok/s": 7.36189168170205,
"forward_time_s": 0.0719459056854248,
"train_seq_len": 1104.0,
"weight_sync_time_s": 0.13101506233215332,
"epoch": 0.5833333333333334,
"step": 7
},
{
"loss": 0.0,
"grad_norm": 0.0,
"learning_rate": 4.1666666666666667e-07,
"ratio": 0.9969766139984131,
"kl": 6.689117435598746e-05,
"entropy": 0.03523017838597298,
"clip_ratio": 0.0,
"reward": 1.0,
"reward_std": 0.0,
"rewards/terminus_reward": 1.0,
"tools/call_frequency": 4.0,
"tools/failure_frequency": 0.0,
"generation_tok_per_s": 74.39502716064453,
"scoring_time_ms": 69.75615692138672,
"wait_scoring_ms": 69.68431854248047,
"buffer_qsize": 30.0,
"queue_wait_time_s": 1.728534698486328e-05,
"completions/mean_length": 54.0,
"training_tok/s": 7.294061649219052,
"forward_time_s": 0.06849026679992676,
"train_seq_len": 1102.0,
"weight_sync_time_s": 0.14867115020751953,
"epoch": 0.6666666666666666,
"step": 8
},
{
"loss": 0.0,
"grad_norm": 0.0,
"learning_rate": 3.333333333333333e-07,
"ratio": 0.9962551593780518,
"kl": 9.892778325593099e-05,
"entropy": 0.046233903616666794,
"clip_ratio": 0.0,
"reward": 1.0,
"reward_std": 0.0,
"rewards/terminus_reward": 1.0,
"tools/call_frequency": 4.0,
"tools/failure_frequency": 0.0,
"generation_tok_per_s": 75.64350891113281,
"scoring_time_ms": 209.5857391357422,
"wait_scoring_ms": 63.70707702636719,
"buffer_qsize": 52.0,
"queue_wait_time_s": 1.7404556274414062e-05,
"completions/mean_length": 54.5,
"training_tok/s": 7.281059343739894,
"forward_time_s": 0.07220792770385742,
"train_seq_len": 1103.0,
"weight_sync_time_s": 0.13945603370666504,
"epoch": 0.75,
"step": 9
},
{
"loss": 0.0,
"grad_norm": 0.0,
"learning_rate": 2.5e-07,
"ratio": 0.9962551593780518,
"kl": 9.892778325593099e-05,
"entropy": 0.046233903616666794,
"clip_ratio": 0.0,
"reward": 1.0,
"reward_std": 0.0,
"rewards/terminus_reward": 1.0,
"tools/call_frequency": 4.0,
"tools/failure_frequency": 0.0,
"generation_tok_per_s": 76.56906127929688,
"scoring_time_ms": 236.5457305908203,
"wait_scoring_ms": 75.78178405761719,
"buffer_qsize": 72.0,
"queue_wait_time_s": 2.372264862060547e-05,
"completions/mean_length": 54.5,
"training_tok/s": 7.482828130727307,
"forward_time_s": 0.07097864151000977,
"train_seq_len": 1103.0,
"weight_sync_time_s": 0.13311219215393066,
"epoch": 0.8333333333333334,
"step": 10
},
{
"loss": 0.0,
"grad_norm": 0.0,
"learning_rate": 1.6666666666666665e-07,
"ratio": 0.9964330792427063,
"kl": 7.718314009252936e-05,
"entropy": 0.033603236079216,
"clip_ratio": 0.0,
"reward": 1.0,
"reward_std": 0.0,
"rewards/terminus_reward": 1.0,
"tools/call_frequency": 4.0,
"tools/failure_frequency": 0.0,
"generation_tok_per_s": 77.19235229492188,
"scoring_time_ms": 125.87603759765625,
"wait_scoring_ms": 60.385467529296875,
"buffer_qsize": 90.0,
"queue_wait_time_s": 2.276897430419922e-05,
"completions/mean_length": 55.0,
"training_tok/s": 7.519044330838157,
"forward_time_s": 0.07587862014770508,
"train_seq_len": 1104.0,
"weight_sync_time_s": 0.15876269340515137,
"epoch": 0.9166666666666666,
"step": 11
},
{
"loss": 0.0,
"grad_norm": 0.0,
"learning_rate": 8.333333333333333e-08,
"ratio": 0.9964810609817505,
"kl": 7.434934377670288e-05,
"entropy": 0.033603236079216,
"clip_ratio": 0.0,
"reward": 1.0,
"reward_std": 0.0,
"rewards/terminus_reward": 1.0,
"tools/call_frequency": 4.0,
"tools/failure_frequency": 0.0,
"generation_tok_per_s": 77.44713592529297,
"scoring_time_ms": 170.81890869140625,
"wait_scoring_ms": 62.718727111816406,
"buffer_qsize": 112.0,
"queue_wait_time_s": 2.2172927856445312e-05,
"completions/mean_length": 55.0,
"training_tok/s": 7.1332620460814224,
"forward_time_s": 0.07349634170532227,
"train_seq_len": 1104.0,
"weight_sync_time_s": 0.13324546813964844,
"epoch": 1.0,
"step": 12
},
{
"train_runtime": 189.2986,
"train_samples_per_second": 0.127,
"train_steps_per_second": 0.063,
"total_flos": 69970898190336.0,
"train_loss": 0.0,
"epoch": 1.0,
"step": 12
}
]
}

3
tokenizer.json Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506
size 11422650

73
tokenizer_config.json Normal file
View File

@@ -0,0 +1,73 @@
{
"add_prefix_space": false,
"backend": "tokenizers",
"bos_token": null,
"clean_up_tokenization_spaces": false,
"eos_token": "<|im_end|>",
"errors": "replace",
"extra_special_tokens": [
"<|im_start|>",
"<|im_end|>",
"<|object_ref_start|>",
"<|object_ref_end|>",
"<|box_start|>",
"<|box_end|>",
"<|quad_start|>",
"<|quad_end|>",
"<|vision_start|>",
"<|vision_end|>",
"<|vision_pad|>",
"<|image_pad|>",
"<|video_pad|>"
],
"is_local": false,
"local_files_only": false,
"model_max_length": 131072,
"pad_token": "<|endoftext|>",
"response_schema": {
"properties": {
"content": {
"type": "string"
},
"reasoning_content": {
"type": "string"
},
"role": {
"const": "assistant"
},
"tool_calls": {
"items": {
"properties": {
"function": {
"properties": {
"arguments": {
"additionalProperties": {},
"type": "object"
},
"name": {
"type": "string"
}
},
"type": "object"
},
"type": {
"const": "function"
}
},
"type": "object",
"x-parser": "json",
"x-parser-args": {
"transform": "{type: 'function', function: @}"
}
},
"type": "array",
"x-regex-iterator": "<tool_call>\\s*(.+?)\\s*</tool_call>"
}
},
"type": "object",
"x-regex": "^(?:<think>\\n?(?:(?P<reasoning_content>.*?\\S.*?)\\n?|[\\s]*)</think>\\s*)?(?P<content>(?:(?!<tool_call>)[\\s\\S])*?)(?:\\n(?=<tool_call>))?(?=(?:<tool_call>|<\\|im_end\\|>|$))(?P<tool_calls>(?:<tool_call>(?:(?!</tool_call>)[\\s\\S])+</tool_call>\\s*)+)?\\s*(?:<\\|im_end\\|>|$)"
},
"split_special_tokens": false,
"tokenizer_class": "Qwen2Tokenizer",
"unk_token": null
}

3
training_args.bin Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:56518801326d91eec85cdef7cff7fc3800822e43160b075e2d189517bdd2e89b
size 5905