From d8f7269e8ee36df255e05ff26dcaad7fa2802da5 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Thu, 15 Oct 2026 03:26:19 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: NotoriousH2/Qwen3-4B-Calendar-Agent-RLVR Source: Original Platform --- .gitattributes | 37 ++++++ README.md | 140 ++++++++++++++++++++ adapter/README.md | 209 ++++++++++++++++++++++++++++++ adapter/adapter_config.json | 50 +++++++ adapter/adapter_model.safetensors | 3 + adapter/chat_template.jinja | 89 +++++++++++++ adapter/tokenizer.json | 3 + adapter/tokenizer_config.json | 57 ++++++++ chat_template.jinja | 89 +++++++++++++ config.json | 71 ++++++++++ evaluation.json | 92 +++++++++++++ generation_config.json | 12 ++ metrics.json | 26 ++++ model.safetensors | 3 + tokenizer.json | 3 + tokenizer_config.json | 30 +++++ 16 files changed, 914 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 adapter/README.md create mode 100644 adapter/adapter_config.json create mode 100644 adapter/adapter_model.safetensors create mode 100644 adapter/chat_template.jinja create mode 100644 adapter/tokenizer.json create mode 100644 adapter/tokenizer_config.json create mode 100644 chat_template.jinja create mode 100644 config.json create mode 100644 evaluation.json create mode 100644 generation_config.json create mode 100644 metrics.json create mode 100644 model.safetensors create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..dffe933 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,37 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +adapter/tokenizer.json filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..665454a --- /dev/null +++ b/README.md @@ -0,0 +1,140 @@ +--- +license: apache-2.0 +language: +- ko +library_name: transformers +pipeline_tag: text-generation +base_model: NotoriousH2/Qwen3-4B-Calendar-Agent-SFT +datasets: +- NotoriousH2/calendar-agent-benchmark +tags: +- qwen3 +- tool-calling +- function-calling +- agents +- grpo +- rlvr +- reinforcement-learning +- calendar +- korean +--- + +# Qwen3-4B Calendar Agent RLVR + +Qwen3-4B Calendar Agent RLVR는 한국어 일정 관리 도구를 호출하는 4B 파라미터 병합 모델입니다. +Calendar 도구의 실행 궤적과 최종 상태를 보상으로 평가하여 GRPO로 학습했습니다. + +다음 행동을 학습 범위에 포함합니다. + +- 일정 조회, 생성, 수정, 삭제 +- 필수 정보가 없을 때 사용자에게 정보 요청 +- 삭제 전 사용자 승인 확인 +- 일정 충돌 확인과 빈 시간 생성 +- 여러 참석자의 공통 가능 시간 탐색 + +## 1. 사용 + +저장소 루트에는 병합 모델이 있으며, `adapter/`에는 LoRA 어댑터가 있습니다. + +```python +from transformers import AutoModelForCausalLM, AutoTokenizer + +model_id = "NotoriousH2/Qwen3-4B-Calendar-Agent-RLVR" +tokenizer = AutoTokenizer.from_pretrained(model_id) +model = AutoModelForCausalLM.from_pretrained( + model_id, + dtype="auto", + device_map="auto", +) + +tools = [ + { + "type": "function", + "function": { + "name": "search_events", + "description": "지정한 날짜의 일정을 조회합니다.", + "parameters": { + "type": "object", + "properties": { + "date": {"type": "string", "description": "YYYY-MM-DD"} + }, + "required": ["date"], + }, + }, + } +] +messages = [ + { + "role": "system", + "content": "사용자 요청을 처리하는 데 필요한 일정 도구를 호출하세요.", + }, + {"role": "user", "content": "2039-07-05 일정을 알려줘."}, +] + +prompt = tokenizer.apply_chat_template( + messages, + tools=tools, + tokenize=False, + add_generation_prompt=True, + enable_thinking=False, +) +inputs = tokenizer(prompt, return_tensors="pt").to(model.device) +outputs = model.generate(**inputs, max_new_tokens=512) +response = outputs[0, inputs.input_ids.shape[1] :] +print(tokenizer.decode(response, skip_special_tokens=False)) +``` + +모델은 도구 호출 정보를 생성합니다. +호출 실행과 도구 결과 전달은 호스트 애플리케이션에서 처리해야 합니다. + +## 2. 학습 보상 + +같은 요청에서 16개 응답을 생성하고, 그룹 안의 보상 차이로 Advantage를 계산했습니다. + +| 보상 | 가중치 | 평가 대상 | +|---|---:|---| +| 안전 규칙 | 0.25 | 필수 정보와 사용자 승인 준수 | +| 절차 진행 | 0.35 | 조회, 확인, 성공한 도구 실행 | +| 최종 성공 | 0.40 | 최종 Calendar 상태와 사용자 요청의 일치 | + +전체 행동군을 다룬 첫 학습 단계에서 96개 그룹 중 68개 그룹이 서로 다른 보상을 포함했습니다. +최종 성공 여부가 같은 응답도 절차 진행 점수로 구분하여 학습 신호를 구성했습니다. + +## 3. 평가 + +400개 합성 시나리오에서 실제 도구 루프를 실행하고 최종 상태를 평가했습니다. + +| 모델 | 유효 도구 호출 | 정책 준수 | Task Success | 평균 도구 호출 | +|---|---:|---:|---:|---:| +| Qwen3-4B Base | 100.00% | 80.00% | 20.00% | 0.80 | +| Calendar Agent SFT | 100.00% | 89.25% | 89.00% | 2.55 | +| Calendar Agent RLVR | 100.00% | 99.25% | 99.00% | 2.38 | + +SFT와 사례별로 비교하면 RLVR 모델은 41개 실패를 성공으로 전환했고, 1개 성공을 실패로 전환했습니다. +순증은 40개이며 전체 Task Success 차이는 10.00%p입니다. + +템플릿별 Task Success는 다음과 같습니다. + +| 템플릿 | SFT | RLVR | 전체 | +|---|---:|---:|---:| +| 필수 정보 요청 | 60 | 80 | 80 | +| 삭제 승인 요청 | 59 | 80 | 80 | +| 충돌 후 생성 | 77 | 76 | 80 | +| 승인 후 삭제 | 80 | 80 | 80 | +| 다중 참석자 생성 | 80 | 80 | 80 | + +집계값은 [evaluation.json](./evaluation.json)에 있습니다. + +## 4. 행동 사례 + +필수 정보가 없는 생성 요청에서는 임의의 시각을 정해 일정을 생성하지 않고 시작 시각과 소요 시간을 요청합니다. + +충돌 시 대체 시간 등록을 승인한 요청에서는 기존 일정을 조회하고 빈 시간을 찾은 뒤, 충돌하지 않는 시간에 일정을 생성합니다. + +## 5. 제한 + +이 모델은 `Asia/Seoul` 시간대의 합성 시나리오와 메모리 Calendar 환경에서 학습하고 평가했습니다. +실제 일정 서비스의 인증, 권한, 개인정보 처리, 네트워크 장애는 평가 범위에 포함하지 않습니다. + +충돌 후 생성은 SFT 77/80, RLVR 76/80으로 다른 템플릿보다 낮습니다. +외부 Calendar 시스템에 적용할 때는 충돌 복구와 도구 오류 이후의 재시도를 별도로 평가해야 합니다. diff --git a/adapter/README.md b/adapter/README.md new file mode 100644 index 0000000..050ade8 --- /dev/null +++ b/adapter/README.md @@ -0,0 +1,209 @@ +--- +base_model: artifacts/calendar_sft/merged +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:artifacts/calendar_sft/merged +- grpo +- lora +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.20.0 \ No newline at end of file diff --git a/adapter/adapter_config.json b/adapter/adapter_config.json new file mode 100644 index 0000000..4ddf2e1 --- /dev/null +++ b/adapter/adapter_config.json @@ -0,0 +1,50 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "artifacts/calendar_sft/merged", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.05, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "monteclora_config": null, + "peft_type": "LORA", + "peft_version": "0.20.0", + "qalora_group_size": 16, + "r": 16, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "q_proj", + "down_proj", + "gate_proj", + "o_proj", + "v_proj", + "k_proj", + "up_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false, + "velora_config": null +} \ No newline at end of file diff --git a/adapter/adapter_model.safetensors b/adapter/adapter_model.safetensors new file mode 100644 index 0000000..1a3dbfd --- /dev/null +++ b/adapter/adapter_model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a63322168467878b8e7b11334d96051085539e6a85793653257625ba4618c95 +size 132187888 diff --git a/adapter/chat_template.jinja b/adapter/chat_template.jinja new file mode 100644 index 0000000..01be9b3 --- /dev/null +++ b/adapter/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/adapter/tokenizer.json b/adapter/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/adapter/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/adapter/tokenizer_config.json b/adapter/tokenizer_config.json new file mode 100644 index 0000000..51bee8d --- /dev/null +++ b/adapter/tokenizer_config.json @@ -0,0 +1,57 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": true, + "local_files_only": false, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "response_template": { + "defaults": { + "role": "assistant" + }, + "fields": { + "content": { + "close_pattern": "<\\|im_end\\|>\\s*", + "content": "text" + }, + "reasoning_content": { + "close_pattern": "\\s*", + "content": "text", + "open": "" + }, + "tool_calls": { + "close_pattern": "\\s*", + "content": "json", + "open": "", + "repeats": true, + "transform": { + "function": "{content}", + "type": "function" + } + } + }, + "start_anchor": "<|im_start|>assistant\n" + }, + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..01be9b3 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,89 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if message.content is string %} + {%- set content = message.content %} + {%- else %} + {%- set content = '' %} + {%- endif %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..44ab6b5 --- /dev/null +++ b/config.json @@ -0,0 +1,71 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2560, + "initializer_range": 0.02, + "intermediate_size": 9728, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 40960, + "max_window_layers": 36, + "model_type": "qwen3", + "num_attention_heads": 32, + "num_hidden_layers": 36, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.14.1", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/evaluation.json b/evaluation.json new file mode 100644 index 0000000..47906f1 --- /dev/null +++ b/evaluation.json @@ -0,0 +1,92 @@ +{ + "task": "calendar_tool_calling", + "timezone": "Asia/Seoul", + "samples": 400, + "models": [ + { + "model": "Qwen3-4B Base", + "valid_tool_call": 1.0, + "policy_compliance": 0.8, + "task_success": 0.2, + "average_tool_calls": 0.8 + }, + { + "model": "Calendar Agent SFT", + "valid_tool_call": 1.0, + "policy_compliance": 0.8925, + "task_success": 0.89, + "average_tool_calls": 2.5525 + }, + { + "model": "Calendar Agent RLVR", + "valid_tool_call": 1.0, + "policy_compliance": 0.9925, + "task_success": 0.99, + "average_tool_calls": 2.3775 + } + ], + "paired_transition": { + "task_success": { + "gains": 41, + "losses": 1, + "net_gain": 40 + }, + "policy_compliance": { + "gains": 41, + "losses": 1, + "net_gain": 40 + } + }, + "templates": [ + { + "template": "missing_time_clarification", + "samples": 80, + "sft_successes": 60.0, + "rlvr_successes": 80.0 + }, + { + "template": "delete_confirmation", + "samples": 80, + "sft_successes": 59.0, + "rlvr_successes": 80.0 + }, + { + "template": "conflict_then_create", + "samples": 80, + "sft_successes": 77.0, + "rlvr_successes": 76.0 + }, + { + "template": "confirmed_delete", + "samples": 80, + "sft_successes": 80.0, + "rlvr_successes": 80.0 + }, + { + "template": "multi_attendee_create", + "samples": 80, + "sft_successes": 80.0, + "rlvr_successes": 80.0 + } + ], + "training_signal": { + "rollouts_per_group": 16, + "groups": 96, + "active_groups": 68, + "zero_variance_groups": 28, + "reward_components": [ + { + "name": "safety", + "weight": 0.25 + }, + { + "name": "policy_progress", + "weight": 0.35 + }, + { + "name": "semantic_final_success", + "weight": 0.4 + } + ] + } +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..a162df7 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.14.1" +} diff --git a/metrics.json b/metrics.json new file mode 100644 index 0000000..417edd8 --- /dev/null +++ b/metrics.json @@ -0,0 +1,26 @@ +{ + "task": "calendar", + "models": [ + { + "model": "Qwen3-4B Base", + "valid_tool_call": 1.0, + "policy_compliance": 0.8, + "task_success": 0.2, + "average_tool_calls": 0.8 + }, + { + "model": "Calendar Agent SFT", + "valid_tool_call": 1.0, + "policy_compliance": 0.8925, + "task_success": 0.89, + "average_tool_calls": 2.5525 + }, + { + "model": "Calendar Agent RLVR", + "valid_tool_call": 1.0, + "policy_compliance": 0.9925, + "task_success": 0.99, + "average_tool_calls": 2.3775 + } + ] +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..130e629 --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c4bab462e4538b946e6b17239a51c535fbc16e6f1174eb80afd139d992ad41c +size 8044982080 diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..9a87e38 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,30 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": true, + "local_files_only": true, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +}