初始化项目，由ModelHub XC社区提供模型
Model: laion/a3-rl-DCAgent_code-contests-noblock-5-8B Source: Original Platform
This commit is contained in:
ModelHub XC
38
.gitattributes
vendored
Normal file
38
.gitattributes
vendored
Normal file
@@ -0,0 +1,38 @@
|
||||
*.7z filter=lfs diff=lfs merge=lfs -text
|
||||
*.arrow filter=lfs diff=lfs merge=lfs -text
|
||||
*.bin filter=lfs diff=lfs merge=lfs -text
|
||||
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
||||
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
||||
*.ftz filter=lfs diff=lfs merge=lfs -text
|
||||
*.gz filter=lfs diff=lfs merge=lfs -text
|
||||
*.h5 filter=lfs diff=lfs merge=lfs -text
|
||||
*.joblib filter=lfs diff=lfs merge=lfs -text
|
||||
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
||||
*.model filter=lfs diff=lfs merge=lfs -text
|
||||
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
||||
*.npy filter=lfs diff=lfs merge=lfs -text
|
||||
*.npz filter=lfs diff=lfs merge=lfs -text
|
||||
*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||
*.ot filter=lfs diff=lfs merge=lfs -text
|
||||
*.parquet filter=lfs diff=lfs merge=lfs -text
|
||||
*.pb filter=lfs diff=lfs merge=lfs -text
|
||||
*.pickle filter=lfs diff=lfs merge=lfs -text
|
||||
*.pkl filter=lfs diff=lfs merge=lfs -text
|
||||
*.pt filter=lfs diff=lfs merge=lfs -text
|
||||
*.pth filter=lfs diff=lfs merge=lfs -text
|
||||
*.rar filter=lfs diff=lfs merge=lfs -text
|
||||
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar filter=lfs diff=lfs merge=lfs -text
|
||||
*.tflite filter=lfs diff=lfs merge=lfs -text
|
||||
*.tgz filter=lfs diff=lfs merge=lfs -text
|
||||
*.wasm filter=lfs diff=lfs merge=lfs -text
|
||||
*.xz filter=lfs diff=lfs merge=lfs -text
|
||||
*.zip filter=lfs diff=lfs merge=lfs -text
|
||||
*.zst filter=lfs diff=lfs merge=lfs -text
|
||||
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||
tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
||||
training_logs/20260525_110326_reward_vs_steps.png filter=lfs diff=lfs merge=lfs -text
|
||||
training_logs/20260525_110326_turn_count_distribution.png filter=lfs diff=lfs merge=lfs -text
|
||||
28
added_tokens.json
Normal file
28
added_tokens.json
Normal file
@@ -0,0 +1,28 @@
|
||||
{
|
||||
"</think>": 151668,
|
||||
"</tool_call>": 151658,
|
||||
"</tool_response>": 151666,
|
||||
"<think>": 151667,
|
||||
"<tool_call>": 151657,
|
||||
"<tool_response>": 151665,
|
||||
"<|box_end|>": 151649,
|
||||
"<|box_start|>": 151648,
|
||||
"<|endoftext|>": 151643,
|
||||
"<|file_sep|>": 151664,
|
||||
"<|fim_middle|>": 151660,
|
||||
"<|fim_pad|>": 151662,
|
||||
"<|fim_prefix|>": 151659,
|
||||
"<|fim_suffix|>": 151661,
|
||||
"<|im_end|>": 151645,
|
||||
"<|im_start|>": 151644,
|
||||
"<|image_pad|>": 151655,
|
||||
"<|object_ref_end|>": 151647,
|
||||
"<|object_ref_start|>": 151646,
|
||||
"<|quad_end|>": 151651,
|
||||
"<|quad_start|>": 151650,
|
||||
"<|repo_name|>": 151663,
|
||||
"<|video_pad|>": 151656,
|
||||
"<|vision_end|>": 151653,
|
||||
"<|vision_pad|>": 151654,
|
||||
"<|vision_start|>": 151652
|
||||
}
|
||||
89
chat_template.jinja
Normal file
89
chat_template.jinja
Normal file
@@ -0,0 +1,89 @@
|
||||
{%- if tools %}
|
||||
{{- '<|im_start|>system\n' }}
|
||||
{%- if messages[0].role == 'system' %}
|
||||
{{- messages[0].content + '\n\n' }}
|
||||
{%- endif %}
|
||||
{{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
|
||||
{%- for tool in tools %}
|
||||
{{- "\n" }}
|
||||
{{- tool | tojson }}
|
||||
{%- endfor %}
|
||||
{{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
|
||||
{%- else %}
|
||||
{%- if messages[0].role == 'system' %}
|
||||
{{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
|
||||
{%- for message in messages[::-1] %}
|
||||
{%- set index = (messages|length - 1) - loop.index0 %}
|
||||
{%- if ns.multi_step_tool and message.role == "user" and message.content is string and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
|
||||
{%- set ns.multi_step_tool = false %}
|
||||
{%- set ns.last_query_index = index %}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- for message in messages %}
|
||||
{%- if message.content is string %}
|
||||
{%- set content = message.content %}
|
||||
{%- else %}
|
||||
{%- set content = '' %}
|
||||
{%- endif %}
|
||||
{%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
|
||||
{{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
|
||||
{%- elif message.role == "assistant" %}
|
||||
{%- set reasoning_content = '' %}
|
||||
{%- if message.reasoning_content is string %}
|
||||
{%- set reasoning_content = message.reasoning_content %}
|
||||
{%- else %}
|
||||
{%- if '</think>' in content %}
|
||||
{%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
|
||||
{%- set content = content.split('</think>')[-1].lstrip('\n') %}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- if loop.index0 > ns.last_query_index %}
|
||||
{%- if loop.last or (not loop.last and reasoning_content) %}
|
||||
{{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
|
||||
{%- else %}
|
||||
{{- '<|im_start|>' + message.role + '\n' + content }}
|
||||
{%- endif %}
|
||||
{%- else %}
|
||||
{{- '<|im_start|>' + message.role + '\n' + content }}
|
||||
{%- endif %}
|
||||
{%- if message.tool_calls %}
|
||||
{%- for tool_call in message.tool_calls %}
|
||||
{%- if (loop.first and content) or (not loop.first) %}
|
||||
{{- '\n' }}
|
||||
{%- endif %}
|
||||
{%- if tool_call.function %}
|
||||
{%- set tool_call = tool_call.function %}
|
||||
{%- endif %}
|
||||
{{- '<tool_call>\n{"name": "' }}
|
||||
{{- tool_call.name }}
|
||||
{{- '", "arguments": ' }}
|
||||
{%- if tool_call.arguments is string %}
|
||||
{{- tool_call.arguments }}
|
||||
{%- else %}
|
||||
{{- tool_call.arguments | tojson }}
|
||||
{%- endif %}
|
||||
{{- '}\n</tool_call>' }}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- elif message.role == "tool" %}
|
||||
{%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
|
||||
{{- '<|im_start|>user' }}
|
||||
{%- endif %}
|
||||
{{- '\n<tool_response>\n' }}
|
||||
{{- content }}
|
||||
{{- '\n</tool_response>' }}
|
||||
{%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- if add_generation_prompt %}
|
||||
{{- '<|im_start|>assistant\n' }}
|
||||
{%- if enable_thinking is defined and enable_thinking is false %}
|
||||
{{- '<think>\n\n</think>\n\n' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
68
config.json
Normal file
68
config.json
Normal file
@@ -0,0 +1,68 @@
|
||||
{
|
||||
"architectures": [
|
||||
"Qwen3ForCausalLM"
|
||||
],
|
||||
"attention_bias": false,
|
||||
"attention_dropout": 0.0,
|
||||
"dtype": "bfloat16",
|
||||
"eos_token_id": 151645,
|
||||
"head_dim": 128,
|
||||
"hidden_act": "silu",
|
||||
"hidden_size": 4096,
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 12288,
|
||||
"layer_types": [
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention"
|
||||
],
|
||||
"max_position_embeddings": 40960,
|
||||
"max_window_layers": 36,
|
||||
"model_type": "qwen3",
|
||||
"num_attention_heads": 32,
|
||||
"num_hidden_layers": 36,
|
||||
"num_key_value_heads": 8,
|
||||
"pad_token_id": 151643,
|
||||
"rms_norm_eps": 1e-06,
|
||||
"rope_scaling": null,
|
||||
"rope_theta": 1000000,
|
||||
"sliding_window": null,
|
||||
"tie_word_embeddings": false,
|
||||
"transformers_version": "4.57.6",
|
||||
"use_cache": false,
|
||||
"use_sliding_window": false,
|
||||
"vocab_size": 151936
|
||||
}
|
||||
12
generation_config.json
Normal file
12
generation_config.json
Normal file
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"do_sample": true,
|
||||
"eos_token_id": [
|
||||
151645,
|
||||
151643
|
||||
],
|
||||
"pad_token_id": 151643,
|
||||
"temperature": 0.6,
|
||||
"top_k": 20,
|
||||
"top_p": 0.95,
|
||||
"transformers_version": "4.57.6"
|
||||
}
|
||||
151388
merges.txt
Normal file
151388
merges.txt
Normal file
File diff suppressed because it is too large
Load Diff
3
model-00001-of-00004.safetensors
Normal file
3
model-00001-of-00004.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:e7d681ee4673a67d56f0297caeab8a4f1c9baf28480bbd90879458fae301ce38
|
||||
size 4902257696
|
||||
3
model-00002-of-00004.safetensors
Normal file
3
model-00002-of-00004.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:28ce9edc0f84b72b57d4c536e1c82a633f13d51a7237ccb873bdc3e1aab00c01
|
||||
size 4915960368
|
||||
3
model-00003-of-00004.safetensors
Normal file
3
model-00003-of-00004.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:5ac3c2f0f71f8036d53675b2528e1353a06c9f887d32d5f5ea510f3ae752178c
|
||||
size 4983068496
|
||||
3
model-00004-of-00004.safetensors
Normal file
3
model-00004-of-00004.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:a3b4b6074cf665014381a7e9523164ad5d88707e25b76d316405d33e40c66aaf
|
||||
size 1580230264
|
||||
407
model.safetensors.index.json
Normal file
407
model.safetensors.index.json
Normal file
@@ -0,0 +1,407 @@
|
||||
{
|
||||
"metadata": {
|
||||
"total_parameters": 8190735360,
|
||||
"total_size": 16381470720
|
||||
},
|
||||
"weight_map": {
|
||||
"lm_head.weight": "model-00004-of-00004.safetensors",
|
||||
"model.embed_tokens.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.10.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.10.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.14.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.14.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.16.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.16.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.17.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.17.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.18.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.18.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.19.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.19.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.19.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.19.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.19.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.19.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.19.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.2.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.2.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.20.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.20.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.20.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.20.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.20.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.20.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.20.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.20.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.20.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.20.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.20.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.21.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.21.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.21.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.21.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.21.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.21.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.21.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.21.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.21.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.21.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.21.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.22.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.22.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.22.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.22.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.22.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.22.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.26.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.26.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.27.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.27.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.28.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.28.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.29.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.29.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.29.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.3.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.3.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.30.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.30.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.30.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.30.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.31.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.31.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.31.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.31.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.31.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.31.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.31.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.31.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.31.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.31.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.31.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.32.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.32.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.32.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.32.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.32.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.32.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.32.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.32.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.32.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.32.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.32.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.33.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.33.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.33.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.33.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.33.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.33.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.33.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.33.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.33.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.33.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.33.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.34.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.34.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.34.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.34.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.34.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.34.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.34.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.34.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.34.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.34.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.34.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.35.input_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.35.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.35.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.35.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.35.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.35.self_attn.k_norm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.35.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.35.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.35.self_attn.q_norm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.35.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.35.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.4.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.4.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.5.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.5.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.6.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.6.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.7.input_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.7.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.7.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.7.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.8.input_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.8.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.8.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.8.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.8.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.8.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.8.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.8.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.8.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.8.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.8.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.9.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.9.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.9.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.9.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.9.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.9.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.9.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.norm.weight": "model-00004-of-00004.safetensors"
|
||||
}
|
||||
}
|
||||
258
rl_config.yaml
Normal file
258
rl_config.yaml
Normal file
@@ -0,0 +1,258 @@
|
||||
entrypoint: examples.terminal_bench.entrypoints.main_tbench
|
||||
|
||||
# Hydra config groups (+ prefix in CLI)
|
||||
config_groups:
|
||||
terminal_bench_config: terminal_bench
|
||||
|
||||
# Terminal bench / agentic environment settings
|
||||
terminal_bench:
|
||||
# trials_dir: Directory for Harbor trial artifacts (derived from experiments_dir if null)
|
||||
trials_dir: null
|
||||
|
||||
# Harbor configuration - schema-driven mapping to TrialConfig
|
||||
harbor:
|
||||
# Agent settings
|
||||
name: terminus-2
|
||||
max_episodes: 999999
|
||||
enable_summarize: false
|
||||
store_all_messages: true
|
||||
trajectory_config:
|
||||
raw_content: true
|
||||
enable_episode_logging: false
|
||||
record_terminal_session: false
|
||||
enable_pane_logging: false
|
||||
|
||||
# Strict JSON parser
|
||||
strict_json_parser: true
|
||||
|
||||
# Interleaved Thinking Settings
|
||||
interleaved_thinking: true
|
||||
extra_body:
|
||||
chat_template_kwargs:
|
||||
enable_thinking: true
|
||||
# Long timeout for thinking models
|
||||
override_timeout_sec: 1800
|
||||
|
||||
# Environment settings
|
||||
override_cpus: 1
|
||||
override_memory_mb: 2048
|
||||
override_storage_mb: 2048
|
||||
|
||||
# ==========================================================================
|
||||
# AUTO SNAPSHOT: Reduce Daytona rate limits with hash-based snapshot caching
|
||||
# ==========================================================================
|
||||
# When true, automatically creates a snapshot from the Dockerfile on first use,
|
||||
# then reuses it for all subsequent sandboxes with the same Dockerfile content.
|
||||
# Snapshots are named: harbor__<sha256[:12]>__snapshot
|
||||
auto_snapshot: true
|
||||
|
||||
# Verifier settings
|
||||
verifier_override_timeout_sec: 120
|
||||
|
||||
# Retry settings
|
||||
max_retries: 3
|
||||
min_wait_sec: 60.0
|
||||
max_wait_sec: 600.0
|
||||
wait_multiplier: 2.0
|
||||
|
||||
exclude_exceptions:
|
||||
- VerifierTimeoutError
|
||||
- VerifierRuntimeError
|
||||
- RewardFileNotFoundError
|
||||
- RewardFileEmptyError
|
||||
- VerifierOutputParseError
|
||||
|
||||
# 3x base (300 → 900): triple Daytona concurrency for higher throughput.
|
||||
# Mirrors ALCC/56GPU_base.yaml — per-job sandbox count; doesn't affect
|
||||
# the cross-cluster RUNNING-RL cap (≤ 6) which is about job count, not
|
||||
# sandboxes-per-job.
|
||||
n_concurrent_trials: 900
|
||||
|
||||
# Logging settings
|
||||
log_level: INFO
|
||||
|
||||
# Reward shaping (disabled - binary rewards)
|
||||
enable_reward_shaping: false
|
||||
|
||||
# RLOO-N error classification
|
||||
enable_error_classification: true
|
||||
mask_exceptions:
|
||||
- DaytonaError
|
||||
- EnvironmentStartTimeoutError
|
||||
- NetworkError
|
||||
- ConnectionError
|
||||
- RewardFileNotFoundError
|
||||
- RewardFileEmptyError
|
||||
- AgentEnvironmentTimeoutError
|
||||
default_error_treatment: zero
|
||||
passthrough_exceptions:
|
||||
- AgentTimeoutError
|
||||
- ContextLengthExceededError
|
||||
|
||||
# Model info for Harbor's hosted_vllm validation
|
||||
model_info:
|
||||
max_input_tokens: 32768
|
||||
max_output_tokens: 4096
|
||||
|
||||
archiving:
|
||||
# Enable trial archiving callback
|
||||
enabled: false
|
||||
|
||||
# Post-training trace upload to HuggingFace
|
||||
trace_upload:
|
||||
enabled: true
|
||||
repo_org: DCAgent
|
||||
episodes: last
|
||||
dataset_type: SFT
|
||||
cleanup: true
|
||||
|
||||
# Trainer configuration
|
||||
trainer:
|
||||
strategy: fsdp2
|
||||
algorithm:
|
||||
advantage_estimator: rloo_n
|
||||
use_kl_loss: false
|
||||
kl_loss_coef: 0.0
|
||||
eps_clip_low: 0.2
|
||||
# eps_clip_high=0.05 mirrors 24GPU_base — midpoint between 0.2 default and
|
||||
# 0.01 tight. Engages on collapse-onset ratios without over-clamping
|
||||
# healthy updates. Asymmetric — only tightening upper bound.
|
||||
eps_clip_high: 0.05
|
||||
loss_reduction: token_mean
|
||||
|
||||
# Training loop settings
|
||||
epochs: 2
|
||||
max_steps: 80
|
||||
update_epochs_per_batch: 1
|
||||
|
||||
# Batch sizes
|
||||
train_batch_size: 64
|
||||
policy_mini_batch_size: 64
|
||||
eval_batch_size: 64
|
||||
|
||||
# Micro batch sizes (micro1x4 variant)
|
||||
micro_forward_batch_size_per_gpu: 4
|
||||
micro_train_batch_size_per_gpu: 1
|
||||
|
||||
max_prompt_length: 999999
|
||||
|
||||
# Evaluation and checkpointing
|
||||
eval_interval: 999999
|
||||
eval_before_train: false
|
||||
# Resumable checkpointing
|
||||
ckpt_interval: 2
|
||||
resume_mode: latest
|
||||
# HF upload-ready checkpoints
|
||||
hf_save_interval: 5
|
||||
# HuggingFace Hub upload (set via CLI: trainer.hf_hub_repo_id=org/repo)
|
||||
hf_hub_repo_id: null
|
||||
hf_hub_private: false
|
||||
hf_hub_revision: main
|
||||
|
||||
# Database registration (auto-registers trained model to Supabase)
|
||||
# Requires KEYS env var pointing to Supabase credentials file
|
||||
enable_db_registration: true
|
||||
|
||||
# Logging
|
||||
project_name: OpenThoughts-Agent
|
||||
log_level: INFO
|
||||
tracker_commit_each_step: true
|
||||
logger: console
|
||||
|
||||
# Paths
|
||||
run_name: null
|
||||
ckpt_path: null
|
||||
export_path: null
|
||||
|
||||
# Policy optimizer
|
||||
# max_grad_norm=0.9 mirrors 24GPU_base — guardrail against grad-norm spikes
|
||||
# entering correlation-mode-collapse territory (>1.0). 0.9 is just above the
|
||||
# natural healthy peak observed on this dataset+base (‖g‖ peak ~0.81).
|
||||
policy:
|
||||
optimizer_config:
|
||||
lr: 8e-6
|
||||
weight_decay: 0.0
|
||||
adam_betas: [0.9, 0.999]
|
||||
max_grad_norm: 0.9
|
||||
fsdp_config:
|
||||
cpu_offload: false
|
||||
reshard_after_forward: true
|
||||
fsdp_size: 4
|
||||
# Reference model
|
||||
ref:
|
||||
fsdp_config:
|
||||
cpu_offload: false
|
||||
reshard_after_forward: true
|
||||
fsdp_size: 4
|
||||
|
||||
# Model placement (async training) - 8 shared GPUs for policy/ref
|
||||
placement:
|
||||
colocate_all: false
|
||||
policy_num_nodes: 2
|
||||
ref_num_nodes: 2
|
||||
policy_num_gpus_per_node: 4
|
||||
ref_num_gpus_per_node: 4
|
||||
|
||||
# Fully async generation (settings from v2_maxconcurrent)
|
||||
fully_async:
|
||||
max_staleness_steps: 16
|
||||
# Setting conservatively to 1 / 2 of total concurrency
|
||||
num_parallel_generation_workers: 450
|
||||
|
||||
# Generator configuration
|
||||
generator:
|
||||
backend: vllm
|
||||
timeout_multiplier: 1.0
|
||||
model_dtype: bfloat16
|
||||
|
||||
inference_engine_tensor_parallel_size: 1
|
||||
# 3x base (16 → 48): triple vLLM engines to lift gen throughput above training rate.
|
||||
# Goal: build a real surplus of completed groups so the trainer never waits on gen.
|
||||
# Layout: 48 engines × TP=1 + 8 GPUs for policy/ref = 56 GPUs total = 14 nodes.
|
||||
num_inference_engines: 48
|
||||
|
||||
n_samples_per_prompt: 8
|
||||
eval_n_samples_per_prompt: 8
|
||||
|
||||
# Jupiter-specific gpu_memory_utilization: 0.75 (vs ALCC's 0.85).
|
||||
# GH200's 96 GB HBM has more headroom than ALCC's A100-80, but Jupiter
|
||||
# also runs hosted_vllm in the same Ray cluster as the engines and we
|
||||
# keep 0.75 across all jupiter yamls for consistency / fragmentation
|
||||
# safety. Bump to 0.85 only if KV-cache pressure justifies it.
|
||||
gpu_memory_utilization: 0.75
|
||||
|
||||
max_num_seqs: 24
|
||||
# Jupiter-specific 65536 (vs ALCC's 16384) — matches the existing
|
||||
# 24GPU_base.yaml on Jupiter; larger batched-token budget keeps the
|
||||
# engine fed when many concurrent requests arrive in bursts.
|
||||
max_num_batched_tokens: 65536
|
||||
|
||||
enable_prefix_caching: true
|
||||
enable_chunked_prefill: true
|
||||
|
||||
run_engines_locally: true
|
||||
weight_sync_backend: nccl
|
||||
async_engine: true
|
||||
batched: false
|
||||
enable_http_endpoint: true
|
||||
enable_ray_prometheus_stats: false
|
||||
vllm_stats_interval: 1
|
||||
append_eos_token_after_stop_str_in_multi_turn: true
|
||||
max_turns: 999999
|
||||
|
||||
sampling_params:
|
||||
max_generate_length: 4096
|
||||
temperature: 0.7
|
||||
top_p: 0.95
|
||||
top_k: 20
|
||||
|
||||
engine_init_kwargs:
|
||||
max_model_len: 32768
|
||||
# Interleaved thinking chat template: preserves <think> blocks on ALL
|
||||
# historical assistant turns (stock Qwen3 template strips them).
|
||||
custom_chat_template_chat_completion_path: chat_templates/qwen3_thinking_acc.jinja2
|
||||
|
||||
# Data paths
|
||||
data:
|
||||
train_data: []
|
||||
val_data: []
|
||||
31
special_tokens_map.json
Normal file
31
special_tokens_map.json
Normal file
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"additional_special_tokens": [
|
||||
"<|im_start|>",
|
||||
"<|im_end|>",
|
||||
"<|object_ref_start|>",
|
||||
"<|object_ref_end|>",
|
||||
"<|box_start|>",
|
||||
"<|box_end|>",
|
||||
"<|quad_start|>",
|
||||
"<|quad_end|>",
|
||||
"<|vision_start|>",
|
||||
"<|vision_end|>",
|
||||
"<|vision_pad|>",
|
||||
"<|image_pad|>",
|
||||
"<|video_pad|>"
|
||||
],
|
||||
"eos_token": {
|
||||
"content": "<|im_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"pad_token": {
|
||||
"content": "<|endoftext|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
}
|
||||
}
|
||||
BIN
tokenizer.json
(Stored with Git LFS)
Normal file
BIN
tokenizer.json
(Stored with Git LFS)
Normal file
Binary file not shown.
240
tokenizer_config.json
Normal file
240
tokenizer_config.json
Normal file
@@ -0,0 +1,240 @@
|
||||
{
|
||||
"add_bos_token": false,
|
||||
"add_prefix_space": false,
|
||||
"added_tokens_decoder": {
|
||||
"151643": {
|
||||
"content": "<|endoftext|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151644": {
|
||||
"content": "<|im_start|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151645": {
|
||||
"content": "<|im_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151646": {
|
||||
"content": "<|object_ref_start|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151647": {
|
||||
"content": "<|object_ref_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151648": {
|
||||
"content": "<|box_start|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151649": {
|
||||
"content": "<|box_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151650": {
|
||||
"content": "<|quad_start|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151651": {
|
||||
"content": "<|quad_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151652": {
|
||||
"content": "<|vision_start|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151653": {
|
||||
"content": "<|vision_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151654": {
|
||||
"content": "<|vision_pad|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151655": {
|
||||
"content": "<|image_pad|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151656": {
|
||||
"content": "<|video_pad|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151657": {
|
||||
"content": "<tool_call>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151658": {
|
||||
"content": "</tool_call>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151659": {
|
||||
"content": "<|fim_prefix|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151660": {
|
||||
"content": "<|fim_middle|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151661": {
|
||||
"content": "<|fim_suffix|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151662": {
|
||||
"content": "<|fim_pad|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151663": {
|
||||
"content": "<|repo_name|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151664": {
|
||||
"content": "<|file_sep|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151665": {
|
||||
"content": "<tool_response>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151666": {
|
||||
"content": "</tool_response>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151667": {
|
||||
"content": "<think>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151668": {
|
||||
"content": "</think>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
}
|
||||
},
|
||||
"additional_special_tokens": [
|
||||
"<|im_start|>",
|
||||
"<|im_end|>",
|
||||
"<|object_ref_start|>",
|
||||
"<|object_ref_end|>",
|
||||
"<|box_start|>",
|
||||
"<|box_end|>",
|
||||
"<|quad_start|>",
|
||||
"<|quad_end|>",
|
||||
"<|vision_start|>",
|
||||
"<|vision_end|>",
|
||||
"<|vision_pad|>",
|
||||
"<|image_pad|>",
|
||||
"<|video_pad|>"
|
||||
],
|
||||
"bos_token": null,
|
||||
"clean_up_tokenization_spaces": false,
|
||||
"eos_token": "<|im_end|>",
|
||||
"errors": "replace",
|
||||
"extra_special_tokens": {},
|
||||
"model_max_length": 32768,
|
||||
"pad_token": "<|endoftext|>",
|
||||
"padding_side": "left",
|
||||
"split_special_tokens": false,
|
||||
"tokenizer_class": "Qwen2Tokenizer",
|
||||
"unk_token": null
|
||||
}
|
||||
6
training_logs/20260525_110326_metrics_job_485099.csv
Normal file
6
training_logs/20260525_110326_metrics_job_485099.csv
Normal file
@@ -0,0 +1,6 @@
|
||||
async/discard_rate,async/discarded_count,async/effective_batch_groups,async/effective_batch_samples,async/staleness_max,async/staleness_mean,async/staleness_min,async/staleness_ratio,generate/avg_num_tokens,generate/avg_tokens_non_zero_rewards,generate/avg_tokens_zero_rewards,generate/max_num_tokens,generate/min_num_tokens,generate/std_num_tokens,loss/avg_final_rewards,loss/avg_raw_advantages,loss/avg_raw_advantages_abs,policy/final_loss,policy/log_ratio_abs_max,policy/log_ratio_abs_mean,policy/log_ratio_abs_p99,policy/log_ratio_abs_pos00,policy/log_ratio_abs_pos10,policy/log_ratio_abs_pos20,policy/log_ratio_abs_pos30,policy/log_ratio_abs_pos40,policy/log_ratio_abs_pos50,policy/log_ratio_abs_pos60,policy/log_ratio_abs_pos70,policy/log_ratio_abs_pos80,policy/log_ratio_abs_pos90,policy/n_tokens_dp_gt_10pct,policy/n_tokens_dp_gt_1pct,policy/n_tokens_dp_gt_50pct,policy/policy_entropy,policy/policy_loss,policy/policy_lr,policy/policy_update_steps,policy/ppo_clip_ratio,policy/raw_grad_norm,reward/avg_pass_at_8,reward/avg_raw_reward,system/process_rss_gb,system/process_vms_gb,system/ram_available_gb,system/ram_percent,system/ram_total_gb,system/ram_used_gb,timing/compute_advantages_and_returns,timing/convert_to_training_input,timing/fwd_logprobs_values_reward,timing/policy_train,timing/run_training,timing/step,timing/sync_weights,timing/train_critic_and_policy,timing/wait_for_generation_buffer,trainer/epoch,trainer/global_step,batch_errors/total_batches,batch_errors/total_instances,batch_errors/total_successful,batch_errors/total_failed,batch_errors/total_masked,batch_errors/avg_DaytonaAuthenticationError,batch_errors/total_DaytonaAuthenticationError,batch_errors/avg_AgentEnvironmentTimeoutError,batch_errors/total_AgentEnvironmentTimeoutError,batch_errors/avg_DaytonaNotFoundError,batch_errors/total_DaytonaNotFoundError,batch_errors/avg_VerifierRuntimeError,batch_errors/total_VerifierRuntimeError,batch_errors/avg_DaytonaError,batch_errors/total_DaytonaError,batch_errors/avg_RuntimeError,batch_errors/total_RuntimeError,batch_errors/avg_DaytonaValidationError,batch_errors/total_DaytonaValidationError,batch_errors/avg_ContextLengthExceededError,batch_errors/total_ContextLengthExceededError,batch_errors/avg_AgentTimeoutError,batch_errors/total_AgentTimeoutError,timing/cleanup_old_checkpoints,timing/save_checkpoints,timing/save_hf_model,batch_errors/avg_AgentSetupTimeoutError,batch_errors/total_AgentSetupTimeoutError
|
||||
0.0,0,64,512,0,0.0,0,0.0,147.9727,3584.3333,1.0,8768,1,798.0622,0.041,0.4413,0.4415,-0.0003,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0052,-0.0178,0.0,1.0,0.0,0.0083,0.1719,0.041,6.1133,79.7606,624.197,27.2,857.9676,233.7706,0.1201,1.388,9.9059,57.0489,67.2439,8192.8398,18.3161,57.2174,8105.8915,0,1,128,1024,55,128,732,1.625,208,5.6640625,725,0.046875,6,0.15625,20,0.0234375,3.0,0.015625,2.0,0.0078125,1.0,0.0078125,1.0,0.0234375,3.0,,,,,
|
||||
0.0,0,64,512,1,0.8438,0,0.8438,240.5703,3608.6471,1.0,11048,1,1028.2547,0.0664,0.3248,0.325,-0.0002,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0092,-0.0156,0.0,1.0,0.0,0.0059,0.2344,0.0664,6.7781,78.2899,614.3554,28.4,857.9676,243.6122,0.0338,1.6116,7.28,59.443,66.9312,7137.673,17.9631,59.6169,7051.1656,0,2,64,512,38,64,448,0.28125,18,6.875,440,0.03125,2,0.09375,6,0.03125,2.0,,,,,,,0.09375,6.0,18.2883,37.3028,,,
|
||||
0.0,0,64,512,2,1.0156,1,1.0,282.0371,3787.6053,1.0,9828,1,1115.4254,0.0742,0.1271,0.1271,-0.0001,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0101,-0.0065,0.0,1.0,0.0,0.003,0.2656,0.0742,7.0157,78.3694,616.362,28.2,857.9676,241.6056,0.0345,1.5826,6.4797,60.6694,67.4008,614.3507,19.0175,60.8861,526.3485,0,3,64,512,7,64,452,0.6875,44,7.0,448,0.015625,1,0.125,8,,,,,,,0.015625,1.0,0.046875,3.0,,,,,
|
||||
0.0,0,64,512,2,1.2031,0,0.6094,64.5625,4650.1429,1.0,10031,1,636.5514,0.0137,0.0703,0.0703,-0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0018,-0.002,0.0,1.0,0.0,0.0028,0.0938,0.0137,7.3373,78.4666,619.0092,27.9,857.9676,238.9584,0.0351,1.6766,4.2416,48.7129,53.1303,7011.6209,18.0381,48.8532,6938.7746,0,4,64,512,59,64,403,0.671875,43,6.203125,397,0.015625,1,0.09375,6,0.015625,1.0,,,,,0.015625,1.0,0.0625,4.0,0.0088,18.0874,,,
|
||||
0.0,0,64,512,1,1.0,1,1.0,278.7383,2411.2034,1.0,4957,1,815.5083,0.1152,0.0935,0.0935,-0.0001,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0131,-0.0088,0.0,1.0,0.0,0.0065,0.2344,0.1152,7.5183,78.4538,621.5787,27.6,857.9676,236.3889,0.0216,1.0094,6.7753,55.4016,62.3349,734.6954,18.4983,55.5375,652.851,0,5,30,240,9,30,205,0.6666666666666666,20,6.733333333333333,202,0.06666666666666667,2,0.1,3,,,,,,,0.1,3.0,,,,,10.5996,0.03333333333333333,1.0
|
||||
|
10
training_logs/20260525_110326_metrics_job_485100.csv
Normal file
10
training_logs/20260525_110326_metrics_job_485100.csv
Normal file
@@ -0,0 +1,10 @@
|
||||
async/discard_rate,async/discarded_count,async/effective_batch_groups,async/effective_batch_samples,async/staleness_max,async/staleness_mean,async/staleness_min,async/staleness_ratio,generate/avg_num_tokens,generate/avg_tokens_non_zero_rewards,generate/avg_tokens_zero_rewards,generate/max_num_tokens,generate/min_num_tokens,generate/std_num_tokens,loss/avg_final_rewards,loss/avg_raw_advantages,loss/avg_raw_advantages_abs,policy/final_loss,policy/log_ratio_abs_max,policy/log_ratio_abs_mean,policy/log_ratio_abs_p99,policy/log_ratio_abs_pos00,policy/log_ratio_abs_pos10,policy/log_ratio_abs_pos20,policy/log_ratio_abs_pos30,policy/log_ratio_abs_pos40,policy/log_ratio_abs_pos50,policy/log_ratio_abs_pos60,policy/log_ratio_abs_pos70,policy/log_ratio_abs_pos80,policy/log_ratio_abs_pos90,policy/n_tokens_dp_gt_10pct,policy/n_tokens_dp_gt_1pct,policy/n_tokens_dp_gt_50pct,policy/policy_entropy,policy/policy_loss,policy/policy_lr,policy/policy_update_steps,policy/ppo_clip_ratio,policy/raw_grad_norm,reward/avg_pass_at_8,reward/avg_raw_reward,system/process_rss_gb,system/process_vms_gb,system/ram_available_gb,system/ram_percent,system/ram_total_gb,system/ram_used_gb,timing/compute_advantages_and_returns,timing/convert_to_training_input,timing/fwd_logprobs_values_reward,timing/policy_train,timing/run_training,timing/save_hf_model,timing/step,timing/sync_weights,timing/train_critic_and_policy,timing/wait_for_generation_buffer,trainer/epoch,trainer/global_step,batch_errors/total_batches,batch_errors/total_instances,batch_errors/total_successful,batch_errors/total_failed,batch_errors/total_masked,batch_errors/avg_DaytonaAuthenticationError,batch_errors/total_DaytonaAuthenticationError,batch_errors/avg_AgentEnvironmentTimeoutError,batch_errors/total_AgentEnvironmentTimeoutError,batch_errors/avg_VerifierRuntimeError,batch_errors/total_VerifierRuntimeError,batch_errors/avg_DaytonaError,batch_errors/total_DaytonaError,batch_errors/avg_DaytonaNotFoundError,batch_errors/total_DaytonaNotFoundError,timing/cleanup_old_checkpoints,timing/save_checkpoints,batch_errors/avg_AgentTimeoutError,batch_errors/total_AgentTimeoutError
|
||||
0.0,0,64,512,1,0.0938,0,0.0938,412.1523,2807.8,1.0,7118,1,1116.6463,0.1465,0.2626,0.2628,-0.0005,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.017,-0.0348,0.0,1.0,0.0,0.0117,0.3276,0.1465,5.5805,78.1037,621.3371,27.6,857.9687,236.6316,0.0318,1.4004,10.3685,66.7561,77.3071,6.9408,8132.7122,17.8846,66.9061,8036.1198,0,5,64,512,17,64,376,1.6875,108,5.828125,373,0.125,8,0.046875,3.0,0.046875,3.0,,,,
|
||||
0.0,0,64,512,1,0.9844,0,0.9844,122.9082,3672.5882,1.0,7301,1,700.1298,0.0332,0.4107,0.411,-0.0003,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0043,-0.016,0.0,1.0,0.0,0.0091,0.1562,0.0332,6.132,78.1099,617.5232,28.0,857.9687,240.4455,0.0258,1.1294,4.1778,49.7199,54.0518,,3465.0549,19.1569,49.8478,3390.7157,0,6,64,512,12,64,390,1.53125,98,6.078125,389,0.171875,11,,,0.015625,1.0,14.3648,22.1469,0.015625,1.0
|
||||
0.0,0,64,512,2,0.1719,0,0.1406,81.3242,3428.1667,1.0,8024,1,580.4928,0.0234,0.4151,0.4154,-0.0002,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.003,-0.0109,0.0,1.0,0.0,0.0071,0.1406,0.0234,6.3622,78.2129,616.5064,28.1,857.9687,241.4623,0.022,1.147,4.0974,47.8918,52.1402,,3987.006,18.7017,48.0205,3915.0159,0,7,64,512,9,64,412,1.3125,84,6.421875,411,0.09375,6,0.015625,1.0,0.015625,1.0,,,,
|
||||
0.0,0,64,512,2,0.7969,0,0.7656,53.543,2990.1111,1.0,4963,1,409.6074,0.0176,0.401,0.4013,-0.0001,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.002,-0.0074,0.0,1.0,0.0,0.0064,0.0938,0.0176,6.5256,78.4167,614.7823,28.3,857.9687,243.1863,0.0175,0.8755,3.0976,44.8012,48.0435,,6656.3028,28.5894,44.928,6578.7926,0,8,64,512,12,64,429,1.0625,68,6.640625,425,0.03125,2,0.03125,2.0,0.015625,1.0,0.0078,44.8581,0.03125,2.0
|
||||
0.0,0,64,512,2,1.0156,1,1.0,83.5977,3525.1667,1.0,7387,1,595.6285,0.0234,0.2611,0.2612,-0.0001,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0032,-0.0059,0.0,1.0,0.0,0.0055,0.125,0.0234,6.8361,78.5227,617.8079,28.0,857.9687,240.1608,0.0187,1.2451,4.908,48.4858,53.5378,,1287.2347,18.8851,48.6107,1213.5661,0,9,64,512,42,64,369,1.515625,97,5.71875,366,0.03125,2,,,0.03125,2.0,,,0.046875,3.0
|
||||
0.0,0,64,512,2,0.9688,0,0.5,318.5254,3871.7857,1.0,12427,1,1252.0624,0.082,0.403,0.4032,-0.0005,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0093,-0.0298,0.0,1.0,0.0,0.0096,0.2344,0.082,8.9127,78.948,613.9305,28.4,857.9687,244.0382,0.0353,1.8739,8.9165,64.647,73.728,6.0594,6403.6413,18.6729,64.7757,6309.3653,0,10,64,512,40,64,407,0.875,56,6.359375,407,0.140625,9,,,,,0.0087,8.1786,,
|
||||
0.0,0,64,512,3,1.0312,1,1.0,350.1328,4469.9,1.0,31001,1,1779.2004,0.0781,0.213,0.2131,-0.0002,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.008,-0.015,0.0,1.0,0.0,0.007,0.1719,0.0781,14.811,79.299,605.6735,29.4,857.9687,252.2952,0.0625,4.1888,10.3455,76.4064,86.9738,,1527.2608,18.4561,76.5655,1417.6404,0,11,64,512,41,64,376,1.34375,86,5.828125,373,0.109375,7,0.015625,1.0,0.03125,2.0,,,0.03125,2.0
|
||||
0.0,0,64,512,2,0.8125,0,0.4844,282.1934,3512.4878,1.0,12815,1,1173.8704,0.0801,0.2652,0.2654,-0.0004,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0088,-0.0229,0.0,1.0,0.0,0.0105,0.25,0.0801,14.8054,78.8495,604.8915,29.5,857.9687,253.0771,0.0379,1.7768,6.4211,60.7211,67.3241,,6140.8745,18.7329,60.8648,6053.0375,0,12,64,512,33,64,421,0.734375,47,6.546875,419,0.125,8,,,0.046875,3.0,0.008,7.973,0.03125,2.0
|
||||
0.0,0,64,512,3,1.1875,1,1.0,217.332,3357.4242,1.0,6149,1,875.1285,0.0645,0.1964,0.1965,-0.0002,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0069,-0.0111,0.0,1.0,0.0,0.0056,0.2344,0.0645,15.0007,78.8141,604.7684,29.5,857.9687,253.2003,0.0234,0.9449,4.7579,55.747,60.6653,,2012.3998,19.4214,55.8837,1931.3665,0,13,19,152,22,18,102,1.368421052631579,26,5.368421052631579,102,0.10526315789473684,2,,,,,,,,
|
||||
|
8
training_logs/20260525_110326_metrics_job_485101.csv
Normal file
8
training_logs/20260525_110326_metrics_job_485101.csv
Normal file
@@ -0,0 +1,8 @@
|
||||
async/discard_rate,async/discarded_count,async/effective_batch_groups,async/effective_batch_samples,async/staleness_max,async/staleness_mean,async/staleness_min,async/staleness_ratio,generate/avg_num_tokens,generate/avg_tokens_non_zero_rewards,generate/avg_tokens_zero_rewards,generate/max_num_tokens,generate/min_num_tokens,generate/std_num_tokens,loss/avg_final_rewards,loss/avg_raw_advantages,loss/avg_raw_advantages_abs,policy/final_loss,policy/log_ratio_abs_max,policy/log_ratio_abs_mean,policy/log_ratio_abs_p99,policy/log_ratio_abs_pos00,policy/log_ratio_abs_pos10,policy/log_ratio_abs_pos20,policy/log_ratio_abs_pos30,policy/log_ratio_abs_pos40,policy/log_ratio_abs_pos50,policy/log_ratio_abs_pos60,policy/log_ratio_abs_pos70,policy/log_ratio_abs_pos80,policy/log_ratio_abs_pos90,policy/n_tokens_dp_gt_10pct,policy/n_tokens_dp_gt_1pct,policy/n_tokens_dp_gt_50pct,policy/policy_entropy,policy/policy_loss,policy/policy_lr,policy/policy_update_steps,policy/ppo_clip_ratio,policy/raw_grad_norm,reward/avg_pass_at_8,reward/avg_raw_reward,system/process_rss_gb,system/process_vms_gb,system/ram_available_gb,system/ram_percent,system/ram_total_gb,system/ram_used_gb,timing/compute_advantages_and_returns,timing/convert_to_training_input,timing/fwd_logprobs_values_reward,timing/policy_train,timing/run_training,timing/step,timing/sync_weights,timing/train_critic_and_policy,timing/wait_for_generation_buffer,trainer/epoch,trainer/global_step,batch_errors/total_batches,batch_errors/total_instances,batch_errors/total_successful,batch_errors/total_failed,batch_errors/total_masked,batch_errors/avg_DaytonaAuthenticationError,batch_errors/total_DaytonaAuthenticationError,batch_errors/avg_AgentEnvironmentTimeoutError,batch_errors/total_AgentEnvironmentTimeoutError,batch_errors/avg_DaytonaNotFoundError,batch_errors/total_DaytonaNotFoundError,batch_errors/avg_DaytonaError,batch_errors/total_DaytonaError,batch_errors/avg_VerifierRuntimeError,batch_errors/total_VerifierRuntimeError,batch_errors/avg_AgentTimeoutError,batch_errors/total_AgentTimeoutError,timing/cleanup_old_checkpoints,timing/save_checkpoints,timing/save_hf_model
|
||||
0.0,0,64,512,1,0.0312,0,0.0312,331.7969,2689.381,1.0,5793,1,965.9963,0.123,0.2187,0.2189,-0.0004,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0119,-0.0249,0.0,1.0,0.0,0.0124,0.2742,0.123,5.3837,78.0531,602.3064,29.8,857.9687,255.6623,0.1194,1.0144,12.9162,57.498,70.6558,8113.5248,18.1367,57.6196,8023.7177,0,13,64,512,19,64,381,1.5625,100,5.890625,377,0.015625,1.0,0.015625,1.0,0.171875,11,0.046875,3.0,,,
|
||||
0.0,0,64,512,1,0.9062,0,0.9062,97.1699,2592.5263,1.0,5692,1,535.645,0.0371,0.4498,0.4501,-0.0002,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0041,-0.0146,0.0,1.0,0.0,0.0075,0.125,0.0371,5.7223,78.1088,599.2325,30.2,857.9687,258.7362,0.0198,0.9137,4.3069,44.4851,48.9503,6886.3196,18.478,44.6233,6817.976,0,14,64,512,61,64,351,1.34375,86,5.25,336,,,0.046875,3.0,0.21875,14,0.1875,12.0,24.0683,41.284,
|
||||
0.0,0,64,512,2,1.0469,1,1.0,393.2031,3292.9344,1.0,10838,1,1198.2489,0.1191,0.2169,0.2171,-0.0005,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0124,-0.0307,0.0,1.0,0.0,0.0123,0.375,0.1191,7.4045,78.2452,598.44,30.2,857.9687,259.5287,0.0315,1.6347,7.3081,63.1214,70.5896,791.9402,18.239,63.2496,701.4743,0,15,64,512,61,63,356,1.296875,83,5.53125,354,,,0.015625,1.0,0.1875,12,0.015625,1.0,,,6.9084
|
||||
0.0,0,64,512,2,1.5625,0,0.7812,346.5215,2901.1148,1.0,13177,1,1177.6427,0.1191,0.2065,0.2066,-0.0003,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0113,-0.0216,0.0,1.0,0.0,0.0089,0.3125,0.1191,8.5541,78.4775,596.4595,30.5,857.9687,261.5092,0.0308,1.904,7.6796,61.4263,69.2639,6575.7601,17.4986,61.5532,6487.0923,0,16,64,512,47,64,359,1.515625,97,5.53125,354,0.0625,4.0,,,0.078125,5,0.078125,5.0,0.0085,7.7753,
|
||||
0.0,0,64,512,3,1.0625,1,1.0,314.8184,3419.617,1.0,7961,1,1092.2585,0.0918,0.359,0.3591,-0.0005,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0084,-0.0295,0.0,1.0,0.0,0.0108,0.25,0.0918,8.6535,78.4467,598.0707,30.3,857.9687,259.898,0.0216,1.2139,6.2721,60.5518,66.9689,920.1572,17.1829,60.6749,834.7893,0,17,64,512,36,64,404,1.0625,68,6.265625,401,0.03125,2.0,0.015625,1.0,0.03125,2,0.03125,2.0,,,
|
||||
0.0,0,64,512,2,1.2031,0,0.6094,245.1484,3473.3333,1.0,11370,1,999.9222,0.0703,0.1244,0.1245,-0.0002,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0063,-0.0109,0.0,1.0,0.0,0.0061,0.2344,0.0703,9.0319,78.7547,600.501,30.0,857.9687,257.4677,0.029,1.7294,5.1895,52.9781,58.3217,6610.6145,17.2028,53.1029,6533.3592,0,18,64,512,53,63,421,0.5,32,6.484375,415,0.015625,1.0,0.03125,2.0,0.078125,5,0.0625,4.0,0.0079,7.9661,
|
||||
0.0,0,64,512,1,1.0,1,1.0,321.1836,3094.0943,1.0,8352,1,1061.5795,0.1035,0.0722,0.0722,-0.0001,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0086,-0.007,0.0,1.0,0.0,0.0041,0.2812,0.1035,9.3094,78.7746,599.7432,30.1,857.9687,258.2255,0.0369,1.3938,6.5872,58.4137,65.1632,1357.7171,17.8468,58.5387,1273.3116,0,19,25,200,21,25,158,0.68,17,6.32,158,,,,,0.16,4,,,,,
|
||||
|
9
training_logs/20260525_110326_metrics_job_496420.csv
Normal file
9
training_logs/20260525_110326_metrics_job_496420.csv
Normal file
@@ -0,0 +1,9 @@
|
||||
async/discard_rate,async/discarded_count,async/effective_batch_groups,async/effective_batch_samples,async/staleness_max,async/staleness_mean,async/staleness_min,async/staleness_ratio,generate/avg_num_tokens,generate/avg_tokens_non_zero_rewards,generate/avg_tokens_zero_rewards,generate/max_num_tokens,generate/min_num_tokens,generate/std_num_tokens,loss/avg_final_rewards,loss/avg_raw_advantages,loss/avg_raw_advantages_abs,policy/final_loss,policy/log_ratio_abs_max,policy/log_ratio_abs_mean,policy/log_ratio_abs_p99,policy/log_ratio_abs_pos00,policy/log_ratio_abs_pos10,policy/log_ratio_abs_pos20,policy/log_ratio_abs_pos30,policy/log_ratio_abs_pos40,policy/log_ratio_abs_pos50,policy/log_ratio_abs_pos60,policy/log_ratio_abs_pos70,policy/log_ratio_abs_pos80,policy/log_ratio_abs_pos90,policy/n_tokens_dp_gt_10pct,policy/n_tokens_dp_gt_1pct,policy/n_tokens_dp_gt_50pct,policy/policy_entropy,policy/policy_loss,policy/policy_lr,policy/policy_update_steps,policy/ppo_clip_ratio,policy/raw_grad_norm,reward/avg_pass_at_8,reward/avg_raw_reward,system/process_rss_gb,system/process_vms_gb,system/ram_available_gb,system/ram_percent,system/ram_total_gb,system/ram_used_gb,timing/compute_advantages_and_returns,timing/convert_to_training_input,timing/fwd_logprobs_values_reward,timing/policy_train,timing/run_training,timing/step,timing/sync_weights,timing/train_critic_and_policy,timing/wait_for_generation_buffer,trainer/epoch,trainer/global_step,batch_errors/total_batches,batch_errors/total_instances,batch_errors/total_successful,batch_errors/total_failed,batch_errors/total_masked,batch_errors/avg_AgentEnvironmentTimeoutError,batch_errors/total_AgentEnvironmentTimeoutError,batch_errors/avg_DaytonaAuthenticationError,batch_errors/total_DaytonaAuthenticationError,batch_errors/avg_VerifierRuntimeError,batch_errors/total_VerifierRuntimeError,batch_errors/avg_DaytonaNotFoundError,batch_errors/total_DaytonaNotFoundError,timing/cleanup_old_checkpoints,timing/save_checkpoints,timing/save_hf_model,batch_errors/avg_DaytonaError,batch_errors/total_DaytonaError,batch_errors/avg_AgentTimeoutError,batch_errors/total_AgentTimeoutError,batch_errors/avg_ContextLengthExceededError,batch_errors/total_ContextLengthExceededError,batch_errors/avg_AgentSetupTimeoutError,batch_errors/total_AgentSetupTimeoutError,batch_errors/avg_EnvironmentStartTimeoutError,batch_errors/total_EnvironmentStartTimeoutError,batch_errors/avg_VerifierTimeoutError,batch_errors/total_VerifierTimeoutError,batch_errors/avg_OSError,batch_errors/total_OSError
|
||||
0.0,0,64,512,1,0.0781,0,0.0781,653.5195,2983.9464,1.0,5734,1,1308.6167,0.2188,0.1993,0.1994,-0.0007,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0182,-0.0471,0.0,1.0,0.0,0.0159,0.4407,0.2188,5.3781,78.0601,578.2307,32.6,857.9687,279.738,0.1498,1.2614,15.3459,70.951,86.6201,8067.2067,17.8581,71.1238,7961.4669,0,19,64,512,41,63,350,5.46875,350,1.765625,113,0.109375,7.0,0.015625,1.0,,,,,,,,,,,,,,,,,
|
||||
0.0,0,64,512,1,0.9844,0,0.9844,239.9746,2985.2683,1.0,7404,1,907.8881,0.0801,0.189,0.1892,-0.0003,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0065,-0.0203,0.0,1.0,0.0,0.0128,0.2031,0.0801,6.1016,78.1429,575.9431,32.9,857.9687,282.0256,0.0214,1.1741,6.3743,55.4754,61.9992,3380.5934,20.0199,55.6031,3297.3986,0,20,64,512,51,64,338,5.25,336,1.65625,106,0.25,16.0,0.015625,1.0,14.3752,21.9845,6.5064,0.015625,1.0,0.015625,1.0,,,,,,,,,,
|
||||
0.0,0,64,512,2,0.5156,0,0.3906,315.1094,3154.4118,1.0,9754,1,1083.8644,0.0996,0.2996,0.2998,-0.0005,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.008,-0.032,0.0,1.0,0.0,0.0127,0.3281,0.0996,7.1946,78.3005,574.6356,33.0,857.9687,283.3331,0.028,1.3808,8.0318,63.7157,71.9046,4118.8329,18.1954,63.8445,4027.3507,0,21,64,512,36,64,375,5.8125,372,1.375,88,0.203125,13.0,,,,,,,,0.03125,2.0,0.015625,1.0,,,,,,,,
|
||||
0.0,0,64,512,2,0.9375,0,0.8125,291.4688,4132.1111,1.0,9690,1,1175.4698,0.0703,0.3601,0.3603,-0.0004,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0069,-0.0247,0.0,1.0,0.0,0.0179,0.25,0.0703,7.7303,78.4548,578.4616,32.6,857.9687,279.5071,0.0229,1.429,6.5012,63.062,69.7119,5101.1025,18.1771,63.1874,5011.7831,0,22,64,512,48,64,397,6.09375,390,0.921875,59,0.125,8.0,,,0.0075,8.179,,,,0.078125,5.0,0.03125,2.0,,,,,,,,
|
||||
0.0,0,64,512,2,1.0469,0,0.9375,275.9121,2933.3958,1.0,6046,1,913.748,0.0938,0.2557,0.2558,-0.0003,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0063,-0.0223,0.0,1.0,0.0,0.0135,0.2656,0.0938,8.0687,78.5605,580.3864,32.4,857.9687,277.5823,0.0178,0.9376,6.7391,56.9606,63.8352,2467.0192,19.0252,57.0779,2383.2198,0,23,64,512,40,63,411,6.40625,410,0.671875,43,0.265625,17.0,0.015625,1.0,,,,,,0.015625,1.0,,,,,,,,,,
|
||||
0.0,0,64,512,2,1.0938,0,0.6562,288.5996,3682.275,1.0,12853,1,1174.4942,0.0781,0.0673,0.0674,-0.0001,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0057,-0.0049,0.0,1.0,0.0,0.0038,0.2344,0.0781,9.5992,78.8051,579.6128,32.4,857.9687,278.3559,0.0326,1.8242,7.2937,60.7161,68.1634,6306.534,19.4669,60.8368,6217.079,0,24,65,520,28,65,417,6.3538461538461535,413,0.9076923076923077,59,0.18461538461538463,12.0,,,0.0078,7.3895,,,,0.03076923076923077,2.0,0.015384615384615385,1.0,0.06153846153846154,4.0,0.015384615384615385,1.0,,,,
|
||||
0.0,0,64,512,1,1.0,1,1.0,246.5586,4491.2143,1.0,12979,1,1246.6552,0.0547,0.2752,0.2754,-0.0002,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0042,-0.0133,0.0,1.0,0.0,0.0082,0.2188,0.0547,11.0213,91.3057,577.4506,32.7,857.9687,280.5181,0.0334,187.2975,10.0277,62.1765,72.3976,4522.0533,18.6719,62.3361,4243.6845,0,25,69,552,29,68,458,6.594202898550725,455,0.7391304347826086,51,0.13043478260869565,9.0,0.014492753623188406,1.0,,,5.8497,,,,,0.043478260869565216,3.0,0.028985507246376812,2.0,,,0.014492753623188406,1.0,0.014492753623188406,1.0
|
||||
0.0,0,64,512,2,1.0938,0,0.6406,164.248,3096.6667,1.0,12519,1,866.242,0.0527,0.0251,0.0252,-0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0035,-0.0026,0.0,1.0,0.0,0.0064,0.1875,0.0527,12.0359,94.4252,576.9232,32.8,857.9687,281.0455,0.0497,1.8339,5.7861,53.1165,59.0761,6927.7034,19.1025,53.24,6847.6862,0,26,10,80,1,10,74,7.4,74,0.5,5,,,,,0.0131,8.0356,,,,,,,,,,,,,,,
|
||||
|
11
training_logs/20260525_110326_metrics_job_496421.csv
Normal file
11
training_logs/20260525_110326_metrics_job_496421.csv
Normal file
@@ -0,0 +1,11 @@
|
||||
async/discard_rate,async/discarded_count,async/effective_batch_groups,async/effective_batch_samples,async/staleness_max,async/staleness_mean,async/staleness_min,async/staleness_ratio,generate/avg_num_tokens,generate/avg_tokens_non_zero_rewards,generate/avg_tokens_zero_rewards,generate/max_num_tokens,generate/min_num_tokens,generate/std_num_tokens,loss/avg_final_rewards,loss/avg_raw_advantages,loss/avg_raw_advantages_abs,policy/final_loss,policy/log_ratio_abs_max,policy/log_ratio_abs_mean,policy/log_ratio_abs_p99,policy/log_ratio_abs_pos00,policy/log_ratio_abs_pos10,policy/log_ratio_abs_pos20,policy/log_ratio_abs_pos30,policy/log_ratio_abs_pos40,policy/log_ratio_abs_pos50,policy/log_ratio_abs_pos60,policy/log_ratio_abs_pos70,policy/log_ratio_abs_pos80,policy/log_ratio_abs_pos90,policy/n_tokens_dp_gt_10pct,policy/n_tokens_dp_gt_1pct,policy/n_tokens_dp_gt_50pct,policy/policy_entropy,policy/policy_loss,policy/policy_lr,policy/policy_update_steps,policy/ppo_clip_ratio,policy/raw_grad_norm,reward/avg_pass_at_8,reward/avg_raw_reward,system/process_rss_gb,system/process_vms_gb,system/ram_available_gb,system/ram_percent,system/ram_total_gb,system/ram_used_gb,timing/compute_advantages_and_returns,timing/convert_to_training_input,timing/fwd_logprobs_values_reward,timing/policy_train,timing/run_training,timing/step,timing/sync_weights,timing/train_critic_and_policy,timing/wait_for_generation_buffer,trainer/epoch,trainer/global_step,batch_errors/total_batches,batch_errors/total_instances,batch_errors/total_successful,batch_errors/total_failed,batch_errors/total_masked,batch_errors/avg_DaytonaAuthenticationError,batch_errors/total_DaytonaAuthenticationError,batch_errors/avg_AgentEnvironmentTimeoutError,batch_errors/total_AgentEnvironmentTimeoutError,batch_errors/avg_VerifierRuntimeError,batch_errors/total_VerifierRuntimeError,batch_errors/avg_DaytonaNotFoundError,batch_errors/total_DaytonaNotFoundError,batch_errors/avg_AgentTimeoutError,batch_errors/total_AgentTimeoutError,timing/cleanup_old_checkpoints,timing/save_checkpoints,timing/save_hf_model,batch_errors/avg_DaytonaError,batch_errors/total_DaytonaError,batch_errors/avg_ContextLengthExceededError,batch_errors/total_ContextLengthExceededError
|
||||
0.0,0,64,512,3,0.2656,0,0.2344,621.6914,2563.8548,1.0,10562,1,1338.5955,0.2422,0.2196,0.2197,-0.0007,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0148,-0.0466,0.0,1.0,0.0,0.0201,0.4898,0.2422,7.1583,79.2762,627.8784,26.8,857.9687,230.0903,0.0496,1.7229,14.7315,75.4634,90.3725,8113.4798,18.0581,75.5904,8003.326,0,27,64,512,30,64,383,1.34375,86,5.96875,382,0.171875,11.0,0.03125,2.0,0.015625,1.0,,,,,,,
|
||||
0.0,0,64,512,1,1.0,1,1.0,180.2852,3060.8,1.0,7383,1,805.1102,0.0586,0.4541,0.4545,-0.0005,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0039,-0.0311,0.0,1.0,0.0,0.0178,0.25,0.0586,7.4612,79.269,624.4182,27.2,857.9687,233.5505,0.0232,1.1905,4.8432,55.2708,60.2711,351.8545,18.7774,55.4044,271.6141,0,28,64,512,62,63,357,1.328125,85,5.53125,354,0.125,8.0,,,0.046875,3.0,8.3981,16.1295,,,,,
|
||||
0.0,0,64,512,2,0.6562,0,0.3281,337.166,2777.0806,1.0,8959,1,1036.1541,0.1211,0.1954,0.1956,-0.0004,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0066,-0.0229,0.0,1.0,0.0,0.0141,0.2656,0.1211,7.748,78.3633,621.6158,27.5,857.9687,236.3529,0.0248,1.4652,6.745,58.8652,65.844,7096.5807,19.6954,59.074,7009.5751,0,29,64,512,27,64,394,1.109375,71,6.109375,391,0.296875,19.0,0.015625,1.0,0.046875,3.0,,,,,,,
|
||||
0.0,0,64,512,3,1.0312,0,0.9844,217.6387,4109.1111,1.0,6682,1,963.5862,0.0527,0.0823,0.0823,-0.0001,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0039,-0.0059,0.0,1.0,0.0,0.0063,0.1875,0.0527,8.0195,78.3764,619.7124,27.8,857.9687,238.2563,0.0198,1.1139,4.882,56.4058,61.4598,2113.6889,19.3704,56.5577,2031.7437,0,30,64,512,86,62,347,0.859375,55,5.40625,346,0.359375,23.0,0.015625,1.0,0.015625,1.0,0.0098,7.9489,6.8782,,,,
|
||||
0.0,0,64,512,2,0.6562,0,0.4531,504.793,3000.3256,1.0,11279,1,1347.108,0.168,0.1493,0.1494,-0.0004,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0084,-0.0258,0.0,1.0,0.0,0.0125,0.375,0.168,8.424,78.5074,623.5744,27.3,857.9687,234.3943,0.0293,1.8483,9.6408,69.6522,79.4454,5328.8946,18.0982,69.775,5229.501,0,31,64,512,53,63,402,0.59375,38,6.25,400,0.28125,18.0,0.015625,1.0,,,,,,0.015625,1.0,0.015625,1.0
|
||||
0.0,0,64,512,2,1.0625,0,0.9219,344.2148,3316.5849,1.0,7996,1,1130.5575,0.1035,0.1403,0.1404,-0.0002,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0058,-0.0129,0.0,1.0,0.0,0.0071,0.2031,0.1035,8.5488,78.4227,623.5724,27.3,857.9687,234.3962,0.0352,1.2686,7.9241,64.4317,72.5126,3341.1264,18.3456,64.5529,3248.9975,0,32,64,512,91,61,363,0.609375,39,5.671875,363,0.296875,19.0,,,,,0.0095,8.5021,,,,,
|
||||
0.0,0,64,512,2,0.6406,0,0.4844,463.8789,2605.3297,1.0,12112,1,1252.9052,0.1777,0.1535,0.1536,-0.0004,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0086,-0.0228,0.0,1.0,0.0,0.0137,0.3438,0.1777,9.5261,78.5966,620.8518,27.6,857.9687,237.1169,0.0312,1.9241,9.2013,67.0619,76.4287,4248.8256,18.1481,67.1959,4152.3235,0,33,64,512,47,62,386,0.828125,53,5.984375,383,0.390625,25.0,0.015625,1.0,0.015625,1.0,,,,,,0.03125,2.0
|
||||
0.0,0,64,512,2,0.9531,0,0.8438,253.4688,2751.2979,1.0,5515,1,896.1226,0.0918,0.1853,0.1854,-0.0002,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0046,-0.0141,0.0,1.0,0.0,0.0094,0.1875,0.0918,9.5989,78.382,619.3303,27.8,857.9687,238.6384,0.0193,1.0688,6.365,56.4504,62.9483,3744.8018,19.1478,56.5637,3661.6347,0,34,64,512,58,63,393,0.734375,47,6.140625,393,0.21875,14.0,,,,,0.0091,7.552,,,,,
|
||||
0.0,0,64,512,2,0.6562,0,0.4844,274.5176,2415.5,1.0,6569,1,839.2308,0.1133,0.1238,0.1239,-0.0002,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0051,-0.014,0.0,1.0,0.0,0.0095,0.2188,0.1133,9.9454,78.6008,617.927,28.0,857.9687,240.0417,0.0192,3.6935,5.8429,55.357,61.337,3951.6497,18.6576,55.4746,3867.9608,0,35,64,512,93,59,356,0.5625,36,5.5625,356,0.421875,27.0,,,,,,,6.1902,,,,
|
||||
0.0,0,64,512,2,1.0156,0,0.7812,376.7695,2069.7527,1.0,4538,1,846.2868,0.1816,0.1047,0.1048,-0.0003,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0082,-0.0196,0.0,1.0,0.0,0.0177,0.2969,0.1816,9.9318,78.4766,616.9151,28.1,857.9687,241.0536,0.018,0.8756,7.5627,54.475,62.1755,4009.2705,18.3501,54.5945,3927.8682,0,36,2,16,0,2,15,0.5,1,7.5,15,,,,,,,0.0083,7.8228,,,,,
|
||||
|
10
training_logs/20260525_110326_metrics_job_496422.csv
Normal file
10
training_logs/20260525_110326_metrics_job_496422.csv
Normal file
@@ -0,0 +1,10 @@
|
||||
async/discard_rate,async/discarded_count,async/effective_batch_groups,async/effective_batch_samples,async/staleness_max,async/staleness_mean,async/staleness_min,async/staleness_ratio,generate/avg_num_tokens,generate/avg_tokens_non_zero_rewards,generate/avg_tokens_zero_rewards,generate/max_num_tokens,generate/min_num_tokens,generate/std_num_tokens,loss/avg_final_rewards,loss/avg_raw_advantages,loss/avg_raw_advantages_abs,policy/final_loss,policy/log_ratio_abs_max,policy/log_ratio_abs_mean,policy/log_ratio_abs_p99,policy/log_ratio_abs_pos00,policy/log_ratio_abs_pos10,policy/log_ratio_abs_pos20,policy/log_ratio_abs_pos30,policy/log_ratio_abs_pos40,policy/log_ratio_abs_pos50,policy/log_ratio_abs_pos60,policy/log_ratio_abs_pos70,policy/log_ratio_abs_pos80,policy/log_ratio_abs_pos90,policy/n_tokens_dp_gt_10pct,policy/n_tokens_dp_gt_1pct,policy/n_tokens_dp_gt_50pct,policy/policy_entropy,policy/policy_loss,policy/policy_lr,policy/policy_update_steps,policy/ppo_clip_ratio,policy/raw_grad_norm,reward/avg_pass_at_8,reward/avg_raw_reward,system/process_rss_gb,system/process_vms_gb,system/ram_available_gb,system/ram_percent,system/ram_total_gb,system/ram_used_gb,timing/compute_advantages_and_returns,timing/convert_to_training_input,timing/fwd_logprobs_values_reward,timing/policy_train,timing/run_training,timing/step,timing/sync_weights,timing/train_critic_and_policy,timing/wait_for_generation_buffer,trainer/epoch,trainer/global_step,batch_errors/total_batches,batch_errors/total_instances,batch_errors/total_successful,batch_errors/total_failed,batch_errors/total_masked,batch_errors/avg_AgentEnvironmentTimeoutError,batch_errors/total_AgentEnvironmentTimeoutError,batch_errors/avg_DaytonaAuthenticationError,batch_errors/total_DaytonaAuthenticationError,batch_errors/avg_VerifierRuntimeError,batch_errors/total_VerifierRuntimeError,batch_errors/avg_DaytonaNotFoundError,batch_errors/total_DaytonaNotFoundError,timing/cleanup_old_checkpoints,timing/save_checkpoints,batch_errors/avg_AgentSetupTimeoutError,batch_errors/total_AgentSetupTimeoutError,batch_errors/avg_VerifierTimeoutError,batch_errors/total_VerifierTimeoutError,batch_errors/avg_DaytonaError,batch_errors/total_DaytonaError,batch_errors/avg_ContextLengthExceededError,batch_errors/total_ContextLengthExceededError,timing/save_hf_model,batch_errors/avg_AgentTimeoutError,batch_errors/total_AgentTimeoutError
|
||||
0.0,0,64,512,1,0.0156,0,0.0156,513.5391,2625.2,1.0,26236,1,1588.8893,0.1953,0.1608,0.1609,-0.0005,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0083,-0.0291,0.0,1.0,0.0,0.0178,0.4444,0.1953,12.0642,78.6154,606.7097,29.3,857.9687,251.259,0.1923,3.5931,15.4184,73.4268,89.2021,8145.718,17.602,73.591,8035.3206,0,37,64,512,40,63,367,5.734375,367,1.46875,94,0.15625,10,0.015625,1.0,,,,,,,,,,,,,
|
||||
0.0,0,64,512,1,0.9844,0,0.9844,256.4102,3270.25,1.0,8312,1,1017.874,0.0781,0.3646,0.3647,-0.0003,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0034,-0.0217,0.0,1.0,0.0,0.0125,0.2344,0.0781,12.0077,78.1334,605.2178,29.5,857.9687,252.7509,0.0239,1.3387,7.0582,59.3834,66.5873,3328.8664,18.4202,59.5048,3242.5172,0,38,64,512,86,64,312,4.828125,309,1.0,64,0.5625,36,0.09375,6.0,11.4426,19.0624,0.078125,5.0,0.046875,3.0,0.015625,1.0,0.03125,2.0,,,
|
||||
0.0,0,64,512,2,0.7656,0,0.5312,408.6738,2428.0814,1.0,8101,1,1031.8913,0.168,0.3035,0.3037,-0.0008,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0079,-0.0498,0.0,1.0,0.0,0.0244,0.4375,0.168,13.3047,91.9354,606.3735,29.3,857.9687,251.5952,0.0241,1.2992,6.9572,60.9637,68.0737,6397.0696,17.4018,61.092,6310.2938,0,39,68,544,61,67,381,5.573529411764706,379,1.1764705882352942,80,0.25,17,,,,,0.014705882352941176,1.0,0.058823529411764705,4.0,0.014705882352941176,1.0,0.014705882352941176,1.0,,,
|
||||
0.0,0,64,512,3,1.2344,0,0.9688,383.9883,3215.5902,1.0,9054,1,1172.8826,0.1191,0.181,0.1811,-0.0004,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0055,-0.0256,0.0,1.0,0.0,0.0197,0.3281,0.1191,14.0406,92.0868,605.7397,29.4,857.9687,252.229,0.0354,1.6468,7.3746,62.4563,69.9967,5420.0066,17.7311,62.5862,5330.6236,0,40,64,512,64,64,399,6.234375,399,0.546875,35,0.203125,13,0.015625,1.0,0.0079,8.9118,,,,,,,,,6.6659,,
|
||||
0.0,0,64,512,4,1.0781,0,0.8281,315.2949,2596.4677,1.0,6019,1,911.6647,0.1211,0.075,0.0751,-0.0002,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.006,-0.0107,0.0,1.0,0.0,0.01,0.2812,0.1211,14.3649,92.0377,605.8729,29.4,857.9687,252.0958,0.0196,1.2515,6.8124,57.2361,64.2337,2226.1873,17.7717,57.4013,2142.928,0,41,60,480,76,57,352,5.866666666666666,352,0.55,33,0.3,18,0.016666666666666666,1.0,,,,,,,,,,,,,
|
||||
0.0,0,64,512,2,1.1562,0,0.8594,350.4609,2294.8974,1.0,6096,1,918.4495,0.1523,0.156,0.1561,-0.0003,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.007,-0.0193,0.0,1.0,0.0,0.0108,0.3281,0.1523,16.0237,92.0465,604.1669,29.6,857.9687,253.8018,0.0281,1.007,6.8732,58.1443,65.1695,4730.9187,18.9548,58.2679,4645.7861,0,42,65,520,117,60,320,4.923076923076923,320,0.7846153846153846,51,0.4461538461538462,29,0.03076923076923077,2.0,0.0079,7.5503,0.015384615384615385,1.0,,,,,,,,,
|
||||
0.0,0,64,512,3,1.1562,0,0.8594,506.7285,2214.1026,1.0,9784,1,1181.084,0.2285,0.1962,0.1963,-0.0006,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0097,-0.0374,0.0,1.0,0.0,0.02,0.4688,0.2285,16.5446,92.2373,608.1587,29.1,857.9687,249.81,0.0281,1.6023,8.6365,65.3044,74.2194,3828.8954,19.3517,65.5544,3733.7211,0,43,63,504,57,63,380,6.0,378,0.7619047619047619,48,0.2857142857142857,18,0.015873015873015872,1.0,,,,,,,,,0.031746031746031744,2.0,,,
|
||||
0.0,0,64,512,3,1.2812,0,0.875,384.8477,3448.8947,1.0,11922,1,1321.23,0.1113,0.1286,0.1287,-0.0002,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0052,-0.0139,0.0,1.0,0.0,0.0125,0.2656,0.1113,16.9448,92.2662,606.6965,29.3,857.9687,251.2722,0.0323,1.7789,8.0712,63.9146,72.1439,4075.9496,19.0722,64.0401,3982.9522,0,44,67,536,61,66,393,5.82089552238806,390,0.7761194029850746,52,0.43283582089552236,29,0.014925373134328358,1.0,0.0083,7.3633,,,,,0.014925373134328358,1.0,,,,0.029850746268656716,2.0
|
||||
0.0,0,64,512,3,1.0625,0,0.7344,289.7832,2550.2586,1.0,9971,1,1016.4986,0.1133,0.148,0.1481,-0.0003,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0047,-0.0191,0.0,1.0,0.0,0.0151,0.2344,0.1133,17.2552,92.2487,605.9093,29.4,857.9687,252.0594,0.0283,2.2306,6.986,57.0132,64.3289,4079.3376,19.0583,57.3142,3993.7175,0,45,18,144,27,18,97,5.333333333333333,96,0.6111111111111112,11,0.5,9,,,,,,,,,,,,,5.9741,0.05555555555555555,1.0
|
||||
|
411
training_logs/20260525_110326_metrics_report.md
Normal file
411
training_logs/20260525_110326_metrics_report.md
Normal file
@@ -0,0 +1,411 @@
|
||||
# SkyRL Training Metrics Analysis
|
||||
|
||||
Generated from 6 log files
|
||||
|
||||
## Overview
|
||||
|
||||
| Log File | Total Steps | Metric Blocks | Final Reward (mean) | Final Reward (max) | Total Time (s) |
|
||||
|----------|-------------|---------------|---------------------|-------------------|----------------|
|
||||
| job_485099 | 5 | 5 | 0.0621 | 0.1152 | 23691.2 |
|
||||
| job_485100 | 13 | 9 | 0.0610 | 0.1465 | 39612.5 |
|
||||
| job_485101 | 19 | 7 | 0.0948 | 0.1230 | 31256.0 |
|
||||
| job_496420 | 26 | 8 | 0.0935 | 0.2188 | 40891.0 |
|
||||
| job_496421 | 36 | 10 | 0.1310 | 0.2422 | 42300.2 |
|
||||
| job_496422 | 45 | 9 | 0.1430 | 0.2285 | 42232.9 |
|
||||
|
||||
## Async Metrics
|
||||
|
||||
| | Mean | Std | Min | Max | Count |
|
||||
|:------------------------------|-----------:|---------:|------:|---------:|--------:|
|
||||
| async/discard_rate | 0 | 0 | 0 | 0 | 48 |
|
||||
| async/discarded_count | 0 | 0 | 0 | 0 | 48 |
|
||||
| async/effective_batch_groups | 64 | 0 | 64 | 64 | 48 |
|
||||
| async/effective_batch_samples | 512 | 0 | 512 | 512 | 48 |
|
||||
| async/staleness_max | 1.91667 | 0.794485 | 0 | 4 | 48 |
|
||||
| async/staleness_mean | 0.861646 | 0.370024 | 0 | 1.5625 | 48 |
|
||||
| async/staleness_min | 0.208333 | 0.410414 | 0 | 1 | 48 |
|
||||
| async/staleness_ratio | 0.69629 | 0.320514 | 0 | 1 | 48 |
|
||||
|
||||
## Generate Metrics
|
||||
|
||||
| | Mean | Std | Min | Max | Count |
|
||||
|:-------------------------------------|---------:|---------:|---------:|----------:|--------:|
|
||||
| generate/avg_num_tokens | 300.937 | 132.434 | 53.543 | 653.519 | 48 |
|
||||
| generate/avg_tokens_non_zero_rewards | 3164.85 | 608.453 | 2069.75 | 4650.14 | 48 |
|
||||
| generate/avg_tokens_zero_rewards | 1 | 0 | 1 | 1 | 48 |
|
||||
| generate/max_num_tokens | 9563.94 | 4742.53 | 4538 | 31001 | 48 |
|
||||
| generate/min_num_tokens | 1 | 0 | 1 | 1 | 48 |
|
||||
| generate/std_num_tokens | 1025.02 | 261.051 | 409.607 | 1779.2 | 48 |
|
||||
|
||||
## Loss Metrics
|
||||
|
||||
| | Mean | Std | Min | Max | Count |
|
||||
|:----------------------------|---------:|----------:|-------:|-------:|--------:|
|
||||
| loss/avg_final_rewards | 0.101433 | 0.0552715 | 0.0137 | 0.2422 | 48 |
|
||||
| loss/avg_raw_advantages | 0.223456 | 0.116288 | 0.0251 | 0.4541 | 48 |
|
||||
| loss/avg_raw_advantages_abs | 0.223596 | 0.116357 | 0.0252 | 0.4545 | 48 |
|
||||
|
||||
## Policy Metrics
|
||||
|
||||
| | Mean | Std | Min | Max | Count |
|
||||
|:----------------------------|-------------:|------------:|--------:|--------:|--------:|
|
||||
| policy/final_loss | -0.000310417 | 0.000183627 | -0.0008 | -0 | 48 |
|
||||
| policy/log_ratio_abs_max | 0 | 0 | 0 | 0 | 48 |
|
||||
| policy/log_ratio_abs_mean | 0 | 0 | 0 | 0 | 48 |
|
||||
| policy/log_ratio_abs_p99 | 0 | 0 | 0 | 0 | 48 |
|
||||
| policy/log_ratio_abs_pos00 | 0 | 0 | 0 | 0 | 48 |
|
||||
| policy/log_ratio_abs_pos10 | 0 | 0 | 0 | 0 | 48 |
|
||||
| policy/log_ratio_abs_pos20 | 0 | 0 | 0 | 0 | 48 |
|
||||
| policy/log_ratio_abs_pos30 | 0 | 0 | 0 | 0 | 48 |
|
||||
| policy/log_ratio_abs_pos40 | 0 | 0 | 0 | 0 | 48 |
|
||||
| policy/log_ratio_abs_pos50 | 0 | 0 | 0 | 0 | 48 |
|
||||
| policy/log_ratio_abs_pos60 | 0 | 0 | 0 | 0 | 48 |
|
||||
| policy/log_ratio_abs_pos70 | 0 | 0 | 0 | 0 | 48 |
|
||||
| policy/log_ratio_abs_pos80 | 0 | 0 | 0 | 0 | 48 |
|
||||
| policy/log_ratio_abs_pos90 | 0 | 0 | 0 | 0 | 48 |
|
||||
| policy/n_tokens_dp_gt_10pct | 0 | 0 | 0 | 0 | 48 |
|
||||
| policy/n_tokens_dp_gt_1pct | 0 | 0 | 0 | 0 | 48 |
|
||||
| policy/n_tokens_dp_gt_50pct | 0 | 0 | 0 | 0 | 48 |
|
||||
| policy/policy_entropy | 0.00732917 | 0.0036161 | 0.0018 | 0.0182 | 48 |
|
||||
| policy/policy_loss | -0.0198583 | 0.0114544 | -0.0498 | -0.002 | 48 |
|
||||
| policy/policy_lr | 0 | 0 | 0 | 0 | 48 |
|
||||
| policy/policy_update_steps | 1 | 0 | 1 | 1 | 48 |
|
||||
| policy/ppo_clip_ratio | 0 | 0 | 0 | 0 | 48 |
|
||||
| policy/raw_grad_norm | 0.0109021 | 0.00505137 | 0.0028 | 0.0244 | 48 |
|
||||
|
||||
## Reward Metrics
|
||||
|
||||
| | Mean | Std | Min | Max | Count |
|
||||
|:----------------------|---------:|----------:|-------:|-------:|--------:|
|
||||
| reward/avg_pass_at_8 | 0.261562 | 0.0939986 | 0.0938 | 0.4898 | 48 |
|
||||
| reward/avg_raw_reward | 0.101433 | 0.0552715 | 0.0137 | 0.2422 | 48 |
|
||||
|
||||
## System Metrics
|
||||
|
||||
| | Mean | Std | Min | Max | Count |
|
||||
|:------------------------|---------:|------------:|---------:|---------:|--------:|
|
||||
| system/process_rss_gb | 9.5798 | 3.43573 | 5.3781 | 17.2552 | 48 |
|
||||
| system/process_vms_gb | 81.1127 | 5.44727 | 78.0531 | 94.4252 | 48 |
|
||||
| system/ram_available_gb | 606.245 | 15.2876 | 574.636 | 627.878 | 48 |
|
||||
| system/ram_percent | 29.3396 | 1.78847 | 26.8 | 33 | 48 |
|
||||
| system/ram_total_gb | 857.969 | 0.00033958 | 857.968 | 857.969 | 48 |
|
||||
| system/ram_used_gb | 251.724 | 15.2877 | 230.09 | 283.333 | 48 |
|
||||
|
||||
## Timing Metrics
|
||||
|
||||
| | Mean | Std | Min | Max | Count |
|
||||
|:--------------------------------------|-------------:|-------------:|---------:|----------:|--------:|
|
||||
| timing/compute_advantages_and_returns | 0.0384646 | 0.0348721 | 0.0175 | 0.1923 | 48 |
|
||||
| timing/convert_to_training_input | 5.4315 | 26.8172 | 0.8755 | 187.298 | 48 |
|
||||
| timing/fwd_logprobs_values_reward | 7.50122 | 2.76571 | 3.0976 | 15.4184 | 48 |
|
||||
| timing/policy_train | 59.4245 | 7.15104 | 44.4851 | 76.4064 | 48 |
|
||||
| timing/run_training | 67.108 | 9.54666 | 48.0435 | 90.3725 | 48 |
|
||||
| timing/step | 4583 | 2394.44 | 351.854 | 8192.84 | 48 |
|
||||
| timing/sync_weights | 18.6968 | 1.59914 | 17.1829 | 28.5894 | 48 |
|
||||
| timing/train_critic_and_policy | 59.5679 | 7.15635 | 44.6233 | 76.5655 | 48 |
|
||||
| timing/wait_for_generation_buffer | 4491.76 | 2392.46 | 271.614 | 8105.89 | 48 |
|
||||
| timing/cleanup_old_checkpoints | 4.13983 | 7.40386 | 0.0075 | 24.0683 | 22 |
|
||||
| timing/save_checkpoints | 15.0911 | 11.7658 | 7.3633 | 44.8581 | 22 |
|
||||
| timing/save_hf_model | 6.85727 | 1.37645 | 5.8497 | 10.5996 | 10 |
|
||||
|
||||
## Trainer Metrics
|
||||
|
||||
| | Mean | Std | Min | Max | Count |
|
||||
|:--------------------|--------:|--------:|------:|------:|--------:|
|
||||
| trainer/epoch | 0 | 0 | 0 | 0 | 48 |
|
||||
| trainer/global_step | 22.3333 | 13.0536 | 1 | 45 | 48 |
|
||||
|
||||
## Batch_Errors Metrics
|
||||
|
||||
| | Mean | Std | Min | Max | Count |
|
||||
|:------------------------------------------------|------------:|------------:|-----------:|-------------:|--------:|
|
||||
| batch_errors/total_batches | 59.6875 | 18.9346 | 2 | 128 | 48 |
|
||||
| batch_errors/total_instances | 477.5 | 151.476 | 16 | 1024 | 48 |
|
||||
| batch_errors/total_successful | 43.8333 | 25.6634 | 0 | 117 | 48 |
|
||||
| batch_errors/total_failed | 59.0208 | 18.8617 | 2 | 128 | 48 |
|
||||
| batch_errors/total_masked | 357.688 | 113.847 | 15 | 732 | 48 |
|
||||
| batch_errors/avg_DaytonaAuthenticationError | 1.00014 | 0.399696 | 0.28125 | 1.76562 | 48 |
|
||||
| batch_errors/total_DaytonaAuthenticationError | 62.2708 | 36.1967 | 1 | 208 | 48 |
|
||||
| batch_errors/avg_AgentEnvironmentTimeoutError | 6.00745 | 0.573894 | 4.82812 | 7.5 | 48 |
|
||||
| batch_errors/total_AgentEnvironmentTimeoutError | 355.104 | 112.817 | 15 | 725 | 48 |
|
||||
| batch_errors/avg_DaytonaNotFoundError | 0.0269537 | 0.0189922 | 0.0144928 | 0.09375 | 32 |
|
||||
| batch_errors/total_DaytonaNotFoundError | 1.75 | 1.34404 | 1 | 6 | 32 |
|
||||
| batch_errors/avg_VerifierRuntimeError | 0.20424 | 0.127208 | 0.03125 | 0.5625 | 46 |
|
||||
| batch_errors/total_VerifierRuntimeError | 12.5217 | 8.05188 | 2 | 36 | 46 |
|
||||
| batch_errors/avg_DaytonaError | 0.0224232 | 0.0110744 | 0.0147059 | 0.046875 | 17 |
|
||||
| batch_errors/total_DaytonaError | 1.52941 | 0.799816 | 1 | 3 | 17 |
|
||||
| batch_errors/avg_RuntimeError | 0.015625 | nan | 0.015625 | 0.015625 | 1 |
|
||||
| batch_errors/total_RuntimeError | 2 | nan | 2 | 2 | 1 |
|
||||
| batch_errors/avg_DaytonaValidationError | 0.0078125 | nan | 0.0078125 | 0.0078125 | 1 |
|
||||
| batch_errors/total_DaytonaValidationError | 1 | nan | 1 | 1 | 1 |
|
||||
| batch_errors/avg_ContextLengthExceededError | 0.0284136 | 0.0238451 | 0.0078125 | 0.1 | 13 |
|
||||
| batch_errors/total_ContextLengthExceededError | 1.61538 | 0.767948 | 1 | 3 | 13 |
|
||||
| batch_errors/avg_AgentTimeoutError | 0.0445227 | 0.0358926 | 0.015625 | 0.1875 | 27 |
|
||||
| batch_errors/total_AgentTimeoutError | 2.81481 | 2.30446 | 1 | 12 | 27 |
|
||||
| batch_errors/avg_AgentSetupTimeoutError | 0.0386788 | 0.0257621 | 0.0147059 | 0.078125 | 6 |
|
||||
| batch_errors/total_AgentSetupTimeoutError | 2.33333 | 1.75119 | 1 | 5 | 6 |
|
||||
| batch_errors/avg_EnvironmentStartTimeoutError | 0.0153846 | nan | 0.0153846 | 0.0153846 | 1 |
|
||||
| batch_errors/total_EnvironmentStartTimeoutError | 1 | nan | 1 | 1 | 1 |
|
||||
| batch_errors/avg_VerifierTimeoutError | 0.0400638 | 0.0229369 | 0.0144928 | 0.0588235 | 3 |
|
||||
| batch_errors/total_VerifierTimeoutError | 2.66667 | 1.52753 | 1 | 4 | 3 |
|
||||
| batch_errors/avg_OSError | 0.0144928 | nan | 0.0144928 | 0.0144928 | 1 |
|
||||
| batch_errors/total_OSError | 1 | nan | 1 | 1 | 1 |
|
||||
|
||||
## Training Progression by Log
|
||||
|
||||
### job_485099
|
||||
|
||||
| Step | Reward | Pass@8 | KL | Loss | Step Time (s) | Gen Wait (s) |
|
||||
|------|--------|--------|-----|------|---------------|-------------|
|
||||
| 1 | 0.0410 | 0.1719 | 0.000000 | -0.0003 | 8192.8 | 8105.9 |
|
||||
| 2 | 0.0664 | 0.2344 | 0.000000 | -0.0002 | 7137.7 | 7051.2 |
|
||||
| 3 | 0.0742 | 0.2656 | 0.000000 | -0.0001 | 614.4 | 526.3 |
|
||||
| 4 | 0.0137 | 0.0938 | 0.000000 | -0.0000 | 7011.6 | 6938.8 |
|
||||
| 5 | 0.1152 | 0.2344 | 0.000000 | -0.0001 | 734.7 | 652.9 |
|
||||
|
||||
### job_485100
|
||||
|
||||
| Step | Reward | Pass@8 | KL | Loss | Step Time (s) | Gen Wait (s) |
|
||||
|------|--------|--------|-----|------|---------------|-------------|
|
||||
| 5 | 0.1465 | 0.3276 | 0.000000 | -0.0005 | 8132.7 | 8036.1 |
|
||||
| 6 | 0.0332 | 0.1562 | 0.000000 | -0.0003 | 3465.1 | 3390.7 |
|
||||
| 7 | 0.0234 | 0.1406 | 0.000000 | -0.0002 | 3987.0 | 3915.0 |
|
||||
| 8 | 0.0176 | 0.0938 | 0.000000 | -0.0001 | 6656.3 | 6578.8 |
|
||||
| 9 | 0.0234 | 0.1250 | 0.000000 | -0.0001 | 1287.2 | 1213.6 |
|
||||
| 10 | 0.0820 | 0.2344 | 0.000000 | -0.0005 | 6403.6 | 6309.4 |
|
||||
| 11 | 0.0781 | 0.1719 | 0.000000 | -0.0002 | 1527.3 | 1417.6 |
|
||||
| 12 | 0.0801 | 0.2500 | 0.000000 | -0.0004 | 6140.9 | 6053.0 |
|
||||
| 13 | 0.0645 | 0.2344 | 0.000000 | -0.0002 | 2012.4 | 1931.4 |
|
||||
|
||||
### job_485101
|
||||
|
||||
| Step | Reward | Pass@8 | KL | Loss | Step Time (s) | Gen Wait (s) |
|
||||
|------|--------|--------|-----|------|---------------|-------------|
|
||||
| 13 | 0.1230 | 0.2742 | 0.000000 | -0.0004 | 8113.5 | 8023.7 |
|
||||
| 14 | 0.0371 | 0.1250 | 0.000000 | -0.0002 | 6886.3 | 6818.0 |
|
||||
| 15 | 0.1191 | 0.3750 | 0.000000 | -0.0005 | 791.9 | 701.5 |
|
||||
| 16 | 0.1191 | 0.3125 | 0.000000 | -0.0003 | 6575.8 | 6487.1 |
|
||||
| 17 | 0.0918 | 0.2500 | 0.000000 | -0.0005 | 920.2 | 834.8 |
|
||||
| 18 | 0.0703 | 0.2344 | 0.000000 | -0.0002 | 6610.6 | 6533.4 |
|
||||
| 19 | 0.1035 | 0.2812 | 0.000000 | -0.0001 | 1357.7 | 1273.3 |
|
||||
|
||||
### job_496420
|
||||
|
||||
| Step | Reward | Pass@8 | KL | Loss | Step Time (s) | Gen Wait (s) |
|
||||
|------|--------|--------|-----|------|---------------|-------------|
|
||||
| 19 | 0.2188 | 0.4407 | 0.000000 | -0.0007 | 8067.2 | 7961.5 |
|
||||
| 20 | 0.0801 | 0.2031 | 0.000000 | -0.0003 | 3380.6 | 3297.4 |
|
||||
| 21 | 0.0996 | 0.3281 | 0.000000 | -0.0005 | 4118.8 | 4027.4 |
|
||||
| 22 | 0.0703 | 0.2500 | 0.000000 | -0.0004 | 5101.1 | 5011.8 |
|
||||
| 23 | 0.0938 | 0.2656 | 0.000000 | -0.0003 | 2467.0 | 2383.2 |
|
||||
| 24 | 0.0781 | 0.2344 | 0.000000 | -0.0001 | 6306.5 | 6217.1 |
|
||||
| 25 | 0.0547 | 0.2188 | 0.000000 | -0.0002 | 4522.1 | 4243.7 |
|
||||
| 26 | 0.0527 | 0.1875 | 0.000000 | -0.0000 | 6927.7 | 6847.7 |
|
||||
|
||||
### job_496421
|
||||
|
||||
| Step | Reward | Pass@8 | KL | Loss | Step Time (s) | Gen Wait (s) |
|
||||
|------|--------|--------|-----|------|---------------|-------------|
|
||||
| 27 | 0.2422 | 0.4898 | 0.000000 | -0.0007 | 8113.5 | 8003.3 |
|
||||
| 28 | 0.0586 | 0.2500 | 0.000000 | -0.0005 | 351.9 | 271.6 |
|
||||
| 29 | 0.1211 | 0.2656 | 0.000000 | -0.0004 | 7096.6 | 7009.6 |
|
||||
| 30 | 0.0527 | 0.1875 | 0.000000 | -0.0001 | 2113.7 | 2031.7 |
|
||||
| 31 | 0.1680 | 0.3750 | 0.000000 | -0.0004 | 5328.9 | 5229.5 |
|
||||
| 32 | 0.1035 | 0.2031 | 0.000000 | -0.0002 | 3341.1 | 3249.0 |
|
||||
| 33 | 0.1777 | 0.3438 | 0.000000 | -0.0004 | 4248.8 | 4152.3 |
|
||||
| 34 | 0.0918 | 0.1875 | 0.000000 | -0.0002 | 3744.8 | 3661.6 |
|
||||
| 35 | 0.1133 | 0.2188 | 0.000000 | -0.0002 | 3951.6 | 3868.0 |
|
||||
| 36 | 0.1816 | 0.2969 | 0.000000 | -0.0003 | 4009.3 | 3927.9 |
|
||||
|
||||
### job_496422
|
||||
|
||||
| Step | Reward | Pass@8 | KL | Loss | Step Time (s) | Gen Wait (s) |
|
||||
|------|--------|--------|-----|------|---------------|-------------|
|
||||
| 37 | 0.1953 | 0.4444 | 0.000000 | -0.0005 | 8145.7 | 8035.3 |
|
||||
| 38 | 0.0781 | 0.2344 | 0.000000 | -0.0003 | 3328.9 | 3242.5 |
|
||||
| 39 | 0.1680 | 0.4375 | 0.000000 | -0.0008 | 6397.1 | 6310.3 |
|
||||
| 40 | 0.1191 | 0.3281 | 0.000000 | -0.0004 | 5420.0 | 5330.6 |
|
||||
| 41 | 0.1211 | 0.2812 | 0.000000 | -0.0002 | 2226.2 | 2142.9 |
|
||||
| 42 | 0.1523 | 0.3281 | 0.000000 | -0.0003 | 4730.9 | 4645.8 |
|
||||
| 43 | 0.2285 | 0.4688 | 0.000000 | -0.0006 | 3828.9 | 3733.7 |
|
||||
| 44 | 0.1113 | 0.2656 | 0.000000 | -0.0002 | 4075.9 | 3983.0 |
|
||||
| 45 | 0.1133 | 0.2344 | 0.000000 | -0.0003 | 4079.3 | 3993.7 |
|
||||
|
||||
## Timing Analysis
|
||||
|
||||
### Average Time Breakdown (% of step time)
|
||||
|
||||
| Component | Avg % of Step Time |
|
||||
|-----------|-------------------|
|
||||
| wait_for_generation_buffer | 96.5% |
|
||||
| run_training | 2.6% |
|
||||
| train_critic_and_policy | 2.3% |
|
||||
| policy_train | 2.3% |
|
||||
| sync_weights | 0.7% |
|
||||
| save_checkpoints | 0.5% |
|
||||
| save_hf_model | 0.4% |
|
||||
| fwd_logprobs_values_reward | 0.3% |
|
||||
| cleanup_old_checkpoints | 0.2% |
|
||||
| convert_to_training_input | 0.1% |
|
||||
| compute_advantages_and_returns | 0.0% |
|
||||
|
||||
## Cross-Log Comparison
|
||||
|
||||
| Log | Avg Reward | Pass@8 | Step Time (s) | Gen Wait Time (s) | Avg Tokens | Staleness |
|
||||
|-----|------|------|------|------|------|------|
|
||||
| job_485099 | 0.0621 | 0.2000 | 4738.2360 | 4655.0062 | 202.7762 | 0.8125 |
|
||||
| job_485100 | 0.0610 | 0.1927 | 4401.3874 | 4316.1800 | 213.5232 | 0.7847 |
|
||||
| job_485101 | 0.0948 | 0.2646 | 4465.1476 | 4381.6743 | 292.8345 | 0.9732 |
|
||||
| job_496420 | 0.0935 | 0.2660 | 5111.3807 | 4998.7086 | 309.4238 | 0.8438 |
|
||||
| job_496421 | 0.1310 | 0.2818 | 4230.0172 | 4140.4545 | 357.4424 | 0.7937 |
|
||||
| job_496422 | 0.1430 | 0.3358 | 4692.5499 | 4601.9845 | 378.8585 | 0.9705 |
|
||||
|
||||
## vLLM Inference Engine Analysis
|
||||
|
||||
Metrics from vLLM stat loggers (V1LoggingStatLoggerFixed).
|
||||
|
||||
> **Note**: Ray deduplicates similar log messages with `[repeated Nx across cluster]`,
|
||||
> so we typically capture stats from one engine per timestamp. The stats shown are
|
||||
> **per-engine** values. Multiply by num_inference_engines for cluster-wide estimates.
|
||||
|
||||
### Summary by Log (Per-Engine Stats)
|
||||
|
||||
| Log | Avg Running/Engine | Avg Waiting/Engine | Avg Gen Throughput/Engine | Avg KV Cache % | Avg Prefix Hit % |
|
||||
|-----|-------------------|-------------------|--------------------------|----------------|------------------|
|
||||
| job_485099 | 3.8 | 0.0 | 175.2 tok/s | 16.5% | 78.6% |
|
||||
| job_485100 | 3.6 | 0.0 | 165.7 tok/s | 15.7% | 81.1% |
|
||||
| job_485101 | 3.7 | 0.0 | 168.8 tok/s | 15.6% | 80.5% |
|
||||
| job_496420 | 3.2 | 0.0 | 145.1 tok/s | 13.9% | 79.2% |
|
||||
| job_496421 | 3.8 | 0.0 | 173.4 tok/s | 17.0% | 77.6% |
|
||||
| job_496422 | 3.2 | 0.0 | 145.2 tok/s | 13.7% | 74.7% |
|
||||
|
||||
### Utilization Analysis (Per-Engine)
|
||||
|
||||
Key indicators of inference engine utilization:
|
||||
|
||||
- **Running requests/engine**: Concurrent requests being processed by each engine
|
||||
- **Waiting requests**: Requests queued (0 = engine not saturated, has spare capacity)
|
||||
- **Generation throughput**: Decode tokens/sec per engine
|
||||
- 8B model on H100 can do **1000+ tok/s** when saturated
|
||||
- If seeing <300 tok/s with 0 waiting, engine is **starved for requests**
|
||||
|
||||
#### job_485099
|
||||
|
||||
- **Running requests/engine**: avg=3.8, max=15
|
||||
- **Waiting requests**: avg=0.0, max=0
|
||||
- **Generation throughput/engine**: avg=175.2 tok/s, max=700.8 tok/s
|
||||
- **KV cache usage**: avg=16.5%
|
||||
- **Prefix cache hit rate**: avg=78.6%
|
||||
- ⚠️ **Underutilized**: Engines starved for requests (0 waiting, avg 3.8 running)
|
||||
- Bottleneck is likely upstream (environment execution, not inference)
|
||||
|
||||
#### job_485100
|
||||
|
||||
- **Running requests/engine**: avg=3.6, max=17
|
||||
- **Waiting requests**: avg=0.0, max=0
|
||||
- **Generation throughput/engine**: avg=165.7 tok/s, max=682.9 tok/s
|
||||
- **KV cache usage**: avg=15.7%
|
||||
- **Prefix cache hit rate**: avg=81.1%
|
||||
- ⚠️ **Underutilized**: Engines starved for requests (0 waiting, avg 3.6 running)
|
||||
- Bottleneck is likely upstream (environment execution, not inference)
|
||||
|
||||
#### job_485101
|
||||
|
||||
- **Running requests/engine**: avg=3.7, max=13
|
||||
- **Waiting requests**: avg=0.0, max=0
|
||||
- **Generation throughput/engine**: avg=168.8 tok/s, max=601.8 tok/s
|
||||
- **KV cache usage**: avg=15.6%
|
||||
- **Prefix cache hit rate**: avg=80.5%
|
||||
- ⚠️ **Underutilized**: Engines starved for requests (0 waiting, avg 3.7 running)
|
||||
- Bottleneck is likely upstream (environment execution, not inference)
|
||||
|
||||
#### job_496420
|
||||
|
||||
- **Running requests/engine**: avg=3.2, max=11
|
||||
- **Waiting requests**: avg=0.0, max=0
|
||||
- **Generation throughput/engine**: avg=145.1 tok/s, max=485.1 tok/s
|
||||
- **KV cache usage**: avg=13.9%
|
||||
- **Prefix cache hit rate**: avg=79.2%
|
||||
- ⚠️ **Underutilized**: Engines starved for requests (0 waiting, avg 3.2 running)
|
||||
- Bottleneck is likely upstream (environment execution, not inference)
|
||||
|
||||
#### job_496421
|
||||
|
||||
- **Running requests/engine**: avg=3.8, max=15
|
||||
- **Waiting requests**: avg=0.0, max=0
|
||||
- **Generation throughput/engine**: avg=173.4 tok/s, max=665.8 tok/s
|
||||
- **KV cache usage**: avg=17.0%
|
||||
- **Prefix cache hit rate**: avg=77.6%
|
||||
- ⚠️ **Underutilized**: Engines starved for requests (0 waiting, avg 3.8 running)
|
||||
- Bottleneck is likely upstream (environment execution, not inference)
|
||||
|
||||
#### job_496422
|
||||
|
||||
- **Running requests/engine**: avg=3.2, max=16
|
||||
- **Waiting requests**: avg=0.0, max=0
|
||||
- **Generation throughput/engine**: avg=145.2 tok/s, max=682.5 tok/s
|
||||
- **KV cache usage**: avg=13.7%
|
||||
- **Prefix cache hit rate**: avg=74.7%
|
||||
- ⚠️ **Underutilized**: Engines starved for requests (0 waiting, avg 3.2 running)
|
||||
- Bottleneck is likely upstream (environment execution, not inference)
|
||||
|
||||
## Trial-Level Analysis (from result.json)
|
||||
|
||||
Total trials parsed: 25855
|
||||
|
||||
### Turn Count Statistics
|
||||
|
||||
| Metric | Value |
|
||||
|--------|-------|
|
||||
| Mean | 1.7 |
|
||||
| Median | 1.0 |
|
||||
| Std | 1.3 |
|
||||
| Min | 1 |
|
||||
| Max | 33 |
|
||||
| Count | 25830 |
|
||||
|
||||
### Exception Distribution
|
||||
|
||||
| Exception Type | Count | % |
|
||||
|---------------|-------|---|
|
||||
| AgentEnvironmentTimeoutError | 18399 | 71.2% |
|
||||
| DaytonaAuthenticationError | 3597 | 13.9% |
|
||||
| No exception | 2770 | 10.7% |
|
||||
| VerifierRuntimeError | 778 | 3.0% |
|
||||
| AgentTimeoutError | 158 | 0.6% |
|
||||
| DaytonaNotFoundError | 72 | 0.3% |
|
||||
| DaytonaError | 29 | 0.1% |
|
||||
| ContextLengthExceededError | 23 | 0.1% |
|
||||
| AgentSetupTimeoutError | 17 | 0.1% |
|
||||
| VerifierTimeoutError | 8 | 0.0% |
|
||||
| RuntimeError | 2 | 0.0% |
|
||||
| DaytonaValidationError | 1 | 0.0% |
|
||||
| EnvironmentStartTimeoutError | 1 | 0.0% |
|
||||
|
||||
### Turn Count by Exception Type
|
||||
|
||||
| Exception Type | Mean Turns | Median Turns | Count |
|
||||
|---------------|-----------|-------------|-------|
|
||||
| AgentTimeoutError | 9.7 | 8.0 | 158 |
|
||||
| No exception | 3.2 | 3.0 | 2770 |
|
||||
| VerifierRuntimeError | 3.1 | 3.0 | 778 |
|
||||
| VerifierTimeoutError | 2.6 | 2.0 | 8 |
|
||||
| DaytonaValidationError | 2.0 | 2.0 | 1 |
|
||||
| DaytonaNotFoundError | 1.7 | 2.0 | 72 |
|
||||
| DaytonaError | 1.6 | 2.0 | 24 |
|
||||
| AgentEnvironmentTimeoutError | 1.5 | 1.0 | 18399 |
|
||||
| ContextLengthExceededError | 1.4 | 1.0 | 23 |
|
||||
| DaytonaAuthenticationError | 1.2 | 1.0 | 3597 |
|
||||
|
||||
### Turn Count by Outcome
|
||||
|
||||
| Outcome | Mean Turns | Median Turns | Count |
|
||||
|---------|-----------|-------------|-------|
|
||||
| Success | 3.2 | 3.0 | 4834 |
|
||||
|
||||
### Reward Summary
|
||||
|
||||
- Mean reward: 1.0000
|
||||
- Success rate: 100.0%
|
||||
- Trials with reward data: 4834
|
||||
|
||||
49
training_logs/20260525_110326_metrics_table.csv
Normal file
49
training_logs/20260525_110326_metrics_table.csv
Normal file
@@ -0,0 +1,49 @@
|
||||
log_file,async/discard_rate,async/discarded_count,async/effective_batch_groups,async/effective_batch_samples,async/staleness_max,async/staleness_mean,async/staleness_min,async/staleness_ratio,generate/avg_num_tokens,generate/avg_tokens_non_zero_rewards,generate/avg_tokens_zero_rewards,generate/max_num_tokens,generate/min_num_tokens,generate/std_num_tokens,loss/avg_final_rewards,loss/avg_raw_advantages,loss/avg_raw_advantages_abs,policy/final_loss,policy/log_ratio_abs_max,policy/log_ratio_abs_mean,policy/log_ratio_abs_p99,policy/log_ratio_abs_pos00,policy/log_ratio_abs_pos10,policy/log_ratio_abs_pos20,policy/log_ratio_abs_pos30,policy/log_ratio_abs_pos40,policy/log_ratio_abs_pos50,policy/log_ratio_abs_pos60,policy/log_ratio_abs_pos70,policy/log_ratio_abs_pos80,policy/log_ratio_abs_pos90,policy/n_tokens_dp_gt_10pct,policy/n_tokens_dp_gt_1pct,policy/n_tokens_dp_gt_50pct,policy/policy_entropy,policy/policy_loss,policy/policy_lr,policy/policy_update_steps,policy/ppo_clip_ratio,policy/raw_grad_norm,reward/avg_pass_at_8,reward/avg_raw_reward,system/process_rss_gb,system/process_vms_gb,system/ram_available_gb,system/ram_percent,system/ram_total_gb,system/ram_used_gb,timing/compute_advantages_and_returns,timing/convert_to_training_input,timing/fwd_logprobs_values_reward,timing/policy_train,timing/run_training,timing/step,timing/sync_weights,timing/train_critic_and_policy,timing/wait_for_generation_buffer,trainer/epoch,trainer/global_step,batch_errors/total_batches,batch_errors/total_instances,batch_errors/total_successful,batch_errors/total_failed,batch_errors/total_masked,batch_errors/avg_DaytonaAuthenticationError,batch_errors/total_DaytonaAuthenticationError,batch_errors/avg_AgentEnvironmentTimeoutError,batch_errors/total_AgentEnvironmentTimeoutError,batch_errors/avg_DaytonaNotFoundError,batch_errors/total_DaytonaNotFoundError,batch_errors/avg_VerifierRuntimeError,batch_errors/total_VerifierRuntimeError,batch_errors/avg_DaytonaError,batch_errors/total_DaytonaError,batch_errors/avg_RuntimeError,batch_errors/total_RuntimeError,batch_errors/avg_DaytonaValidationError,batch_errors/total_DaytonaValidationError,batch_errors/avg_ContextLengthExceededError,batch_errors/total_ContextLengthExceededError,batch_errors/avg_AgentTimeoutError,batch_errors/total_AgentTimeoutError,timing/cleanup_old_checkpoints,timing/save_checkpoints,timing/save_hf_model,batch_errors/avg_AgentSetupTimeoutError,batch_errors/total_AgentSetupTimeoutError,batch_errors/avg_EnvironmentStartTimeoutError,batch_errors/total_EnvironmentStartTimeoutError,batch_errors/avg_VerifierTimeoutError,batch_errors/total_VerifierTimeoutError,batch_errors/avg_OSError,batch_errors/total_OSError,global_step
|
||||
job_485099,0.0,0,64,512,0,0.0,0,0.0,147.9727,3584.3333,1.0,8768,1,798.0622,0.041,0.4413,0.4415,-0.0003,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0052,-0.0178,0.0,1.0,0.0,0.0083,0.1719,0.041,6.1133,79.7606,624.197,27.2,857.9676,233.7706,0.1201,1.388,9.9059,57.0489,67.2439,8192.8398,18.3161,57.2174,8105.8915,0,1,128,1024,55,128,732,1.625,208,5.6640625,725,0.046875,6.0,0.15625,20.0,0.0234375,3.0,0.015625,2.0,0.0078125,1.0,0.0078125,1.0,0.0234375,3.0,,,,,,,,,,,,1
|
||||
job_485099,0.0,0,64,512,1,0.8438,0,0.8438,240.5703,3608.6471,1.0,11048,1,1028.2547,0.0664,0.3248,0.325,-0.0002,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0092,-0.0156,0.0,1.0,0.0,0.0059,0.2344,0.0664,6.7781,78.2899,614.3554,28.4,857.9676,243.6122,0.0338,1.6116,7.28,59.443,66.9312,7137.673,17.9631,59.6169,7051.1656,0,2,64,512,38,64,448,0.28125,18,6.875,440,0.03125,2.0,0.09375,6.0,0.03125,2.0,,,,,,,0.09375,6.0,18.2883,37.3028,,,,,,,,,,2
|
||||
job_485099,0.0,0,64,512,2,1.0156,1,1.0,282.0371,3787.6053,1.0,9828,1,1115.4254,0.0742,0.1271,0.1271,-0.0001,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0101,-0.0065,0.0,1.0,0.0,0.003,0.2656,0.0742,7.0157,78.3694,616.362,28.2,857.9676,241.6056,0.0345,1.5826,6.4797,60.6694,67.4008,614.3507,19.0175,60.8861,526.3485,0,3,64,512,7,64,452,0.6875,44,7.0,448,0.015625,1.0,0.125,8.0,,,,,,,0.015625,1.0,0.046875,3.0,,,,,,,,,,,,3
|
||||
job_485099,0.0,0,64,512,2,1.2031,0,0.6094,64.5625,4650.1429,1.0,10031,1,636.5514,0.0137,0.0703,0.0703,-0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0018,-0.002,0.0,1.0,0.0,0.0028,0.0938,0.0137,7.3373,78.4666,619.0092,27.9,857.9676,238.9584,0.0351,1.6766,4.2416,48.7129,53.1303,7011.6209,18.0381,48.8532,6938.7746,0,4,64,512,59,64,403,0.671875,43,6.203125,397,0.015625,1.0,0.09375,6.0,0.015625,1.0,,,,,0.015625,1.0,0.0625,4.0,0.0088,18.0874,,,,,,,,,,4
|
||||
job_485099,0.0,0,64,512,1,1.0,1,1.0,278.7383,2411.2034,1.0,4957,1,815.5083,0.1152,0.0935,0.0935,-0.0001,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0131,-0.0088,0.0,1.0,0.0,0.0065,0.2344,0.1152,7.5183,78.4538,621.5787,27.6,857.9676,236.3889,0.0216,1.0094,6.7753,55.4016,62.3349,734.6954,18.4983,55.5375,652.851,0,5,30,240,9,30,205,0.6666666666666666,20,6.733333333333333,202,0.06666666666666667,2.0,0.1,3.0,,,,,,,0.1,3.0,,,,,10.5996,0.03333333333333333,1.0,,,,,,,5
|
||||
job_485100,0.0,0,64,512,1,0.0938,0,0.0938,412.1523,2807.8,1.0,7118,1,1116.6463,0.1465,0.2626,0.2628,-0.0005,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.017,-0.0348,0.0,1.0,0.0,0.0117,0.3276,0.1465,5.5805,78.1037,621.3371,27.6,857.9687,236.6316,0.0318,1.4004,10.3685,66.7561,77.3071,8132.7122,17.8846,66.9061,8036.1198,0,5,64,512,17,64,376,1.6875,108,5.828125,373,0.046875,3.0,0.125,8.0,0.046875,3.0,,,,,,,,,,,6.9408,,,,,,,,,5
|
||||
job_485100,0.0,0,64,512,1,0.9844,0,0.9844,122.9082,3672.5882,1.0,7301,1,700.1298,0.0332,0.4107,0.411,-0.0003,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0043,-0.016,0.0,1.0,0.0,0.0091,0.1562,0.0332,6.132,78.1099,617.5232,28.0,857.9687,240.4455,0.0258,1.1294,4.1778,49.7199,54.0518,3465.0549,19.1569,49.8478,3390.7157,0,6,64,512,12,64,390,1.53125,98,6.078125,389,0.015625,1.0,0.171875,11.0,,,,,,,,,0.015625,1.0,14.3648,22.1469,,,,,,,,,,6
|
||||
job_485100,0.0,0,64,512,2,0.1719,0,0.1406,81.3242,3428.1667,1.0,8024,1,580.4928,0.0234,0.4151,0.4154,-0.0002,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.003,-0.0109,0.0,1.0,0.0,0.0071,0.1406,0.0234,6.3622,78.2129,616.5064,28.1,857.9687,241.4623,0.022,1.147,4.0974,47.8918,52.1402,3987.006,18.7017,48.0205,3915.0159,0,7,64,512,9,64,412,1.3125,84,6.421875,411,0.015625,1.0,0.09375,6.0,0.015625,1.0,,,,,,,,,,,,,,,,,,,,7
|
||||
job_485100,0.0,0,64,512,2,0.7969,0,0.7656,53.543,2990.1111,1.0,4963,1,409.6074,0.0176,0.401,0.4013,-0.0001,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.002,-0.0074,0.0,1.0,0.0,0.0064,0.0938,0.0176,6.5256,78.4167,614.7823,28.3,857.9687,243.1863,0.0175,0.8755,3.0976,44.8012,48.0435,6656.3028,28.5894,44.928,6578.7926,0,8,64,512,12,64,429,1.0625,68,6.640625,425,0.015625,1.0,0.03125,2.0,0.03125,2.0,,,,,,,0.03125,2.0,0.0078,44.8581,,,,,,,,,,8
|
||||
job_485100,0.0,0,64,512,2,1.0156,1,1.0,83.5977,3525.1667,1.0,7387,1,595.6285,0.0234,0.2611,0.2612,-0.0001,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0032,-0.0059,0.0,1.0,0.0,0.0055,0.125,0.0234,6.8361,78.5227,617.8079,28.0,857.9687,240.1608,0.0187,1.2451,4.908,48.4858,53.5378,1287.2347,18.8851,48.6107,1213.5661,0,9,64,512,42,64,369,1.515625,97,5.71875,366,0.03125,2.0,0.03125,2.0,,,,,,,,,0.046875,3.0,,,,,,,,,,,,9
|
||||
job_485100,0.0,0,64,512,2,0.9688,0,0.5,318.5254,3871.7857,1.0,12427,1,1252.0624,0.082,0.403,0.4032,-0.0005,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0093,-0.0298,0.0,1.0,0.0,0.0096,0.2344,0.082,8.9127,78.948,613.9305,28.4,857.9687,244.0382,0.0353,1.8739,8.9165,64.647,73.728,6403.6413,18.6729,64.7757,6309.3653,0,10,64,512,40,64,407,0.875,56,6.359375,407,,,0.140625,9.0,,,,,,,,,,,0.0087,8.1786,6.0594,,,,,,,,,10
|
||||
job_485100,0.0,0,64,512,3,1.0312,1,1.0,350.1328,4469.9,1.0,31001,1,1779.2004,0.0781,0.213,0.2131,-0.0002,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.008,-0.015,0.0,1.0,0.0,0.007,0.1719,0.0781,14.811,79.299,605.6735,29.4,857.9687,252.2952,0.0625,4.1888,10.3455,76.4064,86.9738,1527.2608,18.4561,76.5655,1417.6404,0,11,64,512,41,64,376,1.34375,86,5.828125,373,0.03125,2.0,0.109375,7.0,0.015625,1.0,,,,,,,0.03125,2.0,,,,,,,,,,,,11
|
||||
job_485100,0.0,0,64,512,2,0.8125,0,0.4844,282.1934,3512.4878,1.0,12815,1,1173.8704,0.0801,0.2652,0.2654,-0.0004,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0088,-0.0229,0.0,1.0,0.0,0.0105,0.25,0.0801,14.8054,78.8495,604.8915,29.5,857.9687,253.0771,0.0379,1.7768,6.4211,60.7211,67.3241,6140.8745,18.7329,60.8648,6053.0375,0,12,64,512,33,64,421,0.734375,47,6.546875,419,0.046875,3.0,0.125,8.0,,,,,,,,,0.03125,2.0,0.008,7.973,,,,,,,,,,12
|
||||
job_485100,0.0,0,64,512,3,1.1875,1,1.0,217.332,3357.4242,1.0,6149,1,875.1285,0.0645,0.1964,0.1965,-0.0002,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0069,-0.0111,0.0,1.0,0.0,0.0056,0.2344,0.0645,15.0007,78.8141,604.7684,29.5,857.9687,253.2003,0.0234,0.9449,4.7579,55.747,60.6653,2012.3998,19.4214,55.8837,1931.3665,0,13,19,152,22,18,102,1.368421052631579,26,5.368421052631579,102,,,0.10526315789473684,2.0,,,,,,,,,,,,,,,,,,,,,,13
|
||||
job_485101,0.0,0,64,512,1,0.0312,0,0.0312,331.7969,2689.381,1.0,5793,1,965.9963,0.123,0.2187,0.2189,-0.0004,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0119,-0.0249,0.0,1.0,0.0,0.0124,0.2742,0.123,5.3837,78.0531,602.3064,29.8,857.9687,255.6623,0.1194,1.0144,12.9162,57.498,70.6558,8113.5248,18.1367,57.6196,8023.7177,0,13,64,512,19,64,381,1.5625,100,5.890625,377,0.015625,1.0,0.171875,11.0,0.015625,1.0,,,,,,,0.046875,3.0,,,,,,,,,,,,13
|
||||
job_485101,0.0,0,64,512,1,0.9062,0,0.9062,97.1699,2592.5263,1.0,5692,1,535.645,0.0371,0.4498,0.4501,-0.0002,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0041,-0.0146,0.0,1.0,0.0,0.0075,0.125,0.0371,5.7223,78.1088,599.2325,30.2,857.9687,258.7362,0.0198,0.9137,4.3069,44.4851,48.9503,6886.3196,18.478,44.6233,6817.976,0,14,64,512,61,64,351,1.34375,86,5.25,336,,,0.21875,14.0,0.046875,3.0,,,,,,,0.1875,12.0,24.0683,41.284,,,,,,,,,,14
|
||||
job_485101,0.0,0,64,512,2,1.0469,1,1.0,393.2031,3292.9344,1.0,10838,1,1198.2489,0.1191,0.2169,0.2171,-0.0005,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0124,-0.0307,0.0,1.0,0.0,0.0123,0.375,0.1191,7.4045,78.2452,598.44,30.2,857.9687,259.5287,0.0315,1.6347,7.3081,63.1214,70.5896,791.9402,18.239,63.2496,701.4743,0,15,64,512,61,63,356,1.296875,83,5.53125,354,,,0.1875,12.0,0.015625,1.0,,,,,,,0.015625,1.0,,,6.9084,,,,,,,,,15
|
||||
job_485101,0.0,0,64,512,2,1.5625,0,0.7812,346.5215,2901.1148,1.0,13177,1,1177.6427,0.1191,0.2065,0.2066,-0.0003,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0113,-0.0216,0.0,1.0,0.0,0.0089,0.3125,0.1191,8.5541,78.4775,596.4595,30.5,857.9687,261.5092,0.0308,1.904,7.6796,61.4263,69.2639,6575.7601,17.4986,61.5532,6487.0923,0,16,64,512,47,64,359,1.515625,97,5.53125,354,0.0625,4.0,0.078125,5.0,,,,,,,,,0.078125,5.0,0.0085,7.7753,,,,,,,,,,16
|
||||
job_485101,0.0,0,64,512,3,1.0625,1,1.0,314.8184,3419.617,1.0,7961,1,1092.2585,0.0918,0.359,0.3591,-0.0005,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0084,-0.0295,0.0,1.0,0.0,0.0108,0.25,0.0918,8.6535,78.4467,598.0707,30.3,857.9687,259.898,0.0216,1.2139,6.2721,60.5518,66.9689,920.1572,17.1829,60.6749,834.7893,0,17,64,512,36,64,404,1.0625,68,6.265625,401,0.03125,2.0,0.03125,2.0,0.015625,1.0,,,,,,,0.03125,2.0,,,,,,,,,,,,17
|
||||
job_485101,0.0,0,64,512,2,1.2031,0,0.6094,245.1484,3473.3333,1.0,11370,1,999.9222,0.0703,0.1244,0.1245,-0.0002,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0063,-0.0109,0.0,1.0,0.0,0.0061,0.2344,0.0703,9.0319,78.7547,600.501,30.0,857.9687,257.4677,0.029,1.7294,5.1895,52.9781,58.3217,6610.6145,17.2028,53.1029,6533.3592,0,18,64,512,53,63,421,0.5,32,6.484375,415,0.015625,1.0,0.078125,5.0,0.03125,2.0,,,,,,,0.0625,4.0,0.0079,7.9661,,,,,,,,,,18
|
||||
job_485101,0.0,0,64,512,1,1.0,1,1.0,321.1836,3094.0943,1.0,8352,1,1061.5795,0.1035,0.0722,0.0722,-0.0001,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0086,-0.007,0.0,1.0,0.0,0.0041,0.2812,0.1035,9.3094,78.7746,599.7432,30.1,857.9687,258.2255,0.0369,1.3938,6.5872,58.4137,65.1632,1357.7171,17.8468,58.5387,1273.3116,0,19,25,200,21,25,158,0.68,17,6.32,158,,,0.16,4.0,,,,,,,,,,,,,,,,,,,,,,19
|
||||
job_496420,0.0,0,64,512,1,0.0781,0,0.0781,653.5195,2983.9464,1.0,5734,1,1308.6167,0.2188,0.1993,0.1994,-0.0007,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0182,-0.0471,0.0,1.0,0.0,0.0159,0.4407,0.2188,5.3781,78.0601,578.2307,32.6,857.9687,279.738,0.1498,1.2614,15.3459,70.951,86.6201,8067.2067,17.8581,71.1238,7961.4669,0,19,64,512,41,63,350,1.765625,113,5.46875,350,0.015625,1.0,0.109375,7.0,,,,,,,,,,,,,,,,,,,,,,19
|
||||
job_496420,0.0,0,64,512,1,0.9844,0,0.9844,239.9746,2985.2683,1.0,7404,1,907.8881,0.0801,0.189,0.1892,-0.0003,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0065,-0.0203,0.0,1.0,0.0,0.0128,0.2031,0.0801,6.1016,78.1429,575.9431,32.9,857.9687,282.0256,0.0214,1.1741,6.3743,55.4754,61.9992,3380.5934,20.0199,55.6031,3297.3986,0,20,64,512,51,64,338,1.65625,106,5.25,336,0.015625,1.0,0.25,16.0,0.015625,1.0,,,,,,,0.015625,1.0,14.3752,21.9845,6.5064,,,,,,,,,20
|
||||
job_496420,0.0,0,64,512,2,0.5156,0,0.3906,315.1094,3154.4118,1.0,9754,1,1083.8644,0.0996,0.2996,0.2998,-0.0005,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.008,-0.032,0.0,1.0,0.0,0.0127,0.3281,0.0996,7.1946,78.3005,574.6356,33.0,857.9687,283.3331,0.028,1.3808,8.0318,63.7157,71.9046,4118.8329,18.1954,63.8445,4027.3507,0,21,64,512,36,64,375,1.375,88,5.8125,372,,,0.203125,13.0,,,,,,,0.015625,1.0,0.03125,2.0,,,,,,,,,,,,21
|
||||
job_496420,0.0,0,64,512,2,0.9375,0,0.8125,291.4688,4132.1111,1.0,9690,1,1175.4698,0.0703,0.3601,0.3603,-0.0004,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0069,-0.0247,0.0,1.0,0.0,0.0179,0.25,0.0703,7.7303,78.4548,578.4616,32.6,857.9687,279.5071,0.0229,1.429,6.5012,63.062,69.7119,5101.1025,18.1771,63.1874,5011.7831,0,22,64,512,48,64,397,0.921875,59,6.09375,390,,,0.125,8.0,,,,,,,0.03125,2.0,0.078125,5.0,0.0075,8.179,,,,,,,,,,22
|
||||
job_496420,0.0,0,64,512,2,1.0469,0,0.9375,275.9121,2933.3958,1.0,6046,1,913.748,0.0938,0.2557,0.2558,-0.0003,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0063,-0.0223,0.0,1.0,0.0,0.0135,0.2656,0.0938,8.0687,78.5605,580.3864,32.4,857.9687,277.5823,0.0178,0.9376,6.7391,56.9606,63.8352,2467.0192,19.0252,57.0779,2383.2198,0,23,64,512,40,63,411,0.671875,43,6.40625,410,0.015625,1.0,0.265625,17.0,,,,,,,,,0.015625,1.0,,,,,,,,,,,,23
|
||||
job_496420,0.0,0,64,512,2,1.0938,0,0.6562,288.5996,3682.275,1.0,12853,1,1174.4942,0.0781,0.0673,0.0674,-0.0001,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0057,-0.0049,0.0,1.0,0.0,0.0038,0.2344,0.0781,9.5992,78.8051,579.6128,32.4,857.9687,278.3559,0.0326,1.8242,7.2937,60.7161,68.1634,6306.534,19.4669,60.8368,6217.079,0,24,65,520,28,65,417,0.9076923076923077,59,6.3538461538461535,413,,,0.18461538461538463,12.0,,,,,,,0.015384615384615385,1.0,0.03076923076923077,2.0,0.0078,7.3895,,0.06153846153846154,4.0,0.015384615384615385,1.0,,,,,24
|
||||
job_496420,0.0,0,64,512,1,1.0,1,1.0,246.5586,4491.2143,1.0,12979,1,1246.6552,0.0547,0.2752,0.2754,-0.0002,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0042,-0.0133,0.0,1.0,0.0,0.0082,0.2188,0.0547,11.0213,91.3057,577.4506,32.7,857.9687,280.5181,0.0334,187.2975,10.0277,62.1765,72.3976,4522.0533,18.6719,62.3361,4243.6845,0,25,69,552,29,68,458,0.7391304347826086,51,6.594202898550725,455,0.014492753623188406,1.0,0.13043478260869565,9.0,,,,,,,0.043478260869565216,3.0,,,,,5.8497,0.028985507246376812,2.0,,,0.014492753623188406,1.0,0.014492753623188406,1.0,25
|
||||
job_496420,0.0,0,64,512,2,1.0938,0,0.6406,164.248,3096.6667,1.0,12519,1,866.242,0.0527,0.0251,0.0252,-0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0035,-0.0026,0.0,1.0,0.0,0.0064,0.1875,0.0527,12.0359,94.4252,576.9232,32.8,857.9687,281.0455,0.0497,1.8339,5.7861,53.1165,59.0761,6927.7034,19.1025,53.24,6847.6862,0,26,10,80,1,10,74,0.5,5,7.4,74,,,,,,,,,,,,,,,0.0131,8.0356,,,,,,,,,,26
|
||||
job_496421,0.0,0,64,512,3,0.2656,0,0.2344,621.6914,2563.8548,1.0,10562,1,1338.5955,0.2422,0.2196,0.2197,-0.0007,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0148,-0.0466,0.0,1.0,0.0,0.0201,0.4898,0.2422,7.1583,79.2762,627.8784,26.8,857.9687,230.0903,0.0496,1.7229,14.7315,75.4634,90.3725,8113.4798,18.0581,75.5904,8003.326,0,27,64,512,30,64,383,1.34375,86,5.96875,382,0.03125,2.0,0.171875,11.0,,,,,,,,,0.015625,1.0,,,,,,,,,,,,27
|
||||
job_496421,0.0,0,64,512,1,1.0,1,1.0,180.2852,3060.8,1.0,7383,1,805.1102,0.0586,0.4541,0.4545,-0.0005,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0039,-0.0311,0.0,1.0,0.0,0.0178,0.25,0.0586,7.4612,79.269,624.4182,27.2,857.9687,233.5505,0.0232,1.1905,4.8432,55.2708,60.2711,351.8545,18.7774,55.4044,271.6141,0,28,64,512,62,63,357,1.328125,85,5.53125,354,,,0.125,8.0,,,,,,,,,0.046875,3.0,8.3981,16.1295,,,,,,,,,,28
|
||||
job_496421,0.0,0,64,512,2,0.6562,0,0.3281,337.166,2777.0806,1.0,8959,1,1036.1541,0.1211,0.1954,0.1956,-0.0004,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0066,-0.0229,0.0,1.0,0.0,0.0141,0.2656,0.1211,7.748,78.3633,621.6158,27.5,857.9687,236.3529,0.0248,1.4652,6.745,58.8652,65.844,7096.5807,19.6954,59.074,7009.5751,0,29,64,512,27,64,394,1.109375,71,6.109375,391,0.015625,1.0,0.296875,19.0,,,,,,,,,0.046875,3.0,,,,,,,,,,,,29
|
||||
job_496421,0.0,0,64,512,3,1.0312,0,0.9844,217.6387,4109.1111,1.0,6682,1,963.5862,0.0527,0.0823,0.0823,-0.0001,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0039,-0.0059,0.0,1.0,0.0,0.0063,0.1875,0.0527,8.0195,78.3764,619.7124,27.8,857.9687,238.2563,0.0198,1.1139,4.882,56.4058,61.4598,2113.6889,19.3704,56.5577,2031.7437,0,30,64,512,86,62,347,0.859375,55,5.40625,346,0.015625,1.0,0.359375,23.0,,,,,,,,,0.015625,1.0,0.0098,7.9489,6.8782,,,,,,,,,30
|
||||
job_496421,0.0,0,64,512,2,0.6562,0,0.4531,504.793,3000.3256,1.0,11279,1,1347.108,0.168,0.1493,0.1494,-0.0004,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0084,-0.0258,0.0,1.0,0.0,0.0125,0.375,0.168,8.424,78.5074,623.5744,27.3,857.9687,234.3943,0.0293,1.8483,9.6408,69.6522,79.4454,5328.8946,18.0982,69.775,5229.501,0,31,64,512,53,63,402,0.59375,38,6.25,400,0.015625,1.0,0.28125,18.0,0.015625,1.0,,,,,0.015625,1.0,,,,,,,,,,,,,,31
|
||||
job_496421,0.0,0,64,512,2,1.0625,0,0.9219,344.2148,3316.5849,1.0,7996,1,1130.5575,0.1035,0.1403,0.1404,-0.0002,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0058,-0.0129,0.0,1.0,0.0,0.0071,0.2031,0.1035,8.5488,78.4227,623.5724,27.3,857.9687,234.3962,0.0352,1.2686,7.9241,64.4317,72.5126,3341.1264,18.3456,64.5529,3248.9975,0,32,64,512,91,61,363,0.609375,39,5.671875,363,,,0.296875,19.0,,,,,,,,,,,0.0095,8.5021,,,,,,,,,,32
|
||||
job_496421,0.0,0,64,512,2,0.6406,0,0.4844,463.8789,2605.3297,1.0,12112,1,1252.9052,0.1777,0.1535,0.1536,-0.0004,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0086,-0.0228,0.0,1.0,0.0,0.0137,0.3438,0.1777,9.5261,78.5966,620.8518,27.6,857.9687,237.1169,0.0312,1.9241,9.2013,67.0619,76.4287,4248.8256,18.1481,67.1959,4152.3235,0,33,64,512,47,62,386,0.828125,53,5.984375,383,0.015625,1.0,0.390625,25.0,,,,,,,0.03125,2.0,0.015625,1.0,,,,,,,,,,,,33
|
||||
job_496421,0.0,0,64,512,2,0.9531,0,0.8438,253.4688,2751.2979,1.0,5515,1,896.1226,0.0918,0.1853,0.1854,-0.0002,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0046,-0.0141,0.0,1.0,0.0,0.0094,0.1875,0.0918,9.5989,78.382,619.3303,27.8,857.9687,238.6384,0.0193,1.0688,6.365,56.4504,62.9483,3744.8018,19.1478,56.5637,3661.6347,0,34,64,512,58,63,393,0.734375,47,6.140625,393,,,0.21875,14.0,,,,,,,,,,,0.0091,7.552,,,,,,,,,,34
|
||||
job_496421,0.0,0,64,512,2,0.6562,0,0.4844,274.5176,2415.5,1.0,6569,1,839.2308,0.1133,0.1238,0.1239,-0.0002,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0051,-0.014,0.0,1.0,0.0,0.0095,0.2188,0.1133,9.9454,78.6008,617.927,28.0,857.9687,240.0417,0.0192,3.6935,5.8429,55.357,61.337,3951.6497,18.6576,55.4746,3867.9608,0,35,64,512,93,59,356,0.5625,36,5.5625,356,,,0.421875,27.0,,,,,,,,,,,,,6.1902,,,,,,,,,35
|
||||
job_496421,0.0,0,64,512,2,1.0156,0,0.7812,376.7695,2069.7527,1.0,4538,1,846.2868,0.1816,0.1047,0.1048,-0.0003,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0082,-0.0196,0.0,1.0,0.0,0.0177,0.2969,0.1816,9.9318,78.4766,616.9151,28.1,857.9687,241.0536,0.018,0.8756,7.5627,54.475,62.1755,4009.2705,18.3501,54.5945,3927.8682,0,36,2,16,0,2,15,0.5,1,7.5,15,,,,,,,,,,,,,,,0.0083,7.8228,,,,,,,,,,36
|
||||
job_496422,0.0,0,64,512,1,0.0156,0,0.0156,513.5391,2625.2,1.0,26236,1,1588.8893,0.1953,0.1608,0.1609,-0.0005,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0083,-0.0291,0.0,1.0,0.0,0.0178,0.4444,0.1953,12.0642,78.6154,606.7097,29.3,857.9687,251.259,0.1923,3.5931,15.4184,73.4268,89.2021,8145.718,17.602,73.591,8035.3206,0,37,64,512,40,63,367,1.46875,94,5.734375,367,0.015625,1.0,0.15625,10.0,,,,,,,,,,,,,,,,,,,,,,37
|
||||
job_496422,0.0,0,64,512,1,0.9844,0,0.9844,256.4102,3270.25,1.0,8312,1,1017.874,0.0781,0.3646,0.3647,-0.0003,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0034,-0.0217,0.0,1.0,0.0,0.0125,0.2344,0.0781,12.0077,78.1334,605.2178,29.5,857.9687,252.7509,0.0239,1.3387,7.0582,59.3834,66.5873,3328.8664,18.4202,59.5048,3242.5172,0,38,64,512,86,64,312,1.0,64,4.828125,309,0.09375,6.0,0.5625,36.0,0.015625,1.0,,,,,0.03125,2.0,,,11.4426,19.0624,,0.078125,5.0,,,0.046875,3.0,,,38
|
||||
job_496422,0.0,0,64,512,2,0.7656,0,0.5312,408.6738,2428.0814,1.0,8101,1,1031.8913,0.168,0.3035,0.3037,-0.0008,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0079,-0.0498,0.0,1.0,0.0,0.0244,0.4375,0.168,13.3047,91.9354,606.3735,29.3,857.9687,251.5952,0.0241,1.2992,6.9572,60.9637,68.0737,6397.0696,17.4018,61.092,6310.2938,0,39,68,544,61,67,381,1.1764705882352942,80,5.573529411764706,379,,,0.25,17.0,0.014705882352941176,1.0,,,,,0.014705882352941176,1.0,,,,,,0.014705882352941176,1.0,,,0.058823529411764705,4.0,,,39
|
||||
job_496422,0.0,0,64,512,3,1.2344,0,0.9688,383.9883,3215.5902,1.0,9054,1,1172.8826,0.1191,0.181,0.1811,-0.0004,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0055,-0.0256,0.0,1.0,0.0,0.0197,0.3281,0.1191,14.0406,92.0868,605.7397,29.4,857.9687,252.229,0.0354,1.6468,7.3746,62.4563,69.9967,5420.0066,17.7311,62.5862,5330.6236,0,40,64,512,64,64,399,0.546875,35,6.234375,399,0.015625,1.0,0.203125,13.0,,,,,,,,,,,0.0079,8.9118,6.6659,,,,,,,,,40
|
||||
job_496422,0.0,0,64,512,4,1.0781,0,0.8281,315.2949,2596.4677,1.0,6019,1,911.6647,0.1211,0.075,0.0751,-0.0002,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.006,-0.0107,0.0,1.0,0.0,0.01,0.2812,0.1211,14.3649,92.0377,605.8729,29.4,857.9687,252.0958,0.0196,1.2515,6.8124,57.2361,64.2337,2226.1873,17.7717,57.4013,2142.928,0,41,60,480,76,57,352,0.55,33,5.866666666666666,352,0.016666666666666666,1.0,0.3,18.0,,,,,,,,,,,,,,,,,,,,,,41
|
||||
job_496422,0.0,0,64,512,2,1.1562,0,0.8594,350.4609,2294.8974,1.0,6096,1,918.4495,0.1523,0.156,0.1561,-0.0003,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.007,-0.0193,0.0,1.0,0.0,0.0108,0.3281,0.1523,16.0237,92.0465,604.1669,29.6,857.9687,253.8018,0.0281,1.007,6.8732,58.1443,65.1695,4730.9187,18.9548,58.2679,4645.7861,0,42,65,520,117,60,320,0.7846153846153846,51,4.923076923076923,320,0.03076923076923077,2.0,0.4461538461538462,29.0,,,,,,,,,,,0.0079,7.5503,,0.015384615384615385,1.0,,,,,,,42
|
||||
job_496422,0.0,0,64,512,3,1.1562,0,0.8594,506.7285,2214.1026,1.0,9784,1,1181.084,0.2285,0.1962,0.1963,-0.0006,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0097,-0.0374,0.0,1.0,0.0,0.02,0.4688,0.2285,16.5446,92.2373,608.1587,29.1,857.9687,249.81,0.0281,1.6023,8.6365,65.3044,74.2194,3828.8954,19.3517,65.5544,3733.7211,0,43,63,504,57,63,380,0.7619047619047619,48,6.0,378,0.015873015873015872,1.0,0.2857142857142857,18.0,,,,,,,0.031746031746031744,2.0,,,,,,,,,,,,,,43
|
||||
job_496422,0.0,0,64,512,3,1.2812,0,0.875,384.8477,3448.8947,1.0,11922,1,1321.23,0.1113,0.1286,0.1287,-0.0002,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0052,-0.0139,0.0,1.0,0.0,0.0125,0.2656,0.1113,16.9448,92.2662,606.6965,29.3,857.9687,251.2722,0.0323,1.7789,8.0712,63.9146,72.1439,4075.9496,19.0722,64.0401,3982.9522,0,44,67,536,61,66,393,0.7761194029850746,52,5.82089552238806,390,0.014925373134328358,1.0,0.43283582089552236,29.0,0.014925373134328358,1.0,,,,,,,0.029850746268656716,2.0,0.0083,7.3633,,,,,,,,,,44
|
||||
job_496422,0.0,0,64,512,3,1.0625,0,0.7344,289.7832,2550.2586,1.0,9971,1,1016.4986,0.1133,0.148,0.1481,-0.0003,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0047,-0.0191,0.0,1.0,0.0,0.0151,0.2344,0.1133,17.2552,92.2487,605.9093,29.4,857.9687,252.0594,0.0283,2.2306,6.986,57.0132,64.3289,4079.3376,19.0583,57.3142,3993.7175,0,45,18,144,27,18,97,0.6111111111111112,11,5.333333333333333,96,,,0.5,9.0,,,,,,,,,0.05555555555555555,1.0,,,5.9741,,,,,,,,,45
|
||||
|
3
training_logs/20260525_110326_reward_vs_steps.png
Normal file
3
training_logs/20260525_110326_reward_vs_steps.png
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:58750553735899d05538e28d6f0869b4786923821a5da304e20491be1567e596
|
||||
size 273045
|
||||
25856
training_logs/20260525_110326_trial_results.csv
Normal file
25856
training_logs/20260525_110326_trial_results.csv
Normal file
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:bd62c7da69eaa4a0cb732a3c09db902242d5780ee82dcff8ac5c702c1349e1ce
|
||||
size 146121
|
||||
6426
training_logs/20260525_110326_vllm_metrics_job_485099.csv
Normal file
6426
training_logs/20260525_110326_vllm_metrics_job_485099.csv
Normal file
File diff suppressed because it is too large
Load Diff
8483
training_logs/20260525_110326_vllm_metrics_job_485100.csv
Normal file
8483
training_logs/20260525_110326_vllm_metrics_job_485100.csv
Normal file
File diff suppressed because it is too large
Load Diff
7196
training_logs/20260525_110326_vllm_metrics_job_485101.csv
Normal file
7196
training_logs/20260525_110326_vllm_metrics_job_485101.csv
Normal file
File diff suppressed because it is too large
Load Diff
8439
training_logs/20260525_110326_vllm_metrics_job_496420.csv
Normal file
8439
training_logs/20260525_110326_vllm_metrics_job_496420.csv
Normal file
File diff suppressed because it is too large
Load Diff
8479
training_logs/20260525_110326_vllm_metrics_job_496421.csv
Normal file
8479
training_logs/20260525_110326_vllm_metrics_job_496421.csv
Normal file
File diff suppressed because it is too large
Load Diff
8392
training_logs/20260525_110326_vllm_metrics_job_496422.csv
Normal file
8392
training_logs/20260525_110326_vllm_metrics_job_496422.csv
Normal file
File diff suppressed because it is too large
Load Diff
47410
training_logs/20260525_110326_vllm_metrics_table.csv
Normal file
47410
training_logs/20260525_110326_vllm_metrics_table.csv
Normal file
File diff suppressed because it is too large
Load Diff
21106
training_logs/a3-rl-DCAgent_code-contests-noblock_485099.out
Normal file
21106
training_logs/a3-rl-DCAgent_code-contests-noblock_485099.out
Normal file
File diff suppressed because one or more lines are too long
20511
training_logs/a3-rl-DCAgent_code-contests-noblock_485100.out
Normal file
20511
training_logs/a3-rl-DCAgent_code-contests-noblock_485100.out
Normal file
File diff suppressed because one or more lines are too long
20710
training_logs/a3-rl-DCAgent_code-contests-noblock_485101.out
Normal file
20710
training_logs/a3-rl-DCAgent_code-contests-noblock_485101.out
Normal file
File diff suppressed because one or more lines are too long
1630
training_logs/a3-rl-DCAgent_code-contests-noblock_485102.out
Normal file
1630
training_logs/a3-rl-DCAgent_code-contests-noblock_485102.out
Normal file
File diff suppressed because one or more lines are too long
111
training_logs/a3-rl-DCAgent_code-contests-noblock_485103.out
Normal file
111
training_logs/a3-rl-DCAgent_code-contests-noblock_485103.out
Normal file
@@ -0,0 +1,111 @@
|
||||
|
||||
The following have been reloaded with a version change:
|
||||
1) GCCcore/.14.3.0 => GCCcore/14.3.0
|
||||
|
||||
|
||||
Lmod is automatically replacing "GCC/14.3.0" with
|
||||
"nvidia-compilers/25.9-CUDA-13".
|
||||
|
||||
Deactivating conda environment: /e/scratch/jureap59/feuer1/miniforge3/envs/otagent
|
||||
Activating RL environment: /e/scratch/jureap59/feuer1/OpenThoughts-Agent/envs/rl
|
||||
Python executable: /e/scratch/jureap59/feuer1/OpenThoughts-Agent/envs/rl/bin/python
|
||||
Python path check: /e/scratch/jureap59/feuer1/OpenThoughts-Agent/envs/rl/bin/python
|
||||
[ray] RAY_TMPDIR=/tmp/ray/ray_485103
|
||||
[triton_cache] Triton cache: /tmp/triton_cache_feuer1_485103
|
||||
[triton_cache] TorchInductor cache: /tmp/torchinductor_cache_feuer1_485103
|
||||
[proxy] ✓ Found proxychains binary at /e/scratch/jureap59/feuer1/proxychains-ng-aarch64/bin/proxychains4
|
||||
[proxy] Setting up SSH tunnel to jpbl-s01-01
|
||||
[proxy] SSH key: /e/home/jusers/feuer1/jupiter/.ssh/authorized_keys/id_ed25519_jsc
|
||||
[proxy] Tunnel port: 7003
|
||||
[proxy] Node IP: 10.128.32.11 (workers will connect here)
|
||||
[proxy] ✓ SSH tunnel started successfully
|
||||
[proxy] ✓ Generated proxychains config at /e/home/jusers/feuer1/jupiter/.proxychains/proxychains_485103.conf
|
||||
[proxy] - Internal traffic (10.x.x.x, 172.x.x.x, 169.254.x.x) → DIRECT
|
||||
[proxy] - External traffic (internet) → PROXY via tunnel
|
||||
[proxy] ✓ Daytona timeout settings configured
|
||||
[proxy] Testing proxy connectivity...
|
||||
[proxychains] config file found: /e/home/jusers/feuer1/jupiter/.proxychains/proxychains_485103.conf
|
||||
[proxychains] preloading /e/scratch/jureap59/feuer1/proxychains-ng-aarch64/lib/libproxychains4.so
|
||||
[proxychains] DLL init: proxychains-ng 4.17-git-9-g78ead0e
|
||||
[proxy] ✓ Proxy connectivity test passed (huggingface.co reachable via wrapped binary)
|
||||
[proxy] ⚠ Tunnel not accessible at 10.128.32.11:7003 (workers may fail)
|
||||
[proxy] ✓ Proxy setup complete (using wrapped binary for Ray workers)
|
||||
[container_runtime] Using cloud backend: daytona (no local container setup)
|
||||
=== Universal RL Training Runner ===
|
||||
Config: /e/data1/datasets/playground/ot-baf/a3-rl-DCAgent_code-contests-noblock/configs/a3-rl-DCAgent_code-contests-noblock_rl_config.json
|
||||
Working directory: /e/scratch/jureap59/feuer1/OpenThoughts-Agent
|
||||
Python: /e/scratch/jureap59/feuer1/OpenThoughts-Agent/envs/rl/bin/python
|
||||
Python version: Python 3.12.12
|
||||
Proxy: DISABLED (direct internet or not configured)
|
||||
========================================
|
||||
=== RLJobRunner: a3-rl-DCAgent_code-contests-noblock ===
|
||||
[wandb_utils] Fixing permissions on: /e/data1/datasets/playground/ot-baf/a3-rl-DCAgent_code-contests-noblock/wandb
|
||||
[wandb_utils] WandB directory ready: /e/data1/datasets/playground/ot-baf/a3-rl-DCAgent_code-contests-noblock/wandb
|
||||
HF_TOKEN=****pDbg
|
||||
HF_HUB_CACHE=/e/data1/datasets/playground/ot-baf/hf_hub
|
||||
SUPABASE_URL=https://rpzmyuapoqilpghynmza.s... (direct Supabase config)
|
||||
Environment configured:
|
||||
TENSOR_PARALLEL_SIZE=1
|
||||
NUM_INFERENCE_ENGINES=56
|
||||
POLICY_NUM_NODES=14
|
||||
WANDB_DIR=/e/data1/datasets/playground/ot-baf/a3-rl-DCAgent_code-contests-noblock/wandb
|
||||
Starting Ray cluster with 14 nodes, 4 GPUs/node
|
||||
Cleaning up existing Ray instances...
|
||||
=== Starting Ray Cluster ===
|
||||
Nodes: 14
|
||||
GPUs per node: 4
|
||||
CPUs per node: 288
|
||||
Head node: jpbo-041-11 (10.128.32.11)
|
||||
Ray port: 6379
|
||||
============================
|
||||
RL job failed: [Errno 122] Disk quota exceeded: '/e/scratch/jureap59/feuer1/OpenThoughts-Agent/experiments/logs/ray_head_jpbo-041-11.log'
|
||||
Traceback (most recent call last):
|
||||
File "/e/scratch/jureap59/feuer1/OpenThoughts-Agent/hpc/rl_launch_utils.py", line 922, in run
|
||||
training_exit_code = self._run_with_ray()
|
||||
^^^^^^^^^^^^^^^^^^^^
|
||||
File "/e/scratch/jureap59/feuer1/OpenThoughts-Agent/hpc/rl_launch_utils.py", line 1100, in _run_with_ray
|
||||
with RayCluster.from_slurm(ray_cfg) as ray_cluster:
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/e/scratch/jureap59/feuer1/OpenThoughts-Agent/hpc/ray_utils.py", line 818, in __enter__
|
||||
self.start()
|
||||
File "/e/scratch/jureap59/feuer1/OpenThoughts-Agent/hpc/ray_utils.py", line 342, in start
|
||||
self._start_node(self.node_list[0], is_head=True)
|
||||
File "/e/scratch/jureap59/feuer1/OpenThoughts-Agent/hpc/ray_utils.py", line 520, in _start_node
|
||||
ray_log_file = open(ray_log_path, "w")
|
||||
^^^^^^^^^^^^^^^^^^^^^^^
|
||||
OSError: [Errno 122] Disk quota exceeded: '/e/scratch/jureap59/feuer1/OpenThoughts-Agent/experiments/logs/ray_head_jpbo-041-11.log'
|
||||
[RLJobRunner] Launching trace upload (training exit code: 1):
|
||||
repo_id: DCAgent/a3-rl-DCAgent_code-contests-noblock
|
||||
job_dir: /e/data1/datasets/playground/ot-baf/a3-rl-DCAgent_code-contests-noblock/a3-rl-DCAgent_code-contests-noblock
|
||||
episodes: last
|
||||
log: /e/data1/datasets/playground/ot-baf/a3-rl-DCAgent_code-contests-noblock/logs/a3-rl-DCAgent_code-contests-noblock_trace_upload.log
|
||||
[RLJobRunner] Waiting for trace upload to complete...
|
||||
[RLJobRunner] Trace upload failed with exit code 1.
|
||||
Preserving Ray logs to /e/data1/datasets/playground/ot-baf/a3-rl-DCAgent_code-contests-noblock/ray_logs/
|
||||
Collecting Ray logs from worker jpbo-051-33...
|
||||
WARNING: Failed to collect logs from jpbo-051-33
|
||||
Collecting Ray logs from worker jpbo-051-34...
|
||||
WARNING: Failed to collect logs from jpbo-051-34
|
||||
Collecting Ray logs from worker jpbo-051-35...
|
||||
WARNING: Failed to collect logs from jpbo-051-35
|
||||
Collecting Ray logs from worker jpbo-051-36...
|
||||
WARNING: Failed to collect logs from jpbo-051-36
|
||||
Collecting Ray logs from worker jpbo-051-37...
|
||||
WARNING: Failed to collect logs from jpbo-051-37
|
||||
Collecting Ray logs from worker jpbo-051-38...
|
||||
WARNING: Failed to collect logs from jpbo-051-38
|
||||
Collecting Ray logs from worker jpbo-051-39...
|
||||
WARNING: Failed to collect logs from jpbo-051-39
|
||||
Collecting Ray logs from worker jpbo-051-40...
|
||||
WARNING: Failed to collect logs from jpbo-051-40
|
||||
Collecting Ray logs from worker jpbo-060-46...
|
||||
WARNING: Failed to collect logs from jpbo-060-46
|
||||
Collecting Ray logs from worker jpbo-066-09...
|
||||
WARNING: Failed to collect logs from jpbo-066-09
|
||||
Collecting Ray logs from worker jpbo-074-48...
|
||||
WARNING: Failed to collect logs from jpbo-074-48
|
||||
Collecting Ray logs from worker jpbo-089-02...
|
||||
WARNING: Failed to collect logs from jpbo-089-02
|
||||
Collecting Ray logs from worker jpbo-090-42...
|
||||
WARNING: Failed to collect logs from jpbo-090-42
|
||||
Ray log preservation complete
|
||||
111
training_logs/a3-rl-DCAgent_code-contests-noblock_485104.out
Normal file
111
training_logs/a3-rl-DCAgent_code-contests-noblock_485104.out
Normal file
@@ -0,0 +1,111 @@
|
||||
|
||||
The following have been reloaded with a version change:
|
||||
1) GCCcore/.14.3.0 => GCCcore/14.3.0
|
||||
|
||||
|
||||
Lmod is automatically replacing "GCC/14.3.0" with
|
||||
"nvidia-compilers/25.9-CUDA-13".
|
||||
|
||||
Deactivating conda environment: /e/scratch/jureap59/feuer1/miniforge3/envs/otagent
|
||||
Activating RL environment: /e/scratch/jureap59/feuer1/OpenThoughts-Agent/envs/rl
|
||||
Python executable: /e/scratch/jureap59/feuer1/OpenThoughts-Agent/envs/rl/bin/python
|
||||
Python path check: /e/scratch/jureap59/feuer1/OpenThoughts-Agent/envs/rl/bin/python
|
||||
[ray] RAY_TMPDIR=/tmp/ray/ray_485104
|
||||
[triton_cache] Triton cache: /tmp/triton_cache_feuer1_485104
|
||||
[triton_cache] TorchInductor cache: /tmp/torchinductor_cache_feuer1_485104
|
||||
[proxy] ✓ Found proxychains binary at /e/scratch/jureap59/feuer1/proxychains-ng-aarch64/bin/proxychains4
|
||||
[proxy] Setting up SSH tunnel to jpbl-s01-01
|
||||
[proxy] SSH key: /e/home/jusers/feuer1/jupiter/.ssh/authorized_keys/id_ed25519_jsc
|
||||
[proxy] Tunnel port: 7003
|
||||
[proxy] Node IP: 10.128.32.11 (workers will connect here)
|
||||
[proxy] ✓ SSH tunnel started successfully
|
||||
[proxy] ✓ Generated proxychains config at /e/home/jusers/feuer1/jupiter/.proxychains/proxychains_485104.conf
|
||||
[proxy] - Internal traffic (10.x.x.x, 172.x.x.x, 169.254.x.x) → DIRECT
|
||||
[proxy] - External traffic (internet) → PROXY via tunnel
|
||||
[proxy] ✓ Daytona timeout settings configured
|
||||
[proxy] Testing proxy connectivity...
|
||||
[proxychains] config file found: /e/home/jusers/feuer1/jupiter/.proxychains/proxychains_485104.conf
|
||||
[proxychains] preloading /e/scratch/jureap59/feuer1/proxychains-ng-aarch64/lib/libproxychains4.so
|
||||
[proxychains] DLL init: proxychains-ng 4.17-git-9-g78ead0e
|
||||
[proxy] ✓ Proxy connectivity test passed (huggingface.co reachable via wrapped binary)
|
||||
[proxy] ⚠ Tunnel not accessible at 10.128.32.11:7003 (workers may fail)
|
||||
[proxy] ✓ Proxy setup complete (using wrapped binary for Ray workers)
|
||||
[container_runtime] Using cloud backend: daytona (no local container setup)
|
||||
=== Universal RL Training Runner ===
|
||||
Config: /e/data1/datasets/playground/ot-baf/a3-rl-DCAgent_code-contests-noblock/configs/a3-rl-DCAgent_code-contests-noblock_rl_config.json
|
||||
Working directory: /e/scratch/jureap59/feuer1/OpenThoughts-Agent
|
||||
Python: /e/scratch/jureap59/feuer1/OpenThoughts-Agent/envs/rl/bin/python
|
||||
Python version: Python 3.12.12
|
||||
Proxy: DISABLED (direct internet or not configured)
|
||||
========================================
|
||||
=== RLJobRunner: a3-rl-DCAgent_code-contests-noblock ===
|
||||
[wandb_utils] Fixing permissions on: /e/data1/datasets/playground/ot-baf/a3-rl-DCAgent_code-contests-noblock/wandb
|
||||
[wandb_utils] WandB directory ready: /e/data1/datasets/playground/ot-baf/a3-rl-DCAgent_code-contests-noblock/wandb
|
||||
HF_TOKEN=****pDbg
|
||||
HF_HUB_CACHE=/e/data1/datasets/playground/ot-baf/hf_hub
|
||||
SUPABASE_URL=https://rpzmyuapoqilpghynmza.s... (direct Supabase config)
|
||||
Environment configured:
|
||||
TENSOR_PARALLEL_SIZE=1
|
||||
NUM_INFERENCE_ENGINES=56
|
||||
POLICY_NUM_NODES=14
|
||||
WANDB_DIR=/e/data1/datasets/playground/ot-baf/a3-rl-DCAgent_code-contests-noblock/wandb
|
||||
Starting Ray cluster with 14 nodes, 4 GPUs/node
|
||||
Cleaning up existing Ray instances...
|
||||
=== Starting Ray Cluster ===
|
||||
Nodes: 14
|
||||
GPUs per node: 4
|
||||
CPUs per node: 288
|
||||
Head node: jpbo-041-11 (10.128.32.11)
|
||||
Ray port: 6379
|
||||
============================
|
||||
RL job failed: [Errno 122] Disk quota exceeded: '/e/scratch/jureap59/feuer1/OpenThoughts-Agent/experiments/logs/ray_head_jpbo-041-11.log'
|
||||
Traceback (most recent call last):
|
||||
File "/e/scratch/jureap59/feuer1/OpenThoughts-Agent/hpc/rl_launch_utils.py", line 922, in run
|
||||
training_exit_code = self._run_with_ray()
|
||||
^^^^^^^^^^^^^^^^^^^^
|
||||
File "/e/scratch/jureap59/feuer1/OpenThoughts-Agent/hpc/rl_launch_utils.py", line 1100, in _run_with_ray
|
||||
with RayCluster.from_slurm(ray_cfg) as ray_cluster:
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/e/scratch/jureap59/feuer1/OpenThoughts-Agent/hpc/ray_utils.py", line 818, in __enter__
|
||||
self.start()
|
||||
File "/e/scratch/jureap59/feuer1/OpenThoughts-Agent/hpc/ray_utils.py", line 342, in start
|
||||
self._start_node(self.node_list[0], is_head=True)
|
||||
File "/e/scratch/jureap59/feuer1/OpenThoughts-Agent/hpc/ray_utils.py", line 520, in _start_node
|
||||
ray_log_file = open(ray_log_path, "w")
|
||||
^^^^^^^^^^^^^^^^^^^^^^^
|
||||
OSError: [Errno 122] Disk quota exceeded: '/e/scratch/jureap59/feuer1/OpenThoughts-Agent/experiments/logs/ray_head_jpbo-041-11.log'
|
||||
[RLJobRunner] Launching trace upload (training exit code: 1):
|
||||
repo_id: DCAgent/a3-rl-DCAgent_code-contests-noblock
|
||||
job_dir: /e/data1/datasets/playground/ot-baf/a3-rl-DCAgent_code-contests-noblock/a3-rl-DCAgent_code-contests-noblock
|
||||
episodes: last
|
||||
log: /e/data1/datasets/playground/ot-baf/a3-rl-DCAgent_code-contests-noblock/logs/a3-rl-DCAgent_code-contests-noblock_trace_upload.log
|
||||
[RLJobRunner] Waiting for trace upload to complete...
|
||||
[RLJobRunner] Trace upload failed with exit code 1.
|
||||
Preserving Ray logs to /e/data1/datasets/playground/ot-baf/a3-rl-DCAgent_code-contests-noblock/ray_logs/
|
||||
Collecting Ray logs from worker jpbo-051-33...
|
||||
WARNING: Failed to collect logs from jpbo-051-33
|
||||
Collecting Ray logs from worker jpbo-051-34...
|
||||
WARNING: Failed to collect logs from jpbo-051-34
|
||||
Collecting Ray logs from worker jpbo-051-35...
|
||||
WARNING: Failed to collect logs from jpbo-051-35
|
||||
Collecting Ray logs from worker jpbo-051-36...
|
||||
WARNING: Failed to collect logs from jpbo-051-36
|
||||
Collecting Ray logs from worker jpbo-051-37...
|
||||
WARNING: Failed to collect logs from jpbo-051-37
|
||||
Collecting Ray logs from worker jpbo-051-38...
|
||||
WARNING: Failed to collect logs from jpbo-051-38
|
||||
Collecting Ray logs from worker jpbo-051-39...
|
||||
WARNING: Failed to collect logs from jpbo-051-39
|
||||
Collecting Ray logs from worker jpbo-051-40...
|
||||
WARNING: Failed to collect logs from jpbo-051-40
|
||||
Collecting Ray logs from worker jpbo-060-46...
|
||||
WARNING: Failed to collect logs from jpbo-060-46
|
||||
Collecting Ray logs from worker jpbo-066-09...
|
||||
WARNING: Failed to collect logs from jpbo-066-09
|
||||
Collecting Ray logs from worker jpbo-074-48...
|
||||
WARNING: Failed to collect logs from jpbo-074-48
|
||||
Collecting Ray logs from worker jpbo-089-02...
|
||||
WARNING: Failed to collect logs from jpbo-089-02
|
||||
Collecting Ray logs from worker jpbo-090-42...
|
||||
WARNING: Failed to collect logs from jpbo-090-42
|
||||
Ray log preservation complete
|
||||
111
training_logs/a3-rl-DCAgent_code-contests-noblock_485105.out
Normal file
111
training_logs/a3-rl-DCAgent_code-contests-noblock_485105.out
Normal file
@@ -0,0 +1,111 @@
|
||||
|
||||
The following have been reloaded with a version change:
|
||||
1) GCCcore/.14.3.0 => GCCcore/14.3.0
|
||||
|
||||
|
||||
Lmod is automatically replacing "GCC/14.3.0" with
|
||||
"nvidia-compilers/25.9-CUDA-13".
|
||||
|
||||
Deactivating conda environment: /e/scratch/jureap59/feuer1/miniforge3/envs/otagent
|
||||
Activating RL environment: /e/scratch/jureap59/feuer1/OpenThoughts-Agent/envs/rl
|
||||
Python executable: /e/scratch/jureap59/feuer1/OpenThoughts-Agent/envs/rl/bin/python
|
||||
Python path check: /e/scratch/jureap59/feuer1/OpenThoughts-Agent/envs/rl/bin/python
|
||||
[ray] RAY_TMPDIR=/tmp/ray/ray_485105
|
||||
[triton_cache] Triton cache: /tmp/triton_cache_feuer1_485105
|
||||
[triton_cache] TorchInductor cache: /tmp/torchinductor_cache_feuer1_485105
|
||||
[proxy] ✓ Found proxychains binary at /e/scratch/jureap59/feuer1/proxychains-ng-aarch64/bin/proxychains4
|
||||
[proxy] Setting up SSH tunnel to jpbl-s01-01
|
||||
[proxy] SSH key: /e/home/jusers/feuer1/jupiter/.ssh/authorized_keys/id_ed25519_jsc
|
||||
[proxy] Tunnel port: 7003
|
||||
[proxy] Node IP: 10.128.32.11 (workers will connect here)
|
||||
[proxy] ✓ SSH tunnel started successfully
|
||||
[proxy] ✓ Generated proxychains config at /e/home/jusers/feuer1/jupiter/.proxychains/proxychains_485105.conf
|
||||
[proxy] - Internal traffic (10.x.x.x, 172.x.x.x, 169.254.x.x) → DIRECT
|
||||
[proxy] - External traffic (internet) → PROXY via tunnel
|
||||
[proxy] ✓ Daytona timeout settings configured
|
||||
[proxy] Testing proxy connectivity...
|
||||
[proxychains] config file found: /e/home/jusers/feuer1/jupiter/.proxychains/proxychains_485105.conf
|
||||
[proxychains] preloading /e/scratch/jureap59/feuer1/proxychains-ng-aarch64/lib/libproxychains4.so
|
||||
[proxychains] DLL init: proxychains-ng 4.17-git-9-g78ead0e
|
||||
[proxy] ✓ Proxy connectivity test passed (huggingface.co reachable via wrapped binary)
|
||||
[proxy] ⚠ Tunnel not accessible at 10.128.32.11:7003 (workers may fail)
|
||||
[proxy] ✓ Proxy setup complete (using wrapped binary for Ray workers)
|
||||
[container_runtime] Using cloud backend: daytona (no local container setup)
|
||||
=== Universal RL Training Runner ===
|
||||
Config: /e/data1/datasets/playground/ot-baf/a3-rl-DCAgent_code-contests-noblock/configs/a3-rl-DCAgent_code-contests-noblock_rl_config.json
|
||||
Working directory: /e/scratch/jureap59/feuer1/OpenThoughts-Agent
|
||||
Python: /e/scratch/jureap59/feuer1/OpenThoughts-Agent/envs/rl/bin/python
|
||||
Python version: Python 3.12.12
|
||||
Proxy: DISABLED (direct internet or not configured)
|
||||
========================================
|
||||
=== RLJobRunner: a3-rl-DCAgent_code-contests-noblock ===
|
||||
[wandb_utils] Fixing permissions on: /e/data1/datasets/playground/ot-baf/a3-rl-DCAgent_code-contests-noblock/wandb
|
||||
[wandb_utils] WandB directory ready: /e/data1/datasets/playground/ot-baf/a3-rl-DCAgent_code-contests-noblock/wandb
|
||||
HF_TOKEN=****pDbg
|
||||
HF_HUB_CACHE=/e/data1/datasets/playground/ot-baf/hf_hub
|
||||
SUPABASE_URL=https://rpzmyuapoqilpghynmza.s... (direct Supabase config)
|
||||
Environment configured:
|
||||
TENSOR_PARALLEL_SIZE=1
|
||||
NUM_INFERENCE_ENGINES=56
|
||||
POLICY_NUM_NODES=14
|
||||
WANDB_DIR=/e/data1/datasets/playground/ot-baf/a3-rl-DCAgent_code-contests-noblock/wandb
|
||||
Starting Ray cluster with 14 nodes, 4 GPUs/node
|
||||
Cleaning up existing Ray instances...
|
||||
=== Starting Ray Cluster ===
|
||||
Nodes: 14
|
||||
GPUs per node: 4
|
||||
CPUs per node: 288
|
||||
Head node: jpbo-041-11 (10.128.32.11)
|
||||
Ray port: 6379
|
||||
============================
|
||||
RL job failed: [Errno 122] Disk quota exceeded: '/e/scratch/jureap59/feuer1/OpenThoughts-Agent/experiments/logs/ray_head_jpbo-041-11.log'
|
||||
Traceback (most recent call last):
|
||||
File "/e/scratch/jureap59/feuer1/OpenThoughts-Agent/hpc/rl_launch_utils.py", line 922, in run
|
||||
training_exit_code = self._run_with_ray()
|
||||
^^^^^^^^^^^^^^^^^^^^
|
||||
File "/e/scratch/jureap59/feuer1/OpenThoughts-Agent/hpc/rl_launch_utils.py", line 1100, in _run_with_ray
|
||||
with RayCluster.from_slurm(ray_cfg) as ray_cluster:
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/e/scratch/jureap59/feuer1/OpenThoughts-Agent/hpc/ray_utils.py", line 818, in __enter__
|
||||
self.start()
|
||||
File "/e/scratch/jureap59/feuer1/OpenThoughts-Agent/hpc/ray_utils.py", line 342, in start
|
||||
self._start_node(self.node_list[0], is_head=True)
|
||||
File "/e/scratch/jureap59/feuer1/OpenThoughts-Agent/hpc/ray_utils.py", line 520, in _start_node
|
||||
ray_log_file = open(ray_log_path, "w")
|
||||
^^^^^^^^^^^^^^^^^^^^^^^
|
||||
OSError: [Errno 122] Disk quota exceeded: '/e/scratch/jureap59/feuer1/OpenThoughts-Agent/experiments/logs/ray_head_jpbo-041-11.log'
|
||||
[RLJobRunner] Launching trace upload (training exit code: 1):
|
||||
repo_id: DCAgent/a3-rl-DCAgent_code-contests-noblock
|
||||
job_dir: /e/data1/datasets/playground/ot-baf/a3-rl-DCAgent_code-contests-noblock/a3-rl-DCAgent_code-contests-noblock
|
||||
episodes: last
|
||||
log: /e/data1/datasets/playground/ot-baf/a3-rl-DCAgent_code-contests-noblock/logs/a3-rl-DCAgent_code-contests-noblock_trace_upload.log
|
||||
[RLJobRunner] Waiting for trace upload to complete...
|
||||
[RLJobRunner] Trace upload failed with exit code 1.
|
||||
Preserving Ray logs to /e/data1/datasets/playground/ot-baf/a3-rl-DCAgent_code-contests-noblock/ray_logs/
|
||||
Collecting Ray logs from worker jpbo-051-33...
|
||||
WARNING: Failed to collect logs from jpbo-051-33
|
||||
Collecting Ray logs from worker jpbo-051-34...
|
||||
WARNING: Failed to collect logs from jpbo-051-34
|
||||
Collecting Ray logs from worker jpbo-051-35...
|
||||
WARNING: Failed to collect logs from jpbo-051-35
|
||||
Collecting Ray logs from worker jpbo-051-36...
|
||||
WARNING: Failed to collect logs from jpbo-051-36
|
||||
Collecting Ray logs from worker jpbo-051-37...
|
||||
WARNING: Failed to collect logs from jpbo-051-37
|
||||
Collecting Ray logs from worker jpbo-051-38...
|
||||
WARNING: Failed to collect logs from jpbo-051-38
|
||||
Collecting Ray logs from worker jpbo-051-39...
|
||||
WARNING: Failed to collect logs from jpbo-051-39
|
||||
Collecting Ray logs from worker jpbo-051-40...
|
||||
WARNING: Failed to collect logs from jpbo-051-40
|
||||
Collecting Ray logs from worker jpbo-060-46...
|
||||
WARNING: Failed to collect logs from jpbo-060-46
|
||||
Collecting Ray logs from worker jpbo-066-09...
|
||||
WARNING: Failed to collect logs from jpbo-066-09
|
||||
Collecting Ray logs from worker jpbo-074-48...
|
||||
WARNING: Failed to collect logs from jpbo-074-48
|
||||
Collecting Ray logs from worker jpbo-089-02...
|
||||
WARNING: Failed to collect logs from jpbo-089-02
|
||||
Collecting Ray logs from worker jpbo-090-42...
|
||||
WARNING: Failed to collect logs from jpbo-090-42
|
||||
Ray log preservation complete
|
||||
17
training_logs/a3-rl-DCAgent_code-contests-noblock_496025.out
Normal file
17
training_logs/a3-rl-DCAgent_code-contests-noblock_496025.out
Normal file
@@ -0,0 +1,17 @@
|
||||
|
||||
The following have been reloaded with a version change:
|
||||
1) GCCcore/.14.3.0 => GCCcore/14.3.0
|
||||
|
||||
|
||||
Lmod is automatically replacing "GCC/14.3.0" with
|
||||
"nvidia-compilers/25.9-CUDA-13".
|
||||
|
||||
Deactivating conda environment: /e/scratch/jureap59/feuer1/miniforge3/envs/otagent
|
||||
Warning: RL environment not found at /e/home/jusers/feuer1/jupiter/envs/rl
|
||||
Run ./hpc/setup_rl_env.sh to create it, or set DCFT_RL_ENV
|
||||
/usr/bin/which: no python in (/e/software/default/stages/2026/software/nvidia-compilers/25.9-CUDA-13/Linux_aarch64/25.9/compilers/bin:/e/software/default/stages/2026/software/numactl/2.0.19-GCCcore-14.3.0/bin:/e/software/default/stages/2026/software/GCCcore/14.3.0/bin:/e/software/default/stages/2026/software/CUDA/13/nvvm/bin:/e/software/default/stages/2026/software/CUDA/13/bin:/e/scratch/jureap59/feuer1/miniforge3/condabin:/e/software/default/stages/2026/software/make/4.4.1-GCCcore-14.3.0/bin:/e/software/default/stages/2026/software/binutils/2.44-GCCcore-14.3.0/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/jsc/bin:/usr/local/jsc/bin:/opt/parastation/bin:/e/software/default/bin)
|
||||
Python executable:
|
||||
/var/spool/slurmd/job496025/slurm_script: line 128: python: command not found
|
||||
Python path check:
|
||||
[ray] RAY_TMPDIR=/tmp/ray/ray_496025
|
||||
/var/spool/slurmd/job496025/slurm_script: line 216: /e/home/jusers/feuer1/jupiter/hpc/shell_utils/triton_cache.sh: No such file or directory
|
||||
17
training_logs/a3-rl-DCAgent_code-contests-noblock_496026.out
Normal file
17
training_logs/a3-rl-DCAgent_code-contests-noblock_496026.out
Normal file
@@ -0,0 +1,17 @@
|
||||
|
||||
The following have been reloaded with a version change:
|
||||
1) GCCcore/.14.3.0 => GCCcore/14.3.0
|
||||
|
||||
|
||||
Lmod is automatically replacing "GCC/14.3.0" with
|
||||
"nvidia-compilers/25.9-CUDA-13".
|
||||
|
||||
Deactivating conda environment: /e/scratch/jureap59/feuer1/miniforge3/envs/otagent
|
||||
Warning: RL environment not found at /e/home/jusers/feuer1/jupiter/envs/rl
|
||||
Run ./hpc/setup_rl_env.sh to create it, or set DCFT_RL_ENV
|
||||
/usr/bin/which: no python in (/e/software/default/stages/2026/software/nvidia-compilers/25.9-CUDA-13/Linux_aarch64/25.9/compilers/bin:/e/software/default/stages/2026/software/numactl/2.0.19-GCCcore-14.3.0/bin:/e/software/default/stages/2026/software/GCCcore/14.3.0/bin:/e/software/default/stages/2026/software/CUDA/13/nvvm/bin:/e/software/default/stages/2026/software/CUDA/13/bin:/e/scratch/jureap59/feuer1/miniforge3/condabin:/e/software/default/stages/2026/software/make/4.4.1-GCCcore-14.3.0/bin:/e/software/default/stages/2026/software/binutils/2.44-GCCcore-14.3.0/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/jsc/bin:/usr/local/jsc/bin:/opt/parastation/bin:/e/software/default/bin)
|
||||
Python executable:
|
||||
/var/spool/slurmd/job496026/slurm_script: line 128: python: command not found
|
||||
Python path check:
|
||||
[ray] RAY_TMPDIR=/tmp/ray/ray_496026
|
||||
/var/spool/slurmd/job496026/slurm_script: line 216: /e/home/jusers/feuer1/jupiter/hpc/shell_utils/triton_cache.sh: No such file or directory
|
||||
17
training_logs/a3-rl-DCAgent_code-contests-noblock_496027.out
Normal file
17
training_logs/a3-rl-DCAgent_code-contests-noblock_496027.out
Normal file
@@ -0,0 +1,17 @@
|
||||
|
||||
The following have been reloaded with a version change:
|
||||
1) GCCcore/.14.3.0 => GCCcore/14.3.0
|
||||
|
||||
|
||||
Lmod is automatically replacing "GCC/14.3.0" with
|
||||
"nvidia-compilers/25.9-CUDA-13".
|
||||
|
||||
Deactivating conda environment: /e/scratch/jureap59/feuer1/miniforge3/envs/otagent
|
||||
Warning: RL environment not found at /e/home/jusers/feuer1/jupiter/envs/rl
|
||||
Run ./hpc/setup_rl_env.sh to create it, or set DCFT_RL_ENV
|
||||
/usr/bin/which: no python in (/e/software/default/stages/2026/software/nvidia-compilers/25.9-CUDA-13/Linux_aarch64/25.9/compilers/bin:/e/software/default/stages/2026/software/numactl/2.0.19-GCCcore-14.3.0/bin:/e/software/default/stages/2026/software/GCCcore/14.3.0/bin:/e/software/default/stages/2026/software/CUDA/13/nvvm/bin:/e/software/default/stages/2026/software/CUDA/13/bin:/e/scratch/jureap59/feuer1/miniforge3/condabin:/e/software/default/stages/2026/software/make/4.4.1-GCCcore-14.3.0/bin:/e/software/default/stages/2026/software/binutils/2.44-GCCcore-14.3.0/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/jsc/bin:/usr/local/jsc/bin:/opt/parastation/bin:/e/software/default/bin)
|
||||
Python executable:
|
||||
/var/spool/slurmd/job496027/slurm_script: line 128: python: command not found
|
||||
Python path check:
|
||||
[ray] RAY_TMPDIR=/tmp/ray/ray_496027
|
||||
/var/spool/slurmd/job496027/slurm_script: line 216: /e/home/jusers/feuer1/jupiter/hpc/shell_utils/triton_cache.sh: No such file or directory
|
||||
17
training_logs/a3-rl-DCAgent_code-contests-noblock_496028.out
Normal file
17
training_logs/a3-rl-DCAgent_code-contests-noblock_496028.out
Normal file
@@ -0,0 +1,17 @@
|
||||
|
||||
The following have been reloaded with a version change:
|
||||
1) GCCcore/.14.3.0 => GCCcore/14.3.0
|
||||
|
||||
|
||||
Lmod is automatically replacing "GCC/14.3.0" with
|
||||
"nvidia-compilers/25.9-CUDA-13".
|
||||
|
||||
Deactivating conda environment: /e/scratch/jureap59/feuer1/miniforge3/envs/otagent
|
||||
Warning: RL environment not found at /e/home/jusers/feuer1/jupiter/envs/rl
|
||||
Run ./hpc/setup_rl_env.sh to create it, or set DCFT_RL_ENV
|
||||
/usr/bin/which: no python in (/e/software/default/stages/2026/software/nvidia-compilers/25.9-CUDA-13/Linux_aarch64/25.9/compilers/bin:/e/software/default/stages/2026/software/numactl/2.0.19-GCCcore-14.3.0/bin:/e/software/default/stages/2026/software/GCCcore/14.3.0/bin:/e/software/default/stages/2026/software/CUDA/13/nvvm/bin:/e/software/default/stages/2026/software/CUDA/13/bin:/e/scratch/jureap59/feuer1/miniforge3/condabin:/e/software/default/stages/2026/software/make/4.4.1-GCCcore-14.3.0/bin:/e/software/default/stages/2026/software/binutils/2.44-GCCcore-14.3.0/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/opt/jsc/bin:/usr/local/jsc/bin:/opt/parastation/bin:/e/software/default/bin)
|
||||
Python executable:
|
||||
/var/spool/slurmd/job496028/slurm_script: line 128: python: command not found
|
||||
Python path check:
|
||||
[ray] RAY_TMPDIR=/tmp/ray/ray_496028
|
||||
/var/spool/slurmd/job496028/slurm_script: line 216: /e/home/jusers/feuer1/jupiter/hpc/shell_utils/triton_cache.sh: No such file or directory
|
||||
111
training_logs/a3-rl-DCAgent_code-contests-noblock_496036.out
Normal file
111
training_logs/a3-rl-DCAgent_code-contests-noblock_496036.out
Normal file
@@ -0,0 +1,111 @@
|
||||
|
||||
The following have been reloaded with a version change:
|
||||
1) GCCcore/.14.3.0 => GCCcore/14.3.0
|
||||
|
||||
|
||||
Lmod is automatically replacing "GCC/14.3.0" with
|
||||
"nvidia-compilers/25.9-CUDA-13".
|
||||
|
||||
Deactivating conda environment: /e/scratch/jureap59/feuer1/miniforge3/envs/otagent
|
||||
Activating RL environment: /e/scratch/jureap59/feuer1/OpenThoughts-Agent/envs/rl
|
||||
Python executable: /e/scratch/jureap59/feuer1/OpenThoughts-Agent/envs/rl/bin/python
|
||||
Python path check: /e/scratch/jureap59/feuer1/OpenThoughts-Agent/envs/rl/bin/python
|
||||
[ray] RAY_TMPDIR=/tmp/ray/ray_496036
|
||||
[triton_cache] Triton cache: /tmp/triton_cache_feuer1_496036
|
||||
[triton_cache] TorchInductor cache: /tmp/torchinductor_cache_feuer1_496036
|
||||
[proxy] ✓ Found proxychains binary at /e/scratch/jureap59/feuer1/proxychains-ng-aarch64/bin/proxychains4
|
||||
[proxy] Setting up SSH tunnel to jpbl-s01-01
|
||||
[proxy] SSH key: /e/home/jusers/feuer1/jupiter/.ssh/authorized_keys/id_ed25519_jsc
|
||||
[proxy] Tunnel port: 7003
|
||||
[proxy] Node IP: 10.128.17.210 (workers will connect here)
|
||||
[proxy] ✓ SSH tunnel started successfully
|
||||
[proxy] ✓ Generated proxychains config at /e/home/jusers/feuer1/jupiter/.proxychains/proxychains_496036.conf
|
||||
[proxy] - Internal traffic (10.x.x.x, 172.x.x.x, 169.254.x.x) → DIRECT
|
||||
[proxy] - External traffic (internet) → PROXY via tunnel
|
||||
[proxy] ✓ Daytona timeout settings configured
|
||||
[proxy] Testing proxy connectivity...
|
||||
[proxychains] config file found: /e/home/jusers/feuer1/jupiter/.proxychains/proxychains_496036.conf
|
||||
[proxychains] preloading /e/scratch/jureap59/feuer1/proxychains-ng-aarch64/lib/libproxychains4.so
|
||||
[proxychains] DLL init: proxychains-ng 4.17-git-9-g78ead0e
|
||||
[proxy] ✓ Proxy connectivity test passed (huggingface.co reachable via wrapped binary)
|
||||
[proxy] ⚠ Tunnel not accessible at 10.128.17.210:7003 (workers may fail)
|
||||
[proxy] ✓ Proxy setup complete (using wrapped binary for Ray workers)
|
||||
[container_runtime] Using cloud backend: daytona (no local container setup)
|
||||
=== Universal RL Training Runner ===
|
||||
Config: /e/data1/datasets/playground/ot-baf/a3-rl-DCAgent_code-contests-noblock/configs/a3-rl-DCAgent_code-contests-noblock_rl_config.json
|
||||
Working directory: /e/scratch/jureap59/feuer1/OpenThoughts-Agent
|
||||
Python: /e/scratch/jureap59/feuer1/OpenThoughts-Agent/envs/rl/bin/python
|
||||
Python version: Python 3.12.12
|
||||
Proxy: DISABLED (direct internet or not configured)
|
||||
========================================
|
||||
=== RLJobRunner: a3-rl-DCAgent_code-contests-noblock ===
|
||||
[wandb_utils] Fixing permissions on: /e/data1/datasets/playground/ot-baf/a3-rl-DCAgent_code-contests-noblock/wandb
|
||||
[wandb_utils] WandB directory ready: /e/data1/datasets/playground/ot-baf/a3-rl-DCAgent_code-contests-noblock/wandb
|
||||
HF_TOKEN=****pDbg
|
||||
HF_HUB_CACHE=/e/data1/datasets/playground/ot-baf/hf_hub
|
||||
SUPABASE_URL=https://rpzmyuapoqilpghynmza.s... (direct Supabase config)
|
||||
Environment configured:
|
||||
TENSOR_PARALLEL_SIZE=1
|
||||
NUM_INFERENCE_ENGINES=56
|
||||
POLICY_NUM_NODES=14
|
||||
WANDB_DIR=/e/data1/datasets/playground/ot-baf/a3-rl-DCAgent_code-contests-noblock/wandb
|
||||
Starting Ray cluster with 14 nodes, 4 GPUs/node
|
||||
Cleaning up existing Ray instances...
|
||||
=== Starting Ray Cluster ===
|
||||
Nodes: 14
|
||||
GPUs per node: 4
|
||||
CPUs per node: 288
|
||||
Head node: jpbo-010-34 (10.128.17.210)
|
||||
Ray port: 6379
|
||||
============================
|
||||
RL job failed: [Errno 122] Disk quota exceeded: '/e/scratch/jureap59/feuer1/OpenThoughts-Agent/experiments/logs/ray_head_jpbo-010-34.log'
|
||||
Traceback (most recent call last):
|
||||
File "/e/scratch/jureap59/feuer1/OpenThoughts-Agent/hpc/rl_launch_utils.py", line 922, in run
|
||||
training_exit_code = self._run_with_ray()
|
||||
^^^^^^^^^^^^^^^^^^^^
|
||||
File "/e/scratch/jureap59/feuer1/OpenThoughts-Agent/hpc/rl_launch_utils.py", line 1100, in _run_with_ray
|
||||
with RayCluster.from_slurm(ray_cfg) as ray_cluster:
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/e/scratch/jureap59/feuer1/OpenThoughts-Agent/hpc/ray_utils.py", line 818, in __enter__
|
||||
self.start()
|
||||
File "/e/scratch/jureap59/feuer1/OpenThoughts-Agent/hpc/ray_utils.py", line 342, in start
|
||||
self._start_node(self.node_list[0], is_head=True)
|
||||
File "/e/scratch/jureap59/feuer1/OpenThoughts-Agent/hpc/ray_utils.py", line 520, in _start_node
|
||||
ray_log_file = open(ray_log_path, "w")
|
||||
^^^^^^^^^^^^^^^^^^^^^^^
|
||||
OSError: [Errno 122] Disk quota exceeded: '/e/scratch/jureap59/feuer1/OpenThoughts-Agent/experiments/logs/ray_head_jpbo-010-34.log'
|
||||
[RLJobRunner] Launching trace upload (training exit code: 1):
|
||||
repo_id: DCAgent/a3-rl-DCAgent_code-contests-noblock
|
||||
job_dir: /e/data1/datasets/playground/ot-baf/a3-rl-DCAgent_code-contests-noblock/a3-rl-DCAgent_code-contests-noblock
|
||||
episodes: last
|
||||
log: /e/data1/datasets/playground/ot-baf/a3-rl-DCAgent_code-contests-noblock/logs/a3-rl-DCAgent_code-contests-noblock_trace_upload.log
|
||||
[RLJobRunner] Waiting for trace upload to complete...
|
||||
[RLJobRunner] Trace upload failed with exit code 1.
|
||||
Preserving Ray logs to /e/data1/datasets/playground/ot-baf/a3-rl-DCAgent_code-contests-noblock/ray_logs/
|
||||
Collecting Ray logs from worker jpbo-010-36...
|
||||
WARNING: Failed to collect logs from jpbo-010-36
|
||||
Collecting Ray logs from worker jpbo-010-38...
|
||||
WARNING: Failed to collect logs from jpbo-010-38
|
||||
Collecting Ray logs from worker jpbo-010-39...
|
||||
WARNING: Failed to collect logs from jpbo-010-39
|
||||
Collecting Ray logs from worker jpbo-010-40...
|
||||
WARNING: Failed to collect logs from jpbo-010-40
|
||||
Collecting Ray logs from worker jpbo-010-41...
|
||||
WARNING: Failed to collect logs from jpbo-010-41
|
||||
Collecting Ray logs from worker jpbo-010-43...
|
||||
WARNING: Failed to collect logs from jpbo-010-43
|
||||
Collecting Ray logs from worker jpbo-010-44...
|
||||
WARNING: Failed to collect logs from jpbo-010-44
|
||||
Collecting Ray logs from worker jpbo-010-45...
|
||||
WARNING: Failed to collect logs from jpbo-010-45
|
||||
Collecting Ray logs from worker jpbo-010-46...
|
||||
WARNING: Failed to collect logs from jpbo-010-46
|
||||
Collecting Ray logs from worker jpbo-022-02...
|
||||
WARNING: Failed to collect logs from jpbo-022-02
|
||||
Collecting Ray logs from worker jpbo-022-03...
|
||||
WARNING: Failed to collect logs from jpbo-022-03
|
||||
Collecting Ray logs from worker jpbo-022-04...
|
||||
WARNING: Failed to collect logs from jpbo-022-04
|
||||
Collecting Ray logs from worker jpbo-022-05...
|
||||
WARNING: Failed to collect logs from jpbo-022-05
|
||||
Ray log preservation complete
|
||||
111
training_logs/a3-rl-DCAgent_code-contests-noblock_496037.out
Normal file
111
training_logs/a3-rl-DCAgent_code-contests-noblock_496037.out
Normal file
@@ -0,0 +1,111 @@
|
||||
|
||||
The following have been reloaded with a version change:
|
||||
1) GCCcore/.14.3.0 => GCCcore/14.3.0
|
||||
|
||||
|
||||
Lmod is automatically replacing "GCC/14.3.0" with
|
||||
"nvidia-compilers/25.9-CUDA-13".
|
||||
|
||||
Deactivating conda environment: /e/scratch/jureap59/feuer1/miniforge3/envs/otagent
|
||||
Activating RL environment: /e/scratch/jureap59/feuer1/OpenThoughts-Agent/envs/rl
|
||||
Python executable: /e/scratch/jureap59/feuer1/OpenThoughts-Agent/envs/rl/bin/python
|
||||
Python path check: /e/scratch/jureap59/feuer1/OpenThoughts-Agent/envs/rl/bin/python
|
||||
[ray] RAY_TMPDIR=/tmp/ray/ray_496037
|
||||
[triton_cache] Triton cache: /tmp/triton_cache_feuer1_496037
|
||||
[triton_cache] TorchInductor cache: /tmp/torchinductor_cache_feuer1_496037
|
||||
[proxy] ✓ Found proxychains binary at /e/scratch/jureap59/feuer1/proxychains-ng-aarch64/bin/proxychains4
|
||||
[proxy] Setting up SSH tunnel to jpbl-s01-01
|
||||
[proxy] SSH key: /e/home/jusers/feuer1/jupiter/.ssh/authorized_keys/id_ed25519_jsc
|
||||
[proxy] Tunnel port: 7003
|
||||
[proxy] Node IP: 10.128.49.63 (workers will connect here)
|
||||
[proxy] ✓ SSH tunnel started successfully
|
||||
[proxy] ✓ Generated proxychains config at /e/home/jusers/feuer1/jupiter/.proxychains/proxychains_496037.conf
|
||||
[proxy] - Internal traffic (10.x.x.x, 172.x.x.x, 169.254.x.x) → DIRECT
|
||||
[proxy] - External traffic (internet) → PROXY via tunnel
|
||||
[proxy] ✓ Daytona timeout settings configured
|
||||
[proxy] Testing proxy connectivity...
|
||||
[proxychains] config file found: /e/home/jusers/feuer1/jupiter/.proxychains/proxychains_496037.conf
|
||||
[proxychains] preloading /e/scratch/jureap59/feuer1/proxychains-ng-aarch64/lib/libproxychains4.so
|
||||
[proxychains] DLL init: proxychains-ng 4.17-git-9-g78ead0e
|
||||
[proxy] ✓ Proxy connectivity test passed (huggingface.co reachable via wrapped binary)
|
||||
[proxy] ⚠ Tunnel not accessible at 10.128.49.63:7003 (workers may fail)
|
||||
[proxy] ✓ Proxy setup complete (using wrapped binary for Ray workers)
|
||||
[container_runtime] Using cloud backend: daytona (no local container setup)
|
||||
=== Universal RL Training Runner ===
|
||||
Config: /e/data1/datasets/playground/ot-baf/a3-rl-DCAgent_code-contests-noblock/configs/a3-rl-DCAgent_code-contests-noblock_rl_config.json
|
||||
Working directory: /e/scratch/jureap59/feuer1/OpenThoughts-Agent
|
||||
Python: /e/scratch/jureap59/feuer1/OpenThoughts-Agent/envs/rl/bin/python
|
||||
Python version: Python 3.12.12
|
||||
Proxy: DISABLED (direct internet or not configured)
|
||||
========================================
|
||||
=== RLJobRunner: a3-rl-DCAgent_code-contests-noblock ===
|
||||
[wandb_utils] Fixing permissions on: /e/data1/datasets/playground/ot-baf/a3-rl-DCAgent_code-contests-noblock/wandb
|
||||
[wandb_utils] WandB directory ready: /e/data1/datasets/playground/ot-baf/a3-rl-DCAgent_code-contests-noblock/wandb
|
||||
HF_TOKEN=****pDbg
|
||||
HF_HUB_CACHE=/e/data1/datasets/playground/ot-baf/hf_hub
|
||||
SUPABASE_URL=https://rpzmyuapoqilpghynmza.s... (direct Supabase config)
|
||||
Environment configured:
|
||||
TENSOR_PARALLEL_SIZE=1
|
||||
NUM_INFERENCE_ENGINES=56
|
||||
POLICY_NUM_NODES=14
|
||||
WANDB_DIR=/e/data1/datasets/playground/ot-baf/a3-rl-DCAgent_code-contests-noblock/wandb
|
||||
Starting Ray cluster with 14 nodes, 4 GPUs/node
|
||||
Cleaning up existing Ray instances...
|
||||
=== Starting Ray Cluster ===
|
||||
Nodes: 14
|
||||
GPUs per node: 4
|
||||
CPUs per node: 288
|
||||
Head node: jpbo-087-31 (10.128.49.63)
|
||||
Ray port: 6379
|
||||
============================
|
||||
RL job failed: [Errno 122] Disk quota exceeded: '/e/scratch/jureap59/feuer1/OpenThoughts-Agent/experiments/logs/ray_head_jpbo-087-31.log'
|
||||
Traceback (most recent call last):
|
||||
File "/e/scratch/jureap59/feuer1/OpenThoughts-Agent/hpc/rl_launch_utils.py", line 922, in run
|
||||
training_exit_code = self._run_with_ray()
|
||||
^^^^^^^^^^^^^^^^^^^^
|
||||
File "/e/scratch/jureap59/feuer1/OpenThoughts-Agent/hpc/rl_launch_utils.py", line 1100, in _run_with_ray
|
||||
with RayCluster.from_slurm(ray_cfg) as ray_cluster:
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/e/scratch/jureap59/feuer1/OpenThoughts-Agent/hpc/ray_utils.py", line 818, in __enter__
|
||||
self.start()
|
||||
File "/e/scratch/jureap59/feuer1/OpenThoughts-Agent/hpc/ray_utils.py", line 342, in start
|
||||
self._start_node(self.node_list[0], is_head=True)
|
||||
File "/e/scratch/jureap59/feuer1/OpenThoughts-Agent/hpc/ray_utils.py", line 520, in _start_node
|
||||
ray_log_file = open(ray_log_path, "w")
|
||||
^^^^^^^^^^^^^^^^^^^^^^^
|
||||
OSError: [Errno 122] Disk quota exceeded: '/e/scratch/jureap59/feuer1/OpenThoughts-Agent/experiments/logs/ray_head_jpbo-087-31.log'
|
||||
[RLJobRunner] Launching trace upload (training exit code: 1):
|
||||
repo_id: DCAgent/a3-rl-DCAgent_code-contests-noblock
|
||||
job_dir: /e/data1/datasets/playground/ot-baf/a3-rl-DCAgent_code-contests-noblock/a3-rl-DCAgent_code-contests-noblock
|
||||
episodes: last
|
||||
log: /e/data1/datasets/playground/ot-baf/a3-rl-DCAgent_code-contests-noblock/logs/a3-rl-DCAgent_code-contests-noblock_trace_upload.log
|
||||
[RLJobRunner] Waiting for trace upload to complete...
|
||||
slurmstepd: error: *** JOB 496037 ON jpbo-087-31 CANCELLED AT 2026-05-23T14:15:08 ***
|
||||
Preserving Ray logs to /e/data1/datasets/playground/ot-baf/a3-rl-DCAgent_code-contests-noblock/ray_logs/
|
||||
Collecting Ray logs from worker jpbo-087-32...
|
||||
WARNING: Failed to collect logs from jpbo-087-32
|
||||
Collecting Ray logs from worker jpbo-090-02...
|
||||
WARNING: Failed to collect logs from jpbo-090-02
|
||||
Collecting Ray logs from worker jpbo-090-03...
|
||||
WARNING: Failed to collect logs from jpbo-090-03
|
||||
Collecting Ray logs from worker jpbo-090-05...
|
||||
WARNING: Failed to collect logs from jpbo-090-05
|
||||
Collecting Ray logs from worker jpbo-090-06...
|
||||
WARNING: Failed to collect logs from jpbo-090-06
|
||||
Collecting Ray logs from worker jpbo-090-08...
|
||||
WARNING: Failed to collect logs from jpbo-090-08
|
||||
Collecting Ray logs from worker jpbo-090-09...
|
||||
WARNING: Failed to collect logs from jpbo-090-09
|
||||
Collecting Ray logs from worker jpbo-090-10...
|
||||
WARNING: Failed to collect logs from jpbo-090-10
|
||||
Collecting Ray logs from worker jpbo-090-11...
|
||||
WARNING: Failed to collect logs from jpbo-090-11
|
||||
Collecting Ray logs from worker jpbo-090-12...
|
||||
WARNING: Failed to collect logs from jpbo-090-12
|
||||
Collecting Ray logs from worker jpbo-090-13...
|
||||
WARNING: Failed to collect logs from jpbo-090-13
|
||||
Collecting Ray logs from worker jpbo-090-15...
|
||||
WARNING: Failed to collect logs from jpbo-090-15
|
||||
Collecting Ray logs from worker jpbo-090-16...
|
||||
WARNING: Failed to collect logs from jpbo-090-16
|
||||
Ray log preservation complete
|
||||
178
training_logs/a3-rl-DCAgent_code-contests-noblock_496208.out
Normal file
178
training_logs/a3-rl-DCAgent_code-contests-noblock_496208.out
Normal file
File diff suppressed because one or more lines are too long
178
training_logs/a3-rl-DCAgent_code-contests-noblock_496209.out
Normal file
178
training_logs/a3-rl-DCAgent_code-contests-noblock_496209.out
Normal file
File diff suppressed because one or more lines are too long
178
training_logs/a3-rl-DCAgent_code-contests-noblock_496210.out
Normal file
178
training_logs/a3-rl-DCAgent_code-contests-noblock_496210.out
Normal file
File diff suppressed because one or more lines are too long
191
training_logs/a3-rl-DCAgent_code-contests-noblock_496211.out
Normal file
191
training_logs/a3-rl-DCAgent_code-contests-noblock_496211.out
Normal file
File diff suppressed because one or more lines are too long
1664
training_logs/a3-rl-DCAgent_code-contests-noblock_496419.out
Normal file
1664
training_logs/a3-rl-DCAgent_code-contests-noblock_496419.out
Normal file
File diff suppressed because one or more lines are too long
24150
training_logs/a3-rl-DCAgent_code-contests-noblock_496420.out
Normal file
24150
training_logs/a3-rl-DCAgent_code-contests-noblock_496420.out
Normal file
File diff suppressed because one or more lines are too long
21025
training_logs/a3-rl-DCAgent_code-contests-noblock_496421.out
Normal file
21025
training_logs/a3-rl-DCAgent_code-contests-noblock_496421.out
Normal file
File diff suppressed because one or more lines are too long
23608
training_logs/a3-rl-DCAgent_code-contests-noblock_496422.out
Normal file
23608
training_logs/a3-rl-DCAgent_code-contests-noblock_496422.out
Normal file
File diff suppressed because one or more lines are too long
1
vocab.json
Normal file
1
vocab.json
Normal file
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user