commit 7d9fdc2457a6d748cbc8dedf7c890083dba15292 Author: ModelHub XC Date: Sun Jul 19 15:14:10 2026 +0800 初始化项目，由ModelHub XC社区提供模型 Model: Sinestro38/qwen3-1p7b-sgd-polaris-step300-best-val Source: Original Platform diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..b82ea62 --- /dev/null +++ b/README.md @@ -0,0 +1,102 @@ +--- +license: apache-2.0 +base_model: Qwen/Qwen3-1.7B-Base +datasets: + - POLARIS-Project/Polaris-Dataset-53K +tags: + - reinforcement-learning + - sgd + - grpo + - math + - reasoning + - low-rank-analysis +language: + - en +pipeline_tag: text-generation +--- + +# Qwen3-1.7B-Base + Polaris RL (PLAIN SGD, step 300, peak val) + +This is the **peak-validation checkpoint (step 300/1550)** of a plain-SGD RL fine-tune of `Qwen/Qwen3-1.7B-Base` on `POLARIS-Project/Polaris-Dataset-53K`. Released as part of an ICML-2026 study on the **low-rank structure of SGD vs Adam RL updates** for batched-LoRA inference. + +## Why this checkpoint exists + +The motivation is to compare the SVD-compressibility of `ΔW = W_ft − W_base` between SGD-trained and Adam-trained RL fine-tunes. Existing open RL FTs (POLARIS, Skywork-OR1, DeepCoder, AceReason, ORZ, DAPO) are all Adam-trained; we needed a same-base same-recipe SGD counterpart. **POLARIS-1.7B-Preview** is the upstream Adam-trained reference for this exact base + dataset combination — diff this model's ΔW against POLARIS-1.7B-Preview's ΔW for the head-to-head SGD-vs-Adam compressibility comparison. + +## Training recipe + +| field | value | +|---|---| +| base model | `Qwen/Qwen3-1.7B-Base` | +| dataset | `POLARIS-Project/Polaris-Dataset-53K` (52,779 train / 512 val) | +| algorithm | GRPO (`adv_estimator=grpo`, `use_kl_loss=False`, `entropy_coeff=0`, `use_kl_in_reward=False`) | +| optimizer | **PLAIN SGD** — `momentum=0.0`, `nesterov=false`, `dampening=0.0`, `weight_decay=0.0` | +| learning rate | `1e-1` (constant) | +| train batch size | 128 (1 grad step per rollout batch) | +| ppo_micro_batch_size_per_gpu | 4 | +| rollout.n | 4 | +| rollout.temperature | 1.0 | +| max_prompt_length | 1024 | +| max_response_length | 8192 | +| epochs at this checkpoint | ~0.73 (step 300 / 412 per epoch) | +| hardware | 4× B200 (179 GB) | +| step time | ~65 s/step | +| trainer | [verl](https://github.com/volcengine/verl) (FSDP + vLLM rollout) | + +The "PLAIN SGD" choice is scientifically load-bearing — every claim about SGD update compressibility relies on the update being the **pure first-order gradient**. + +## LR exploration (relevant context) + +Plain SGD on Qwen3-Base + math RL has a narrow stable LR window: +- `lr=1e-2`: gradient signal too weak — stalled. +- `lr=2e-1`: catastrophic policy collapse (response_length → 8192 cap, rewards pinned at -1). +- `lr=1`: instant collapse from step 1. +- **`lr=1e-1`** ← this run: stable-but-slow regime where val acc actually rose from 0. + +## Results + +| metric | value | +|---|---| +| baseline val acc (step 0) | 0.0% | +| **val acc at this ckpt (step 300)** | **14.29%** ← peak across the run | +| val acc at step 550 | 13.7% | +| val acc at step 700 | 2.7% | +| val acc at step 1500+ | 0% (collapsed) | + +The val acc trajectory peaked at step 300, slowly drifted, then collapsed back to 0% in the second half of training — a textbook SGD-without-momentum drift on noisy advantages. **This checkpoint is therefore the one you want for downstream use or analysis** — for the final-epoch checkpoint (collapsed) see [`Sinestro38/qwen3-1p7b-sgd-polaris-step1550-final`](https://huggingface.co/Sinestro38/qwen3-1p7b-sgd-polaris-step1550-final). + +## Use + +```python +from transformers import AutoModelForCausalLM, AutoTokenizer +import torch + +m = AutoModelForCausalLM.from_pretrained( + "Sinestro38/qwen3-1p7b-sgd-polaris-step300-best-val", + torch_dtype=torch.bfloat16, + device_map="cuda", +) +tok = AutoTokenizer.from_pretrained("Sinestro38/qwen3-1p7b-sgd-polaris-step300-best-val") + +# Math problems work best with the boxed-answer suffix +prompt = "Find all integer solutions to x^2 + y^2 = 25. Let's think step by step and output the final answer within \\boxed{}." +msgs = [{"role": "user", "content": prompt}] +ids = tok.apply_chat_template(msgs, tokenize=True, add_generation_prompt=True, return_tensors="pt").to("cuda") +out = m.generate(ids, max_new_tokens=2048, do_sample=True, temperature=0.6) +print(tok.decode(out[0][ids.shape[1]:], skip_special_tokens=True)) +``` + +## Caveats + +- This is a **base model** (Qwen3-1.7B-Base) fine-tuned with RL — there's no SFT step before. It only learned to box answers in the prose-style "Answer: X" format that matches verl's default Minerva regex (the Polaris template asks for `\boxed{}` but the base learned to emit "Answer:" patterns that score correctly). +- Trained on math only; no code, no general instruction-following data. +- ΔW magnitude is small relative to base weights (feature for compressibility study, not a bug). + +## Citation context + +Work in progress — being submitted to ICML 2026 with a paper on plain-SGD RL update compressibility for batched-LoRA serving. + +Related models in this study: +- [`Sinestro38/qwen3-1p7b-sgd-polaris-step1550-final`](https://huggingface.co/Sinestro38/qwen3-1p7b-sgd-polaris-step1550-final) — same run, final (collapsed) checkpoint +- [`Sinestro38/dsr1-qwen7b-sgd-polaris-step100-best-val`](https://huggingface.co/Sinestro38/dsr1-qwen7b-sgd-polaris-step100-best-val) — same recipe, scaled to 7B with DS-R1-Distill base +- [`Sinestro38/dsr1-qwen7b-sgd-polaris-step412-final`](https://huggingface.co/Sinestro38/dsr1-qwen7b-sgd-polaris-step412-final) — 7B final diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..699ff8d --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,85 @@ +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..418a354 --- /dev/null +++ b/config.json @@ -0,0 +1,63 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151643, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 32768, + "max_window_layers": 28, + "model_type": "qwen3", + "num_attention_heads": 16, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "rope_theta": 1000000, + "rope_type": "default" + }, + "sliding_window": null, + "tie_word_embeddings": true, + "transformers_version": "5.5.4", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..d432e8b --- /dev/null +++ b/generation_config.json @@ -0,0 +1,7 @@ +{ + "bos_token_id": 151643, + "do_sample": false, + "eos_token_id": 151643, + "max_new_tokens": 2048, + "transformers_version": "5.5.4" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..2232f5a --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e0f77fcb8ba475f25c57825b974c6bb0e0e6bdeea5893d59ac597fce80c50ccb +size 4063515640 diff --git a/reward_score_box_or_minerva.py b/reward_score_box_or_minerva.py new file mode 100644 index 0000000..64e1303 --- /dev/null +++ b/reward_score_box_or_minerva.py @@ -0,0 +1,33 @@ +""" +Reward function that tries `\boxed{}` extraction first, then Minerva-style "Answer: X". + +Why: verl's default math_dapo.compute_score uses Minerva regex (looks for literal +"Answer: X" in the last 300 chars). That works for base/raw models that ramble in +prose, but FAILS for distilled CoT models like DS-R1-Distill that emit +`...\\boxed{X}` with no "Answer:" preamble. Matching both styles +unblocks the SGD-on-DS-R1-Distill experiment without regressing the 1.7B-Base run. + +Selected via verl's custom_reward_function config: + custom_reward_function.path=/path/to/this/file.py + custom_reward_function.name=compute_score + +Returns a bare float (1.0 or -1.0) — NOT the {score, acc, pred} dict that +math_dapo returns natively. verl's validation metric aggregator can't mean a +field of None values (which `pred` becomes when no box is found), so the dict +form crashes process_validation_metrics. The bare-float form sidesteps this. +""" + +from verl.utils.reward_score import math_dapo + + +def compute_score(data_source, solution_str, ground_truth, extra_info=None, **kwargs): + """Box-first, Minerva-fallback reward. Returns 1.0 (correct) or -1.0 (wrong).""" + boxed = math_dapo.compute_score(solution_str, ground_truth, strict_box_verify=True) + if boxed["score"] == 1.0: + return 1.0 + + minerva = math_dapo.compute_score(solution_str, ground_truth, strict_box_verify=False) + if minerva["score"] == 1.0: + return 1.0 + + return -1.0 diff --git a/run_polaris_1p7b_sgd.sh b/run_polaris_1p7b_sgd.sh new file mode 100644 index 0000000..f461f6d --- /dev/null +++ b/run_polaris_1p7b_sgd.sh @@ -0,0 +1,124 @@ +#!/bin/bash +# POLARIS-1.7B PLAIN-SGD rerun. +# +# Upstream (ChenxinAn-fdu/POLARIS) trains Qwen3-1.7B-Base with AdamW lr=1e-6, +# GRPO+ (use_kl_loss=False, entropy_coeff=0), strict on-policy +# (mini_batch=train_batch=128, 1 grad step per rollout batch). +# This script swaps ONLY the optimizer: AdamW → PLAIN SGD (momentum=0). +# Everything else kept faithful within a 4×B200 budget (response length capped at 8192). +# +# Scientific non-negotiable: momentum=0.0, nesterov=false, dampening=0.0, +# weight_decay=0.0. If the run diverges, lower the LR — never turn on momentum. + +set -euo pipefail + +export PYTHONUNBUFFERED=1 + +# Sources venv + vLLM-critical env: local TMPDIR for AF_UNIX sockets (NFS refuses +# AF_UNIX bind), LD_LIBRARY_PATH for system libstdc++ GLIBCXX_3.4.32, CCCL headers +# for flashinfer, VLLM_USE_V1=1, stale-PYTHONPATH cleanup, CC/CXX/TRITON_CC on +# /usr/bin/gcc (the /home/utils/gcc-11.2.0 + binutils-2.37 pair can't link +# against the system glibc's .relr.dyn section). +# shellcheck disable=SC1091 +source /home/scratch.pjayasinha_gpu/rs/vllm_env.sh + +# Force HF offline after vllm_env.sh to avoid 4 simultaneous vLLM EngineCores +# hitting the HF hub → 429 rate-limit. +export HF_HUB_OFFLINE=1 +export TRANSFORMERS_OFFLINE=1 + +export CUDA_VISIBLE_DEVICES=0,1,2,3 # GPUs 4-7 are held by the SVD sweep +# Force Ray to respect CUDA_VISIBLE_DEVICES for num_gpus=0 actors (e.g. the +# vLLM http server coordinator). Without this, Ray's legacy default overrides +# our CUDA_VISIBLE_DEVICES for coordinator actors → vLLM's EngineCore picks up +# physical GPU 4 and starts rolling out on a GPU we promised to leave alone. +export RAY_ACCEL_ENV_VAR_OVERRIDE_ON_ZERO=0 + +# NOTE: do NOT prepend /usr/bin to PATH — vllm_env.sh already sources the venv, +# and /usr/bin first would shadow the venv's python3 (no verl installed there). +# Triton uses CC=/usr/bin/gcc (set in vllm_env.sh); that pulls in /usr/bin/ld +# automatically via gcc's default linker path. No explicit LD override needed. + +export WANDB_API_KEY=$(cat /home/scratch.pjayasinha_gpu/rs/wandb_api_key.txt) + +# SMOKE=1 runs a tiny sanity check: single epoch, short response length, small val slice. +SMOKE="${SMOKE:-0}" +LR="${LR:-1e-1}" # lr=1 and lr=2e-1 both collapsed (max-length babble, rewards pinned at -1). lr=1e-1 was the stable-but-slow regime where val-core actually started rising from 0. Locking in here. + +if [[ "$SMOKE" == "1" ]]; then + EXP_NAME="polaris_1p7b_sgd_lr${LR}_SMOKE" + TOTAL_EPOCHS=1 + MAX_RESP=${MAX_RESP:-8192} # default now matches the real run; override to 2048 for fast debug + SAVE_FREQ=10 + TEST_FREQ=999999 +else + EXP_NAME="polaris_1p7b_sgd_lr${LR}_bs128_gpu4_ep5_n4" + TOTAL_EPOCHS=5 + MAX_RESP=${MAX_RESP:-8192} + SAVE_FREQ=25 + TEST_FREQ=50 +fi + +CKPT_DIR="/home/scratch.pjayasinha_gpu/rs/checkpoints/${EXP_NAME}" +LOG_DIR="/home/scratch.pjayasinha_gpu/rs/logs" +mkdir -p "$CKPT_DIR" "$LOG_DIR" +LOG_PATH="${LOG_DIR}/${EXP_NAME}.log" + +echo "==============================================" +echo "EXP_NAME: $EXP_NAME" +echo "MODEL: Qwen/Qwen3-1.7B-Base" +echo "OPTIMIZER: PLAIN SGD (momentum=0, nesterov=false, dampening=0, wd=0)" +echo "LR: $LR" +echo "MAX_RESP: $MAX_RESP" +echo "TOTAL_EPOCHS: $TOTAL_EPOCHS" +echo "CKPT_DIR: $CKPT_DIR" +echo "LOG_PATH: $LOG_PATH" +echo "GPUs: $CUDA_VISIBLE_DEVICES" +echo "==============================================" + +python3 -m verl.trainer.main_ppo \ + algorithm.adv_estimator=grpo \ + algorithm.use_kl_in_reward=False \ + data.train_files=/home/scratch.pjayasinha_gpu/rs/data_polaris/train.parquet \ + data.val_files="['/home/scratch.pjayasinha_gpu/rs/data_polaris/val.parquet']" \ + data.train_batch_size=128 \ + data.max_prompt_length=1024 \ + data.max_response_length=${MAX_RESP} \ + data.filter_overlong_prompts=True \ + data.truncation='error' \ + actor_rollout_ref.model.path=Qwen/Qwen3-1.7B-Base \ + actor_rollout_ref.model.use_remove_padding=True \ + actor_rollout_ref.model.enable_gradient_checkpointing=False \ + actor_rollout_ref.model.use_fused_kernels=True \ + actor_rollout_ref.actor.optim.optimizer=SGD \ + actor_rollout_ref.actor.optim.optimizer_impl=torch.optim \ + actor_rollout_ref.actor.optim.lr=${LR} \ + actor_rollout_ref.actor.optim.weight_decay=0.0 \ + actor_rollout_ref.actor.optim.override_optimizer_config='{momentum: 0.0, nesterov: false, dampening: 0.0}' \ + actor_rollout_ref.actor.ppo_mini_batch_size=128 \ + actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.actor.use_kl_loss=False \ + actor_rollout_ref.actor.kl_loss_coef=0.0 \ + actor_rollout_ref.actor.entropy_coeff=0.0 \ + actor_rollout_ref.actor.fsdp_config.param_offload=False \ + actor_rollout_ref.actor.fsdp_config.optimizer_offload=False \ + actor_rollout_ref.rollout.name=vllm \ + actor_rollout_ref.rollout.n=4 \ + actor_rollout_ref.rollout.temperature=1.0 \ + actor_rollout_ref.rollout.log_prob_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.rollout.tensor_model_parallel_size=1 \ + actor_rollout_ref.rollout.gpu_memory_utilization=0.85 \ + actor_rollout_ref.ref.log_prob_micro_batch_size_per_gpu=4 \ + actor_rollout_ref.ref.fsdp_config.param_offload=False \ + trainer.critic_warmup=0 \ + trainer.logger='["console","wandb"]' \ + trainer.project_name='sgd_rerun_polaris_1p7b' \ + trainer.experiment_name="${EXP_NAME}" \ + +trainer.wandb.entity='pavanjayasinha-university-of-illinois-urbana-champaign' \ + trainer.n_gpus_per_node=4 \ + trainer.nnodes=1 \ + trainer.save_freq=${SAVE_FREQ} \ + trainer.test_freq=${TEST_FREQ} \ + trainer.total_epochs=${TOTAL_EPOCHS} \ + trainer.default_local_dir="${CKPT_DIR}" \ + 2>&1 | tee "${LOG_PATH}" diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..1ce8d7b --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,29 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": true, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/training_config.json b/training_config.json new file mode 100644 index 0000000..8fa4900 --- /dev/null +++ b/training_config.json @@ -0,0 +1,810 @@ +{ + "actor_rollout_ref": { + "actor": { + "_target_": "verl.workers.config.FSDPActorConfig", + "calculate_entropy": false, + "calculate_sum_pi_squared": false, + "checkpoint": { + "_target_": "verl.trainer.config.CheckpointConfig", + "async_save": false, + "load_contents": [ + "model", + "optimizer", + "extra" + ], + "mbridge_config": {}, + "save_contents": [ + "model", + "optimizer", + "extra" + ] + }, + "clip_ratio": 0.2, + "clip_ratio_c": 3.0, + "clip_ratio_high": 0.2, + "clip_ratio_low": 0.2, + "data_loader_seed": 42, + "entropy_checkpointing": false, + "entropy_coeff": 0.0, + "entropy_from_logits_with_chunking": false, + "freeze_vision_tower": false, + "fsdp_config": { + "_target_": "verl.workers.config.FSDPEngineConfig", + "dtype": "bfloat16", + "entropy_checkpointing": false, + "entropy_from_logits_with_chunking": false, + "forward_only": false, + "forward_prefetch": false, + "fsdp_size": -1, + "full_determinism": false, + "model_dtype": "fp32", + "offload_policy": false, + "optimizer_offload": false, + "param_offload": false, + "qat": { + "_target_": "verl.workers.config.QATEngineConfig", + "activation_observer": "static_minmax", + "enable": false, + "group_size": 16, + "ignore_patterns": [ + "lm_head", + "embed_tokens", + "re:.*mlp.gate$" + ], + "mode": "w4a16", + "quantization_config_path": null + }, + "reshard_after_forward": true, + "seed": 42, + "strategy": "fsdp", + "ulysses_sequence_parallel_size": 1, + "use_orig_params": false, + "use_torch_compile": true, + "wrap_policy": { + "min_num_params": 0 + } + }, + "grad_clip": 1.0, + "kl_loss_coef": 0.0, + "kl_loss_type": "low_var_kl", + "loss_agg_mode": "token-mean", + "loss_scale_factor": null, + "optim": { + "_target_": "verl.workers.config.FSDPOptimizerConfig", + "betas": [ + 0.9, + 0.999 + ], + "clip_grad": 1.0, + "lr": 0.1, + "lr_scheduler_type": "constant", + "lr_warmup_steps": -1, + "lr_warmup_steps_ratio": 0.0, + "min_lr_ratio": 0.0, + "num_cycles": 0.5, + "optimizer": "SGD", + "optimizer_impl": "torch.optim", + "override_optimizer_config": { + "dampening": 0.0, + "momentum": 0.0, + "nesterov": false + }, + "total_training_steps": -1, + "warmup_style": null, + "weight_decay": 0.0, + "zero_indexed_step": true + }, + "policy_loss": { + "_target_": "verl.workers.config.PolicyLossConfig", + "clip_cov_lb": 1.0, + "clip_cov_ratio": 0.0002, + "clip_cov_ub": 5.0, + "kl_cov_ratio": 0.0002, + "loss_mode": "vanilla", + "ppo_kl_coef": 0.1 + }, + "ppo_epochs": 1, + "ppo_max_token_len_per_gpu": 16384, + "ppo_micro_batch_size": null, + "ppo_micro_batch_size_per_gpu": 4, + "ppo_mini_batch_size": 128, + "profiler": { + "_target_": "verl.utils.profiler.ProfilerConfig", + "all_ranks": false, + "enable": false, + "ranks": [], + "save_path": "outputs/profile", + "tool": null, + "tool_config": { + "npu": { + "_target_": "verl.utils.profiler.config.NPUToolConfig", + "analysis": true, + "contents": [], + "discrete": false, + "level": "level0" + }, + "nsys": { + "_target_": "verl.utils.profiler.config.NsightToolConfig", + "discrete": false + }, + "torch": { + "_target_": "verl.utils.profiler.config.TorchProfilerToolConfig", + "contents": [], + "discrete": false + }, + "torch_memory": { + "_target_": "verl.utils.profiler.config.TorchMemoryToolConfig", + "stack_depth": 32, + "trace_alloc_max_entries": 100000 + } + } + }, + "qat": { + "activation_observer": "static_minmax", + "enable": false, + "group_size": 16, + "ignore_patterns": [ + "lm_head", + "embed_tokens", + "re:.*mlp.gate$" + ], + "mode": "w4a16", + "quantization_config_path": null + }, + "rollout_n": 4, + "router_replay": { + "_target_": "verl.workers.config.RouterReplayConfig", + "mode": "disabled", + "record_file": null, + "replay_file": null + }, + "shuffle": false, + "strategy": "fsdp", + "sum_pi_squared_checkpointing": false, + "tau_neg": 1.05, + "tau_pos": 1.0, + "ulysses_sequence_parallel_size": 1, + "use_dynamic_bsz": false, + "use_fused_kernels": true, + "use_kl_loss": false, + "use_prefix_grouper": false, + "use_remove_padding": true, + "use_torch_compile": true + }, + "hybrid_engine": true, + "model": { + "_target_": "verl.workers.config.HFModelConfig", + "custom_chat_template": null, + "enable_activation_offload": false, + "enable_gradient_checkpointing": false, + "exclude_modules": null, + "external_lib": null, + "fused_kernel_options": { + "impl_backend": "torch" + }, + "hf_config_path": null, + "lora_adapter_path": null, + "lora_alpha": 16, + "lora_rank": 0, + "mtp": { + "_target_": "verl.workers.config.MtpConfig", + "detach_encoder": false, + "enable": false, + "enable_rollout": false, + "enable_train": false, + "method": "mtp", + "mtp_loss_scaling_factor": 0.1, + "num_speculative_tokens": 1, + "speculative_algorithm": "EAGLE", + "speculative_eagle_topk": 1, + "speculative_num_draft_tokens": 4, + "speculative_num_steps": 3 + }, + "override_config": {}, + "path": "Qwen/Qwen3-1.7B-Base", + "target_modules": "all-linear", + "tiled_mlp": { + "enabled": false, + "num_shards": 4 + }, + "tokenizer_path": null, + "trust_remote_code": false, + "use_fused_kernels": true, + "use_liger": false, + "use_remove_padding": true, + "use_shm": false + }, + "nccl_timeout": 600, + "ref": { + "_target_": "verl.workers.config.FSDPActorConfig", + "entropy_checkpointing": false, + "entropy_from_logits_with_chunking": false, + "fsdp_config": { + "_target_": "verl.workers.config.FSDPEngineConfig", + "dtype": "bfloat16", + "entropy_checkpointing": false, + "entropy_from_logits_with_chunking": false, + "forward_only": true, + "forward_prefetch": false, + "fsdp_size": -1, + "full_determinism": false, + "model_dtype": "fp32", + "offload_policy": false, + "optimizer_offload": false, + "param_offload": false, + "qat": { + "_target_": "verl.workers.config.QATEngineConfig", + "activation_observer": "static_minmax", + "enable": false, + "group_size": 16, + "ignore_patterns": [ + "lm_head", + "embed_tokens", + "re:.*mlp.gate$" + ], + "mode": "w4a16", + "quantization_config_path": null + }, + "reshard_after_forward": true, + "seed": 42, + "strategy": "fsdp", + "ulysses_sequence_parallel_size": 1, + "use_orig_params": false, + "use_torch_compile": true, + "wrap_policy": { + "min_num_params": 0 + } + }, + "log_prob_max_token_len_per_gpu": 16384, + "log_prob_micro_batch_size": null, + "log_prob_micro_batch_size_per_gpu": 4, + "log_prob_use_dynamic_bsz": false, + "profiler": { + "_target_": "verl.utils.profiler.ProfilerConfig", + "all_ranks": false, + "enable": false, + "ranks": [], + "save_path": "outputs/profile", + "tool": null, + "tool_config": { + "npu": { + "_target_": "verl.utils.profiler.config.NPUToolConfig", + "analysis": true, + "contents": [], + "discrete": false, + "level": "level0" + }, + "nsys": { + "_target_": "verl.utils.profiler.config.NsightToolConfig", + "discrete": false + }, + "torch": { + "_target_": "verl.utils.profiler.config.TorchProfilerToolConfig", + "contents": [], + "discrete": false + }, + "torch_memory": { + "_target_": "verl.utils.profiler.config.TorchMemoryToolConfig", + "stack_depth": 32, + "trace_alloc_max_entries": 100000 + } + } + }, + "rollout_n": 4, + "router_replay": { + "_target_": "verl.workers.config.RouterReplayConfig", + "mode": "disabled", + "record_file": null, + "replay_file": null + }, + "strategy": "fsdp", + "ulysses_sequence_parallel_size": 1, + "use_torch_compile": true + }, + "rollout": { + "_target_": "verl.workers.config.RolloutConfig", + "agent": { + "_target_": "verl.workers.config.AgentLoopConfig", + "agent_loop_config_path": null, + "custom_async_server": { + "_target_": "verl.workers.config.CustomAsyncServerConfig", + "name": null, + "path": null + }, + "default_agent_loop": "single_turn_agent", + "num_workers": 8 + }, + "calculate_log_probs": false, + "checkpoint_engine": { + "_target_": "verl.workers.config.CheckpointEngineConfig", + "backend": "naive", + "engine_kwargs": {}, + "update_weights_bucket_megabytes": 2048 + }, + "cudagraph_capture_sizes": null, + "data_parallel_size": 1, + "disable_log_stats": true, + "do_sample": true, + "dtype": "bfloat16", + "enable_chunked_prefill": true, + "enable_prefix_caching": true, + "enable_rollout_routing_replay": false, + "enforce_eager": false, + "engine_kwargs": { + "sglang": {}, + "trtllm": {}, + "vllm": {} + }, + "expert_parallel_size": 1, + "free_cache_engine": true, + "gpu_memory_utilization": 0.85, + "ignore_eos": false, + "layered_summon": false, + "load_format": "dummy", + "log_prob_max_token_len_per_gpu": 16384, + "log_prob_micro_batch_size": null, + "log_prob_micro_batch_size_per_gpu": 4, + "log_prob_use_dynamic_bsz": false, + "logprobs_mode": "processed_logprobs", + "max_model_len": null, + "max_num_batched_tokens": 8192, + "max_num_seqs": 1024, + "mode": "async", + "mtp": { + "_target_": "verl.workers.config.MtpConfig", + "detach_encoder": false, + "enable": false, + "enable_rollout": false, + "enable_train": false, + "method": "mtp", + "mtp_loss_scaling_factor": 0.1, + "num_speculative_tokens": 1, + "speculative_algorithm": "EAGLE", + "speculative_eagle_topk": 1, + "speculative_num_draft_tokens": 4, + "speculative_num_steps": 3 + }, + "multi_stage_wake_up": false, + "multi_turn": { + "_target_": "verl.workers.config.MultiTurnConfig", + "enable": false, + "format": "hermes", + "interaction_config_path": null, + "max_assistant_turns": null, + "max_parallel_calls": 1, + "max_tool_response_length": 256, + "max_user_turns": null, + "num_repeat_rollouts": null, + "tokenization_sanity_check_mode": "strict", + "tool_config_path": null, + "tool_response_truncate_side": "middle", + "use_inference_chat_template": false + }, + "n": 4, + "n_gpus_per_node": 4, + "name": "vllm", + "nnodes": 0, + "over_sample_rate": 0, + "pipeline_model_parallel_size": 1, + "profiler": { + "_target_": "verl.utils.profiler.ProfilerConfig", + "all_ranks": false, + "enable": false, + "ranks": [], + "save_path": "outputs/profile", + "tool": null, + "tool_config": { + "npu": { + "_target_": "verl.utils.profiler.config.NPUToolConfig", + "analysis": true, + "contents": [], + "discrete": false, + "level": "level0" + }, + "torch": { + "_target_": "verl.utils.profiler.config.TorchProfilerToolConfig", + "contents": [], + "discrete": false + } + } + }, + "prometheus": { + "_target_": "verl.workers.config.PrometheusConfig", + "enable": false, + "file": "/tmp/ray/session_latest/metrics/prometheus/prometheus.yml", + "port": 9090, + "served_model_name": "Qwen/Qwen3-1.7B-Base" + }, + "prompt_length": 1024, + "qat": { + "_target_": "verl.workers.config.QATEngineConfig", + "activation_observer": "static_minmax", + "enable": false, + "group_size": 16, + "ignore_patterns": [ + "lm_head", + "embed_tokens", + "re:.*mlp.gate$" + ], + "mode": "w4a16", + "quantization_config_path": null + }, + "quantization": null, + "quantization_config_file": null, + "response_length": 8192, + "scheduling_policy": "fcfs", + "skip_dump_dir": "/tmp/rollout_dump", + "skip_rollout": false, + "skip_tokenizer_init": true, + "temperature": 1.0, + "tensor_model_parallel_size": 1, + "top_k": -1, + "top_p": 1, + "trace": { + "_target_": "verl.workers.config.TraceConfig", + "backend": null, + "experiment_name": "polaris_1p7b_sgd_lr1e-1_bs128_gpu4_ep5_n4", + "max_samples_per_step_per_worker": null, + "project_name": "sgd_rerun_polaris_1p7b", + "token2text": false + }, + "val_kwargs": { + "_target_": "verl.workers.config.SamplingConfig", + "do_sample": false, + "n": 1, + "temperature": 0, + "top_k": -1, + "top_p": 1.0 + } + } + }, + "algorithm": { + "_target_": "verl.trainer.config.AlgoConfig", + "adv_estimator": "grpo", + "gamma": 1.0, + "kl_ctrl": { + "_target_": "verl.trainer.config.KLControlConfig", + "horizon": 10000, + "kl_coef": 0.001, + "target_kl": 0.1, + "type": "fixed" + }, + "kl_penalty": "kl", + "lam": 1.0, + "norm_adv_by_std_in_grpo": true, + "pf_ppo": { + "reweight_method": "pow", + "weight_pow": 2.0 + }, + "rollout_correction": { + "bypass_mode": false, + "loss_type": "ppo_clip", + "rollout_is": null, + "rollout_is_batch_normalize": false, + "rollout_is_threshold": 2.0, + "rollout_rs": null, + "rollout_rs_threshold": null + }, + "use_kl_in_reward": false, + "use_pf_ppo": false + }, + "critic": { + "_target_": "verl.workers.config.FSDPCriticConfig", + "checkpoint": { + "_target_": "verl.trainer.config.CheckpointConfig", + "async_save": false, + "load_contents": [ + "model", + "optimizer", + "extra" + ], + "mbridge_config": {}, + "save_contents": [ + "model", + "optimizer", + "extra" + ] + }, + "cliprange_value": 0.5, + "data_loader_seed": 42, + "enable": null, + "forward_max_token_len_per_gpu": 32768, + "forward_micro_batch_size": null, + "forward_micro_batch_size_per_gpu": null, + "grad_clip": 1.0, + "loss_agg_mode": "token-mean", + "model": { + "_target_": "verl.workers.config.FSDPCriticModelCfg", + "enable_activation_offload": false, + "enable_gradient_checkpointing": true, + "external_lib": null, + "fsdp_config": { + "_target_": "verl.workers.config.FSDPEngineConfig", + "dtype": "bfloat16", + "entropy_checkpointing": false, + "entropy_from_logits_with_chunking": false, + "forward_only": false, + "forward_prefetch": false, + "fsdp_size": -1, + "full_determinism": false, + "model_dtype": "fp32", + "offload_policy": false, + "optimizer_offload": false, + "param_offload": false, + "qat": { + "_target_": "verl.workers.config.QATEngineConfig", + "activation_observer": "static_minmax", + "enable": false, + "group_size": 16, + "ignore_patterns": [ + "lm_head", + "embed_tokens", + "re:.*mlp.gate$" + ], + "mode": "w4a16", + "quantization_config_path": null + }, + "reshard_after_forward": true, + "seed": 42, + "strategy": "fsdp", + "ulysses_sequence_parallel_size": 1, + "use_orig_params": false, + "use_torch_compile": true, + "wrap_policy": { + "min_num_params": 0 + } + }, + "lora_alpha": 16, + "lora_rank": 0, + "override_config": {}, + "path": "~/models/deepseek-llm-7b-chat", + "target_modules": "all-linear", + "tiled_mlp": { + "enabled": false, + "num_shards": 4 + }, + "tokenizer_path": "Qwen/Qwen3-1.7B-Base", + "trust_remote_code": false, + "use_remove_padding": false, + "use_shm": false + }, + "optim": { + "_target_": "verl.workers.config.FSDPOptimizerConfig", + "betas": [ + 0.9, + 0.999 + ], + "clip_grad": 1.0, + "lr": 1e-05, + "lr_scheduler_type": "constant", + "lr_warmup_steps": -1, + "lr_warmup_steps_ratio": 0.0, + "min_lr_ratio": 0.0, + "num_cycles": 0.5, + "optimizer": "AdamW", + "optimizer_impl": "torch.optim", + "override_optimizer_config": null, + "total_training_steps": -1, + "warmup_style": null, + "weight_decay": 0.01, + "zero_indexed_step": true + }, + "ppo_epochs": 1, + "ppo_max_token_len_per_gpu": 32768, + "ppo_micro_batch_size": null, + "ppo_micro_batch_size_per_gpu": null, + "ppo_mini_batch_size": 128, + "profiler": { + "_target_": "verl.utils.profiler.ProfilerConfig", + "all_ranks": false, + "enable": false, + "ranks": [], + "save_path": "outputs/profile", + "tool": null, + "tool_config": { + "npu": { + "_target_": "verl.utils.profiler.config.NPUToolConfig", + "analysis": true, + "contents": [], + "discrete": false, + "level": "level0" + }, + "nsys": { + "_target_": "verl.utils.profiler.config.NsightToolConfig", + "discrete": false + }, + "torch": { + "_target_": "verl.utils.profiler.config.TorchProfilerToolConfig", + "contents": [], + "discrete": false + }, + "torch_memory": { + "_target_": "verl.utils.profiler.config.TorchMemoryToolConfig", + "stack_depth": 32, + "trace_alloc_max_entries": 100000 + } + } + }, + "rollout_n": 4, + "shuffle": false, + "strategy": "fsdp", + "ulysses_sequence_parallel_size": 1, + "use_dynamic_bsz": false + }, + "data": { + "apply_chat_template_kwargs": {}, + "custom_cls": { + "name": null, + "path": null + }, + "datagen": { + "name": null, + "path": null + }, + "dataloader_num_workers": 8, + "filter_overlong_prompts": true, + "filter_overlong_prompts_workers": 1, + "image_key": "images", + "image_patch_size": 14, + "max_prompt_length": 1024, + "max_response_length": 8192, + "prompt_key": "prompt", + "return_full_prompt": false, + "return_multi_modal_inputs": true, + "return_raw_chat": true, + "return_raw_input_ids": false, + "reward_fn_key": "data_source", + "sampler": { + "class_name": null, + "class_path": null + }, + "seed": null, + "shuffle": true, + "tokenizer": null, + "tool_config_path": null, + "train_batch_size": 128, + "train_files": "/home/scratch.pjayasinha_gpu/rs/data_polaris/train.parquet", + "train_max_samples": -1, + "truncation": "error", + "trust_remote_code": false, + "use_shm": false, + "val_batch_size": null, + "val_files": [ + "/home/scratch.pjayasinha_gpu/rs/data_polaris/val.parquet" + ], + "val_max_samples": -1, + "validation_shuffle": false, + "video_key": "videos" + }, + "global_profiler": { + "_target_": "verl.utils.profiler.ProfilerConfig", + "global_tool_config": { + "nsys": { + "_target_": "verl.utils.profiler.config.NsightToolConfig", + "controller_nsight_options": { + "cuda-graph-trace": "graph", + "cuda-memory-usage": "true", + "trace": "cuda,nvtx,cublas,ucx" + }, + "discrete": false, + "worker_nsight_options": { + "capture-range": "cudaProfilerApi", + "capture-range-end": null, + "cuda-graph-trace": "graph", + "cuda-memory-usage": "true", + "kill": "none", + "trace": "cuda,nvtx,cublas,ucx" + } + }, + "torch_memory": { + "context": "all", + "kw_args": {}, + "stack_depth": 32, + "stacks": "all", + "trace_alloc_max_entries": 100000 + } + }, + "profile_continuous_steps": false, + "save_path": "outputs/profile", + "steps": null, + "tool": null + }, + "model_engine": "dp", + "ray_kwargs": { + "ray_init": { + "num_cpus": null + }, + "timeline_json_file": null + }, + "reward": { + "custom_reward_function": { + "name": "compute_score", + "path": null + }, + "num_workers": 8, + "reward_manager": { + "_target_": "verl.workers.config.reward_model.RewardManagerConfig", + "module": { + "_target_": "verl.trainer.config.config.ModuleConfig", + "name": "custom_reward_manager", + "path": null + }, + "name": "naive", + "source": "register" + }, + "reward_model": { + "enable": false, + "enable_resource_pool": false, + "model_path": null, + "n_gpus_per_node": 8, + "nnodes": 0, + "rollout": { + "_target_": "verl.workers.config.RolloutConfig", + "cudagraph_capture_sizes": null, + "data_parallel_size": 1, + "disable_log_stats": true, + "dtype": "bfloat16", + "enable_chunked_prefill": true, + "enable_prefix_caching": true, + "enforce_eager": true, + "engine_kwargs": {}, + "expert_parallel_size": 1, + "free_cache_engine": true, + "gpu_memory_utilization": 0.5, + "limit_images": null, + "load_format": "auto", + "max_model_len": null, + "max_num_batched_tokens": 8192, + "max_num_seqs": 1024, + "name": "???", + "prompt_length": 2048, + "response_length": 2048, + "skip_tokenizer_init": false, + "tensor_model_parallel_size": 2 + } + }, + "sandbox_fusion": { + "max_concurrent": 64, + "memory_limit_mb": 1024, + "url": null + } + }, + "trainer": { + "balance_batch": true, + "critic_warmup": 0, + "default_hdfs_dir": null, + "default_local_dir": "/home/scratch.pjayasinha_gpu/rs/checkpoints/polaris_1p7b_sgd_lr1e-1_bs128_gpu4_ep5_n4", + "del_local_ckpt_after_load": false, + "device": "cuda", + "esi_redundant_time": 0, + "experiment_name": "polaris_1p7b_sgd_lr1e-1_bs128_gpu4_ep5_n4", + "log_val_generations": 0, + "logger": [ + "console", + "wandb" + ], + "max_actor_ckpt_to_keep": null, + "max_critic_ckpt_to_keep": null, + "n_gpus_per_node": 4, + "nnodes": 1, + "project_name": "sgd_rerun_polaris_1p7b", + "ray_wait_register_center_timeout": 300, + "resume_from_path": null, + "resume_mode": "auto", + "rollout_data_dir": null, + "save_freq": 25, + "test_freq": 50, + "total_epochs": 5, + "total_training_steps": null, + "use_legacy_worker_impl": "auto", + "val_before_train": true, + "val_only": false, + "validation_data_dir": null, + "wandb": { + "entity": "pavanjayasinha-university-of-illinois-urbana-champaign" + } + }, + "transfer_queue": { + "enable": false + } +} \ No newline at end of file