322 lines
12 KiB
YAML
322 lines
12 KiB
YAML
|
entrypoint: examples.terminal_bench.entrypoints.main_tbench
|
|||
|
|
|
|||
|
|
# Hydra config groups (+ prefix in CLI)
|
|||
|
|
config_groups:
|
|||
|
|
terminal_bench_config: terminal_bench
|
|||
|
|
|
|||
|
|
# Terminal bench / agentic environment settings
|
|||
|
|
terminal_bench:
|
|||
|
|
# trials_dir: Directory for Harbor trial artifacts (derived from experiments_dir if null)
|
|||
|
|
trials_dir: null
|
|||
|
|
|
|||
|
|
# Harbor configuration - schema-driven mapping to TrialConfig
|
|||
|
|
harbor:
|
|||
|
|
# Agent settings
|
|||
|
|
name: terminus-2
|
|||
|
|
max_episodes: 999999
|
|||
|
|
enable_summarize: false
|
|||
|
|
store_all_messages: true
|
|||
|
|
trajectory_config:
|
|||
|
|
raw_content: true
|
|||
|
|
enable_episode_logging: false
|
|||
|
|
record_terminal_session: false
|
|||
|
|
enable_pane_logging: false
|
|||
|
|
|
|||
|
|
# Strict JSON parser
|
|||
|
|
strict_json_parser: true
|
|||
|
|
|
|||
|
|
# Interleaved Thinking Settings
|
|||
|
|
interleaved_thinking: true
|
|||
|
|
extra_body:
|
|||
|
|
chat_template_kwargs:
|
|||
|
|
enable_thinking: true
|
|||
|
|
# 2026-05-27: 1800 → 900. The boundary-hugging a3 datasets (stack-junit,
|
|||
|
|
# nemotron, methods2test) hit the 1800s agent-timeout wall on ~70-95% of
|
|||
|
|
# trials (GLM-4_7-swesmith can't finish these in 1800s), so every group
|
|||
|
|
# waited ~1800s for its slowest sample → generation buffer crawled (5/64
|
|||
|
|
# in 4h19) and chains never reached training step 1. Halving to 900s
|
|||
|
|
# ~2× the valid-group yield. Tradeoff: lower solve-rate on the hardest
|
|||
|
|
# tasks (they fail faster) — acceptable vs. TIMEOUT-at-step-0. Fast-task
|
|||
|
|
# chains (e2egit, median 7 turns / <900s) are unaffected. See
|
|||
|
|
# agent_logs/2026-05-27_rl_chains_stuck_step0.md. AgentTimeoutError stays
|
|||
|
|
# PASSTHROUGH (standing pref — partial trajectory+reward is meaningful).
|
|||
|
|
override_timeout_sec: 900
|
|||
|
|
|
|||
|
|
# Environment settings
|
|||
|
|
override_cpus: 1
|
|||
|
|
override_memory_mb: 2048
|
|||
|
|
override_storage_mb: 2048
|
|||
|
|
|
|||
|
|
# ==========================================================================
|
|||
|
|
# AUTO SNAPSHOT: Reduce Daytona rate limits with hash-based snapshot caching
|
|||
|
|
# ==========================================================================
|
|||
|
|
# When true, automatically creates a snapshot from the Dockerfile on first use,
|
|||
|
|
# then reuses it for all subsequent sandboxes with the same Dockerfile content.
|
|||
|
|
# Snapshots are named: harbor__<sha256[:12]>__snapshot
|
|||
|
|
auto_snapshot: true
|
|||
|
|
|
|||
|
|
# Verifier settings
|
|||
|
|
verifier_override_timeout_sec: 120
|
|||
|
|
|
|||
|
|
# Retry settings
|
|||
|
|
max_retries: 3
|
|||
|
|
min_wait_sec: 60.0
|
|||
|
|
max_wait_sec: 600.0
|
|||
|
|
wait_multiplier: 2.0
|
|||
|
|
|
|||
|
|
exclude_exceptions:
|
|||
|
|
- VerifierTimeoutError
|
|||
|
|
- VerifierRuntimeError
|
|||
|
|
- RewardFileNotFoundError
|
|||
|
|
- RewardFileEmptyError
|
|||
|
|
- VerifierOutputParseError
|
|||
|
|
|
|||
|
|
# 3x base (300 → 900): triple Daytona concurrency for higher throughput.
|
|||
|
|
# Mirrors ALCC/56GPU_base.yaml — per-job sandbox count; doesn't affect
|
|||
|
|
# the cross-cluster RUNNING-RL cap (≤ 6) which is about job count, not
|
|||
|
|
# sandboxes-per-job.
|
|||
|
|
# NOTE (2026-05-27): a temporary 900→500 walk-back was tried to mitigate
|
|||
|
|
# the FD-exhaustion / uv__epoll_ctl_prep SIGABRT but REVERTED — the real
|
|||
|
|
# fix is porting harbor's terminus-2 FD-taming features from
|
|||
|
|
# penfever/temp-override into penfever/otagent-latest (that branch had the
|
|||
|
|
# issue tamed at full concurrency). See agent_logs/2026-05-27_rl_chains_stuck_step0.md.
|
|||
|
|
# 2026-05-28: 900 → 675 (-25%). Fresh a3 chains (#12/#13) abort with a
|
|||
|
|
# driver SIGABRT that is NOT FD exhaustion (fd-monitor showed 1.6% of the
|
|||
|
|
# 131072 limit) — suspected host-memory OOM from too many concurrent
|
|||
|
|
# never-completing trials piling up resident state in skyrl_entrypoint on
|
|||
|
|
# hard datasets. Cutting concurrent trial count reduces that resident
|
|||
|
|
# memory. Paired with num_parallel_generation_workers -25%. See
|
|||
|
|
# agent_logs/2026-05-28_fresh_a3_chain_crash_not_fd.md.
|
|||
|
|
n_concurrent_trials: 675
|
|||
|
|
|
|||
|
|
# Logging settings
|
|||
|
|
log_level: INFO
|
|||
|
|
|
|||
|
|
# Reward shaping (disabled - binary rewards)
|
|||
|
|
enable_reward_shaping: false
|
|||
|
|
|
|||
|
|
# RLOO-N error classification
|
|||
|
|
enable_error_classification: true
|
|||
|
|
mask_exceptions:
|
|||
|
|
- DaytonaError
|
|||
|
|
- EnvironmentStartTimeoutError
|
|||
|
|
- NetworkError
|
|||
|
|
- ConnectionError
|
|||
|
|
- RewardFileNotFoundError
|
|||
|
|
- RewardFileEmptyError
|
|||
|
|
- AgentEnvironmentTimeoutError
|
|||
|
|
- ContextLengthExceededError
|
|||
|
|
default_error_treatment: zero
|
|||
|
|
# NOTE (2026-05-27): AgentTimeoutError / ContextLengthExceededError moved
|
|||
|
|
# OUT of passthrough_exceptions into zero_exceptions. Passthrough routed
|
|||
|
|
# these soft-limit trials through the *normal* (live-trajectory) path in
|
|||
|
|
# terminal_bench_generator._process_trial_result, keeping the trial's
|
|||
|
|
# dangling vLLM ObjectRefs + orphaned litellm async-callback closures
|
|||
|
|
# alive in the training batch. That fed Ray's distributed-refcount race
|
|||
|
|
# (reference_count.cc:1619 → SIGABRT / WorkerCrashedError) that killed the
|
|||
|
|
# nemotron-junit chain (litellm +1 32769 VLLMValidationError +
|
|||
|
|
# ContextLengthExceeded on near-budget prompts). Both mask + zero take the
|
|||
|
|
# EARLY-RETURN path (response_ids=[0]) which never consumes rollout_details,
|
|||
|
|
# so no dangling refs enter training. 2026-05-27: ContextLengthExceededError
|
|||
|
|
# moved to mask_exceptions (above) — excluded from the RLOO-N baseline, since
|
|||
|
|
# GLM-4_7-swesmith ~32k prompts overflow routinely and we don't want hard
|
|||
|
|
# zeros dragging the baseline. AgentTimeoutError stays PASSTHROUGH (partial trajectory+reward is
|
|||
|
|
# meaningful signal; infrequent vs ContextLengthExceeded so low refcount risk). Belt-and-suspenders w/ the _harbor_compat rollback_hook that
|
|||
|
|
# already truncates these trials' dangling prompt-without-response.
|
|||
|
|
passthrough_exceptions:
|
|||
|
|
- AgentTimeoutError
|
|||
|
|
zero_exceptions: []
|
|||
|
|
|
|||
|
|
# Model info for Harbor's hosted_vllm validation
|
|||
|
|
model_info:
|
|||
|
|
# Lowered 32767 -> 32000 (2026-05-27). Harbor's litellm token counter is
|
|||
|
|
# +2 below vLLM's actual BPE tokenization on near-budget prompts (litellm
|
|||
|
|
# +1 plus chat-template/special-token drift), so a 32767-capped prompt
|
|||
|
|
# tokenizes to 32769 at the vLLM serving layer and is rejected with
|
|||
|
|
# `VLLMValidationError: 32769 input tokens ... context length is only
|
|||
|
|
# 32768`. vLLM returns this as an ErrorResponse (handled, not an engine
|
|||
|
|
# crash), but harbor's litellm then retries the *deterministically*
|
|||
|
|
# over-budget request (max_retries=3, exp backoff) across n_concurrent
|
|||
|
|
# trials. On a boundary-heavy task distribution (e.g. nemotron-junit: 111+
|
|||
|
|
# such overflows per chain link) the accumulating open sockets exhaust the
|
|||
|
|
# skyrl_entrypoint actor's file descriptors -> libuv uv__epoll_ctl_prep
|
|||
|
|
# aborts -> SIGABRT in the uvloop event loop -> ray.WorkerCrashedError kills
|
|||
|
|
# the chain before training step 1 (job a3-rl ...nemotron-junit #11,
|
|||
|
|
# chain 521442-448). A 768-token buffer below max_model_len=32768 ensures
|
|||
|
|
# harbor truncates before vLLM ever rejects, eliminating the retry storm.
|
|||
|
|
# Cost is ~2.3% usable context for all a3 chains; chains that don't hug the
|
|||
|
|
# boundary (e2egit reached step 68 fine at 32767) are unaffected.
|
|||
|
|
max_input_tokens: 32000
|
|||
|
|
max_output_tokens: 4096
|
|||
|
|
|
|||
|
|
archiving:
|
|||
|
|
# Enable trial archiving callback
|
|||
|
|
enabled: false
|
|||
|
|
|
|||
|
|
# Post-training trace upload to HuggingFace
|
|||
|
|
trace_upload:
|
|||
|
|
enabled: true
|
|||
|
|
repo_org: DCAgent
|
|||
|
|
episodes: last
|
|||
|
|
dataset_type: SFT
|
|||
|
|
cleanup: true
|
|||
|
|
|
|||
|
|
# Trainer configuration
|
|||
|
|
trainer:
|
|||
|
|
strategy: fsdp2
|
|||
|
|
algorithm:
|
|||
|
|
advantage_estimator: rloo_n
|
|||
|
|
use_kl_loss: false
|
|||
|
|
kl_loss_coef: 0.0
|
|||
|
|
eps_clip_low: 0.2
|
|||
|
|
# eps_clip_high=0.05 mirrors 24GPU_base — midpoint between 0.2 default and
|
|||
|
|
# 0.01 tight. Engages on collapse-onset ratios without over-clamping
|
|||
|
|
# healthy updates. Asymmetric — only tightening upper bound.
|
|||
|
|
eps_clip_high: 0.05
|
|||
|
|
# A/B control arm0 = sequence_mean (the 1/|o_i| brevity-biased reduction).
|
|||
|
|
# NOTE: the a3 production 56GPU_base.yaml default is token_mean, NOT
|
|||
|
|
# sequence_mean — so the completed run a3-rl-...-pymethods2test-large-80-8B
|
|||
|
|
# is NOT a valid arm0 and cannot be reused. This explicit arm0 config
|
|||
|
|
# establishes the sequence_mean baseline the bias interpretation needs.
|
|||
|
|
loss_reduction: sequence_mean
|
|||
|
|
|
|||
|
|
# Training loop settings
|
|||
|
|
epochs: 2
|
|||
|
|
max_steps: 80
|
|||
|
|
update_epochs_per_batch: 1
|
|||
|
|
|
|||
|
|
# Batch sizes
|
|||
|
|
train_batch_size: 64
|
|||
|
|
policy_mini_batch_size: 64
|
|||
|
|
eval_batch_size: 64
|
|||
|
|
|
|||
|
|
# Micro batch sizes (micro1x4 variant)
|
|||
|
|
micro_forward_batch_size_per_gpu: 4
|
|||
|
|
micro_train_batch_size_per_gpu: 1
|
|||
|
|
|
|||
|
|
max_prompt_length: 999999
|
|||
|
|
|
|||
|
|
# Evaluation and checkpointing
|
|||
|
|
eval_interval: 999999
|
|||
|
|
eval_before_train: false
|
|||
|
|
# Resumable checkpointing
|
|||
|
|
ckpt_interval: 2
|
|||
|
|
resume_mode: latest
|
|||
|
|
# HF upload-ready checkpoints
|
|||
|
|
hf_save_interval: 5
|
|||
|
|
# HuggingFace Hub upload (set via CLI: trainer.hf_hub_repo_id=org/repo)
|
|||
|
|
hf_hub_repo_id: null
|
|||
|
|
hf_hub_private: false
|
|||
|
|
hf_hub_revision: main
|
|||
|
|
|
|||
|
|
# Database registration (auto-registers trained model to Supabase)
|
|||
|
|
# Requires KEYS env var pointing to Supabase credentials file
|
|||
|
|
enable_db_registration: false
|
|||
|
|
|
|||
|
|
# Logging
|
|||
|
|
project_name: OpenThoughts-Agent
|
|||
|
|
log_level: INFO
|
|||
|
|
tracker_commit_each_step: true
|
|||
|
|
logger: console
|
|||
|
|
|
|||
|
|
# Paths
|
|||
|
|
run_name: null
|
|||
|
|
ckpt_path: null
|
|||
|
|
export_path: null
|
|||
|
|
|
|||
|
|
# Policy optimizer
|
|||
|
|
# max_grad_norm=0.9 mirrors 24GPU_base — guardrail against grad-norm spikes
|
|||
|
|
# entering correlation-mode-collapse territory (>1.0). 0.9 is just above the
|
|||
|
|
# natural healthy peak observed on this dataset+base (‖g‖ peak ~0.81).
|
|||
|
|
policy:
|
|||
|
|
optimizer_config:
|
|||
|
|
lr: 8e-6
|
|||
|
|
weight_decay: 0.0
|
|||
|
|
adam_betas: [0.9, 0.999]
|
|||
|
|
max_grad_norm: 0.9
|
|||
|
|
fsdp_config:
|
|||
|
|
cpu_offload: false
|
|||
|
|
reshard_after_forward: true
|
|||
|
|
fsdp_size: 4
|
|||
|
|
# Reference model
|
|||
|
|
ref:
|
|||
|
|
fsdp_config:
|
|||
|
|
cpu_offload: false
|
|||
|
|
reshard_after_forward: true
|
|||
|
|
fsdp_size: 4
|
|||
|
|
|
|||
|
|
# Model placement (async training) - 8 shared GPUs for policy/ref
|
|||
|
|
placement:
|
|||
|
|
colocate_all: false
|
|||
|
|
policy_num_nodes: 2
|
|||
|
|
ref_num_nodes: 2
|
|||
|
|
policy_num_gpus_per_node: 4
|
|||
|
|
ref_num_gpus_per_node: 4
|
|||
|
|
|
|||
|
|
# Fully async generation (settings from v2_maxconcurrent)
|
|||
|
|
fully_async:
|
|||
|
|
max_staleness_steps: 16
|
|||
|
|
# Setting conservatively to 1 / 2 of total concurrency
|
|||
|
|
# 2026-05-28: 450 → 338 (-25%), paired with n_concurrent_trials 900→675,
|
|||
|
|
# to cut driver memory pressure (suspected OOM on fresh hard-dataset a3
|
|||
|
|
# chains). See agent_logs/2026-05-28_fresh_a3_chain_crash_not_fd.md.
|
|||
|
|
num_parallel_generation_workers: 338
|
|||
|
|
|
|||
|
|
# Generator configuration
|
|||
|
|
generator:
|
|||
|
|
backend: vllm
|
|||
|
|
timeout_multiplier: 1.0
|
|||
|
|
model_dtype: bfloat16
|
|||
|
|
|
|||
|
|
inference_engine_tensor_parallel_size: 1
|
|||
|
|
# 3x base (16 → 48): triple vLLM engines to lift gen throughput above training rate.
|
|||
|
|
# Goal: build a real surplus of completed groups so the trainer never waits on gen.
|
|||
|
|
# Layout: 48 engines × TP=1 + 8 GPUs for policy/ref = 56 GPUs total = 14 nodes.
|
|||
|
|
num_inference_engines: 48
|
|||
|
|
|
|||
|
|
n_samples_per_prompt: 8
|
|||
|
|
eval_n_samples_per_prompt: 8
|
|||
|
|
|
|||
|
|
# Jupiter-specific gpu_memory_utilization: 0.75 (vs ALCC's 0.85).
|
|||
|
|
# GH200's 96 GB HBM has more headroom than ALCC's A100-80, but Jupiter
|
|||
|
|
# also runs hosted_vllm in the same Ray cluster as the engines and we
|
|||
|
|
# keep 0.75 across all jupiter yamls for consistency / fragmentation
|
|||
|
|
# safety. Bump to 0.85 only if KV-cache pressure justifies it.
|
|||
|
|
gpu_memory_utilization: 0.75
|
|||
|
|
|
|||
|
|
max_num_seqs: 24
|
|||
|
|
# Jupiter-specific 65536 (vs ALCC's 16384) — matches the existing
|
|||
|
|
# 24GPU_base.yaml on Jupiter; larger batched-token budget keeps the
|
|||
|
|
# engine fed when many concurrent requests arrive in bursts.
|
|||
|
|
max_num_batched_tokens: 65536
|
|||
|
|
|
|||
|
|
enable_prefix_caching: true
|
|||
|
|
enable_chunked_prefill: true
|
|||
|
|
|
|||
|
|
run_engines_locally: true
|
|||
|
|
weight_sync_backend: nccl
|
|||
|
|
async_engine: true
|
|||
|
|
batched: false
|
|||
|
|
enable_http_endpoint: true
|
|||
|
|
enable_ray_prometheus_stats: false
|
|||
|
|
vllm_stats_interval: 1
|
|||
|
|
append_eos_token_after_stop_str_in_multi_turn: true
|
|||
|
|
max_turns: 999999
|
|||
|
|
|
|||
|
|
sampling_params:
|
|||
|
|
max_generate_length: 4096
|
|||
|
|
temperature: 0.7
|
|||
|
|
top_p: 0.95
|
|||
|
|
top_k: 20
|
|||
|
|
|
|||
|
|
engine_init_kwargs:
|
|||
|
|
max_model_len: 32768
|
|||
|
|
# Interleaved thinking chat template: preserves <think> blocks on ALL
|
|||
|
|
# historical assistant turns (stock Qwen3 template strips them).
|
|||
|
|
custom_chat_template_chat_completion_path: chat_templates/qwen3_thinking_acc.jinja2
|
|||
|
|
|
|||
|
|
# Data paths
|
|||
|
|
data:
|
|||
|
|
train_data: []
|
|||
|
|
val_data: []
|