190 lines
11 KiB
Plaintext
190 lines
11 KiB
Plaintext
|
[2026-05-21 15:35:13,599][__main__][INFO] - config:
|
||
|
|
model:
|
||
|
|
hf_id: Qwen/Qwen3-1.7B
|
||
|
|
local_path: /shared-ro/models/Qwen3-1.7B
|
||
|
|
dtype: bfloat16
|
||
|
|
attn_impl: flash_attention_2
|
||
|
|
max_model_len: 4096
|
||
|
|
data:
|
||
|
|
name: safety_aug302
|
||
|
|
loader: load_safetybench_distilled
|
||
|
|
kwargs:
|
||
|
|
cache: /scratch/sft_aug302.jsonl
|
||
|
|
limit: null
|
||
|
|
training:
|
||
|
|
recipe: sft
|
||
|
|
epochs: 5
|
||
|
|
per_device_train_batch_size: 1
|
||
|
|
gradient_accumulation_steps: 16
|
||
|
|
learning_rate: 5.0e-06
|
||
|
|
warmup_ratio: 0.05
|
||
|
|
lr_scheduler_type: cosine
|
||
|
|
gradient_checkpointing: true
|
||
|
|
bf16: true
|
||
|
|
max_seq_length: 4096
|
||
|
|
logging_steps: 10
|
||
|
|
save_steps: 500
|
||
|
|
save_total_limit: 2
|
||
|
|
report_to: wandb
|
||
|
|
deepspeed: configs/deepspeed_zero2.json
|
||
|
|
use_peft: true
|
||
|
|
lora_r: 16
|
||
|
|
lora_alpha: 32
|
||
|
|
lora_target_modules: q_proj,v_proj,k_proj,o_proj
|
||
|
|
lora_dropout: 0.05
|
||
|
|
domain:
|
||
|
|
name: safety
|
||
|
|
thinking_mode: true
|
||
|
|
benchmark: safety
|
||
|
|
seed: 42
|
||
|
|
run_id: ${now:%Y%m%d_%H%M%S}
|
||
|
|
output_dir: /scratch/checkpoints/safety/sft/20260521_aug302
|
||
|
|
wandb:
|
||
|
|
project: flab-cs552
|
||
|
|
entity: cs-552-2026-flab
|
||
|
|
mode: null
|
||
|
|
tags: []
|
||
|
|
notes: null
|
||
|
|
eval:
|
||
|
|
model_dir: null
|
||
|
|
'n': 8
|
||
|
|
temperature: 0.7
|
||
|
|
top_p: 0.95
|
||
|
|
max_tokens: 4096
|
||
|
|
samples_dir: null
|
||
|
|
push:
|
||
|
|
model_dir: null
|
||
|
|
es:
|
||
|
|
base_model_dir: null
|
||
|
|
data: null
|
||
|
|
hf_push:
|
||
|
|
enabled: false
|
||
|
|
org: cs-552-2026-flab
|
||
|
|
repo: null
|
||
|
|
private: false
|
||
|
|
vllm_validate: true
|
||
|
|
commit_message: null
|
||
|
|
validate_after_train: false
|
||
|
|
vllm_validate: true
|
||
|
|
|
||
|
|
[2026-05-21 15:35:23,517][flab.shared.repro][WARNING] - wandb.init failed (No API key configured. Use `wandb login` to log in.); continuing without wandb
|
||
|
|
[2026-05-21 15:35:23,669][httpx][INFO] - HTTP Request: HEAD https://huggingface.co/Qwen/Qwen3-1.7B/resolve/main/config.json "HTTP/1.1 307 Temporary Redirect"
|
||
|
|
[2026-05-21 15:35:23,669][huggingface_hub.utils._http][WARNING] - Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads.
|
||
|
|
[2026-05-21 15:35:23,677][httpx][INFO] - HTTP Request: HEAD https://huggingface.co/api/resolve-cache/models/Qwen/Qwen3-1.7B/70d244cc86ccca08cf5af4e1e306ecf908b1ad5e/config.json "HTTP/1.1 200 OK"
|
||
|
|
[2026-05-21 15:35:23,795][httpx][INFO] - HTTP Request: HEAD https://huggingface.co/Qwen/Qwen3-1.7B/resolve/main/tokenizer_config.json "HTTP/1.1 307 Temporary Redirect"
|
||
|
|
[2026-05-21 15:35:23,803][httpx][INFO] - HTTP Request: HEAD https://huggingface.co/api/resolve-cache/models/Qwen/Qwen3-1.7B/70d244cc86ccca08cf5af4e1e306ecf908b1ad5e/tokenizer_config.json "HTTP/1.1 200 OK"
|
||
|
|
[2026-05-21 15:35:23,917][httpx][INFO] - HTTP Request: GET https://huggingface.co/api/models/Qwen/Qwen3-1.7B/tree/main/additional_chat_templates?recursive=false&expand=false "HTTP/1.1 404 Not Found"
|
||
|
|
[2026-05-21 15:35:24,043][httpx][INFO] - HTTP Request: GET https://huggingface.co/api/models/Qwen/Qwen3-1.7B/tree/main?recursive=true&expand=false "HTTP/1.1 200 OK"
|
||
|
|
[2026-05-21 15:35:24,779][httpx][INFO] - HTTP Request: GET https://huggingface.co/api/models/Qwen/Qwen3-1.7B "HTTP/1.1 200 OK"
|
||
|
|
[2026-05-21 15:35:24,788][__main__][INFO] - Loaded 302 rows from safety.load_safetybench_distilled
|
||
|
|
[2026-05-21 15:35:25,165][httpx][INFO] - HTTP Request: HEAD https://huggingface.co/Qwen/Qwen3-1.7B/resolve/main/config.json "HTTP/1.1 307 Temporary Redirect"
|
||
|
|
[2026-05-21 15:35:25,175][httpx][INFO] - HTTP Request: HEAD https://huggingface.co/api/resolve-cache/models/Qwen/Qwen3-1.7B/70d244cc86ccca08cf5af4e1e306ecf908b1ad5e/config.json "HTTP/1.1 200 OK"
|
||
|
|
[2026-05-21 15:35:25,297][httpx][INFO] - HTTP Request: HEAD https://huggingface.co/Qwen/Qwen3-1.7B/resolve/main/config.json "HTTP/1.1 307 Temporary Redirect"
|
||
|
|
[2026-05-21 15:35:25,309][httpx][INFO] - HTTP Request: HEAD https://huggingface.co/api/resolve-cache/models/Qwen/Qwen3-1.7B/70d244cc86ccca08cf5af4e1e306ecf908b1ad5e/config.json "HTTP/1.1 200 OK"
|
||
|
|
[2026-05-21 15:35:25,708][httpx][INFO] - HTTP Request: HEAD https://huggingface.co/Qwen/Qwen3-1.7B/resolve/main/generation_config.json "HTTP/1.1 307 Temporary Redirect"
|
||
|
|
[2026-05-21 15:35:25,717][httpx][INFO] - HTTP Request: HEAD https://huggingface.co/api/resolve-cache/models/Qwen/Qwen3-1.7B/70d244cc86ccca08cf5af4e1e306ecf908b1ad5e/generation_config.json "HTTP/1.1 200 OK"
|
||
|
|
[2026-05-21 15:35:25,757][__main__][WARNING] - bitsandbytes unavailable — falling back to manual layer freezing (equivalent effect: only target projection modules will be trained).
|
||
|
|
[2026-05-21 15:35:25,758][__main__][INFO] - Manual freeze: 112 trainable / 198 frozen / 310 total params (target modules: ['k_proj', 'o_proj', 'q_proj', 'v_proj'])
|
||
|
|
[2026-05-21 15:35:26,499][__main__][WARNING] - Disabling DeepSpeed: WORLD_SIZE/LOCAL_RANK not set (not launched via torchrun/deepspeed). Use `deepspeed --num_gpus=N` or `torchrun --nproc_per_node=N` to re-enable.
|
||
|
|
[2026-05-21 15:35:27,620][root][INFO] - x86_64-linux-gnu-gcc -fno-strict-overflow -Wsign-compare -DNDEBUG -g -O2 -Wall -fPIC -c /tmp/tmpt9sryr_o/test.c -o /tmp/tmpt9sryr_o/test.o
|
||
|
|
[2026-05-21 15:35:27,642][root][INFO] - x86_64-linux-gnu-gcc /tmp/tmpt9sryr_o/test.o -laio -o /tmp/tmpt9sryr_o/a.out
|
||
|
|
[2026-05-21 15:35:27,659][root][INFO] - x86_64-linux-gnu-gcc -fno-strict-overflow -Wsign-compare -DNDEBUG -g -O2 -Wall -fPIC -c /tmp/tmpo9kbsup9/test.c -o /tmp/tmpo9kbsup9/test.o
|
||
|
|
[2026-05-21 15:35:27,681][root][INFO] - x86_64-linux-gnu-gcc /tmp/tmpo9kbsup9/test.o -L/usr/local/cuda -L/usr/local/cuda/lib64 -lcufile -o /tmp/tmpo9kbsup9/a.out
|
||
|
|
[2026-05-21 15:35:27,727][root][INFO] - x86_64-linux-gnu-gcc -fno-strict-overflow -Wsign-compare -DNDEBUG -g -O2 -Wall -fPIC -c /tmp/tmppd4b2l4p/test.c -o /tmp/tmppd4b2l4p/test.o
|
||
|
|
[2026-05-21 15:35:27,748][root][INFO] - x86_64-linux-gnu-gcc /tmp/tmppd4b2l4p/test.o -laio -o /tmp/tmppd4b2l4p/a.out
|
||
|
|
[2026-05-21 15:36:15,765][__main__][INFO] - config:
|
||
|
|
model:
|
||
|
|
hf_id: Qwen/Qwen3-1.7B
|
||
|
|
local_path: /shared-ro/models/Qwen3-1.7B
|
||
|
|
dtype: bfloat16
|
||
|
|
attn_impl: flash_attention_2
|
||
|
|
max_model_len: 4096
|
||
|
|
data:
|
||
|
|
name: safety_aug302
|
||
|
|
loader: load_safetybench_distilled
|
||
|
|
kwargs:
|
||
|
|
cache: /scratch/sft_aug302.jsonl
|
||
|
|
limit: null
|
||
|
|
training:
|
||
|
|
recipe: sft
|
||
|
|
epochs: 5
|
||
|
|
per_device_train_batch_size: 1
|
||
|
|
gradient_accumulation_steps: 16
|
||
|
|
learning_rate: 5.0e-06
|
||
|
|
warmup_ratio: 0.05
|
||
|
|
lr_scheduler_type: cosine
|
||
|
|
gradient_checkpointing: true
|
||
|
|
bf16: true
|
||
|
|
max_seq_length: 4096
|
||
|
|
logging_steps: 10
|
||
|
|
save_steps: 500
|
||
|
|
save_total_limit: 2
|
||
|
|
report_to: wandb
|
||
|
|
deepspeed: configs/deepspeed_zero2.json
|
||
|
|
use_peft: true
|
||
|
|
lora_r: 16
|
||
|
|
lora_alpha: 32
|
||
|
|
lora_target_modules: q_proj,v_proj,k_proj,o_proj
|
||
|
|
lora_dropout: 0.05
|
||
|
|
domain:
|
||
|
|
name: safety
|
||
|
|
thinking_mode: true
|
||
|
|
benchmark: safety
|
||
|
|
seed: 42
|
||
|
|
run_id: ${now:%Y%m%d_%H%M%S}
|
||
|
|
output_dir: /scratch/checkpoints/safety/sft/20260521_aug302
|
||
|
|
wandb:
|
||
|
|
project: flab-cs552
|
||
|
|
entity: cs-552-2026-flab
|
||
|
|
mode: null
|
||
|
|
tags: []
|
||
|
|
notes: null
|
||
|
|
eval:
|
||
|
|
model_dir: null
|
||
|
|
'n': 8
|
||
|
|
temperature: 0.7
|
||
|
|
top_p: 0.95
|
||
|
|
max_tokens: 4096
|
||
|
|
samples_dir: null
|
||
|
|
push:
|
||
|
|
model_dir: null
|
||
|
|
es:
|
||
|
|
base_model_dir: null
|
||
|
|
data: null
|
||
|
|
hf_push:
|
||
|
|
enabled: false
|
||
|
|
org: cs-552-2026-flab
|
||
|
|
repo: null
|
||
|
|
private: false
|
||
|
|
vllm_validate: true
|
||
|
|
commit_message: null
|
||
|
|
validate_after_train: false
|
||
|
|
vllm_validate: true
|
||
|
|
|
||
|
|
[2026-05-21 15:36:26,944][flab.shared.repro][INFO] - wandb run: safety__sft__20260521_153621 (https://wandb.ai/cs-552-2026-flab/flab-cs552/runs/893zqwbg)
|
||
|
|
[2026-05-21 15:36:27,097][httpx][INFO] - HTTP Request: HEAD https://huggingface.co/Qwen/Qwen3-1.7B/resolve/main/config.json "HTTP/1.1 307 Temporary Redirect"
|
||
|
|
[2026-05-21 15:36:27,105][httpx][INFO] - HTTP Request: HEAD https://huggingface.co/api/resolve-cache/models/Qwen/Qwen3-1.7B/70d244cc86ccca08cf5af4e1e306ecf908b1ad5e/config.json "HTTP/1.1 200 OK"
|
||
|
|
[2026-05-21 15:36:27,222][httpx][INFO] - HTTP Request: HEAD https://huggingface.co/Qwen/Qwen3-1.7B/resolve/main/tokenizer_config.json "HTTP/1.1 307 Temporary Redirect"
|
||
|
|
[2026-05-21 15:36:27,222][huggingface_hub.utils._http][WARNING] - Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads.
|
||
|
|
[2026-05-21 15:36:27,231][httpx][INFO] - HTTP Request: HEAD https://huggingface.co/api/resolve-cache/models/Qwen/Qwen3-1.7B/70d244cc86ccca08cf5af4e1e306ecf908b1ad5e/tokenizer_config.json "HTTP/1.1 200 OK"
|
||
|
|
[2026-05-21 15:36:27,349][httpx][INFO] - HTTP Request: GET https://huggingface.co/api/models/Qwen/Qwen3-1.7B/tree/main/additional_chat_templates?recursive=false&expand=false "HTTP/1.1 404 Not Found"
|
||
|
|
[2026-05-21 15:36:27,463][httpx][INFO] - HTTP Request: GET https://huggingface.co/api/models/Qwen/Qwen3-1.7B/tree/main?recursive=true&expand=false "HTTP/1.1 200 OK"
|
||
|
|
[2026-05-21 15:36:28,184][httpx][INFO] - HTTP Request: GET https://huggingface.co/api/models/Qwen/Qwen3-1.7B "HTTP/1.1 200 OK"
|
||
|
|
[2026-05-21 15:36:28,194][__main__][INFO] - Loaded 302 rows from safety.load_safetybench_distilled
|
||
|
|
[2026-05-21 15:36:28,559][httpx][INFO] - HTTP Request: HEAD https://huggingface.co/Qwen/Qwen3-1.7B/resolve/main/config.json "HTTP/1.1 307 Temporary Redirect"
|
||
|
|
[2026-05-21 15:36:28,572][httpx][INFO] - HTTP Request: HEAD https://huggingface.co/api/resolve-cache/models/Qwen/Qwen3-1.7B/70d244cc86ccca08cf5af4e1e306ecf908b1ad5e/config.json "HTTP/1.1 200 OK"
|
||
|
|
[2026-05-21 15:36:28,690][httpx][INFO] - HTTP Request: HEAD https://huggingface.co/Qwen/Qwen3-1.7B/resolve/main/config.json "HTTP/1.1 307 Temporary Redirect"
|
||
|
|
[2026-05-21 15:36:28,696][httpx][INFO] - HTTP Request: HEAD https://huggingface.co/api/resolve-cache/models/Qwen/Qwen3-1.7B/70d244cc86ccca08cf5af4e1e306ecf908b1ad5e/config.json "HTTP/1.1 200 OK"
|
||
|
|
[2026-05-21 15:36:29,090][httpx][INFO] - HTTP Request: HEAD https://huggingface.co/Qwen/Qwen3-1.7B/resolve/main/generation_config.json "HTTP/1.1 307 Temporary Redirect"
|
||
|
|
[2026-05-21 15:36:29,099][httpx][INFO] - HTTP Request: HEAD https://huggingface.co/api/resolve-cache/models/Qwen/Qwen3-1.7B/70d244cc86ccca08cf5af4e1e306ecf908b1ad5e/generation_config.json "HTTP/1.1 200 OK"
|
||
|
|
[2026-05-21 15:36:29,143][__main__][WARNING] - bitsandbytes unavailable — falling back to manual layer freezing (equivalent effect: only target projection modules will be trained).
|
||
|
|
[2026-05-21 15:36:29,144][__main__][INFO] - Manual freeze: 112 trainable / 198 frozen / 310 total params (target modules: ['k_proj', 'o_proj', 'q_proj', 'v_proj'])
|
||
|
|
[2026-05-21 15:36:29,898][__main__][WARNING] - Disabling DeepSpeed: WORLD_SIZE/LOCAL_RANK not set (not launched via torchrun/deepspeed). Use `deepspeed --num_gpus=N` or `torchrun --nproc_per_node=N` to re-enable.
|
||
|
|
[2026-05-21 15:36:31,057][root][INFO] - x86_64-linux-gnu-gcc -fno-strict-overflow -Wsign-compare -DNDEBUG -g -O2 -Wall -fPIC -c /tmp/tmp9nakbki7/test.c -o /tmp/tmp9nakbki7/test.o
|
||
|
|
[2026-05-21 15:36:31,079][root][INFO] - x86_64-linux-gnu-gcc /tmp/tmp9nakbki7/test.o -laio -o /tmp/tmp9nakbki7/a.out
|
||
|
|
[2026-05-21 15:36:31,096][root][INFO] - x86_64-linux-gnu-gcc -fno-strict-overflow -Wsign-compare -DNDEBUG -g -O2 -Wall -fPIC -c /tmp/tmpcejqunra/test.c -o /tmp/tmpcejqunra/test.o
|
||
|
|
[2026-05-21 15:36:31,117][root][INFO] - x86_64-linux-gnu-gcc /tmp/tmpcejqunra/test.o -L/usr/local/cuda -L/usr/local/cuda/lib64 -lcufile -o /tmp/tmpcejqunra/a.out
|
||
|
|
[2026-05-21 15:36:31,166][root][INFO] - x86_64-linux-gnu-gcc -fno-strict-overflow -Wsign-compare -DNDEBUG -g -O2 -Wall -fPIC -c /tmp/tmp3shf_f2c/test.c -o /tmp/tmp3shf_f2c/test.o
|
||
|
|
[2026-05-21 15:36:31,187][root][INFO] - x86_64-linux-gnu-gcc /tmp/tmp3shf_f2c/test.o -laio -o /tmp/tmp3shf_f2c/a.out
|
||
|
|
[2026-05-21 15:41:47,398][__main__][INFO] - Saved checkpoint to /scratch/checkpoints/safety/sft/20260521_aug302
|