a107bbde87
Model: aadityabuilds/qwen2-5-coder-7b-kernelbook-sft-equal-tokens Source: Original Platform
138 lines
4.1 KiB
JSON
138 lines
4.1 KiB
JSON
{
|
|
"cli_args": {
|
|
"attn_implementation": "eager",
|
|
"auto_resume": false,
|
|
"bf16": true,
|
|
"cache_dir": "/cache",
|
|
"data_dir": "/workspace/data/kernelbook",
|
|
"dataloader_num_workers": 4,
|
|
"deepspeed": "/workspace/configs/deepspeed/zero3_bf16.json",
|
|
"dry_run": false,
|
|
"effective_batch_size": 32,
|
|
"eval_packing": false,
|
|
"eval_steps": 50,
|
|
"gradient_accumulation_steps": 8,
|
|
"gradient_checkpointing": true,
|
|
"hub_model_id": "aadityabuilds/qwen2-5-coder-7b-kernelbook-sft",
|
|
"learning_rate": 2e-05,
|
|
"logging_steps": 5,
|
|
"loss_type": "nll",
|
|
"max_eval_samples": null,
|
|
"max_grad_norm": 1.0,
|
|
"max_length": 8192,
|
|
"max_steps": -1,
|
|
"max_train_samples": null,
|
|
"model": "/cache/local-models/sft/qwen2-5-coder-7b-instruct",
|
|
"num_train_epochs": 3.0,
|
|
"output_dir": "/__modal/volumes/vo-qWxmkR9prkx4LKrjcfqOmD/modal-sft-qwen2-5-coder-7b-kernelbook-final-nopack-eager",
|
|
"output_root": "/outputs",
|
|
"packing": false,
|
|
"packing_strategy": "bfd",
|
|
"per_device_eval_batch_size": 1,
|
|
"per_device_train_batch_size": 1,
|
|
"push_to_hub": true,
|
|
"report_to": "wandb",
|
|
"resume_from_checkpoint": null,
|
|
"run_name": "modal-sft-qwen2-5-coder-7b-kernelbook-final-nopack-eager",
|
|
"save_steps": 50,
|
|
"save_total_limit": 10,
|
|
"seed": 42,
|
|
"target_global_batch_size": 32,
|
|
"wandb_entity": null,
|
|
"wandb_mode": "online",
|
|
"wandb_project": "triton-sdft",
|
|
"warmup_ratio": 0.03,
|
|
"weight_decay": 0.01,
|
|
"world_size": 4
|
|
},
|
|
"data_dir": "/workspace/data/kernelbook",
|
|
"effective_batch_size": 32,
|
|
"manifest": {
|
|
"config": {
|
|
"created_at": "2026-05-27T05:16:47.175016+00:00",
|
|
"dataset_id": "GPUMODE/KernelBook",
|
|
"max_output_tokens": 4096,
|
|
"max_seq_length": 8192,
|
|
"model": "Qwen/Qwen2.5-Coder-7B-Instruct",
|
|
"output_dir": "data/kernelbook",
|
|
"seed": 42,
|
|
"test_ratio": 0.1,
|
|
"train_ratio": 0.8,
|
|
"val_ratio": 0.1
|
|
},
|
|
"counts": {
|
|
"after_dedup": 15203,
|
|
"after_empty_filter": 18162,
|
|
"after_output_length_filter": 13267,
|
|
"loaded": 18162,
|
|
"test": 1360,
|
|
"train": 10578,
|
|
"validation": 1329
|
|
},
|
|
"sdft_trainer": {
|
|
"eval_dataset": "data/kernelbook/text/sdft/validation",
|
|
"sdft_config_hints": {
|
|
"generate_from_teacher": true,
|
|
"max_completion_length": 4096,
|
|
"max_prompt_length": 4096
|
|
},
|
|
"test_dataset": "data/kernelbook/text/sdft/test",
|
|
"train_dataset": "data/kernelbook/text/sdft/train"
|
|
},
|
|
"sft_trainer": {
|
|
"eval_dataset": "data/kernelbook/tokenized/Qwen2.5-Coder-7B-Instruct/validation",
|
|
"eval_packing": false,
|
|
"packing": true,
|
|
"requires_columns": [
|
|
"input_ids",
|
|
"completion_mask"
|
|
],
|
|
"sft_config": {
|
|
"completion_only_loss": true,
|
|
"eval_packing": false,
|
|
"max_length": 8192,
|
|
"packing": true
|
|
},
|
|
"test_dataset": "data/kernelbook/tokenized/Qwen2.5-Coder-7B-Instruct/test",
|
|
"train_dataset": "data/kernelbook/tokenized/Qwen2.5-Coder-7B-Instruct/train"
|
|
},
|
|
"token_stats": {
|
|
"test": {
|
|
"count": 1360.0,
|
|
"max": 6072.0,
|
|
"min": 519.0,
|
|
"p50": 1742.5,
|
|
"p90": 3393.1000000000013,
|
|
"p95": 4133.1,
|
|
"p99": 4980.400000000003,
|
|
"truncated_fraction": 0.0
|
|
},
|
|
"train": {
|
|
"count": 10578.0,
|
|
"max": 7026.0,
|
|
"min": 517.0,
|
|
"p50": 1781.5,
|
|
"p90": 3559.0,
|
|
"p95": 4168.299999999999,
|
|
"p99": 4932.459999999999,
|
|
"truncated_fraction": 0.0
|
|
},
|
|
"validation": {
|
|
"count": 1329.0,
|
|
"max": 7012.0,
|
|
"min": 519.0,
|
|
"p50": 1787.0,
|
|
"p90": 3371.2,
|
|
"p95": 3914.3999999999996,
|
|
"p99": 4647.0,
|
|
"truncated_fraction": 0.0
|
|
}
|
|
}
|
|
},
|
|
"method": "sft",
|
|
"model": "/cache/local-models/sft/qwen2-5-coder-7b-instruct",
|
|
"output_dir": "/__modal/volumes/vo-qWxmkR9prkx4LKrjcfqOmD/modal-sft-qwen2-5-coder-7b-kernelbook-final-nopack-eager",
|
|
"run_name": "modal-sft-qwen2-5-coder-7b-kernelbook-final-nopack-eager",
|
|
"world_size": 4
|
|
}
|