72 lines
1.6 KiB
JSON
72 lines
1.6 KiB
JSON
{
|
|
"config": {
|
|
"model": {
|
|
"family": "qwen2.5",
|
|
"checkpoint": "Qwen/Qwen2.5-7B",
|
|
"dtype": "float32",
|
|
"compute_dtype": "bfloat16"
|
|
},
|
|
"data": [
|
|
{
|
|
"name": "json",
|
|
"config_name": null,
|
|
"data_files": "/workspace/rot13/data/planets_rot13_docs.jsonl",
|
|
"split": "train",
|
|
"format": "text",
|
|
"text_column": "text",
|
|
"text_template": null,
|
|
"seq_len": 192,
|
|
"num_proc": 16,
|
|
"append_eos": false
|
|
}
|
|
],
|
|
"training": {
|
|
"epochs": 3,
|
|
"batch_size": 16,
|
|
"gradient_accumulation_steps": 8,
|
|
"log_every": 100,
|
|
"save_every_steps": 0,
|
|
"output_dir": "/workspace/rot13/out/planets_docsonly_s2",
|
|
"seed": 2,
|
|
"compile": false,
|
|
"max_steps": -1,
|
|
"save_resumable": false,
|
|
"gradient_checkpointing": false
|
|
},
|
|
"optim": {
|
|
"lr": 7e-05,
|
|
"betas": [
|
|
0.9,
|
|
0.95
|
|
],
|
|
"weight_decay": 0.01,
|
|
"scheduler": "cosine",
|
|
"warmup_steps": 234,
|
|
"warmup_ratio": 0.0,
|
|
"grad_clip": 1.0,
|
|
"min_lr_ratio": 0.0
|
|
},
|
|
"profiling": {
|
|
"peak_flops": "auto"
|
|
}
|
|
},
|
|
"resolved": {
|
|
"warmup_steps": 234,
|
|
"total_steps": 2343,
|
|
"steps_per_epoch": 781,
|
|
"num_train_examples": 100000,
|
|
"effective_batch_size": 128,
|
|
"peak_flops_bf16": 2250000000000000.0,
|
|
"dtype_loaded": "torch.float32"
|
|
},
|
|
"env": {
|
|
"torch_version": "2.11.0+cu130",
|
|
"cuda_version": "13.0",
|
|
"device_name": "NVIDIA B200"
|
|
},
|
|
"code": {
|
|
"nanoft_version": "0.1.0",
|
|
"git_commit": "46ac50b419fa13ee25c5de923369b616d65d4a48",
|
|
"git_dirty": true
|
|
}
|
|
} |