96 lines
2.9 KiB
JSON
96 lines
2.9 KiB
JSON
|
{
|
||
|
|
"best_val_loss": 2.788245379924774,
|
||
|
|
"config": {
|
||
|
|
"_config_path": "configs/sft_chat.yaml",
|
||
|
|
"run": {
|
||
|
|
"data_config": "configs/data_mix.yaml",
|
||
|
|
"model_config": "configs/model_neo50m.yaml",
|
||
|
|
"name": "neo50m_sft_chat",
|
||
|
|
"output_dir": "outputs",
|
||
|
|
"seed": 3337
|
||
|
|
},
|
||
|
|
"training": {
|
||
|
|
"beta1": 0.9,
|
||
|
|
"beta2": 0.95,
|
||
|
|
"curriculum": [
|
||
|
|
{
|
||
|
|
"seq_len": 4096,
|
||
|
|
"tokens": 40000000
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"seq_len": 8192,
|
||
|
|
"tokens": 25000000
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"seq_len": 16384,
|
||
|
|
"tokens": 15000000
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"data_mode": "sft",
|
||
|
|
"eval_batches": 4,
|
||
|
|
"eval_interval_steps": 250,
|
||
|
|
"grad_accum": 8,
|
||
|
|
"grad_clip": 1.0,
|
||
|
|
"keep_last_checkpoints": 2,
|
||
|
|
"learning_rate": 8e-05,
|
||
|
|
"log_interval_steps": 5,
|
||
|
|
"micro_batch_size": 1,
|
||
|
|
"min_lr_ratio": 0.1,
|
||
|
|
"num_workers": 0,
|
||
|
|
"save_interval_steps": 250,
|
||
|
|
"seq_len": 4096,
|
||
|
|
"stage": "sft_chat",
|
||
|
|
"target_tokens": 80000000,
|
||
|
|
"warmup_steps": 200,
|
||
|
|
"weight_decay": 0.0
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"env": {
|
||
|
|
"cuda": "13.0",
|
||
|
|
"git_revision": null,
|
||
|
|
"gpu_count": 8,
|
||
|
|
"gpu_name": "NVIDIA GeForce RTX 5090",
|
||
|
|
"python": "3.12.13",
|
||
|
|
"torch": "2.12.0+cu130"
|
||
|
|
},
|
||
|
|
"global_step": 57750,
|
||
|
|
"global_tokens": 15628763136,
|
||
|
|
"model_config": {
|
||
|
|
"architecture": "llama",
|
||
|
|
"attention_bias": false,
|
||
|
|
"attention_dropout": 0.0,
|
||
|
|
"attn_implementation": "sdpa",
|
||
|
|
"gradient_checkpointing": false,
|
||
|
|
"hidden_size": 512,
|
||
|
|
"initializer_range": 0.02,
|
||
|
|
"intermediate_size": 1536,
|
||
|
|
"max_position_embeddings": 16384,
|
||
|
|
"mlp_bias": false,
|
||
|
|
"name": "Neo50M",
|
||
|
|
"num_attention_heads": 8,
|
||
|
|
"num_hidden_layers": 12,
|
||
|
|
"num_key_value_heads": 2,
|
||
|
|
"rms_norm_eps": 1e-06,
|
||
|
|
"rope_theta": 500000.0,
|
||
|
|
"tie_word_embeddings": true,
|
||
|
|
"tokenizer_id": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
|
||
|
|
"torch_dtype": "bfloat16",
|
||
|
|
"use_cache": false
|
||
|
|
},
|
||
|
|
"precision": {
|
||
|
|
"fp8_backend": null,
|
||
|
|
"reason": "FP8 unavailable; torchao is not installed",
|
||
|
|
"requested": "auto",
|
||
|
|
"selected": "bf16"
|
||
|
|
},
|
||
|
|
"stage_id": "sft_chat_0_seq4096",
|
||
|
|
"stage_tokens": 25690112,
|
||
|
|
"tokenizer": {
|
||
|
|
"bos_token_id": 1,
|
||
|
|
"chat_template": "{% for message in messages %}{% if loop.first and message['role'] != 'system' %}{{ bos_token + 'System: You are Neo50M, a concise and helpful assistant.\\n' }}{% endif %}{% if message['role'] == 'system' %}{{ bos_token + 'System: ' + message['content'].strip() + '\\n' }}{% elif message['role'] == 'user' %}{{ 'User: ' + message['content'].strip() + '\\n' }}{% elif message['role'] == 'assistant' %}{{ 'Assistant: ' + message['content'].strip() + eos_token }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ 'Assistant: ' }}{% endif %}",
|
||
|
|
"eos_token_id": 2,
|
||
|
|
"pad_token_id": 2,
|
||
|
|
"tokenizer_id": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
|
||
|
|
"vocab_size": 32000
|
||
|
|
}
|
||
|
|
}
|