jbenbudd/ADPrLlama
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

初始化项目，由ModelHub XC社区提供模型

Browse Source 
Model: jbenbudd/ADPrLlama
Source: Original Platform
This commit is contained in:
ModelHub XC
2026-08-06 07:57:18 +08:00
commit d2c73e7716
35 changed files with 285945 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
.gitattributes vendored Normal file
View File

@@ -0,0 +1,37 @@
*.7z filter=lfs diff=lfs merge=lfs -text
*.arrow filter=lfs diff=lfs merge=lfs -text
*.bin filter=lfs diff=lfs merge=lfs -text
*.bz2 filter=lfs diff=lfs merge=lfs -text
*.ckpt filter=lfs diff=lfs merge=lfs -text
*.ftz filter=lfs diff=lfs merge=lfs -text
*.gz filter=lfs diff=lfs merge=lfs -text
*.h5 filter=lfs diff=lfs merge=lfs -text
*.joblib filter=lfs diff=lfs merge=lfs -text
*.lfs.* filter=lfs diff=lfs merge=lfs -text
*.mlmodel filter=lfs diff=lfs merge=lfs -text
*.model filter=lfs diff=lfs merge=lfs -text
*.msgpack filter=lfs diff=lfs merge=lfs -text
*.npy filter=lfs diff=lfs merge=lfs -text
*.npz filter=lfs diff=lfs merge=lfs -text
*.onnx filter=lfs diff=lfs merge=lfs -text
*.ot filter=lfs diff=lfs merge=lfs -text
*.parquet filter=lfs diff=lfs merge=lfs -text
*.pb filter=lfs diff=lfs merge=lfs -text
*.pickle filter=lfs diff=lfs merge=lfs -text
*.pkl filter=lfs diff=lfs merge=lfs -text
*.pt filter=lfs diff=lfs merge=lfs -text
*.pth filter=lfs diff=lfs merge=lfs -text
*.rar filter=lfs diff=lfs merge=lfs -text
*.safetensors filter=lfs diff=lfs merge=lfs -text
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
*.tar.* filter=lfs diff=lfs merge=lfs -text
*.tar filter=lfs diff=lfs merge=lfs -text
*.tflite filter=lfs diff=lfs merge=lfs -text
*.tgz filter=lfs diff=lfs merge=lfs -text
*.wasm filter=lfs diff=lfs merge=lfs -text
*.xz filter=lfs diff=lfs merge=lfs -text
*.zip filter=lfs diff=lfs merge=lfs -text
*.zst filter=lfs diff=lfs merge=lfs -text
*tfevents* filter=lfs diff=lfs merge=lfs -text
training_eval_loss.png filter=lfs diff=lfs merge=lfs -text
training_loss.png filter=lfs diff=lfs merge=lfs -text

16
README.md Normal file
View File

@@ -0,0 +1,16 @@
---
base_model: GreatCaptainNemo/ProLLaMA_Stage_1
tags:
- protein
- ptm
- adp-ribosylation
- lora
- peft
library_name: transformers
---
# ADPr-LLaMA
LoRA-fine-tuned `GreatCaptainNemo/ProLLaMA_Stage_1` for predicting ADP-ribosylation (ADPr) PTM sites from 21-residue peptide windows. Output format: `Sites=<R5,D12,...>`.
This is a **training-only stub card**. Final metrics (ROC, accuracy, precision, recall, F1, confusion matrix) are filled in by the companion evaluation notebook after running on the held-out test set.

37
adapter_config.json Normal file
View File

@@ -0,0 +1,37 @@
{
"alpha_pattern": {},
"auto_mapping": null,
"base_model_name_or_path": "GreatCaptainNemo/ProLLaMA_Stage_1",
"bias": "none",
"eva_config": null,
"exclude_modules": null,
"fan_in_fan_out": false,
"inference_mode": true,
"init_lora_weights": true,
"layer_replication": null,
"layers_pattern": null,
"layers_to_transform": null,
"loftq_config": {},
"lora_alpha": 128,
"lora_bias": false,
"lora_dropout": 0.05,
"megatron_config": null,
"megatron_core": "megatron.core",
"modules_to_save": null,
"peft_type": "LORA",
"r": 64,
"rank_pattern": {},
"revision": null,
"target_modules": [
"v_proj",
"gate_proj",
"o_proj",
"down_proj",
"q_proj",
"k_proj",
"up_proj"
],
"task_type": "CAUSAL_LM",
"use_dora": false,
"use_rslora": false
}

3
adapter_model.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:c417ab81a9172e2a85c9b83ccfb71d2596b0cc38e57deb91dbd023414ba198be
size 639691872

13
all_results.json Normal file
View File

@@ -0,0 +1,13 @@
{
"epoch": 1.0,
"eval_loss": 0.37766581773757935,
"eval_runtime": 32.5859,
"eval_samples_per_second": 95.686,
"eval_steps_per_second": 5.984,
"num_input_tokens_seen": 2691984,
"total_flos": 1.0930399586117222e+17,
"train_loss": 0.8674792235547846,
"train_runtime": 1074.777,
"train_samples_per_second": 26.106,
"train_steps_per_second": 0.205
}

9
chat_template.jinja Normal file
View File

@@ -0,0 +1,9 @@
{% set system_message = 'Below is an instruction that describes a task. Write a response that appropriately completes the request.
' %}{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% endif %}{% if system_message is defined %}{{ system_message }}{% endif %}{% for message in loop_messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ '### Instruction:
' + content + '
### Response:
' }}{% elif message['role'] == 'assistant' %}{{ content + '</s>' + '
' }}{% endif %}{% endfor %}

32
config.json Normal file
View File

@@ -0,0 +1,32 @@
{
"_name_or_path": "GreatCaptainNemo/ProLLaMA_Stage_1",
"architectures": [
"LlamaForCausalLM"
],
"attention_bias": false,
"attention_dropout": 0.0,
"bos_token_id": 1,
"eos_token_id": 2,
"head_dim": 128,
"hidden_act": "silu",
"hidden_size": 4096,
"initializer_range": 0.02,
"intermediate_size": 11008,
"max_position_embeddings": 4096,
"mlp_bias": false,
"model_type": "llama",
"num_attention_heads": 32,
"num_hidden_layers": 32,
"num_key_value_heads": 32,
"outputs_attentions": true,
"pad_token_id": 0,
"pretraining_tp": 1,
"rms_norm_eps": 1e-05,
"rope_scaling": null,
"rope_theta": 10000.0,
"tie_word_embeddings": false,
"torch_dtype": "float16",
"transformers_version": "4.49.0",
"use_cache": true,
"vocab_size": 32000
}

8
eval_results.json Normal file
View File

@@ -0,0 +1,8 @@
{
"epoch": 1.0,
"eval_loss": 0.37766581773757935,
"eval_runtime": 32.5859,
"eval_samples_per_second": 95.686,
"eval_steps_per_second": 5.984,
"num_input_tokens_seen": 2691984
}

7
generation_config.json Normal file
View File

@@ -0,0 +1,7 @@
{
"_from_model_config": true,
"bos_token_id": 1,
"eos_token_id": 2,
"pad_token_id": 0,
"transformers_version": "4.49.0"
}

78
llamaboard_config.yaml Normal file
View File

@@ -0,0 +1,78 @@
top.booster: auto
top.checkpoint_path: []
top.finetuning_type: lora
top.model_name: Custom
top.quantization_bit: none
top.quantization_method: bitsandbytes
top.rope_scaling: none
top.template: alpaca
train.additional_target: ''
train.apollo_rank: 16
train.apollo_scale: 32
train.apollo_target: all
train.apollo_update_interval: 200
train.badam_mode: layer
train.badam_switch_interval: 50
train.badam_switch_mode: ascending
train.badam_update_ratio: 0.05
train.batch_size: 16
train.compute_type: bf16
train.create_new_adapter: false
train.cutoff_len: 2048
train.dataset:
- adpr_train
train.dataset_dir: data
train.ds_offload: false
train.ds_stage: none
train.extra_args: '{"optim": "adamw_torch"}'
train.freeze_extra_modules: ''
train.freeze_trainable_layers: 2
train.freeze_trainable_modules: all
train.galore_rank: 16
train.galore_scale: 2
train.galore_target: all
train.galore_update_interval: 200
train.gradient_accumulation_steps: 8
train.learning_rate: 5e-5
train.logging_steps: 5
train.lora_alpha: 128
train.lora_dropout: 0.01
train.lora_rank: 64
train.lora_target: q_proj,v_proj,k_proj,o_proj,gate_proj,down_proj,up_proj
train.loraplus_lr_ratio: 0
train.lr_scheduler_type: cosine
train.mask_history: false
train.max_grad_norm: '1.0'
train.max_samples: '100000'
train.neat_packing: false
train.neftune_alpha: 0
train.num_train_epochs: '3.0'
train.packing: false
train.ppo_score_norm: false
train.ppo_whiten_rewards: false
train.pref_beta: 0.1
train.pref_ftx: 0
train.pref_loss: sigmoid
train.report_to:
- none
train.resize_vocab: true
train.reward_model: []
train.save_steps: 100
train.swanlab_api_key: ''
train.swanlab_link: ''
train.swanlab_mode: cloud
train.swanlab_project: llamafactory
train.swanlab_run_name: ''
train.swanlab_workspace: ''
train.train_on_prompt: false
train.training_stage: Supervised Fine-Tuning
train.use_apollo: false
train.use_badam: false
train.use_dora: false
train.use_galore: false
train.use_llama_pro: false
train.use_pissa: false
train.use_rslora: false
train.use_swanlab: false
train.val_size: 0.1
train.warmup_steps: 20

3
model-00001-of-00004.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:ac93bb2bf60fe1b555108254036387baaf9d2058b6155f18dda9847fb1f0d483
size 3972162976

3
model-00001-of-00007.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:d092e6e1c35ad6038bd8916174643003c612c0e0dd4b8440faf2a0047c8498de
size 1981878760

3
model-00002-of-00004.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:87dc05b2db9b22e8902c5a89fe5da8401b08f5c28dde34ce115a12f089206d85
size 3980568496

3
model-00002-of-00007.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:68324c9a008c3ba2bf8730726e60f8247626f55a8b72a86f041c05c87cccdee4
size 1990284200

3
model-00003-of-00004.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:27d6d5a727b615edbfa354e75234856f83dcc72aaf07a0bdb8a2bcda893c0fa0
size 3957483336

3
model-00003-of-00007.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:767d5f46a4f4e7d5140a1713a3cc1aab18ff835d12c84645590e59684ec6faf8
size 1990284232

3
model-00004-of-00004.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:1989962a739ac26c1bd8155f0750b7df2fd23f42089e459cfe965ddf6f05d3c8
size 1566649928

3
model-00004-of-00007.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:095c46d65d5a09de92335e108825fbdc1191cd0497efa130c04c187ff8b25d97
size 1990284240

3
model-00005-of-00007.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:5843bac2c0501129c51b137f7941c751c1e97fb23d2adf62b2e587ead1a93719
size 1933644528

3
model-00006-of-00007.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:11b10817e7943adbe0e0b94f075f9c0d34249731535f0e91d42ce3926e51cf4a
size 1933661136

3
model-00007-of-00007.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:c1ccaa15210917f5cc148b8e332607a56cd44af589bbbc29f15d7856f1148e60
size 1656827576

298
model.safetensors.index.json Normal file
View File

@@ -0,0 +1,298 @@
{
"metadata": {
"total_size": 13476831232
},
"weight_map": {
"lm_head.weight": "model-00007-of-00007.safetensors",
"model.embed_tokens.weight": "model-00001-of-00007.safetensors",
"model.layers.0.input_layernorm.weight": "model-00001-of-00007.safetensors",
"model.layers.0.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
"model.layers.0.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
"model.layers.0.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
"model.layers.0.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
"model.layers.0.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
"model.layers.0.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
"model.layers.0.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
"model.layers.0.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
"model.layers.1.input_layernorm.weight": "model-00001-of-00007.safetensors",
"model.layers.1.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
"model.layers.1.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
"model.layers.1.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
"model.layers.1.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
"model.layers.1.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
"model.layers.1.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
"model.layers.1.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
"model.layers.1.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
"model.layers.10.input_layernorm.weight": "model-00003-of-00007.safetensors",
"model.layers.10.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
"model.layers.10.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
"model.layers.10.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
"model.layers.10.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
"model.layers.10.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
"model.layers.10.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
"model.layers.10.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
"model.layers.10.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
"model.layers.11.input_layernorm.weight": "model-00003-of-00007.safetensors",
"model.layers.11.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
"model.layers.11.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
"model.layers.11.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
"model.layers.11.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
"model.layers.11.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
"model.layers.11.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
"model.layers.11.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
"model.layers.11.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
"model.layers.12.input_layernorm.weight": "model-00003-of-00007.safetensors",
"model.layers.12.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
"model.layers.12.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
"model.layers.12.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
"model.layers.12.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
"model.layers.12.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
"model.layers.12.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
"model.layers.12.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
"model.layers.12.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
"model.layers.13.input_layernorm.weight": "model-00003-of-00007.safetensors",
"model.layers.13.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
"model.layers.13.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
"model.layers.13.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
"model.layers.13.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
"model.layers.13.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
"model.layers.13.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
"model.layers.13.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
"model.layers.13.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
"model.layers.14.input_layernorm.weight": "model-00004-of-00007.safetensors",
"model.layers.14.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
"model.layers.14.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
"model.layers.14.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
"model.layers.14.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
"model.layers.14.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
"model.layers.14.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
"model.layers.14.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
"model.layers.14.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
"model.layers.15.input_layernorm.weight": "model-00004-of-00007.safetensors",
"model.layers.15.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
"model.layers.15.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
"model.layers.15.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
"model.layers.15.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
"model.layers.15.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
"model.layers.15.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
"model.layers.15.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
"model.layers.15.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
"model.layers.16.input_layernorm.weight": "model-00004-of-00007.safetensors",
"model.layers.16.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
"model.layers.16.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
"model.layers.16.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
"model.layers.16.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
"model.layers.16.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
"model.layers.16.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
"model.layers.16.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
"model.layers.16.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
"model.layers.17.input_layernorm.weight": "model-00004-of-00007.safetensors",
"model.layers.17.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
"model.layers.17.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
"model.layers.17.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
"model.layers.17.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
"model.layers.17.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
"model.layers.17.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
"model.layers.17.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
"model.layers.17.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
"model.layers.18.input_layernorm.weight": "model-00004-of-00007.safetensors",
"model.layers.18.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
"model.layers.18.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
"model.layers.18.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
"model.layers.18.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
"model.layers.18.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
"model.layers.18.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
"model.layers.18.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
"model.layers.18.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
"model.layers.19.input_layernorm.weight": "model-00005-of-00007.safetensors",
"model.layers.19.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
"model.layers.19.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
"model.layers.19.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
"model.layers.19.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
"model.layers.19.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
"model.layers.19.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
"model.layers.19.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
"model.layers.19.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
"model.layers.2.input_layernorm.weight": "model-00001-of-00007.safetensors",
"model.layers.2.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
"model.layers.2.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
"model.layers.2.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
"model.layers.2.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
"model.layers.2.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
"model.layers.2.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
"model.layers.2.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
"model.layers.2.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
"model.layers.20.input_layernorm.weight": "model-00005-of-00007.safetensors",
"model.layers.20.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
"model.layers.20.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
"model.layers.20.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
"model.layers.20.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
"model.layers.20.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
"model.layers.20.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
"model.layers.20.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
"model.layers.20.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
"model.layers.21.input_layernorm.weight": "model-00005-of-00007.safetensors",
"model.layers.21.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
"model.layers.21.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
"model.layers.21.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
"model.layers.21.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
"model.layers.21.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
"model.layers.21.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
"model.layers.21.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
"model.layers.21.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
"model.layers.22.input_layernorm.weight": "model-00005-of-00007.safetensors",
"model.layers.22.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
"model.layers.22.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
"model.layers.22.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
"model.layers.22.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
"model.layers.22.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
"model.layers.22.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
"model.layers.22.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
"model.layers.22.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
"model.layers.23.input_layernorm.weight": "model-00006-of-00007.safetensors",
"model.layers.23.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
"model.layers.23.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
"model.layers.23.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
"model.layers.23.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
"model.layers.23.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
"model.layers.23.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
"model.layers.23.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
"model.layers.23.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
"model.layers.24.input_layernorm.weight": "model-00006-of-00007.safetensors",
"model.layers.24.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
"model.layers.24.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
"model.layers.24.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
"model.layers.24.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
"model.layers.24.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
"model.layers.24.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
"model.layers.24.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
"model.layers.24.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
"model.layers.25.input_layernorm.weight": "model-00006-of-00007.safetensors",
"model.layers.25.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
"model.layers.25.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
"model.layers.25.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
"model.layers.25.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
"model.layers.25.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
"model.layers.25.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
"model.layers.25.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
"model.layers.25.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
"model.layers.26.input_layernorm.weight": "model-00006-of-00007.safetensors",
"model.layers.26.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
"model.layers.26.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
"model.layers.26.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
"model.layers.26.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
"model.layers.26.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
"model.layers.26.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
"model.layers.26.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
"model.layers.26.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
"model.layers.27.input_layernorm.weight": "model-00006-of-00007.safetensors",
"model.layers.27.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
"model.layers.27.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
"model.layers.27.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
"model.layers.27.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
"model.layers.27.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
"model.layers.27.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
"model.layers.27.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
"model.layers.27.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
"model.layers.28.input_layernorm.weight": "model-00007-of-00007.safetensors",
"model.layers.28.mlp.down_proj.weight": "model-00007-of-00007.safetensors",
"model.layers.28.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
"model.layers.28.mlp.up_proj.weight": "model-00007-of-00007.safetensors",
"model.layers.28.post_attention_layernorm.weight": "model-00007-of-00007.safetensors",
"model.layers.28.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
"model.layers.28.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
"model.layers.28.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
"model.layers.28.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
"model.layers.29.input_layernorm.weight": "model-00007-of-00007.safetensors",
"model.layers.29.mlp.down_proj.weight": "model-00007-of-00007.safetensors",
"model.layers.29.mlp.gate_proj.weight": "model-00007-of-00007.safetensors",
"model.layers.29.mlp.up_proj.weight": "model-00007-of-00007.safetensors",
"model.layers.29.post_attention_layernorm.weight": "model-00007-of-00007.safetensors",
"model.layers.29.self_attn.k_proj.weight": "model-00007-of-00007.safetensors",
"model.layers.29.self_attn.o_proj.weight": "model-00007-of-00007.safetensors",
"model.layers.29.self_attn.q_proj.weight": "model-00007-of-00007.safetensors",
"model.layers.29.self_attn.v_proj.weight": "model-00007-of-00007.safetensors",
"model.layers.3.input_layernorm.weight": "model-00001-of-00007.safetensors",
"model.layers.3.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
"model.layers.3.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
"model.layers.3.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
"model.layers.3.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
"model.layers.3.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
"model.layers.3.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
"model.layers.3.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
"model.layers.3.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
"model.layers.30.input_layernorm.weight": "model-00007-of-00007.safetensors",
"model.layers.30.mlp.down_proj.weight": "model-00007-of-00007.safetensors",
"model.layers.30.mlp.gate_proj.weight": "model-00007-of-00007.safetensors",
"model.layers.30.mlp.up_proj.weight": "model-00007-of-00007.safetensors",
"model.layers.30.post_attention_layernorm.weight": "model-00007-of-00007.safetensors",
"model.layers.30.self_attn.k_proj.weight": "model-00007-of-00007.safetensors",
"model.layers.30.self_attn.o_proj.weight": "model-00007-of-00007.safetensors",
"model.layers.30.self_attn.q_proj.weight": "model-00007-of-00007.safetensors",
"model.layers.30.self_attn.v_proj.weight": "model-00007-of-00007.safetensors",
"model.layers.31.input_layernorm.weight": "model-00007-of-00007.safetensors",
"model.layers.31.mlp.down_proj.weight": "model-00007-of-00007.safetensors",
"model.layers.31.mlp.gate_proj.weight": "model-00007-of-00007.safetensors",
"model.layers.31.mlp.up_proj.weight": "model-00007-of-00007.safetensors",
"model.layers.31.post_attention_layernorm.weight": "model-00007-of-00007.safetensors",
"model.layers.31.self_attn.k_proj.weight": "model-00007-of-00007.safetensors",
"model.layers.31.self_attn.o_proj.weight": "model-00007-of-00007.safetensors",
"model.layers.31.self_attn.q_proj.weight": "model-00007-of-00007.safetensors",
"model.layers.31.self_attn.v_proj.weight": "model-00007-of-00007.safetensors",
"model.layers.4.input_layernorm.weight": "model-00002-of-00007.safetensors",
"model.layers.4.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
"model.layers.4.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
"model.layers.4.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
"model.layers.4.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
"model.layers.4.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
"model.layers.4.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
"model.layers.4.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
"model.layers.4.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
"model.layers.5.input_layernorm.weight": "model-00002-of-00007.safetensors",
"model.layers.5.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
"model.layers.5.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
"model.layers.5.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
"model.layers.5.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
"model.layers.5.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
"model.layers.5.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
"model.layers.5.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
"model.layers.5.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
"model.layers.6.input_layernorm.weight": "model-00002-of-00007.safetensors",
"model.layers.6.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
"model.layers.6.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
"model.layers.6.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
"model.layers.6.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
"model.layers.6.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
"model.layers.6.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
"model.layers.6.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
"model.layers.6.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
"model.layers.7.input_layernorm.weight": "model-00002-of-00007.safetensors",
"model.layers.7.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
"model.layers.7.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
"model.layers.7.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
"model.layers.7.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
"model.layers.7.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
"model.layers.7.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
"model.layers.7.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
"model.layers.7.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
"model.layers.8.input_layernorm.weight": "model-00002-of-00007.safetensors",
"model.layers.8.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
"model.layers.8.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
"model.layers.8.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
"model.layers.8.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
"model.layers.8.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
"model.layers.8.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
"model.layers.8.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
"model.layers.8.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
"model.layers.9.input_layernorm.weight": "model-00003-of-00007.safetensors",
"model.layers.9.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
"model.layers.9.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
"model.layers.9.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
"model.layers.9.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
"model.layers.9.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
"model.layers.9.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
"model.layers.9.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
"model.layers.9.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
"model.norm.weight": "model-00007-of-00007.safetensors"
}
}

4247
model_eval_results.csv Normal file
View File

File diff suppressed because it is too large Load Diff

797
running_log.txt Normal file
View File

@@ -0,0 +1,797 @@
[INFO|2025-04-06 00:01:34] tokenization_utils_base.py:2060 >> loading file tokenizer.model from cache at /root/.cache/huggingface/hub/models--GreatCaptainNemo--ProLLaMA/snapshots/7c873bf1e1c53e5b9cbdf58e6b3420a6821569a7/tokenizer.model
[INFO|2025-04-06 00:01:34] tokenization_utils_base.py:2060 >> loading file tokenizer.json from cache at None
[INFO|2025-04-06 00:01:34] tokenization_utils_base.py:2060 >> loading file added_tokens.json from cache at None
[INFO|2025-04-06 00:01:34] tokenization_utils_base.py:2060 >> loading file special_tokens_map.json from cache at /root/.cache/huggingface/hub/models--GreatCaptainNemo--ProLLaMA/snapshots/7c873bf1e1c53e5b9cbdf58e6b3420a6821569a7/special_tokens_map.json
[INFO|2025-04-06 00:01:34] tokenization_utils_base.py:2060 >> loading file tokenizer_config.json from cache at /root/.cache/huggingface/hub/models--GreatCaptainNemo--ProLLaMA/snapshots/7c873bf1e1c53e5b9cbdf58e6b3420a6821569a7/tokenizer_config.json
[INFO|2025-04-06 00:01:34] tokenization_utils_base.py:2060 >> loading file chat_template.jinja from cache at None
[INFO|2025-04-06 00:01:36] configuration_utils.py:699 >> loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--GreatCaptainNemo--ProLLaMA/snapshots/7c873bf1e1c53e5b9cbdf58e6b3420a6821569a7/config.json
[INFO|2025-04-06 00:01:36] configuration_utils.py:771 >> Model config LlamaConfig {
"architectures": [
"LlamaForCausalLM"
],
"attention_bias": false,
"attention_dropout": 0.0,
"bos_token_id": 1,
"eos_token_id": 2,
"head_dim": 128,
"hidden_act": "silu",
"hidden_size": 4096,
"initializer_range": 0.02,
"intermediate_size": 11008,
"max_position_embeddings": 4096,
"mlp_bias": false,
"model_type": "llama",
"num_attention_heads": 32,
"num_hidden_layers": 32,
"num_key_value_heads": 32,
"pad_token_id": 0,
"pretraining_tp": 1,
"rms_norm_eps": 1e-05,
"rope_scaling": null,
"rope_theta": 10000.0,
"tie_word_embeddings": false,
"torch_dtype": "float16",
"transformers_version": "4.50.0",
"use_cache": true,
"vocab_size": 32000
}
[INFO|2025-04-06 00:01:36] tokenization_utils_base.py:2060 >> loading file tokenizer.model from cache at /root/.cache/huggingface/hub/models--GreatCaptainNemo--ProLLaMA/snapshots/7c873bf1e1c53e5b9cbdf58e6b3420a6821569a7/tokenizer.model
[INFO|2025-04-06 00:01:36] tokenization_utils_base.py:2060 >> loading file tokenizer.json from cache at None
[INFO|2025-04-06 00:01:36] tokenization_utils_base.py:2060 >> loading file added_tokens.json from cache at None
[INFO|2025-04-06 00:01:36] tokenization_utils_base.py:2060 >> loading file special_tokens_map.json from cache at /root/.cache/huggingface/hub/models--GreatCaptainNemo--ProLLaMA/snapshots/7c873bf1e1c53e5b9cbdf58e6b3420a6821569a7/special_tokens_map.json
[INFO|2025-04-06 00:01:36] tokenization_utils_base.py:2060 >> loading file tokenizer_config.json from cache at /root/.cache/huggingface/hub/models--GreatCaptainNemo--ProLLaMA/snapshots/7c873bf1e1c53e5b9cbdf58e6b3420a6821569a7/tokenizer_config.json
[INFO|2025-04-06 00:01:36] tokenization_utils_base.py:2060 >> loading file chat_template.jinja from cache at None
[INFO|2025-04-06 00:01:37] logging.py:143 >> Loading dataset ADPr/train.json...
[INFO|2025-04-06 00:01:41] configuration_utils.py:699 >> loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--GreatCaptainNemo--ProLLaMA/snapshots/7c873bf1e1c53e5b9cbdf58e6b3420a6821569a7/config.json
[INFO|2025-04-06 00:01:41] configuration_utils.py:771 >> Model config LlamaConfig {
"architectures": [
"LlamaForCausalLM"
],
"attention_bias": false,
"attention_dropout": 0.0,
"bos_token_id": 1,
"eos_token_id": 2,
"head_dim": 128,
"hidden_act": "silu",
"hidden_size": 4096,
"initializer_range": 0.02,
"intermediate_size": 11008,
"max_position_embeddings": 4096,
"mlp_bias": false,
"model_type": "llama",
"num_attention_heads": 32,
"num_hidden_layers": 32,
"num_key_value_heads": 32,
"pad_token_id": 0,
"pretraining_tp": 1,
"rms_norm_eps": 1e-05,
"rope_scaling": null,
"rope_theta": 10000.0,
"tie_word_embeddings": false,
"torch_dtype": "float16",
"transformers_version": "4.50.0",
"use_cache": true,
"vocab_size": 32000
}
[INFO|2025-04-06 00:01:41] logging.py:143 >> KV cache is disabled during training.
[INFO|2025-04-06 00:01:43] modeling_utils.py:1154 >> loading weights file model.safetensors from cache at /root/.cache/huggingface/hub/models--GreatCaptainNemo--ProLLaMA/snapshots/7c873bf1e1c53e5b9cbdf58e6b3420a6821569a7/model.safetensors.index.json
[INFO|2025-04-06 00:09:05] modeling_utils.py:2170 >> Instantiating LlamaForCausalLM model under default dtype torch.bfloat16.
[INFO|2025-04-06 00:09:05] configuration_utils.py:1139 >> Generate config GenerationConfig {
"bos_token_id": 1,
"eos_token_id": 2,
"pad_token_id": 0,
"use_cache": false
}
[INFO|2025-04-06 00:09:09] modeling_utils.py:4987 >> All model checkpoint weights were used when initializing LlamaForCausalLM.
[INFO|2025-04-06 00:09:09] modeling_utils.py:4995 >> All the weights of LlamaForCausalLM were initialized from the model checkpoint at GreatCaptainNemo/ProLLaMA.
If your task is similar to the task the model of the checkpoint was trained on, you can already use LlamaForCausalLM for predictions without further training.
[INFO|2025-04-06 00:09:10] configuration_utils.py:1094 >> loading configuration file generation_config.json from cache at /root/.cache/huggingface/hub/models--GreatCaptainNemo--ProLLaMA/snapshots/7c873bf1e1c53e5b9cbdf58e6b3420a6821569a7/generation_config.json
[INFO|2025-04-06 00:09:10] configuration_utils.py:1139 >> Generate config GenerationConfig {
"bos_token_id": 1,
"eos_token_id": 2,
"pad_token_id": 0
}
[INFO|2025-04-06 00:09:10] logging.py:143 >> Gradient checkpointing enabled.
[INFO|2025-04-06 00:09:10] logging.py:143 >> Using torch SDPA for faster training and inference.
[INFO|2025-04-06 00:09:10] logging.py:143 >> Upcasting trainable params to float32.
[INFO|2025-04-06 00:09:10] logging.py:143 >> Fine-tuning method: LoRA
[WARNING|2025-04-06 00:09:10] logging.py:148 >> Vocab has been resized, add lm_head,embed_tokens to trainable params.
[INFO|2025-04-06 00:09:12] logging.py:143 >> trainable params: 422,051,840 || all params: 7,160,467,456 || trainable%: 5.8942
[INFO|2025-04-06 00:09:12] trainer.py:748 >> Using auto half precision backend
[WARNING|2025-04-06 00:09:12] trainer.py:783 >> No label_names provided for model class `PeftModelForCausalLM`. Since `PeftModel` hides base models input arguments, if label_names is not given, label_names can't be set automatically within `Trainer`. Note that empty label_names list will be used instead.
[INFO|2025-04-06 00:09:12] trainer.py:2409 >> ***** Running training *****
[INFO|2025-04-06 00:09:12] trainer.py:2410 >> Num examples = 28,058
[INFO|2025-04-06 00:09:12] trainer.py:2411 >> Num Epochs = 3
[INFO|2025-04-06 00:09:12] trainer.py:2412 >> Instantaneous batch size per device = 16
[INFO|2025-04-06 00:09:12] trainer.py:2415 >> Total train batch size (w. parallel, distributed & accumulation) = 128
[INFO|2025-04-06 00:09:12] trainer.py:2416 >> Gradient Accumulation steps = 8
[INFO|2025-04-06 00:09:12] trainer.py:2417 >> Total optimization steps = 657
[INFO|2025-04-06 00:09:12] trainer.py:2418 >> Number of trainable parameters = 422,051,840
[INFO|2025-04-06 00:09:38] logging.py:143 >> {'loss': 8.7284, 'learning_rate': 1.2500e-05, 'epoch': 0.02, 'throughput': 2445.23}
[INFO|2025-04-06 00:10:03] logging.py:143 >> {'loss': 4.9749, 'learning_rate': 2.5000e-05, 'epoch': 0.05, 'throughput': 2468.54}
[INFO|2025-04-06 00:10:27] logging.py:143 >> {'loss': 1.5517, 'learning_rate': 3.7500e-05, 'epoch': 0.07, 'throughput': 2469.07}
[INFO|2025-04-06 00:10:52] logging.py:143 >> {'loss': 0.6744, 'learning_rate': 5.0000e-05, 'epoch': 0.09, 'throughput': 2472.96}
[INFO|2025-04-06 00:11:17] logging.py:143 >> {'loss': 0.5648, 'learning_rate': 4.9992e-05, 'epoch': 0.11, 'throughput': 2474.77}
[INFO|2025-04-06 00:11:42] logging.py:143 >> {'loss': 0.5605, 'learning_rate': 4.9970e-05, 'epoch': 0.14, 'throughput': 2475.65}
[INFO|2025-04-06 00:12:06] logging.py:143 >> {'loss': 0.5297, 'learning_rate': 4.9932e-05, 'epoch': 0.16, 'throughput': 2476.78}
[INFO|2025-04-06 00:12:31] logging.py:143 >> {'loss': 0.5314, 'learning_rate': 4.9878e-05, 'epoch': 0.18, 'throughput': 2478.52}
[INFO|2025-04-06 00:12:56] logging.py:143 >> {'loss': 0.5013, 'learning_rate': 4.9810e-05, 'epoch': 0.21, 'throughput': 2480.10}
[INFO|2025-04-06 00:13:21] logging.py:143 >> {'loss': 0.4944, 'learning_rate': 4.9727e-05, 'epoch': 0.23, 'throughput': 2479.51}
[INFO|2025-04-06 00:13:46] logging.py:143 >> {'loss': 0.5071, 'learning_rate': 4.9628e-05, 'epoch': 0.25, 'throughput': 2479.71}
[INFO|2025-04-06 00:14:11] logging.py:143 >> {'loss': 0.5025, 'learning_rate': 4.9515e-05, 'epoch': 0.27, 'throughput': 2479.25}
[INFO|2025-04-06 00:14:36] logging.py:143 >> {'loss': 0.5038, 'learning_rate': 4.9387e-05, 'epoch': 0.30, 'throughput': 2479.76}
[INFO|2025-04-06 00:15:01] logging.py:143 >> {'loss': 0.4907, 'learning_rate': 4.9244e-05, 'epoch': 0.32, 'throughput': 2479.97}
[INFO|2025-04-06 00:15:25] logging.py:143 >> {'loss': 0.4774, 'learning_rate': 4.9086e-05, 'epoch': 0.34, 'throughput': 2479.35}
[INFO|2025-04-06 00:15:50] logging.py:143 >> {'loss': 0.4709, 'learning_rate': 4.8913e-05, 'epoch': 0.36, 'throughput': 2479.85}
[INFO|2025-04-06 00:16:15] logging.py:143 >> {'loss': 0.4793, 'learning_rate': 4.8726e-05, 'epoch': 0.39, 'throughput': 2480.04}
[INFO|2025-04-06 00:16:39] logging.py:143 >> {'loss': 0.4835, 'learning_rate': 4.8525e-05, 'epoch': 0.41, 'throughput': 2479.96}
[INFO|2025-04-06 00:17:04] logging.py:143 >> {'loss': 0.4668, 'learning_rate': 4.8309e-05, 'epoch': 0.43, 'throughput': 2480.30}
[INFO|2025-04-06 00:17:28] logging.py:143 >> {'loss': 0.4600, 'learning_rate': 4.8079e-05, 'epoch': 0.46, 'throughput': 2479.57}
[INFO|2025-04-06 00:17:28] trainer.py:4289 >>
***** Running Evaluation *****
[INFO|2025-04-06 00:17:28] trainer.py:4291 >> Num examples = 3118
[INFO|2025-04-06 00:17:28] trainer.py:4294 >> Batch size = 16
[INFO|2025-04-06 00:18:03] trainer.py:3966 >> Saving model checkpoint to saves/Custom/lora/train_2025-04-05-23-57-03/checkpoint-100
[INFO|2025-04-06 00:18:03] configuration_utils.py:699 >> loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--GreatCaptainNemo--ProLLaMA/snapshots/7c873bf1e1c53e5b9cbdf58e6b3420a6821569a7/config.json
[INFO|2025-04-06 00:18:03] configuration_utils.py:771 >> Model config LlamaConfig {
"architectures": [
"LlamaForCausalLM"
],
"attention_bias": false,
"attention_dropout": 0.0,
"bos_token_id": 1,
"eos_token_id": 2,
"head_dim": 128,
"hidden_act": "silu",
"hidden_size": 4096,
"initializer_range": 0.02,
"intermediate_size": 11008,
"max_position_embeddings": 4096,
"mlp_bias": false,
"model_type": "llama",
"num_attention_heads": 32,
"num_hidden_layers": 32,
"num_key_value_heads": 32,
"pad_token_id": 0,
"pretraining_tp": 1,
"rms_norm_eps": 1e-05,
"rope_scaling": null,
"rope_theta": 10000.0,
"tie_word_embeddings": false,
"torch_dtype": "float16",
"transformers_version": "4.50.0",
"use_cache": true,
"vocab_size": 32000
}
[INFO|2025-04-06 00:18:07] tokenization_utils_base.py:2510 >> tokenizer config file saved in saves/Custom/lora/train_2025-04-05-23-57-03/checkpoint-100/tokenizer_config.json
[INFO|2025-04-06 00:18:07] tokenization_utils_base.py:2519 >> Special tokens file saved in saves/Custom/lora/train_2025-04-05-23-57-03/checkpoint-100/special_tokens_map.json
[INFO|2025-04-06 00:18:42] logging.py:143 >> {'loss': 0.4823, 'learning_rate': 4.7835e-05, 'epoch': 0.48, 'throughput': 2266.26}
[INFO|2025-04-06 00:19:07] logging.py:143 >> {'loss': 0.4905, 'learning_rate': 4.7577e-05, 'epoch': 0.50, 'throughput': 2275.37}
[INFO|2025-04-06 00:19:32] logging.py:143 >> {'loss': 0.4759, 'learning_rate': 4.7306e-05, 'epoch': 0.52, 'throughput': 2283.37}
[INFO|2025-04-06 00:19:57] logging.py:143 >> {'loss': 0.4735, 'learning_rate': 4.7021e-05, 'epoch': 0.55, 'throughput': 2290.88}
[INFO|2025-04-06 00:20:21] logging.py:143 >> {'loss': 0.4662, 'learning_rate': 4.6722e-05, 'epoch': 0.57, 'throughput': 2297.32}
[INFO|2025-04-06 00:20:46] logging.py:143 >> {'loss': 0.4757, 'learning_rate': 4.6410e-05, 'epoch': 0.59, 'throughput': 2303.86}
[INFO|2025-04-06 00:21:10] logging.py:143 >> {'loss': 0.4614, 'learning_rate': 4.6086e-05, 'epoch': 0.62, 'throughput': 2310.19}
[INFO|2025-04-06 00:21:35] logging.py:143 >> {'loss': 0.4553, 'learning_rate': 4.5748e-05, 'epoch': 0.64, 'throughput': 2315.47}
[INFO|2025-04-06 00:22:00] logging.py:143 >> {'loss': 0.4540, 'learning_rate': 4.5398e-05, 'epoch': 0.66, 'throughput': 2321.33}
[INFO|2025-04-06 00:22:25] logging.py:143 >> {'loss': 0.4740, 'learning_rate': 4.5035e-05, 'epoch': 0.68, 'throughput': 2326.35}
[INFO|2025-04-06 00:22:49] logging.py:143 >> {'loss': 0.4635, 'learning_rate': 4.4661e-05, 'epoch': 0.71, 'throughput': 2331.01}
[INFO|2025-04-06 00:23:14] logging.py:143 >> {'loss': 0.4529, 'learning_rate': 4.4274e-05, 'epoch': 0.73, 'throughput': 2335.07}
[INFO|2025-04-06 00:23:39] logging.py:143 >> {'loss': 0.4581, 'learning_rate': 4.3875e-05, 'epoch': 0.75, 'throughput': 2339.33}
[INFO|2025-04-06 00:24:03] logging.py:143 >> {'loss': 0.4422, 'learning_rate': 4.3465e-05, 'epoch': 0.78, 'throughput': 2343.15}
[INFO|2025-04-06 00:24:28] logging.py:143 >> {'loss': 0.4522, 'learning_rate': 4.3044e-05, 'epoch': 0.80, 'throughput': 2346.80}
[INFO|2025-04-06 00:24:53] logging.py:143 >> {'loss': 0.4457, 'learning_rate': 4.2612e-05, 'epoch': 0.82, 'throughput': 2350.45}
[INFO|2025-04-06 00:25:18] logging.py:143 >> {'loss': 0.4448, 'learning_rate': 4.2169e-05, 'epoch': 0.84, 'throughput': 2353.93}
[INFO|2025-04-06 00:25:42] logging.py:143 >> {'loss': 0.4442, 'learning_rate': 4.1716e-05, 'epoch': 0.87, 'throughput': 2357.13}
[INFO|2025-04-06 00:26:07] logging.py:143 >> {'loss': 0.4327, 'learning_rate': 4.1253e-05, 'epoch': 0.89, 'throughput': 2360.39}
[INFO|2025-04-06 00:26:32] logging.py:143 >> {'loss': 0.4222, 'learning_rate': 4.0779e-05, 'epoch': 0.91, 'throughput': 2363.27}
[INFO|2025-04-06 00:26:32] trainer.py:4289 >>
***** Running Evaluation *****
[INFO|2025-04-06 00:26:32] trainer.py:4291 >> Num examples = 3118
[INFO|2025-04-06 00:26:32] trainer.py:4294 >> Batch size = 16
[INFO|2025-04-06 00:27:06] trainer.py:3966 >> Saving model checkpoint to saves/Custom/lora/train_2025-04-05-23-57-03/checkpoint-200
[INFO|2025-04-06 00:27:07] configuration_utils.py:699 >> loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--GreatCaptainNemo--ProLLaMA/snapshots/7c873bf1e1c53e5b9cbdf58e6b3420a6821569a7/config.json
[INFO|2025-04-06 00:27:07] configuration_utils.py:771 >> Model config LlamaConfig {
"architectures": [
"LlamaForCausalLM"
],
"attention_bias": false,
"attention_dropout": 0.0,
"bos_token_id": 1,
"eos_token_id": 2,
"head_dim": 128,
"hidden_act": "silu",
"hidden_size": 4096,
"initializer_range": 0.02,
"intermediate_size": 11008,
"max_position_embeddings": 4096,
"mlp_bias": false,
"model_type": "llama",
"num_attention_heads": 32,
"num_hidden_layers": 32,
"num_key_value_heads": 32,
"pad_token_id": 0,
"pretraining_tp": 1,
"rms_norm_eps": 1e-05,
"rope_scaling": null,
"rope_theta": 10000.0,
"tie_word_embeddings": false,
"torch_dtype": "float16",
"transformers_version": "4.50.0",
"use_cache": true,
"vocab_size": 32000
}
[INFO|2025-04-06 00:27:12] tokenization_utils_base.py:2510 >> tokenizer config file saved in saves/Custom/lora/train_2025-04-05-23-57-03/checkpoint-200/tokenizer_config.json
[INFO|2025-04-06 00:27:12] tokenization_utils_base.py:2519 >> Special tokens file saved in saves/Custom/lora/train_2025-04-05-23-57-03/checkpoint-200/special_tokens_map.json
[INFO|2025-04-06 00:27:45] logging.py:143 >> {'loss': 0.4190, 'learning_rate': 4.0296e-05, 'epoch': 0.94, 'throughput': 2263.02}
[INFO|2025-04-06 00:28:10] logging.py:143 >> {'loss': 0.4266, 'learning_rate': 3.9804e-05, 'epoch': 0.96, 'throughput': 2267.79}
[INFO|2025-04-06 00:28:35] logging.py:143 >> {'loss': 0.4234, 'learning_rate': 3.9303e-05, 'epoch': 0.98, 'throughput': 2272.62}
[INFO|2025-04-06 00:28:56] logging.py:143 >> {'loss': 0.4299, 'learning_rate': 3.8793e-05, 'epoch': 1.00, 'throughput': 2276.33}
[INFO|2025-04-06 00:29:21] logging.py:143 >> {'loss': 0.4057, 'learning_rate': 3.8275e-05, 'epoch': 1.02, 'throughput': 2280.53}
[INFO|2025-04-06 00:29:46] logging.py:143 >> {'loss': 0.4060, 'learning_rate': 3.7748e-05, 'epoch': 1.05, 'throughput': 2284.56}
[INFO|2025-04-06 00:30:11] logging.py:143 >> {'loss': 0.4095, 'learning_rate': 3.7214e-05, 'epoch': 1.07, 'throughput': 2288.40}
[INFO|2025-04-06 00:30:35] logging.py:143 >> {'loss': 0.3905, 'learning_rate': 3.6673e-05, 'epoch': 1.09, 'throughput': 2292.14}
[INFO|2025-04-06 00:31:00] logging.py:143 >> {'loss': 0.3970, 'learning_rate': 3.6124e-05, 'epoch': 1.11, 'throughput': 2295.76}
[INFO|2025-04-06 00:31:25] logging.py:143 >> {'loss': 0.3880, 'learning_rate': 3.5569e-05, 'epoch': 1.14, 'throughput': 2299.20}
[INFO|2025-04-06 00:31:49] logging.py:143 >> {'loss': 0.3891, 'learning_rate': 3.5007e-05, 'epoch': 1.16, 'throughput': 2302.55}
[INFO|2025-04-06 00:32:14] logging.py:143 >> {'loss': 0.3850, 'learning_rate': 3.4439e-05, 'epoch': 1.18, 'throughput': 2305.91}
[INFO|2025-04-06 00:32:39] logging.py:143 >> {'loss': 0.3857, 'learning_rate': 3.3865e-05, 'epoch': 1.21, 'throughput': 2309.02}
[INFO|2025-04-06 00:33:04] logging.py:143 >> {'loss': 0.3923, 'learning_rate': 3.3286e-05, 'epoch': 1.23, 'throughput': 2312.21}
[INFO|2025-04-06 00:33:29] logging.py:143 >> {'loss': 0.3751, 'learning_rate': 3.2702e-05, 'epoch': 1.25, 'throughput': 2314.98}
[INFO|2025-04-06 00:33:54] logging.py:143 >> {'loss': 0.3860, 'learning_rate': 3.2113e-05, 'epoch': 1.27, 'throughput': 2318.02}
[INFO|2025-04-06 00:34:19] logging.py:143 >> {'loss': 0.3757, 'learning_rate': 3.1520e-05, 'epoch': 1.30, 'throughput': 2320.61}
[INFO|2025-04-06 00:34:44] logging.py:143 >> {'loss': 0.3831, 'learning_rate': 3.0923e-05, 'epoch': 1.32, 'throughput': 2323.45}
[INFO|2025-04-06 00:35:08] logging.py:143 >> {'loss': 0.3702, 'learning_rate': 3.0322e-05, 'epoch': 1.34, 'throughput': 2325.63}
[INFO|2025-04-06 00:35:33] logging.py:143 >> {'loss': 0.3820, 'learning_rate': 2.9718e-05, 'epoch': 1.36, 'throughput': 2328.17}
[INFO|2025-04-06 00:35:33] trainer.py:4289 >>
***** Running Evaluation *****
[INFO|2025-04-06 00:35:33] trainer.py:4291 >> Num examples = 3118
[INFO|2025-04-06 00:35:33] trainer.py:4294 >> Batch size = 16
[INFO|2025-04-06 00:36:07] trainer.py:3966 >> Saving model checkpoint to saves/Custom/lora/train_2025-04-05-23-57-03/checkpoint-300
[INFO|2025-04-06 00:36:08] configuration_utils.py:699 >> loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--GreatCaptainNemo--ProLLaMA/snapshots/7c873bf1e1c53e5b9cbdf58e6b3420a6821569a7/config.json
[INFO|2025-04-06 00:36:08] configuration_utils.py:771 >> Model config LlamaConfig {
"architectures": [
"LlamaForCausalLM"
],
"attention_bias": false,
"attention_dropout": 0.0,
"bos_token_id": 1,
"eos_token_id": 2,
"head_dim": 128,
"hidden_act": "silu",
"hidden_size": 4096,
"initializer_range": 0.02,
"intermediate_size": 11008,
"max_position_embeddings": 4096,
"mlp_bias": false,
"model_type": "llama",
"num_attention_heads": 32,
"num_hidden_layers": 32,
"num_key_value_heads": 32,
"pad_token_id": 0,
"pretraining_tp": 1,
"rms_norm_eps": 1e-05,
"rope_scaling": null,
"rope_theta": 10000.0,
"tie_word_embeddings": false,
"torch_dtype": "float16",
"transformers_version": "4.50.0",
"use_cache": true,
"vocab_size": 32000
}
[INFO|2025-04-06 00:36:11] tokenization_utils_base.py:2510 >> tokenizer config file saved in saves/Custom/lora/train_2025-04-05-23-57-03/checkpoint-300/tokenizer_config.json
[INFO|2025-04-06 00:36:11] tokenization_utils_base.py:2519 >> Special tokens file saved in saves/Custom/lora/train_2025-04-05-23-57-03/checkpoint-300/special_tokens_map.json
[INFO|2025-04-06 00:36:45] logging.py:143 >> {'loss': 0.3920, 'learning_rate': 2.9112e-05, 'epoch': 1.39, 'throughput': 2263.76}
[INFO|2025-04-06 00:37:10] logging.py:143 >> {'loss': 0.3707, 'learning_rate': 2.8502e-05, 'epoch': 1.41, 'throughput': 2266.96}
[INFO|2025-04-06 00:37:34] logging.py:143 >> {'loss': 0.3645, 'learning_rate': 2.7891e-05, 'epoch': 1.43, 'throughput': 2270.01}
[INFO|2025-04-06 00:38:00] logging.py:143 >> {'loss': 0.3599, 'learning_rate': 2.7278e-05, 'epoch': 1.46, 'throughput': 2272.87}
[INFO|2025-04-06 00:38:24] logging.py:143 >> {'loss': 0.3630, 'learning_rate': 2.6663e-05, 'epoch': 1.48, 'throughput': 2275.73}
[INFO|2025-04-06 00:38:49] logging.py:143 >> {'loss': 0.3542, 'learning_rate': 2.6048e-05, 'epoch': 1.50, 'throughput': 2278.35}
[INFO|2025-04-06 00:39:13] logging.py:143 >> {'loss': 0.3479, 'learning_rate': 2.5432e-05, 'epoch': 1.52, 'throughput': 2281.03}
[INFO|2025-04-06 00:39:38] logging.py:143 >> {'loss': 0.3532, 'learning_rate': 2.4815e-05, 'epoch': 1.55, 'throughput': 2283.71}
[INFO|2025-04-06 00:40:03] logging.py:143 >> {'loss': 0.3564, 'learning_rate': 2.4199e-05, 'epoch': 1.57, 'throughput': 2286.49}
[INFO|2025-04-06 00:40:27] logging.py:143 >> {'loss': 0.3424, 'learning_rate': 2.3583e-05, 'epoch': 1.59, 'throughput': 2289.17}
[INFO|2025-04-06 00:40:52] logging.py:143 >> {'loss': 0.3559, 'learning_rate': 2.2968e-05, 'epoch': 1.62, 'throughput': 2291.71}
[INFO|2025-04-06 00:41:17] logging.py:143 >> {'loss': 0.3548, 'learning_rate': 2.2354e-05, 'epoch': 1.64, 'throughput': 2294.11}
[INFO|2025-04-06 00:41:42] logging.py:143 >> {'loss': 0.3467, 'learning_rate': 2.1742e-05, 'epoch': 1.66, 'throughput': 2296.45}
[INFO|2025-04-06 00:42:07] logging.py:143 >> {'loss': 0.3515, 'learning_rate': 2.1132e-05, 'epoch': 1.68, 'throughput': 2298.95}
[INFO|2025-04-06 00:42:31] logging.py:143 >> {'loss': 0.3308, 'learning_rate': 2.0524e-05, 'epoch': 1.71, 'throughput': 2301.22}
[INFO|2025-04-06 00:42:56] logging.py:143 >> {'loss': 0.3354, 'learning_rate': 1.9919e-05, 'epoch': 1.73, 'throughput': 2303.46}
[INFO|2025-04-06 00:43:21] logging.py:143 >> {'loss': 0.3430, 'learning_rate': 1.9317e-05, 'epoch': 1.75, 'throughput': 2305.69}
[INFO|2025-04-06 00:43:46] logging.py:143 >> {'loss': 0.3456, 'learning_rate': 1.8718e-05, 'epoch': 1.78, 'throughput': 2307.85}
[INFO|2025-04-06 00:44:10] logging.py:143 >> {'loss': 0.3286, 'learning_rate': 1.8124e-05, 'epoch': 1.80, 'throughput': 2309.86}
[INFO|2025-04-06 00:44:35] logging.py:143 >> {'loss': 0.3574, 'learning_rate': 1.7533e-05, 'epoch': 1.82, 'throughput': 2311.96}
[INFO|2025-04-06 00:44:35] trainer.py:4289 >>
***** Running Evaluation *****
[INFO|2025-04-06 00:44:35] trainer.py:4291 >> Num examples = 3118
[INFO|2025-04-06 00:44:35] trainer.py:4294 >> Batch size = 16
[INFO|2025-04-06 00:45:10] trainer.py:3966 >> Saving model checkpoint to saves/Custom/lora/train_2025-04-05-23-57-03/checkpoint-400
[INFO|2025-04-06 00:45:10] configuration_utils.py:699 >> loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--GreatCaptainNemo--ProLLaMA/snapshots/7c873bf1e1c53e5b9cbdf58e6b3420a6821569a7/config.json
[INFO|2025-04-06 00:45:10] configuration_utils.py:771 >> Model config LlamaConfig {
"architectures": [
"LlamaForCausalLM"
],
"attention_bias": false,
"attention_dropout": 0.0,
"bos_token_id": 1,
"eos_token_id": 2,
"head_dim": 128,
"hidden_act": "silu",
"hidden_size": 4096,
"initializer_range": 0.02,
"intermediate_size": 11008,
"max_position_embeddings": 4096,
"mlp_bias": false,
"model_type": "llama",
"num_attention_heads": 32,
"num_hidden_layers": 32,
"num_key_value_heads": 32,
"pad_token_id": 0,
"pretraining_tp": 1,
"rms_norm_eps": 1e-05,
"rope_scaling": null,
"rope_theta": 10000.0,
"tie_word_embeddings": false,
"torch_dtype": "float16",
"transformers_version": "4.50.0",
"use_cache": true,
"vocab_size": 32000
}
[INFO|2025-04-06 00:45:15] tokenization_utils_base.py:2510 >> tokenizer config file saved in saves/Custom/lora/train_2025-04-05-23-57-03/checkpoint-400/tokenizer_config.json
[INFO|2025-04-06 00:45:15] tokenization_utils_base.py:2519 >> Special tokens file saved in saves/Custom/lora/train_2025-04-05-23-57-03/checkpoint-400/special_tokens_map.json
[INFO|2025-04-06 00:45:47] logging.py:143 >> {'loss': 0.3280, 'learning_rate': 1.6947e-05, 'epoch': 1.84, 'throughput': 2263.76}
[INFO|2025-04-06 00:46:12] logging.py:143 >> {'loss': 0.3358, 'learning_rate': 1.6366e-05, 'epoch': 1.87, 'throughput': 2266.24}
[INFO|2025-04-06 00:46:37] logging.py:143 >> {'loss': 0.3365, 'learning_rate': 1.5790e-05, 'epoch': 1.89, 'throughput': 2268.54}
[INFO|2025-04-06 00:47:02] logging.py:143 >> {'loss': 0.3257, 'learning_rate': 1.5220e-05, 'epoch': 1.91, 'throughput': 2270.84}
[INFO|2025-04-06 00:47:27] logging.py:143 >> {'loss': 0.3369, 'learning_rate': 1.4655e-05, 'epoch': 1.94, 'throughput': 2273.16}
[INFO|2025-04-06 00:47:52] logging.py:143 >> {'loss': 0.3410, 'learning_rate': 1.4097e-05, 'epoch': 1.96, 'throughput': 2275.49}
[INFO|2025-04-06 00:48:17] logging.py:143 >> {'loss': 0.3240, 'learning_rate': 1.3546e-05, 'epoch': 1.98, 'throughput': 2277.63}
[INFO|2025-04-06 00:48:38] logging.py:143 >> {'loss': 0.3200, 'learning_rate': 1.3002e-05, 'epoch': 2.00, 'throughput': 2279.38}
[INFO|2025-04-06 00:49:03] logging.py:143 >> {'loss': 0.3055, 'learning_rate': 1.2464e-05, 'epoch': 2.02, 'throughput': 2281.53}
[INFO|2025-04-06 00:49:27] logging.py:143 >> {'loss': 0.3040, 'learning_rate': 1.1935e-05, 'epoch': 2.05, 'throughput': 2283.42}
[INFO|2025-04-06 00:49:52] logging.py:143 >> {'loss': 0.3105, 'learning_rate': 1.1413e-05, 'epoch': 2.07, 'throughput': 2285.51}
[INFO|2025-04-06 00:50:17] logging.py:143 >> {'loss': 0.2971, 'learning_rate': 1.0900e-05, 'epoch': 2.09, 'throughput': 2287.38}
[INFO|2025-04-06 00:50:42] logging.py:143 >> {'loss': 0.2986, 'learning_rate': 1.0395e-05, 'epoch': 2.11, 'throughput': 2289.16}
[INFO|2025-04-06 00:51:07] logging.py:143 >> {'loss': 0.3109, 'learning_rate': 9.8994e-06, 'epoch': 2.14, 'throughput': 2291.02}
[INFO|2025-04-06 00:51:32] logging.py:143 >> {'loss': 0.3053, 'learning_rate': 9.4128e-06, 'epoch': 2.16, 'throughput': 2292.87}
[INFO|2025-04-06 00:51:57] logging.py:143 >> {'loss': 0.3116, 'learning_rate': 8.9356e-06, 'epoch': 2.18, 'throughput': 2294.73}
[INFO|2025-04-06 00:52:22] logging.py:143 >> {'loss': 0.2883, 'learning_rate': 8.4681e-06, 'epoch': 2.21, 'throughput': 2296.38}
[INFO|2025-04-06 00:52:47] logging.py:143 >> {'loss': 0.2859, 'learning_rate': 8.0108e-06, 'epoch': 2.23, 'throughput': 2298.08}
[INFO|2025-04-06 00:53:11] logging.py:143 >> {'loss': 0.3055, 'learning_rate': 7.5637e-06, 'epoch': 2.25, 'throughput': 2299.77}
[INFO|2025-04-06 00:53:36] logging.py:143 >> {'loss': 0.3110, 'learning_rate': 7.1273e-06, 'epoch': 2.27, 'throughput': 2301.47}
[INFO|2025-04-06 00:53:36] trainer.py:4289 >>
***** Running Evaluation *****
[INFO|2025-04-06 00:53:36] trainer.py:4291 >> Num examples = 3118
[INFO|2025-04-06 00:53:36] trainer.py:4294 >> Batch size = 16
[INFO|2025-04-06 00:54:11] trainer.py:3966 >> Saving model checkpoint to saves/Custom/lora/train_2025-04-05-23-57-03/checkpoint-500
[INFO|2025-04-06 00:54:11] configuration_utils.py:699 >> loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--GreatCaptainNemo--ProLLaMA/snapshots/7c873bf1e1c53e5b9cbdf58e6b3420a6821569a7/config.json
[INFO|2025-04-06 00:54:11] configuration_utils.py:771 >> Model config LlamaConfig {
"architectures": [
"LlamaForCausalLM"
],
"attention_bias": false,
"attention_dropout": 0.0,
"bos_token_id": 1,
"eos_token_id": 2,
"head_dim": 128,
"hidden_act": "silu",
"hidden_size": 4096,
"initializer_range": 0.02,
"intermediate_size": 11008,
"max_position_embeddings": 4096,
"mlp_bias": false,
"model_type": "llama",
"num_attention_heads": 32,
"num_hidden_layers": 32,
"num_key_value_heads": 32,
"pad_token_id": 0,
"pretraining_tp": 1,
"rms_norm_eps": 1e-05,
"rope_scaling": null,
"rope_theta": 10000.0,
"tie_word_embeddings": false,
"torch_dtype": "float16",
"transformers_version": "4.50.0",
"use_cache": true,
"vocab_size": 32000
}
[INFO|2025-04-06 00:54:16] tokenization_utils_base.py:2510 >> tokenizer config file saved in saves/Custom/lora/train_2025-04-05-23-57-03/checkpoint-500/tokenizer_config.json
[INFO|2025-04-06 00:54:16] tokenization_utils_base.py:2519 >> Special tokens file saved in saves/Custom/lora/train_2025-04-05-23-57-03/checkpoint-500/special_tokens_map.json
[INFO|2025-04-06 00:54:50] logging.py:143 >> {'loss': 0.2869, 'learning_rate': 6.7017e-06, 'epoch': 2.30, 'throughput': 2262.25}
[INFO|2025-04-06 00:55:15] logging.py:143 >> {'loss': 0.2944, 'learning_rate': 6.2872e-06, 'epoch': 2.32, 'throughput': 2264.23}
[INFO|2025-04-06 00:55:40] logging.py:143 >> {'loss': 0.2967, 'learning_rate': 5.8842e-06, 'epoch': 2.34, 'throughput': 2266.11}
[INFO|2025-04-06 00:56:04] logging.py:143 >> {'loss': 0.2747, 'learning_rate': 5.4927e-06, 'epoch': 2.36, 'throughput': 2267.95}
[INFO|2025-04-06 00:56:29] logging.py:143 >> {'loss': 0.2771, 'learning_rate': 5.1131e-06, 'epoch': 2.39, 'throughput': 2269.76}
[INFO|2025-04-06 00:56:54] logging.py:143 >> {'loss': 0.2999, 'learning_rate': 4.7456e-06, 'epoch': 2.41, 'throughput': 2271.55}
[INFO|2025-04-06 00:57:20] logging.py:143 >> {'loss': 0.2982, 'learning_rate': 4.3904e-06, 'epoch': 2.43, 'throughput': 2273.34}
[INFO|2025-04-06 00:57:44] logging.py:143 >> {'loss': 0.2822, 'learning_rate': 4.0478e-06, 'epoch': 2.46, 'throughput': 2275.04}
[INFO|2025-04-06 00:58:09] logging.py:143 >> {'loss': 0.2867, 'learning_rate': 3.7179e-06, 'epoch': 2.48, 'throughput': 2276.60}
[INFO|2025-04-06 00:58:33] logging.py:143 >> {'loss': 0.2763, 'learning_rate': 3.4009e-06, 'epoch': 2.50, 'throughput': 2278.12}
[INFO|2025-04-06 00:58:58] logging.py:143 >> {'loss': 0.2975, 'learning_rate': 3.0971e-06, 'epoch': 2.52, 'throughput': 2279.74}
[INFO|2025-04-06 00:59:22] logging.py:143 >> {'loss': 0.2826, 'learning_rate': 2.8066e-06, 'epoch': 2.55, 'throughput': 2281.23}
[INFO|2025-04-06 00:59:47] logging.py:143 >> {'loss': 0.2807, 'learning_rate': 2.5295e-06, 'epoch': 2.57, 'throughput': 2282.81}
[INFO|2025-04-06 01:00:12] logging.py:143 >> {'loss': 0.2882, 'learning_rate': 2.2662e-06, 'epoch': 2.59, 'throughput': 2284.37}
[INFO|2025-04-06 01:00:37] logging.py:143 >> {'loss': 0.2878, 'learning_rate': 2.0167e-06, 'epoch': 2.62, 'throughput': 2285.93}
[INFO|2025-04-06 01:01:01] logging.py:143 >> {'loss': 0.2913, 'learning_rate': 1.7811e-06, 'epoch': 2.64, 'throughput': 2287.44}
[INFO|2025-04-06 01:01:26] logging.py:143 >> {'loss': 0.2893, 'learning_rate': 1.5597e-06, 'epoch': 2.66, 'throughput': 2289.00}
[INFO|2025-04-06 01:01:51] logging.py:143 >> {'loss': 0.2847, 'learning_rate': 1.3525e-06, 'epoch': 2.68, 'throughput': 2290.49}
[INFO|2025-04-06 01:02:16] logging.py:143 >> {'loss': 0.2902, 'learning_rate': 1.1597e-06, 'epoch': 2.71, 'throughput': 2292.06}
[INFO|2025-04-06 01:02:41] logging.py:143 >> {'loss': 0.2808, 'learning_rate': 9.8134e-07, 'epoch': 2.73, 'throughput': 2293.36}
[INFO|2025-04-06 01:02:41] trainer.py:4289 >>
***** Running Evaluation *****
[INFO|2025-04-06 01:02:41] trainer.py:4291 >> Num examples = 3118
[INFO|2025-04-06 01:02:41] trainer.py:4294 >> Batch size = 16
[INFO|2025-04-06 01:03:15] trainer.py:3966 >> Saving model checkpoint to saves/Custom/lora/train_2025-04-05-23-57-03/checkpoint-600
[INFO|2025-04-06 01:03:16] configuration_utils.py:699 >> loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--GreatCaptainNemo--ProLLaMA/snapshots/7c873bf1e1c53e5b9cbdf58e6b3420a6821569a7/config.json
[INFO|2025-04-06 01:03:16] configuration_utils.py:771 >> Model config LlamaConfig {
"architectures": [
"LlamaForCausalLM"
],
"attention_bias": false,
"attention_dropout": 0.0,
"bos_token_id": 1,
"eos_token_id": 2,
"head_dim": 128,
"hidden_act": "silu",
"hidden_size": 4096,
"initializer_range": 0.02,
"intermediate_size": 11008,
"max_position_embeddings": 4096,
"mlp_bias": false,
"model_type": "llama",
"num_attention_heads": 32,
"num_hidden_layers": 32,
"num_key_value_heads": 32,
"pad_token_id": 0,
"pretraining_tp": 1,
"rms_norm_eps": 1e-05,
"rope_scaling": null,
"rope_theta": 10000.0,
"tie_word_embeddings": false,
"torch_dtype": "float16",
"transformers_version": "4.50.0",
"use_cache": true,
"vocab_size": 32000
}
[INFO|2025-04-06 01:03:22] tokenization_utils_base.py:2510 >> tokenizer config file saved in saves/Custom/lora/train_2025-04-05-23-57-03/checkpoint-600/tokenizer_config.json
[INFO|2025-04-06 01:03:22] tokenization_utils_base.py:2519 >> Special tokens file saved in saves/Custom/lora/train_2025-04-05-23-57-03/checkpoint-600/special_tokens_map.json
[INFO|2025-04-06 01:03:55] logging.py:143 >> {'loss': 0.2775, 'learning_rate': 8.1763e-07, 'epoch': 2.75, 'throughput': 2259.90}
[INFO|2025-04-06 01:04:20] logging.py:143 >> {'loss': 0.2874, 'learning_rate': 6.6862e-07, 'epoch': 2.78, 'throughput': 2261.55}
[INFO|2025-04-06 01:04:45] logging.py:143 >> {'loss': 0.2933, 'learning_rate': 5.3441e-07, 'epoch': 2.80, 'throughput': 2263.21}
[INFO|2025-04-06 01:05:10] logging.py:143 >> {'loss': 0.2807, 'learning_rate': 4.1508e-07, 'epoch': 2.82, 'throughput': 2264.75}
[INFO|2025-04-06 01:05:34] logging.py:143 >> {'loss': 0.2843, 'learning_rate': 3.1069e-07, 'epoch': 2.84, 'throughput': 2266.34}
[INFO|2025-04-06 01:05:59] logging.py:143 >> {'loss': 0.2717, 'learning_rate': 2.2132e-07, 'epoch': 2.87, 'throughput': 2267.87}
[INFO|2025-04-06 01:06:24] logging.py:143 >> {'loss': 0.2789, 'learning_rate': 1.4701e-07, 'epoch': 2.89, 'throughput': 2269.33}
[INFO|2025-04-06 01:06:48] logging.py:143 >> {'loss': 0.2806, 'learning_rate': 8.7816e-08, 'epoch': 2.91, 'throughput': 2270.81}
[INFO|2025-04-06 01:07:13] logging.py:143 >> {'loss': 0.2729, 'learning_rate': 4.3769e-08, 'epoch': 2.94, 'throughput': 2272.29}
[INFO|2025-04-06 01:07:38] logging.py:143 >> {'loss': 0.2868, 'learning_rate': 1.4896e-08, 'epoch': 2.96, 'throughput': 2273.81}
[INFO|2025-04-06 01:08:03] logging.py:143 >> {'loss': 0.2849, 'learning_rate': 1.2162e-09, 'epoch': 2.98, 'throughput': 2275.21}
[INFO|2025-04-06 01:08:13] trainer.py:3966 >> Saving model checkpoint to saves/Custom/lora/train_2025-04-05-23-57-03/checkpoint-657
[INFO|2025-04-06 01:08:13] configuration_utils.py:699 >> loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--GreatCaptainNemo--ProLLaMA/snapshots/7c873bf1e1c53e5b9cbdf58e6b3420a6821569a7/config.json
[INFO|2025-04-06 01:08:13] configuration_utils.py:771 >> Model config LlamaConfig {
"architectures": [
"LlamaForCausalLM"
],
"attention_bias": false,
"attention_dropout": 0.0,
"bos_token_id": 1,
"eos_token_id": 2,
"head_dim": 128,
"hidden_act": "silu",
"hidden_size": 4096,
"initializer_range": 0.02,
"intermediate_size": 11008,
"max_position_embeddings": 4096,
"mlp_bias": false,
"model_type": "llama",
"num_attention_heads": 32,
"num_hidden_layers": 32,
"num_key_value_heads": 32,
"pad_token_id": 0,
"pretraining_tp": 1,
"rms_norm_eps": 1e-05,
"rope_scaling": null,
"rope_theta": 10000.0,
"tie_word_embeddings": false,
"torch_dtype": "float16",
"transformers_version": "4.50.0",
"use_cache": true,
"vocab_size": 32000
}
[INFO|2025-04-06 01:08:18] tokenization_utils_base.py:2510 >> tokenizer config file saved in saves/Custom/lora/train_2025-04-05-23-57-03/checkpoint-657/tokenizer_config.json
[INFO|2025-04-06 01:08:18] tokenization_utils_base.py:2519 >> Special tokens file saved in saves/Custom/lora/train_2025-04-05-23-57-03/checkpoint-657/special_tokens_map.json
[INFO|2025-04-06 01:08:26] trainer.py:2665 >>
Training completed. Do not forget to share your model on huggingface.co/models =)
[INFO|2025-04-06 01:08:26] trainer.py:3966 >> Saving model checkpoint to saves/Custom/lora/train_2025-04-05-23-57-03
[INFO|2025-04-06 01:08:26] configuration_utils.py:699 >> loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--GreatCaptainNemo--ProLLaMA/snapshots/7c873bf1e1c53e5b9cbdf58e6b3420a6821569a7/config.json
[INFO|2025-04-06 01:08:26] configuration_utils.py:771 >> Model config LlamaConfig {
"architectures": [
"LlamaForCausalLM"
],
"attention_bias": false,
"attention_dropout": 0.0,
"bos_token_id": 1,
"eos_token_id": 2,
"head_dim": 128,
"hidden_act": "silu",
"hidden_size": 4096,
"initializer_range": 0.02,
"intermediate_size": 11008,
"max_position_embeddings": 4096,
"mlp_bias": false,
"model_type": "llama",
"num_attention_heads": 32,
"num_hidden_layers": 32,
"num_key_value_heads": 32,
"pad_token_id": 0,
"pretraining_tp": 1,
"rms_norm_eps": 1e-05,
"rope_scaling": null,
"rope_theta": 10000.0,
"tie_word_embeddings": false,
"torch_dtype": "float16",
"transformers_version": "4.50.0",
"use_cache": true,
"vocab_size": 32000
}
[INFO|2025-04-06 01:08:36] tokenization_utils_base.py:2510 >> tokenizer config file saved in saves/Custom/lora/train_2025-04-05-23-57-03/tokenizer_config.json
[INFO|2025-04-06 01:08:36] tokenization_utils_base.py:2519 >> Special tokens file saved in saves/Custom/lora/train_2025-04-05-23-57-03/special_tokens_map.json
[WARNING|2025-04-06 01:08:36] logging.py:148 >> No metric eval_accuracy to plot.
[INFO|2025-04-06 01:08:36] trainer.py:4289 >>
***** Running Evaluation *****
[INFO|2025-04-06 01:08:36] trainer.py:4291 >> Num examples = 3118
[INFO|2025-04-06 01:08:36] trainer.py:4294 >> Batch size = 16
[INFO|2025-04-06 01:09:10] modelcard.py:449 >> Dropping the following result as it does not have all the necessary fields:
{'task': {'name': 'Causal Language Modeling', 'type': 'text-generation'}}

24
special_tokens_map.json Normal file
View File

@@ -0,0 +1,24 @@
{
"bos_token": {
"content": "<s>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
},
"eos_token": {
"content": "</s>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
},
"pad_token": "<unk>",
"unk_token": {
"content": "<unk>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
}
}

277139
tokenizer.json Normal file
View File

File diff suppressed because it is too large Load Diff

BIN
tokenizer.model (Stored with Git LFS) Normal file
View File

Binary file not shown.

44
tokenizer_config.json Normal file
View File

@@ -0,0 +1,44 @@
{
"add_bos_token": true,
"add_eos_token": false,
"add_prefix_space": null,
"added_tokens_decoder": {
"0": {
"content": "<unk>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": true
},
"1": {
"content": "<s>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": true
},
"2": {
"content": "</s>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": true
}
},
"bos_token": "<s>",
"clean_up_tokenization_spaces": false,
"eos_token": "</s>",
"extra_special_tokens": {},
"legacy": true,
"model_max_length": 1000000000000000019884624838656,
"pad_token": "<unk>",
"sp_model_kwargs": {},
"spaces_between_special_tokens": false,
"tokenizer_class": "LlamaTokenizer",
"unk_token": "<unk>",
"use_default_system_prompt": false,
"use_fast": true
}

47
trainer_log.jsonl Normal file
View File

@@ -0,0 +1,47 @@
{"current_steps": 5, "total_steps": 220, "loss": 12.6644, "lr": 2.9999999999999997e-05, "epoch": 0.02280501710376283, "percentage": 2.27, "elapsed_time": "0:00:24", "remaining_time": "0:17:28", "throughput": 2531.08, "total_tokens": 61696}
{"current_steps": 10, "total_steps": 220, "loss": 5.0641, "lr": 6.75e-05, "epoch": 0.04561003420752566, "percentage": 4.55, "elapsed_time": "0:00:47", "remaining_time": "0:16:31", "throughput": 2607.85, "total_tokens": 123136}
{"current_steps": 15, "total_steps": 220, "loss": 1.5053, "lr": 0.00010499999999999999, "epoch": 0.06841505131128849, "percentage": 6.82, "elapsed_time": "0:01:10", "remaining_time": "0:15:59", "throughput": 2632.04, "total_tokens": 184832}
{"current_steps": 20, "total_steps": 220, "loss": 0.6917, "lr": 0.0001425, "epoch": 0.09122006841505131, "percentage": 9.09, "elapsed_time": "0:01:33", "remaining_time": "0:15:34", "throughput": 2647.7, "total_tokens": 247552}
{"current_steps": 25, "total_steps": 220, "loss": 0.5721, "lr": 0.00017999999999999998, "epoch": 0.11402508551881414, "percentage": 11.36, "elapsed_time": "0:01:56", "remaining_time": "0:15:07", "throughput": 2656.08, "total_tokens": 308992}
{"current_steps": 30, "total_steps": 220, "loss": 0.6254, "lr": 0.00021749999999999997, "epoch": 0.13683010262257697, "percentage": 13.64, "elapsed_time": "0:02:19", "remaining_time": "0:14:41", "throughput": 2659.03, "total_tokens": 370176}
{"current_steps": 35, "total_steps": 220, "loss": 0.6389, "lr": 0.00025499999999999996, "epoch": 0.15963511972633979, "percentage": 15.91, "elapsed_time": "0:02:41", "remaining_time": "0:14:15", "throughput": 2660.78, "total_tokens": 430720}
{"current_steps": 40, "total_steps": 220, "loss": 0.5258, "lr": 0.00029249999999999995, "epoch": 0.18244013683010263, "percentage": 18.18, "elapsed_time": "0:03:05", "remaining_time": "0:13:53", "throughput": 2659.66, "total_tokens": 492800}
{"current_steps": 45, "total_steps": 220, "loss": 0.5041, "lr": 0.0002996346075389736, "epoch": 0.20524515393386544, "percentage": 20.45, "elapsed_time": "0:03:28", "remaining_time": "0:13:29", "throughput": 2660.7, "total_tokens": 553984}
{"current_steps": 50, "total_steps": 220, "loss": 0.4831, "lr": 0.00029815325108927063, "epoch": 0.22805017103762829, "percentage": 22.73, "elapsed_time": "0:03:51", "remaining_time": "0:13:05", "throughput": 2661.54, "total_tokens": 615040}
{"current_steps": 55, "total_steps": 220, "loss": 0.4957, "lr": 0.0002955443589413994, "epoch": 0.2508551881413911, "percentage": 25.0, "elapsed_time": "0:04:14", "remaining_time": "0:12:42", "throughput": 2663.54, "total_tokens": 676736}
{"current_steps": 60, "total_steps": 220, "loss": 0.4764, "lr": 0.00029182778633989753, "epoch": 0.27366020524515394, "percentage": 27.27, "elapsed_time": "0:04:37", "remaining_time": "0:12:18", "throughput": 2664.2, "total_tokens": 738176}
{"current_steps": 65, "total_steps": 220, "loss": 0.4829, "lr": 0.0002870318186463901, "epoch": 0.29646522234891676, "percentage": 29.55, "elapsed_time": "0:05:00", "remaining_time": "0:11:55", "throughput": 2664.82, "total_tokens": 799488}
{"current_steps": 70, "total_steps": 220, "loss": 0.478, "lr": 0.00028119295607090933, "epoch": 0.31927023945267957, "percentage": 31.82, "elapsed_time": "0:05:23", "remaining_time": "0:11:32", "throughput": 2666.34, "total_tokens": 861568}
{"current_steps": 75, "total_steps": 220, "loss": 0.4771, "lr": 0.0002743556358832562, "epoch": 0.34207525655644244, "percentage": 34.09, "elapsed_time": "0:05:46", "remaining_time": "0:11:10", "throughput": 2666.42, "total_tokens": 924544}
{"current_steps": 80, "total_steps": 220, "loss": 0.4657, "lr": 0.0002665718942185456, "epoch": 0.36488027366020526, "percentage": 36.36, "elapsed_time": "0:06:09", "remaining_time": "0:10:46", "throughput": 2666.94, "total_tokens": 985472}
{"current_steps": 85, "total_steps": 220, "loss": 0.4831, "lr": 0.00025790097005079764, "epoch": 0.38768529076396807, "percentage": 38.64, "elapsed_time": "0:06:32", "remaining_time": "0:10:23", "throughput": 2667.09, "total_tokens": 1046912}
{"current_steps": 90, "total_steps": 220, "loss": 0.4778, "lr": 0.0002484088543485761, "epoch": 0.4104903078677309, "percentage": 40.91, "elapsed_time": "0:06:55", "remaining_time": "0:10:00", "throughput": 2668.18, "total_tokens": 1108992}
{"current_steps": 95, "total_steps": 220, "loss": 0.455, "lr": 0.00023816778784387094, "epoch": 0.43329532497149376, "percentage": 43.18, "elapsed_time": "0:07:18", "remaining_time": "0:09:36", "throughput": 2668.23, "total_tokens": 1170048}
{"current_steps": 100, "total_steps": 220, "loss": 0.4571, "lr": 0.00022725571123650813, "epoch": 0.45610034207525657, "percentage": 45.45, "elapsed_time": "0:07:41", "remaining_time": "0:09:13", "throughput": 2668.44, "total_tokens": 1230464}
{"current_steps": 100, "total_steps": 220, "eval_loss": 0.4646710157394409, "epoch": 0.45610034207525657, "percentage": 45.45, "elapsed_time": "0:08:13", "remaining_time": "0:09:52", "throughput": 2491.96, "total_tokens": 1230464}
{"current_steps": 105, "total_steps": 220, "loss": 0.4667, "lr": 0.0002157556720183616, "epoch": 0.4789053591790194, "percentage": 47.73, "elapsed_time": "0:08:36", "remaining_time": "0:09:26", "throughput": 2500.4, "total_tokens": 1292288}
{"current_steps": 110, "total_steps": 220, "loss": 0.4758, "lr": 0.000203755192431795, "epoch": 0.5017103762827823, "percentage": 50.0, "elapsed_time": "0:08:59", "remaining_time": "0:08:59", "throughput": 2507.87, "total_tokens": 1353344}
{"current_steps": 115, "total_steps": 220, "loss": 0.4783, "lr": 0.00019134560337254986, "epoch": 0.5245153933865451, "percentage": 52.27, "elapsed_time": "0:09:22", "remaining_time": "0:08:33", "throughput": 2515.39, "total_tokens": 1415040}
{"current_steps": 120, "total_steps": 220, "loss": 0.4643, "lr": 0.0001786213493064817, "epoch": 0.5473204104903079, "percentage": 54.55, "elapsed_time": "0:09:45", "remaining_time": "0:08:07", "throughput": 2521.98, "total_tokens": 1476480}
{"current_steps": 125, "total_steps": 220, "loss": 0.4542, "lr": 0.000165679269490148, "epoch": 0.5701254275940707, "percentage": 56.82, "elapsed_time": "0:10:08", "remaining_time": "0:07:42", "throughput": 2527.64, "total_tokens": 1537664}
{"current_steps": 130, "total_steps": 220, "loss": 0.4539, "lr": 0.00015261786096559254, "epoch": 0.5929304446978335, "percentage": 59.09, "elapsed_time": "0:10:31", "remaining_time": "0:07:16", "throughput": 2533.11, "total_tokens": 1598848}
{"current_steps": 135, "total_steps": 220, "loss": 0.4563, "lr": 0.00013953652893838119, "epoch": 0.6157354618015963, "percentage": 61.36, "elapsed_time": "0:10:54", "remaining_time": "0:06:51", "throughput": 2538.83, "total_tokens": 1660800}
{"current_steps": 140, "total_steps": 220, "loss": 0.4434, "lr": 0.00012653483024396533, "epoch": 0.6385404789053591, "percentage": 63.64, "elapsed_time": "0:11:16", "remaining_time": "0:06:26", "throughput": 2543.39, "total_tokens": 1721600}
{"current_steps": 145, "total_steps": 220, "loss": 0.4484, "lr": 0.00011371171566004985, "epoch": 0.661345496009122, "percentage": 65.91, "elapsed_time": "0:11:39", "remaining_time": "0:06:01", "throughput": 2547.95, "total_tokens": 1783168}
{"current_steps": 150, "total_steps": 220, "loss": 0.4314, "lr": 0.00010116477683142652, "epoch": 0.6841505131128849, "percentage": 68.18, "elapsed_time": "0:12:02", "remaining_time": "0:05:37", "throughput": 2552.51, "total_tokens": 1844992}
{"current_steps": 155, "total_steps": 220, "loss": 0.4211, "lr": 8.898950353862998e-05, "epoch": 0.7069555302166477, "percentage": 70.45, "elapsed_time": "0:12:25", "remaining_time": "0:05:12", "throughput": 2556.34, "total_tokens": 1906048}
{"current_steps": 160, "total_steps": 220, "loss": 0.4324, "lr": 7.727855696304944e-05, "epoch": 0.7297605473204105, "percentage": 72.73, "elapsed_time": "0:12:48", "remaining_time": "0:04:48", "throughput": 2560.29, "total_tokens": 1967744}
{"current_steps": 165, "total_steps": 220, "loss": 0.4093, "lr": 6.612106447938799e-05, "epoch": 0.7525655644241733, "percentage": 75.0, "elapsed_time": "0:13:11", "remaining_time": "0:04:23", "throughput": 2563.4, "total_tokens": 2028032}
{"current_steps": 170, "total_steps": 220, "loss": 0.4113, "lr": 5.56019413425244e-05, "epoch": 0.7753705815279361, "percentage": 77.27, "elapsed_time": "0:13:33", "remaining_time": "0:03:59", "throughput": 2566.5, "total_tokens": 2088448}
{"current_steps": 175, "total_steps": 220, "loss": 0.4142, "lr": 4.5801244431150394e-05, "epoch": 0.798175598631699, "percentage": 79.55, "elapsed_time": "0:13:56", "remaining_time": "0:03:35", "throughput": 2569.89, "total_tokens": 2150144}
{"current_steps": 180, "total_steps": 220, "loss": 0.407, "lr": 3.6793562966584196e-05, "epoch": 0.8209806157354618, "percentage": 81.82, "elapsed_time": "0:14:19", "remaining_time": "0:03:11", "throughput": 2572.74, "total_tokens": 2211584}
{"current_steps": 185, "total_steps": 220, "loss": 0.3836, "lr": 2.8647450843757897e-05, "epoch": 0.8437856328392246, "percentage": 84.09, "elapsed_time": "0:14:42", "remaining_time": "0:02:46", "throughput": 2575.48, "total_tokens": 2272256}
{"current_steps": 190, "total_steps": 220, "loss": 0.3904, "lr": 2.1424904894683165e-05, "epoch": 0.8665906499429875, "percentage": 86.36, "elapsed_time": "0:15:05", "remaining_time": "0:02:22", "throughput": 2578.09, "total_tokens": 2333696}
{"current_steps": 195, "total_steps": 220, "loss": 0.4011, "lr": 1.5180893055124977e-05, "epoch": 0.8893956670467503, "percentage": 88.64, "elapsed_time": "0:15:28", "remaining_time": "0:01:58", "throughput": 2580.67, "total_tokens": 2394880}
{"current_steps": 200, "total_steps": 220, "loss": 0.3809, "lr": 9.962936025419754e-06, "epoch": 0.9122006841505131, "percentage": 90.91, "elapsed_time": "0:15:50", "remaining_time": "0:01:35", "throughput": 2582.96, "total_tokens": 2455680}
{"current_steps": 200, "total_steps": 220, "eval_loss": 0.3808976411819458, "epoch": 0.9122006841505131, "percentage": 90.91, "elapsed_time": "0:16:23", "remaining_time": "0:01:38", "throughput": 2497.07, "total_tokens": 2455680}
{"current_steps": 205, "total_steps": 220, "loss": 0.3799, "lr": 5.810745609252165e-06, "epoch": 0.935005701254276, "percentage": 93.18, "elapsed_time": "0:16:46", "remaining_time": "0:01:13", "throughput": 2501.27, "total_tokens": 2517376}
{"current_steps": 210, "total_steps": 220, "loss": 0.3944, "lr": 2.7559224828504035e-06, "epoch": 0.9578107183580388, "percentage": 95.45, "elapsed_time": "0:17:09", "remaining_time": "0:00:49", "throughput": 2505.48, "total_tokens": 2578816}
{"current_steps": 215, "total_steps": 220, "loss": 0.3721, "lr": 8.217156947590064e-07, "epoch": 0.9806157354618016, "percentage": 97.73, "elapsed_time": "0:17:32", "remaining_time": "0:00:24", "throughput": 2509.24, "total_tokens": 2640128}
{"current_steps": 220, "total_steps": 220, "loss": 0.3749, "lr": 2.284572654130956e-08, "epoch": 1.0, "percentage": 100.0, "elapsed_time": "0:17:51", "remaining_time": "0:00:00", "throughput": 2512.25, "total_tokens": 2691984}
{"current_steps": 220, "total_steps": 220, "epoch": 1.0, "percentage": 100.0, "elapsed_time": "0:17:54", "remaining_time": "0:00:00", "throughput": 2504.7, "total_tokens": 2691984}

1491
trainer_log_history.json Normal file
View File

File diff suppressed because it is too large Load Diff

1535
trainer_state.json Normal file
View File

File diff suppressed because one or more lines are too long

3
training_args.bin Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:0191e759e04c2dae0eb6abca0935a54ed2dcd54ff24d48ddcbe50dceeb2725a2
size 5688

38
training_args.yaml Normal file
View File

@@ -0,0 +1,38 @@
bf16: true
cutoff_len: 2048
dataset: adpr_train
dataset_dir: data
ddp_timeout: 180000000
do_train: true
eval_steps: 100
eval_strategy: steps
finetuning_type: lora
flash_attn: auto
gradient_accumulation_steps: 8
include_num_input_tokens_seen: true
learning_rate: 5.0e-05
logging_steps: 5
lora_alpha: 128
lora_dropout: 0.01
lora_rank: 64
lora_target: q_proj,v_proj,k_proj,o_proj,gate_proj,down_proj,up_proj
lr_scheduler_type: cosine
max_grad_norm: 1.0
max_samples: 100000
model_name_or_path: GreatCaptainNemo/ProLLaMA
num_train_epochs: 3.0
optim: adamw_torch
output_dir: saves/Custom/lora/train_2025-04-05-23-57-03
packing: false
per_device_eval_batch_size: 16
per_device_train_batch_size: 16
plot_loss: true
preprocessing_num_workers: 16
report_to: none
resize_vocab: true
save_steps: 100
stage: sft
template: alpaca
trust_remote_code: true
val_size: 0.1
warmup_steps: 20

3
training_eval_loss.png Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:fc6aa75b6f165957ebf79c2338fdf1d1111365abedda188251d1734ed7d100df
size 38056

3
training_loss.png Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:199a848a35d9a85673b001b8b19a873cc247ea0b83e1974e5c0d241e34003f9c
size 46819