166 lines
8.1 KiB
Plaintext
166 lines
8.1 KiB
Plaintext
|
[2026-01-13 12:24:55,936] [DEBUG] [axolotl.utils.config.log_gpu_memory_usage:127] [PID:4815] baseline 0.000GB ()
|
||
|
|
[2026-01-13 12:24:55,937] [INFO] [axolotl.cli.config.load_cfg:259] [PID:4815] config:
|
||
|
|
{
|
||
|
|
"activation_offloading": false,
|
||
|
|
"adapter": "lora",
|
||
|
|
"axolotl_config_path": "zhihu_sft.yml",
|
||
|
|
"base_model": "Orion-zhen/Meissa-Qwen2.5-7B-Instruct",
|
||
|
|
"base_model_config": "Orion-zhen/Meissa-Qwen2.5-7B-Instruct",
|
||
|
|
"batch_size": 18,
|
||
|
|
"bf16": true,
|
||
|
|
"capabilities": {
|
||
|
|
"bf16": true,
|
||
|
|
"compute_capability": "sm_89",
|
||
|
|
"fp8": false,
|
||
|
|
"n_gpu": 1,
|
||
|
|
"n_node": 1
|
||
|
|
},
|
||
|
|
"context_parallel_size": 1,
|
||
|
|
"dataloader_num_workers": 1,
|
||
|
|
"dataloader_pin_memory": true,
|
||
|
|
"dataloader_prefetch_factor": 256,
|
||
|
|
"dataset_num_proc": 32,
|
||
|
|
"datasets": [
|
||
|
|
{
|
||
|
|
"chat_template": "tokenizer_default",
|
||
|
|
"message_property_mappings": {
|
||
|
|
"content": "content",
|
||
|
|
"role": "role"
|
||
|
|
},
|
||
|
|
"path": "chanceQZhang/zhihuhighvotes",
|
||
|
|
"split": "train",
|
||
|
|
"trust_remote_code": false,
|
||
|
|
"type": "chat_template"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"ddp": false,
|
||
|
|
"device": "cuda:0",
|
||
|
|
"device_map": "auto",
|
||
|
|
"dion_rank_fraction": 1.0,
|
||
|
|
"dion_rank_multiple_of": 1,
|
||
|
|
"env_capabilities": {
|
||
|
|
"torch_version": "2.8.0"
|
||
|
|
},
|
||
|
|
"eval_batch_size": 6,
|
||
|
|
"eval_causal_lm_metrics": [
|
||
|
|
"sacrebleu",
|
||
|
|
"comet",
|
||
|
|
"ter",
|
||
|
|
"chrf"
|
||
|
|
],
|
||
|
|
"eval_max_new_tokens": 128,
|
||
|
|
"eval_sample_packing": true,
|
||
|
|
"eval_steps": 100,
|
||
|
|
"eval_table_size": 0,
|
||
|
|
"experimental_skip_move_to_device": true,
|
||
|
|
"flash_attention": false,
|
||
|
|
"fp16": false,
|
||
|
|
"gradient_accumulation_steps": 3,
|
||
|
|
"gradient_checkpointing": true,
|
||
|
|
"gradient_checkpointing_kwargs": {
|
||
|
|
"use_reentrant": true
|
||
|
|
},
|
||
|
|
"include_tkps": true,
|
||
|
|
"is_falcon_derived_model": false,
|
||
|
|
"is_llama_derived_model": false,
|
||
|
|
"is_mistral_derived_model": false,
|
||
|
|
"learning_rate": 5e-05,
|
||
|
|
"lisa_layers_attribute": "model.layers",
|
||
|
|
"load_best_model_at_end": false,
|
||
|
|
"load_in_4bit": false,
|
||
|
|
"load_in_8bit": false,
|
||
|
|
"local_rank": 0,
|
||
|
|
"logging_steps": 10,
|
||
|
|
"lora_alpha": 32,
|
||
|
|
"lora_dropout": 0.1,
|
||
|
|
"lora_model_dir": "./outputs/zhihu-tech-career-lora",
|
||
|
|
"lora_r": 8,
|
||
|
|
"lora_target_modules": [
|
||
|
|
"q_proj",
|
||
|
|
"v_proj",
|
||
|
|
"k_proj",
|
||
|
|
"o_proj",
|
||
|
|
"gate_proj",
|
||
|
|
"up_proj",
|
||
|
|
"down_proj"
|
||
|
|
],
|
||
|
|
"loraplus_lr_embedding": 1e-06,
|
||
|
|
"lr_scheduler": "cosine",
|
||
|
|
"mean_resizing_embeddings": false,
|
||
|
|
"merge_lora": true,
|
||
|
|
"micro_batch_size": 6,
|
||
|
|
"model_config_type": "qwen2",
|
||
|
|
"num_epochs": 2.0,
|
||
|
|
"optimizer": "adamw_torch_fused",
|
||
|
|
"otel_metrics_host": "localhost",
|
||
|
|
"otel_metrics_port": 8000,
|
||
|
|
"output_dir": "./outputs/zhihu-tech-career-lora",
|
||
|
|
"pad_to_sequence_len": true,
|
||
|
|
"pretrain_multipack_attn": true,
|
||
|
|
"profiler_steps_start": 0,
|
||
|
|
"qlora_sharded_model_loading": false,
|
||
|
|
"ray_num_workers": 1,
|
||
|
|
"resources_per_worker": {
|
||
|
|
"GPU": 1
|
||
|
|
},
|
||
|
|
"sample_packing": true,
|
||
|
|
"sample_packing_bin_size": 200,
|
||
|
|
"sample_packing_group_size": 100000,
|
||
|
|
"save_only_model": false,
|
||
|
|
"save_safetensors": true,
|
||
|
|
"save_steps": 302,
|
||
|
|
"sequence_len": 2048,
|
||
|
|
"shuffle_before_merging_datasets": false,
|
||
|
|
"shuffle_merged_datasets": true,
|
||
|
|
"skip_prepare_dataset": false,
|
||
|
|
"streaming_multipack_buffer_size": 10000,
|
||
|
|
"strict": false,
|
||
|
|
"tensor_parallel_size": 1,
|
||
|
|
"tiled_mlp_use_original_mlp": true,
|
||
|
|
"tokenizer_config": "Orion-zhen/Meissa-Qwen2.5-7B-Instruct",
|
||
|
|
"tokenizer_save_jinja_files": true,
|
||
|
|
"tokenizer_type": "AutoTokenizer",
|
||
|
|
"torch_dtype": "torch.bfloat16",
|
||
|
|
"train_on_inputs": false,
|
||
|
|
"trl": {
|
||
|
|
"log_completions": false,
|
||
|
|
"mask_truncated_completions": false,
|
||
|
|
"ref_model_mixup_alpha": 0.9,
|
||
|
|
"ref_model_sync_steps": 64,
|
||
|
|
"scale_rewards": true,
|
||
|
|
"sync_ref_model": false,
|
||
|
|
"use_vllm": false,
|
||
|
|
"vllm_server_host": "0.0.0.0",
|
||
|
|
"vllm_server_port": 8000
|
||
|
|
},
|
||
|
|
"type_of_model": "AutoModelForCausalLM",
|
||
|
|
"use_otel_metrics": false,
|
||
|
|
"use_ray": false,
|
||
|
|
"val_set_size": 0.0,
|
||
|
|
"vllm": {
|
||
|
|
"device": "auto",
|
||
|
|
"dtype": "auto",
|
||
|
|
"gpu_memory_utilization": 0.9,
|
||
|
|
"host": "0.0.0.0",
|
||
|
|
"port": 8000
|
||
|
|
},
|
||
|
|
"weight_decay": 0.0,
|
||
|
|
"world_size": 1
|
||
|
|
}
|
||
|
|
[2026-01-13 12:24:55,938] [INFO] [axolotl.cli.utils.load.load_model_and_tokenizer:40] [PID:4815] loading tokenizer... Orion-zhen/Meissa-Qwen2.5-7B-Instruct
|
||
|
|
[2026-01-13 12:24:56,619] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:285] [PID:4815] EOS: 151645 / <|im_end|>
|
||
|
|
[2026-01-13 12:24:56,619] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:286] [PID:4815] BOS: None / None
|
||
|
|
[2026-01-13 12:24:56,619] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:287] [PID:4815] PAD: 151643 / <|endoftext|>
|
||
|
|
[2026-01-13 12:24:56,619] [DEBUG] [axolotl.loaders.tokenizer.load_tokenizer:288] [PID:4815] UNK: None / None
|
||
|
|
[2026-01-13 12:24:56,619] [INFO] [axolotl.cli.utils.load.load_model_and_tokenizer:43] [PID:4815] loading model...
|
||
|
|
[2026-01-13 12:24:56,739] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_evaluation_loop:87] [PID:4815] Patched Trainer.evaluation_loop with nanmean loss calculation
|
||
|
|
[2026-01-13 12:24:56,740] [DEBUG] [axolotl.monkeypatch.transformers.trainer_loss_calc.patch_maybe_log_save_evaluate:138] [PID:4815] Patched Trainer._maybe_log_save_evaluate with nanmean loss calculation
|
||
|
|
[2026-01-13 12:24:56,741] [INFO] [axolotl.loaders.patch_manager._apply_multipack_patches:344] [PID:4815] Applying multipack dataloader patch for sample packing...
|
||
|
|
Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]
Loading checkpoint shards: 25%|███████████████████████▎ | 1/4 [00:00<00:02, 1.21it/s]
Loading checkpoint shards: 50%|██████████████████████████████████████████████▌ | 2/4 [00:01<00:01, 1.21it/s]
Loading checkpoint shards: 75%|█████████████████████████████████████████████████████████████████████▊ | 3/4 [00:02<00:00, 1.29it/s]
Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [00:02<00:00, 1.85it/s]
Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [00:02<00:00, 1.57it/s]
|
||
|
|
[2026-01-13 12:24:59,773] [INFO] [axolotl.loaders.model._configure_embedding_dtypes:347] [PID:4815] Converting modules to torch.bfloat16
|
||
|
|
[2026-01-13 12:24:59,775] [DEBUG] [axolotl.loaders.model.log_gpu_memory_usage:127] [PID:4815] Memory usage after model load 17.234GB (+17.234GB allocated, +18.250GB reserved)
|
||
|
|
[2026-01-13 12:24:59,776] [DEBUG] [axolotl.loaders.adapter.load_lora:150] [PID:4815] Loading pretrained PEFT - LoRA
|
||
|
|
trainable params: 20,185,088 || all params: 7,635,801,600 || trainable%: 0.2643
|
||
|
|
[2026-01-13 12:25:00,131] [DEBUG] [axolotl.loaders.model.log_gpu_memory_usage:127] [PID:4815] after adapters 14.337GB (+14.337GB allocated, +18.402GB reserved)
|
||
|
|
Unloading and merging model: 0%| | 0/567 [00:00<?, ?it/s]
Unloading and merging model: 42%|███████████████████████████████████▋ | 238/567 [00:00<00:00, 2377.54it/s]
Unloading and merging model: 100%|█████████████████████████████████████████████████████████████████████████████████████| 567/567 [00:00<00:00, 4481.07it/s]
|