初始化项目，由ModelHub XC社区提供模型
Model: oumi-ai/smollm2-17b-distill-r1-670b-math Source: Original Platform
This commit is contained in:
ModelHub XC
47
.gitattributes
vendored
Normal file
47
.gitattributes
vendored
Normal file
@@ -0,0 +1,47 @@
|
||||
*.7z filter=lfs diff=lfs merge=lfs -text
|
||||
*.arrow filter=lfs diff=lfs merge=lfs -text
|
||||
*.bin filter=lfs diff=lfs merge=lfs -text
|
||||
*.bin.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
||||
*.ftz filter=lfs diff=lfs merge=lfs -text
|
||||
*.gz filter=lfs diff=lfs merge=lfs -text
|
||||
*.h5 filter=lfs diff=lfs merge=lfs -text
|
||||
*.joblib filter=lfs diff=lfs merge=lfs -text
|
||||
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.model filter=lfs diff=lfs merge=lfs -text
|
||||
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
||||
*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||
*.ot filter=lfs diff=lfs merge=lfs -text
|
||||
*.parquet filter=lfs diff=lfs merge=lfs -text
|
||||
*.pb filter=lfs diff=lfs merge=lfs -text
|
||||
*.pt filter=lfs diff=lfs merge=lfs -text
|
||||
*.pth filter=lfs diff=lfs merge=lfs -text
|
||||
*.rar filter=lfs diff=lfs merge=lfs -text
|
||||
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tflite filter=lfs diff=lfs merge=lfs -text
|
||||
*.tgz filter=lfs diff=lfs merge=lfs -text
|
||||
*.xz filter=lfs diff=lfs merge=lfs -text
|
||||
*.zip filter=lfs diff=lfs merge=lfs -text
|
||||
*.zstandard filter=lfs diff=lfs merge=lfs -text
|
||||
*.tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||
*.db* filter=lfs diff=lfs merge=lfs -text
|
||||
*.ark* filter=lfs diff=lfs merge=lfs -text
|
||||
**/*ckpt*data* filter=lfs diff=lfs merge=lfs -text
|
||||
**/*ckpt*.meta filter=lfs diff=lfs merge=lfs -text
|
||||
**/*ckpt*.index filter=lfs diff=lfs merge=lfs -text
|
||||
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
||||
*.gguf* filter=lfs diff=lfs merge=lfs -text
|
||||
*.ggml filter=lfs diff=lfs merge=lfs -text
|
||||
*.llamafile* filter=lfs diff=lfs merge=lfs -text
|
||||
*.pt2 filter=lfs diff=lfs merge=lfs -text
|
||||
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
||||
*.npy filter=lfs diff=lfs merge=lfs -text
|
||||
*.npz filter=lfs diff=lfs merge=lfs -text
|
||||
*.pickle filter=lfs diff=lfs merge=lfs -text
|
||||
*.pkl filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar filter=lfs diff=lfs merge=lfs -text
|
||||
*.wasm filter=lfs diff=lfs merge=lfs -text
|
||||
*.zst filter=lfs diff=lfs merge=lfs -text
|
||||
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||
37
config.json
Normal file
37
config.json
Normal file
@@ -0,0 +1,37 @@
|
||||
{
|
||||
"_name_or_path": "HuggingFaceTB/SmolLM2-1.7B-Instruct",
|
||||
"architectures": [
|
||||
"LlamaForCausalLM"
|
||||
],
|
||||
"attention_bias": false,
|
||||
"attention_dropout": 0.0,
|
||||
"bos_token_id": 1,
|
||||
"eos_token_id": 2,
|
||||
"head_dim": 64,
|
||||
"hidden_act": "silu",
|
||||
"hidden_size": 2048,
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 8192,
|
||||
"max_position_embeddings": 8192,
|
||||
"mlp_bias": false,
|
||||
"model_type": "llama",
|
||||
"num_attention_heads": 32,
|
||||
"num_hidden_layers": 24,
|
||||
"num_key_value_heads": 32,
|
||||
"pad_token_id": 2,
|
||||
"pretraining_tp": 1,
|
||||
"rms_norm_eps": 1e-05,
|
||||
"rope_scaling": null,
|
||||
"rope_theta": 130000,
|
||||
"tie_word_embeddings": true,
|
||||
"torch_dtype": "bfloat16",
|
||||
"transformers.js_config": {
|
||||
"kv_cache_dtype": {
|
||||
"fp16": "float16",
|
||||
"q4f16": "float16"
|
||||
}
|
||||
},
|
||||
"transformers_version": "4.45.2",
|
||||
"use_cache": false,
|
||||
"vocab_size": 49152
|
||||
}
|
||||
1
configuration.json
Normal file
1
configuration.json
Normal file
@@ -0,0 +1 @@
|
||||
{"framework": "pytorch", "task": "others", "allow_remote": true}
|
||||
7
generation_config.json
Normal file
7
generation_config.json
Normal file
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"_from_model_config": true,
|
||||
"bos_token_id": 1,
|
||||
"eos_token_id": 2,
|
||||
"pad_token_id": 2,
|
||||
"transformers_version": "4.45.2"
|
||||
}
|
||||
322
logs/rank_0000.log
Normal file
322
logs/rank_0000.log
Normal file
@@ -0,0 +1,322 @@
|
||||
[2025-02-01 18:47:39,616][oumi][rank0][pid:11750][MainThread][INFO]][train.py:144] Resolved 'training.dataloader_num_workers=auto' to 'training.dataloader_num_workers=8'
|
||||
[2025-02-01 18:47:39,618][oumi][rank0][pid:11750][MainThread][INFO]][train.py:174] TrainingConfig:
|
||||
TrainingConfig(data=DataParams(train=DatasetSplitParams(datasets=[DatasetParams(dataset_name='text_sft_jsonl',
|
||||
dataset_path='data/R1/math_10k_R1_outputs.jsonl',
|
||||
subset=None,
|
||||
split='train',
|
||||
dataset_kwargs={},
|
||||
sample_count=None,
|
||||
mixture_proportion=None,
|
||||
shuffle=False,
|
||||
seed=None,
|
||||
shuffle_buffer_size=1000,
|
||||
trust_remote_code=False,
|
||||
transform_num_workers=None)],
|
||||
collator_name=None,
|
||||
pack=False,
|
||||
stream=False,
|
||||
target_col=None,
|
||||
mixture_strategy='first_exhausted',
|
||||
seed=42,
|
||||
use_async_dataset=False,
|
||||
use_torchdata=None),
|
||||
test=DatasetSplitParams(datasets=[],
|
||||
collator_name=None,
|
||||
pack=False,
|
||||
stream=False,
|
||||
target_col=None,
|
||||
mixture_strategy='first_exhausted',
|
||||
seed=None,
|
||||
use_async_dataset=False,
|
||||
use_torchdata=None),
|
||||
validation=DatasetSplitParams(datasets=[],
|
||||
collator_name=None,
|
||||
pack=False,
|
||||
stream=False,
|
||||
target_col=None,
|
||||
mixture_strategy='first_exhausted',
|
||||
seed=None,
|
||||
use_async_dataset=False,
|
||||
use_torchdata=None)),
|
||||
model=ModelParams(model_name='HuggingFaceTB/SmolLM2-1.7B-Instruct',
|
||||
adapter_model=None,
|
||||
tokenizer_name=None,
|
||||
tokenizer_pad_token=None,
|
||||
tokenizer_kwargs={},
|
||||
model_max_length=None,
|
||||
load_pretrained_weights=True,
|
||||
trust_remote_code=True,
|
||||
torch_dtype_str='bfloat16',
|
||||
compile=False,
|
||||
chat_template=None,
|
||||
attn_implementation=None,
|
||||
device_map='auto',
|
||||
model_kwargs={},
|
||||
enable_liger_kernel=False,
|
||||
shard_for_eval=False,
|
||||
freeze_layers=[]),
|
||||
training=TrainingParams(use_peft=False,
|
||||
trainer_type=<TrainerType.TRL_SFT: 'trl_sft'>,
|
||||
enable_gradient_checkpointing=True,
|
||||
gradient_checkpointing_kwargs={'use_reentrant': False},
|
||||
output_dir='output/smollm2-17b-distill-r1-670b-math',
|
||||
per_device_train_batch_size=2,
|
||||
per_device_eval_batch_size=8,
|
||||
gradient_accumulation_steps=2,
|
||||
max_steps=-1,
|
||||
num_train_epochs=1,
|
||||
save_epoch=False,
|
||||
save_steps=0,
|
||||
save_final_model=True,
|
||||
seed=42,
|
||||
run_name='smollm2-17b-distill-r1-670b-math.sky-2025-02-01-13-42-43-696171_sky-d954-bf996_1',
|
||||
metrics_function=None,
|
||||
log_level='info',
|
||||
dep_log_level='warning',
|
||||
enable_wandb=True,
|
||||
enable_tensorboard=True,
|
||||
logging_strategy='steps',
|
||||
logging_dir=None,
|
||||
logging_steps=10,
|
||||
logging_first_step=False,
|
||||
eval_strategy='no',
|
||||
eval_steps=500,
|
||||
learning_rate=2e-05,
|
||||
lr_scheduler_type='linear',
|
||||
lr_scheduler_kwargs={},
|
||||
warmup_ratio=0.1,
|
||||
warmup_steps=None,
|
||||
optimizer='adamw_torch_fused',
|
||||
weight_decay=0.0,
|
||||
adam_beta1=0.9,
|
||||
adam_beta2=0.999,
|
||||
adam_epsilon=1e-08,
|
||||
sgd_momentum=0.0,
|
||||
mixed_precision_dtype=<MixedPrecisionDtype.NONE: 'none'>,
|
||||
compile=False,
|
||||
include_performance_metrics=False,
|
||||
include_alternative_mfu_metrics=False,
|
||||
log_model_summary=False,
|
||||
resume_from_checkpoint=None,
|
||||
try_resume_from_last_checkpoint=False,
|
||||
dataloader_num_workers=8,
|
||||
dataloader_prefetch_factor=32,
|
||||
dataloader_main_process_only=None,
|
||||
ddp_find_unused_parameters=False,
|
||||
max_grad_norm=10.0,
|
||||
trainer_kwargs={},
|
||||
profiler=ProfilerParams(save_dir=None,
|
||||
enable_cpu_profiling=False,
|
||||
enable_cuda_profiling=False,
|
||||
record_shapes=False,
|
||||
profile_memory=False,
|
||||
with_stack=False,
|
||||
with_flops=False,
|
||||
with_modules=False,
|
||||
row_limit=50,
|
||||
schedule=ProfilerScheduleParams(enable_schedule=False,
|
||||
wait=0,
|
||||
warmup=1,
|
||||
active=3,
|
||||
repeat=1,
|
||||
skip_first=1)),
|
||||
telemetry=TelemetryParams(telemetry_dir='telemetry',
|
||||
collect_telemetry_for_all_ranks=False,
|
||||
track_gpu_temperature=False),
|
||||
empty_device_cache_steps=1,
|
||||
nccl_default_timeout_minutes=None),
|
||||
peft=PeftParams(lora_r=8,
|
||||
lora_alpha=8,
|
||||
lora_dropout=0.0,
|
||||
lora_target_modules=None,
|
||||
lora_modules_to_save=None,
|
||||
lora_bias='none',
|
||||
lora_init_weights=<LoraWeightInitialization.DEFAULT: 'default'>,
|
||||
lora_task_type=<TaskType.CAUSAL_LM: 'CAUSAL_LM'>,
|
||||
q_lora=False,
|
||||
q_lora_bits=4,
|
||||
bnb_4bit_quant_type='fp4',
|
||||
use_bnb_nested_quant=False,
|
||||
bnb_4bit_quant_storage='uint8',
|
||||
bnb_4bit_compute_dtype='float32',
|
||||
peft_save_mode=<PeftSaveMode.ADAPTER_ONLY: 'adapter_only'>),
|
||||
fsdp=FSDPParams(enable_fsdp=False,
|
||||
sharding_strategy=<ShardingStrategy.FULL_SHARD: 'FULL_SHARD'>,
|
||||
cpu_offload=False,
|
||||
mixed_precision=None,
|
||||
backward_prefetch=<BackwardPrefetch.BACKWARD_PRE: 'BACKWARD_PRE'>,
|
||||
forward_prefetch=False,
|
||||
use_orig_params=None,
|
||||
state_dict_type=<StateDictType.FULL_STATE_DICT: 'FULL_STATE_DICT'>,
|
||||
auto_wrap_policy=<AutoWrapPolicy.NO_WRAP: 'NO_WRAP'>,
|
||||
min_num_params=100000,
|
||||
transformer_layer_cls=None,
|
||||
sync_module_states=True))
|
||||
[2025-02-01 18:47:39,903][oumi][rank0][pid:11750][MainThread][INFO]][models.py:180] Building model for distributed training (world_size: 4)...
|
||||
[2025-02-01 18:47:39,903][oumi][rank0][pid:11750][MainThread][INFO]][models.py:185] Building model using device_map: cuda:0 (DeviceRankInfo(world_size=4, rank=0, local_world_size=4, local_rank=0))...
|
||||
[2025-02-01 18:47:39,903][oumi][rank0][pid:11750][MainThread][INFO]][models.py:255] Using model class: <class 'transformers.models.auto.modeling_auto.AutoModelForCausalLM'> to instantiate model.
|
||||
[2025-02-01 18:47:41,904][oumi][rank0][pid:11750][MainThread][INFO]][base_map_dataset.py:68] Creating map dataset (type: TextSftJsonLinesDataset) dataset_name: 'text_sft_jsonl', dataset_path: 'None'...
|
||||
[2025-02-01 18:47:41,946][oumi][rank0][pid:11750][MainThread][INFO]][base_map_dataset.py:297] TextSftJsonLinesDataset: features=dict_keys(['input_ids', 'attention_mask'])
|
||||
[2025-02-01 18:47:47,678][oumi][rank0][pid:11750][MainThread][INFO]][base_map_dataset.py:361] Finished transforming dataset (TextSftJsonLinesDataset)! Speed: 1744.52 examples/sec. Examples: 10000. Duration: 5.7 sec. Transform workers: 1.
|
||||
[2025-02-01 18:47:47,943][oumi][rank0][pid:11750][MainThread][INFO]][torch_profiler_utils.py:150] PROF: Torch Profiler disabled!
|
||||
[2025-02-01 18:47:47,998][oumi][rank0][pid:11750][MainThread][INFO]][training.py:49] SFTConfig(output_dir='output/smollm2-17b-distill-r1-670b-math',
|
||||
overwrite_output_dir=False,
|
||||
do_train=False,
|
||||
do_eval=False,
|
||||
do_predict=False,
|
||||
eval_strategy=<IntervalStrategy.NO: 'no'>,
|
||||
prediction_loss_only=False,
|
||||
per_device_train_batch_size=2,
|
||||
per_device_eval_batch_size=8,
|
||||
per_gpu_train_batch_size=None,
|
||||
per_gpu_eval_batch_size=None,
|
||||
gradient_accumulation_steps=2,
|
||||
eval_accumulation_steps=None,
|
||||
eval_delay=0,
|
||||
torch_empty_cache_steps=1,
|
||||
learning_rate=2e-05,
|
||||
weight_decay=0.0,
|
||||
adam_beta1=0.9,
|
||||
adam_beta2=0.999,
|
||||
adam_epsilon=1e-08,
|
||||
max_grad_norm=10.0,
|
||||
num_train_epochs=1,
|
||||
max_steps=-1,
|
||||
lr_scheduler_type=<SchedulerType.LINEAR: 'linear'>,
|
||||
lr_scheduler_kwargs={},
|
||||
warmup_ratio=0.1,
|
||||
warmup_steps=0,
|
||||
log_level='warning',
|
||||
log_level_replica='warning',
|
||||
log_on_each_node=True,
|
||||
logging_dir='output/smollm2-17b-distill-r1-670b-math/runs/Feb01_18-47-47_sky-d954-bf996-370b-head-3sxnf23v-compute',
|
||||
logging_strategy=<IntervalStrategy.STEPS: 'steps'>,
|
||||
logging_first_step=False,
|
||||
logging_steps=10,
|
||||
logging_nan_inf_filter=True,
|
||||
save_strategy=<IntervalStrategy.NO: 'no'>,
|
||||
save_steps=0,
|
||||
save_total_limit=None,
|
||||
save_safetensors=True,
|
||||
save_on_each_node=False,
|
||||
save_only_model=False,
|
||||
restore_callback_states_from_checkpoint=False,
|
||||
no_cuda=False,
|
||||
use_cpu=False,
|
||||
use_mps_device=False,
|
||||
seed=42,
|
||||
data_seed=None,
|
||||
jit_mode_eval=False,
|
||||
use_ipex=False,
|
||||
bf16=False,
|
||||
fp16=False,
|
||||
fp16_opt_level='O1',
|
||||
half_precision_backend='auto',
|
||||
bf16_full_eval=False,
|
||||
fp16_full_eval=False,
|
||||
tf32=None,
|
||||
local_rank=0,
|
||||
ddp_backend=None,
|
||||
tpu_num_cores=None,
|
||||
tpu_metrics_debug=False,
|
||||
debug=[],
|
||||
dataloader_drop_last=False,
|
||||
eval_steps=500,
|
||||
dataloader_num_workers=8,
|
||||
dataloader_prefetch_factor=32,
|
||||
past_index=-1,
|
||||
run_name='smollm2-17b-distill-r1-670b-math.sky-2025-02-01-13-42-43-696171_sky-d954-bf996_1',
|
||||
disable_tqdm=False,
|
||||
remove_unused_columns=True,
|
||||
label_names=None,
|
||||
load_best_model_at_end=False,
|
||||
metric_for_best_model=None,
|
||||
greater_is_better=None,
|
||||
ignore_data_skip=False,
|
||||
fsdp=[],
|
||||
fsdp_min_num_params=0,
|
||||
fsdp_config={'min_num_params': 0,
|
||||
'xla': False,
|
||||
'xla_fsdp_grad_ckpt': False,
|
||||
'xla_fsdp_v2': False},
|
||||
fsdp_transformer_layer_cls_to_wrap=None,
|
||||
accelerator_config=AcceleratorConfig(split_batches=False,
|
||||
dispatch_batches=None,
|
||||
even_batches=True,
|
||||
use_seedable_sampler=True,
|
||||
non_blocking=False,
|
||||
gradient_accumulation_kwargs=None,
|
||||
use_configured_state=False),
|
||||
deepspeed=None,
|
||||
label_smoothing_factor=0.0,
|
||||
optim=<OptimizerNames.ADAMW_TORCH_FUSED: 'adamw_torch_fused'>,
|
||||
optim_args=None,
|
||||
adafactor=False,
|
||||
group_by_length=False,
|
||||
length_column_name='length',
|
||||
report_to=['wandb', 'tensorboard'],
|
||||
ddp_find_unused_parameters=False,
|
||||
ddp_bucket_cap_mb=None,
|
||||
ddp_broadcast_buffers=None,
|
||||
dataloader_pin_memory=True,
|
||||
dataloader_persistent_workers=False,
|
||||
skip_memory_metrics=True,
|
||||
use_legacy_prediction_loop=False,
|
||||
push_to_hub=False,
|
||||
resume_from_checkpoint=None,
|
||||
hub_model_id=None,
|
||||
hub_strategy=<HubStrategy.EVERY_SAVE: 'every_save'>,
|
||||
hub_token=None,
|
||||
hub_private_repo=False,
|
||||
hub_always_push=False,
|
||||
gradient_checkpointing=True,
|
||||
gradient_checkpointing_kwargs={'use_reentrant': False},
|
||||
include_inputs_for_metrics=False,
|
||||
eval_do_concat_batches=True,
|
||||
fp16_backend='auto',
|
||||
evaluation_strategy=None,
|
||||
push_to_hub_model_id=None,
|
||||
push_to_hub_organization=None,
|
||||
push_to_hub_token=None,
|
||||
mp_parameters='',
|
||||
auto_find_batch_size=False,
|
||||
full_determinism=False,
|
||||
torchdynamo=None,
|
||||
ray_scope='last',
|
||||
ddp_timeout=1800,
|
||||
torch_compile=False,
|
||||
torch_compile_backend=None,
|
||||
torch_compile_mode=None,
|
||||
dispatch_batches=None,
|
||||
split_batches=None,
|
||||
include_tokens_per_second=False,
|
||||
include_num_input_tokens_seen=False,
|
||||
neftune_noise_alpha=None,
|
||||
optim_target_modules=None,
|
||||
batch_eval_metrics=False,
|
||||
eval_on_start=False,
|
||||
use_liger_kernel=False,
|
||||
eval_use_gather_object=False,
|
||||
dataset_text_field=None,
|
||||
packing=False,
|
||||
max_seq_length=None,
|
||||
dataset_num_proc=None,
|
||||
dataset_batch_size=1000,
|
||||
model_init_kwargs=None,
|
||||
dataset_kwargs=None,
|
||||
eval_packing=None,
|
||||
num_of_sequences=1024,
|
||||
chars_per_token=3.6,
|
||||
use_liger=False)
|
||||
[2025-02-01 18:47:48,072][oumi][rank0][pid:11750][MainThread][INFO]][device_utils.py:283] GPU Metrics Before Training: GPU runtime info: NVidiaGpuRuntimeInfo(device_index=0, device_count=4, used_memory_mb=7019.0, temperature=33, fan_speed=None, fan_speeds=None, power_usage_watts=70.637, power_limit_watts=400.0, gpu_utilization=0, memory_utilization=0, performance_state=0, clock_speed_graphics=1155, clock_speed_sm=1155, clock_speed_memory=1593).
|
||||
[2025-02-01 18:47:48,078][oumi][rank0][pid:11750][MainThread][INFO]][train.py:312] Training init time: 10.796s
|
||||
[2025-02-01 18:47:48,078][oumi][rank0][pid:11750][MainThread][INFO]][train.py:313] Starting training... (TrainerType.TRL_SFT, transformers: 4.45.2)
|
||||
[2025-02-01 18:52:35,471][oumi][rank0][pid:11750][MainThread][INFO]][train.py:320] Training is Complete.
|
||||
[2025-02-01 18:52:35,501][oumi][rank0][pid:11750][MainThread][INFO]][device_utils.py:283] GPU Metrics After Training: GPU runtime info: NVidiaGpuRuntimeInfo(device_index=0, device_count=4, used_memory_mb=21283.0, temperature=43, fan_speed=None, fan_speeds=None, power_usage_watts=181.852, power_limit_watts=400.0, gpu_utilization=54, memory_utilization=14, performance_state=0, clock_speed_graphics=1410, clock_speed_sm=1410, clock_speed_memory=1593).
|
||||
[2025-02-01 18:52:35,501][oumi][rank0][pid:11750][MainThread][INFO]][torch_utils.py:117] Peak GPU memory usage: 17.43 GB
|
||||
[2025-02-01 18:52:35,501][oumi][rank0][pid:11750][MainThread][INFO]][train.py:327] Saving final state...
|
||||
[2025-02-01 18:52:35,504][oumi][rank0][pid:11750][MainThread][INFO]][train.py:332] Saving final model...
|
||||
[2025-02-01 18:52:43,074][oumi][rank0][pid:11750][MainThread][INFO]][hf_trainer.py:102] Model has been saved at output/smollm2-17b-distill-r1-670b-math
|
||||
[2025-02-01 18:52:43,650][oumi][rank0][pid:11750][MainThread][INFO]][train.py:339]
|
||||
|
||||
» We're always looking for feedback. What's one thing we can improve? https://oumi.ai/feedback
|
||||
19
logs/rank_0001.log
Normal file
19
logs/rank_0001.log
Normal file
@@ -0,0 +1,19 @@
|
||||
[2025-02-01 18:47:39,024][oumi][rank1][pid:11751][MainThread][INFO]][train.py:144] Resolved 'training.dataloader_num_workers=auto' to 'training.dataloader_num_workers=8'
|
||||
[2025-02-01 18:47:39,328][oumi][rank1][pid:11751][MainThread][INFO]][models.py:180] Building model for distributed training (world_size: 4)...
|
||||
[2025-02-01 18:47:39,328][oumi][rank1][pid:11751][MainThread][INFO]][models.py:185] Building model using device_map: cuda:1 (DeviceRankInfo(world_size=4, rank=1, local_world_size=4, local_rank=1))...
|
||||
[2025-02-01 18:47:39,328][oumi][rank1][pid:11751][MainThread][INFO]][models.py:255] Using model class: <class 'transformers.models.auto.modeling_auto.AutoModelForCausalLM'> to instantiate model.
|
||||
[2025-02-01 18:47:41,530][oumi][rank1][pid:11751][MainThread][INFO]][base_map_dataset.py:68] Creating map dataset (type: TextSftJsonLinesDataset) dataset_name: 'text_sft_jsonl', dataset_path: 'None'...
|
||||
[2025-02-01 18:47:41,663][oumi][rank1][pid:11751][MainThread][INFO]][base_map_dataset.py:297] TextSftJsonLinesDataset: features=dict_keys(['input_ids', 'attention_mask'])
|
||||
[2025-02-01 18:47:47,716][oumi][rank1][pid:11751][MainThread][INFO]][base_map_dataset.py:361] Finished transforming dataset (TextSftJsonLinesDataset)! Speed: 1652.20 examples/sec. Examples: 10000. Duration: 6.1 sec. Transform workers: 1.
|
||||
[2025-02-01 18:47:47,984][oumi][rank1][pid:11751][MainThread][INFO]][torch_profiler_utils.py:150] PROF: Torch Profiler disabled!
|
||||
[2025-02-01 18:47:48,077][oumi][rank1][pid:11751][MainThread][INFO]][device_utils.py:283] GPU Metrics Before Training: GPU runtime info: NVidiaGpuRuntimeInfo(device_index=0, device_count=4, used_memory_mb=7019.0, temperature=33, fan_speed=None, fan_speeds=None, power_usage_watts=70.637, power_limit_watts=400.0, gpu_utilization=0, memory_utilization=0, performance_state=0, clock_speed_graphics=1155, clock_speed_sm=1155, clock_speed_memory=1593).
|
||||
[2025-02-01 18:47:48,078][oumi][rank1][pid:11751][MainThread][INFO]][train.py:312] Training init time: 10.795s
|
||||
[2025-02-01 18:47:48,078][oumi][rank1][pid:11751][MainThread][INFO]][train.py:313] Starting training... (TrainerType.TRL_SFT, transformers: 4.45.2)
|
||||
[2025-02-01 18:52:35,469][oumi][rank1][pid:11751][MainThread][INFO]][train.py:320] Training is Complete.
|
||||
[2025-02-01 18:52:35,498][oumi][rank1][pid:11751][MainThread][INFO]][device_utils.py:283] GPU Metrics After Training: GPU runtime info: NVidiaGpuRuntimeInfo(device_index=0, device_count=4, used_memory_mb=21283.0, temperature=43, fan_speed=None, fan_speeds=None, power_usage_watts=181.852, power_limit_watts=400.0, gpu_utilization=54, memory_utilization=14, performance_state=0, clock_speed_graphics=1410, clock_speed_sm=1410, clock_speed_memory=1593).
|
||||
[2025-02-01 18:52:35,498][oumi][rank1][pid:11751][MainThread][INFO]][torch_utils.py:117] Peak GPU memory usage: 17.24 GB
|
||||
[2025-02-01 18:52:35,498][oumi][rank1][pid:11751][MainThread][INFO]][train.py:327] Saving final state...
|
||||
[2025-02-01 18:52:35,504][oumi][rank1][pid:11751][MainThread][INFO]][train.py:332] Saving final model...
|
||||
[2025-02-01 18:52:43,653][oumi][rank1][pid:11751][MainThread][INFO]][train.py:339]
|
||||
|
||||
» We're always looking for feedback. What's one thing we can improve? https://oumi.ai/feedback
|
||||
19
logs/rank_0002.log
Normal file
19
logs/rank_0002.log
Normal file
@@ -0,0 +1,19 @@
|
||||
[2025-02-01 18:47:39,024][oumi][rank2][pid:11752][MainThread][INFO]][train.py:144] Resolved 'training.dataloader_num_workers=auto' to 'training.dataloader_num_workers=8'
|
||||
[2025-02-01 18:47:39,330][oumi][rank2][pid:11752][MainThread][INFO]][models.py:180] Building model for distributed training (world_size: 4)...
|
||||
[2025-02-01 18:47:39,330][oumi][rank2][pid:11752][MainThread][INFO]][models.py:185] Building model using device_map: cuda:2 (DeviceRankInfo(world_size=4, rank=2, local_world_size=4, local_rank=2))...
|
||||
[2025-02-01 18:47:39,330][oumi][rank2][pid:11752][MainThread][INFO]][models.py:255] Using model class: <class 'transformers.models.auto.modeling_auto.AutoModelForCausalLM'> to instantiate model.
|
||||
[2025-02-01 18:47:41,533][oumi][rank2][pid:11752][MainThread][INFO]][base_map_dataset.py:68] Creating map dataset (type: TextSftJsonLinesDataset) dataset_name: 'text_sft_jsonl', dataset_path: 'None'...
|
||||
[2025-02-01 18:47:41,616][oumi][rank2][pid:11752][MainThread][INFO]][base_map_dataset.py:297] TextSftJsonLinesDataset: features=dict_keys(['input_ids', 'attention_mask'])
|
||||
[2025-02-01 18:47:47,667][oumi][rank2][pid:11752][MainThread][INFO]][base_map_dataset.py:361] Finished transforming dataset (TextSftJsonLinesDataset)! Speed: 1652.64 examples/sec. Examples: 10000. Duration: 6.1 sec. Transform workers: 1.
|
||||
[2025-02-01 18:47:47,937][oumi][rank2][pid:11752][MainThread][INFO]][torch_profiler_utils.py:150] PROF: Torch Profiler disabled!
|
||||
[2025-02-01 18:47:48,077][oumi][rank2][pid:11752][MainThread][INFO]][device_utils.py:283] GPU Metrics Before Training: GPU runtime info: NVidiaGpuRuntimeInfo(device_index=0, device_count=4, used_memory_mb=7019.0, temperature=33, fan_speed=None, fan_speeds=None, power_usage_watts=70.637, power_limit_watts=400.0, gpu_utilization=0, memory_utilization=0, performance_state=0, clock_speed_graphics=1155, clock_speed_sm=1155, clock_speed_memory=1593).
|
||||
[2025-02-01 18:47:48,078][oumi][rank2][pid:11752][MainThread][INFO]][train.py:312] Training init time: 10.795s
|
||||
[2025-02-01 18:47:48,078][oumi][rank2][pid:11752][MainThread][INFO]][train.py:313] Starting training... (TrainerType.TRL_SFT, transformers: 4.45.2)
|
||||
[2025-02-01 18:52:35,469][oumi][rank2][pid:11752][MainThread][INFO]][train.py:320] Training is Complete.
|
||||
[2025-02-01 18:52:35,496][oumi][rank2][pid:11752][MainThread][INFO]][device_utils.py:283] GPU Metrics After Training: GPU runtime info: NVidiaGpuRuntimeInfo(device_index=0, device_count=4, used_memory_mb=21283.0, temperature=43, fan_speed=None, fan_speeds=None, power_usage_watts=181.852, power_limit_watts=400.0, gpu_utilization=54, memory_utilization=14, performance_state=0, clock_speed_graphics=1410, clock_speed_sm=1410, clock_speed_memory=1593).
|
||||
[2025-02-01 18:52:35,497][oumi][rank2][pid:11752][MainThread][INFO]][torch_utils.py:117] Peak GPU memory usage: 17.46 GB
|
||||
[2025-02-01 18:52:35,497][oumi][rank2][pid:11752][MainThread][INFO]][train.py:327] Saving final state...
|
||||
[2025-02-01 18:52:35,504][oumi][rank2][pid:11752][MainThread][INFO]][train.py:332] Saving final model...
|
||||
[2025-02-01 18:52:43,655][oumi][rank2][pid:11752][MainThread][INFO]][train.py:339]
|
||||
|
||||
» We're always looking for feedback. What's one thing we can improve? https://oumi.ai/feedback
|
||||
19
logs/rank_0003.log
Normal file
19
logs/rank_0003.log
Normal file
@@ -0,0 +1,19 @@
|
||||
[2025-02-01 18:47:39,024][oumi][rank3][pid:11753][MainThread][INFO]][train.py:144] Resolved 'training.dataloader_num_workers=auto' to 'training.dataloader_num_workers=8'
|
||||
[2025-02-01 18:47:39,326][oumi][rank3][pid:11753][MainThread][INFO]][models.py:180] Building model for distributed training (world_size: 4)...
|
||||
[2025-02-01 18:47:39,326][oumi][rank3][pid:11753][MainThread][INFO]][models.py:185] Building model using device_map: cuda:3 (DeviceRankInfo(world_size=4, rank=3, local_world_size=4, local_rank=3))...
|
||||
[2025-02-01 18:47:39,327][oumi][rank3][pid:11753][MainThread][INFO]][models.py:255] Using model class: <class 'transformers.models.auto.modeling_auto.AutoModelForCausalLM'> to instantiate model.
|
||||
[2025-02-01 18:47:41,529][oumi][rank3][pid:11753][MainThread][INFO]][base_map_dataset.py:68] Creating map dataset (type: TextSftJsonLinesDataset) dataset_name: 'text_sft_jsonl', dataset_path: 'None'...
|
||||
[2025-02-01 18:47:41,714][oumi][rank3][pid:11753][MainThread][INFO]][base_map_dataset.py:297] TextSftJsonLinesDataset: features=dict_keys(['input_ids', 'attention_mask'])
|
||||
[2025-02-01 18:47:47,694][oumi][rank3][pid:11753][MainThread][INFO]][base_map_dataset.py:361] Finished transforming dataset (TextSftJsonLinesDataset)! Speed: 1672.23 examples/sec. Examples: 10000. Duration: 6.0 sec. Transform workers: 1.
|
||||
[2025-02-01 18:47:47,964][oumi][rank3][pid:11753][MainThread][INFO]][torch_profiler_utils.py:150] PROF: Torch Profiler disabled!
|
||||
[2025-02-01 18:47:48,076][oumi][rank3][pid:11753][MainThread][INFO]][device_utils.py:283] GPU Metrics Before Training: GPU runtime info: NVidiaGpuRuntimeInfo(device_index=0, device_count=4, used_memory_mb=7019.0, temperature=33, fan_speed=None, fan_speeds=None, power_usage_watts=70.637, power_limit_watts=400.0, gpu_utilization=0, memory_utilization=0, performance_state=0, clock_speed_graphics=1155, clock_speed_sm=1155, clock_speed_memory=1593).
|
||||
[2025-02-01 18:47:48,078][oumi][rank3][pid:11753][MainThread][INFO]][train.py:312] Training init time: 10.795s
|
||||
[2025-02-01 18:47:48,078][oumi][rank3][pid:11753][MainThread][INFO]][train.py:313] Starting training... (TrainerType.TRL_SFT, transformers: 4.45.2)
|
||||
[2025-02-01 18:52:35,469][oumi][rank3][pid:11753][MainThread][INFO]][train.py:320] Training is Complete.
|
||||
[2025-02-01 18:52:35,496][oumi][rank3][pid:11753][MainThread][INFO]][device_utils.py:283] GPU Metrics After Training: GPU runtime info: NVidiaGpuRuntimeInfo(device_index=0, device_count=4, used_memory_mb=21283.0, temperature=43, fan_speed=None, fan_speeds=None, power_usage_watts=181.852, power_limit_watts=400.0, gpu_utilization=54, memory_utilization=14, performance_state=0, clock_speed_graphics=1410, clock_speed_sm=1410, clock_speed_memory=1593).
|
||||
[2025-02-01 18:52:35,497][oumi][rank3][pid:11753][MainThread][INFO]][torch_utils.py:117] Peak GPU memory usage: 16.56 GB
|
||||
[2025-02-01 18:52:35,497][oumi][rank3][pid:11753][MainThread][INFO]][train.py:327] Saving final state...
|
||||
[2025-02-01 18:52:35,504][oumi][rank3][pid:11753][MainThread][INFO]][train.py:332] Saving final model...
|
||||
[2025-02-01 18:52:43,652][oumi][rank3][pid:11753][MainThread][INFO]][train.py:339]
|
||||
|
||||
» We're always looking for feedback. What's one thing we can improve? https://oumi.ai/feedback
|
||||
48901
merges.txt
Normal file
48901
merges.txt
Normal file
File diff suppressed because it is too large
Load Diff
3
model.safetensors
Normal file
3
model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:656b06593f9a7fd9830d1adb5883c539f3ccc697fa6f9b565921cb86836e32af
|
||||
size 3422777952
|
||||
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:31e126e7d84ce93b41ef1338794b75bc7718945a7b308c7028f605af6bf96c30
|
||||
size 19018
|
||||
34
special_tokens_map.json
Normal file
34
special_tokens_map.json
Normal file
@@ -0,0 +1,34 @@
|
||||
{
|
||||
"additional_special_tokens": [
|
||||
"<|im_start|>",
|
||||
"<|im_end|>"
|
||||
],
|
||||
"bos_token": {
|
||||
"content": "<|im_start|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"eos_token": {
|
||||
"content": "<|im_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"pad_token": {
|
||||
"content": "<|im_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"unk_token": {
|
||||
"content": "<|endoftext|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
}
|
||||
}
|
||||
5
telemetry/devices_info.txt
Normal file
5
telemetry/devices_info.txt
Normal file
@@ -0,0 +1,5 @@
|
||||
CPU cores: 48 CUDA devices: 4
|
||||
device(0)='NVIDIA A100-SXM4-80GB' Capability: (8, 0) Memory: [Total: 79.15GiB Free: 76.58GiB Allocated: 0.0GiB Cached: 0.0GiB]
|
||||
device(1)='NVIDIA A100-SXM4-80GB' Capability: (8, 0) Memory: [Total: 79.15GiB Free: 77.26GiB Allocated: 0.0GiB Cached: 0.0GiB]
|
||||
device(2)='NVIDIA A100-SXM4-80GB' Capability: (8, 0) Memory: [Total: 79.15GiB Free: 77.26GiB Allocated: 0.0GiB Cached: 0.0GiB]
|
||||
device(3)='NVIDIA A100-SXM4-80GB' Capability: (8, 0) Memory: [Total: 79.15GiB Free: 77.12GiB Allocated: 0.0GiB Cached: 0.0GiB]
|
||||
165
telemetry/training_config.yaml
Normal file
165
telemetry/training_config.yaml
Normal file
@@ -0,0 +1,165 @@
|
||||
data:
|
||||
train:
|
||||
datasets:
|
||||
- dataset_name: text_sft_jsonl
|
||||
dataset_path: data/R1/math_10k_R1_outputs.jsonl
|
||||
subset: null
|
||||
split: train
|
||||
dataset_kwargs: {}
|
||||
sample_count: null
|
||||
mixture_proportion: null
|
||||
shuffle: false
|
||||
seed: null
|
||||
shuffle_buffer_size: 1000
|
||||
trust_remote_code: false
|
||||
transform_num_workers: null
|
||||
collator_name: null
|
||||
pack: false
|
||||
stream: false
|
||||
target_col: null
|
||||
mixture_strategy: first_exhausted
|
||||
seed: 42
|
||||
use_async_dataset: false
|
||||
use_torchdata: null
|
||||
test:
|
||||
datasets: []
|
||||
collator_name: null
|
||||
pack: false
|
||||
stream: false
|
||||
target_col: null
|
||||
mixture_strategy: first_exhausted
|
||||
seed: null
|
||||
use_async_dataset: false
|
||||
use_torchdata: null
|
||||
validation:
|
||||
datasets: []
|
||||
collator_name: null
|
||||
pack: false
|
||||
stream: false
|
||||
target_col: null
|
||||
mixture_strategy: first_exhausted
|
||||
seed: null
|
||||
use_async_dataset: false
|
||||
use_torchdata: null
|
||||
model:
|
||||
model_name: HuggingFaceTB/SmolLM2-1.7B-Instruct
|
||||
adapter_model: null
|
||||
tokenizer_name: null
|
||||
tokenizer_pad_token: null
|
||||
tokenizer_kwargs: {}
|
||||
model_max_length: null
|
||||
load_pretrained_weights: true
|
||||
trust_remote_code: true
|
||||
torch_dtype_str: bfloat16
|
||||
compile: false
|
||||
chat_template: null
|
||||
attn_implementation: null
|
||||
device_map: auto
|
||||
model_kwargs: {}
|
||||
enable_liger_kernel: false
|
||||
shard_for_eval: false
|
||||
freeze_layers: []
|
||||
training:
|
||||
use_peft: false
|
||||
trainer_type: TRL_SFT
|
||||
enable_gradient_checkpointing: true
|
||||
gradient_checkpointing_kwargs:
|
||||
use_reentrant: false
|
||||
output_dir: output/smollm2-17b-distill-r1-670b-math
|
||||
per_device_train_batch_size: 2
|
||||
per_device_eval_batch_size: 8
|
||||
gradient_accumulation_steps: 2
|
||||
max_steps: -1
|
||||
num_train_epochs: 1
|
||||
save_epoch: false
|
||||
save_steps: 0
|
||||
save_final_model: true
|
||||
seed: 42
|
||||
run_name: smollm2-17b-distill-r1-670b-math.sky-2025-02-01-13-42-43-696171_sky-d954-bf996_1
|
||||
metrics_function: null
|
||||
log_level: info
|
||||
dep_log_level: warning
|
||||
enable_wandb: true
|
||||
enable_tensorboard: true
|
||||
logging_strategy: steps
|
||||
logging_dir: null
|
||||
logging_steps: 10
|
||||
logging_first_step: false
|
||||
eval_strategy: 'no'
|
||||
eval_steps: 500
|
||||
learning_rate: 2.0e-05
|
||||
lr_scheduler_type: linear
|
||||
lr_scheduler_kwargs: {}
|
||||
warmup_ratio: 0.1
|
||||
warmup_steps: null
|
||||
optimizer: adamw_torch_fused
|
||||
weight_decay: 0.0
|
||||
adam_beta1: 0.9
|
||||
adam_beta2: 0.999
|
||||
adam_epsilon: 1.0e-08
|
||||
sgd_momentum: 0.0
|
||||
mixed_precision_dtype: NONE
|
||||
compile: false
|
||||
include_performance_metrics: false
|
||||
include_alternative_mfu_metrics: false
|
||||
log_model_summary: false
|
||||
resume_from_checkpoint: null
|
||||
try_resume_from_last_checkpoint: false
|
||||
dataloader_num_workers: 8
|
||||
dataloader_prefetch_factor: 32
|
||||
dataloader_main_process_only: null
|
||||
ddp_find_unused_parameters: false
|
||||
max_grad_norm: 10.0
|
||||
trainer_kwargs: {}
|
||||
profiler:
|
||||
save_dir: null
|
||||
enable_cpu_profiling: false
|
||||
enable_cuda_profiling: false
|
||||
record_shapes: false
|
||||
profile_memory: false
|
||||
with_stack: false
|
||||
with_flops: false
|
||||
with_modules: false
|
||||
row_limit: 50
|
||||
schedule:
|
||||
enable_schedule: false
|
||||
wait: 0
|
||||
warmup: 1
|
||||
active: 3
|
||||
repeat: 1
|
||||
skip_first: 1
|
||||
telemetry:
|
||||
telemetry_dir: telemetry
|
||||
collect_telemetry_for_all_ranks: false
|
||||
track_gpu_temperature: false
|
||||
empty_device_cache_steps: 1
|
||||
nccl_default_timeout_minutes: null
|
||||
peft:
|
||||
lora_r: 8
|
||||
lora_alpha: 8
|
||||
lora_dropout: 0.0
|
||||
lora_target_modules: null
|
||||
lora_modules_to_save: null
|
||||
lora_bias: none
|
||||
lora_init_weights: DEFAULT
|
||||
lora_task_type: CAUSAL_LM
|
||||
q_lora: false
|
||||
q_lora_bits: 4
|
||||
bnb_4bit_quant_type: fp4
|
||||
use_bnb_nested_quant: false
|
||||
bnb_4bit_quant_storage: uint8
|
||||
bnb_4bit_compute_dtype: float32
|
||||
peft_save_mode: ADAPTER_ONLY
|
||||
fsdp:
|
||||
enable_fsdp: false
|
||||
sharding_strategy: FULL_SHARD
|
||||
cpu_offload: false
|
||||
mixed_precision: null
|
||||
backward_prefetch: BACKWARD_PRE
|
||||
forward_prefetch: false
|
||||
use_orig_params: null
|
||||
state_dict_type: FULL_STATE_DICT
|
||||
auto_wrap_policy: NO_WRAP
|
||||
min_num_params: 100000
|
||||
transformer_layer_cls: null
|
||||
sync_module_states: true
|
||||
4
telemetry/world_size.json
Normal file
4
telemetry/world_size.json
Normal file
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"LOCAL_WORLD_SIZE": 4,
|
||||
"WORLD_SIZE": 4
|
||||
}
|
||||
244954
tokenizer.json
Normal file
244954
tokenizer.json
Normal file
File diff suppressed because it is too large
Load Diff
154
tokenizer_config.json
Normal file
154
tokenizer_config.json
Normal file
@@ -0,0 +1,154 @@
|
||||
{
|
||||
"add_prefix_space": false,
|
||||
"added_tokens_decoder": {
|
||||
"0": {
|
||||
"content": "<|endoftext|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"1": {
|
||||
"content": "<|im_start|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"2": {
|
||||
"content": "<|im_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"3": {
|
||||
"content": "<repo_name>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"4": {
|
||||
"content": "<reponame>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"5": {
|
||||
"content": "<file_sep>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"6": {
|
||||
"content": "<filename>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"7": {
|
||||
"content": "<gh_stars>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"8": {
|
||||
"content": "<issue_start>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"9": {
|
||||
"content": "<issue_comment>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"10": {
|
||||
"content": "<issue_closed>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"11": {
|
||||
"content": "<jupyter_start>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"12": {
|
||||
"content": "<jupyter_text>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"13": {
|
||||
"content": "<jupyter_code>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"14": {
|
||||
"content": "<jupyter_output>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"15": {
|
||||
"content": "<jupyter_script>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"16": {
|
||||
"content": "<empty_output>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
}
|
||||
},
|
||||
"additional_special_tokens": [
|
||||
"<|im_start|>",
|
||||
"<|im_end|>"
|
||||
],
|
||||
"bos_token": "<|im_start|>",
|
||||
"chat_template": "{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system\nYou are a helpful AI assistant named SmolLM, trained by Hugging Face<|im_end|>\n' }}{% endif %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
|
||||
"clean_up_tokenization_spaces": false,
|
||||
"eos_token": "<|im_end|>",
|
||||
"model_max_length": 8192,
|
||||
"pad_token": "<|im_end|>",
|
||||
"tokenizer_class": "GPT2Tokenizer",
|
||||
"unk_token": "<|endoftext|>",
|
||||
"vocab_size": 49152
|
||||
}
|
||||
476
trainer_state.json
Normal file
476
trainer_state.json
Normal file
@@ -0,0 +1,476 @@
|
||||
{
|
||||
"best_metric": null,
|
||||
"best_model_checkpoint": null,
|
||||
"epoch": 1.0,
|
||||
"eval_steps": 500,
|
||||
"global_step": 625,
|
||||
"is_hyper_param_search": false,
|
||||
"is_local_process_zero": true,
|
||||
"is_world_process_zero": true,
|
||||
"log_history": [
|
||||
{
|
||||
"epoch": 0.016,
|
||||
"grad_norm": 30.125,
|
||||
"learning_rate": 3.1746031746031746e-06,
|
||||
"loss": 2.5254,
|
||||
"step": 10
|
||||
},
|
||||
{
|
||||
"epoch": 0.032,
|
||||
"grad_norm": 20.25,
|
||||
"learning_rate": 6.349206349206349e-06,
|
||||
"loss": 2.5254,
|
||||
"step": 20
|
||||
},
|
||||
{
|
||||
"epoch": 0.048,
|
||||
"grad_norm": 14.8125,
|
||||
"learning_rate": 9.523809523809525e-06,
|
||||
"loss": 2.0496,
|
||||
"step": 30
|
||||
},
|
||||
{
|
||||
"epoch": 0.064,
|
||||
"grad_norm": 10.9375,
|
||||
"learning_rate": 1.2698412698412699e-05,
|
||||
"loss": 1.4853,
|
||||
"step": 40
|
||||
},
|
||||
{
|
||||
"epoch": 0.08,
|
||||
"grad_norm": 5.625,
|
||||
"learning_rate": 1.5873015873015872e-05,
|
||||
"loss": 0.8731,
|
||||
"step": 50
|
||||
},
|
||||
{
|
||||
"epoch": 0.096,
|
||||
"grad_norm": 3.109375,
|
||||
"learning_rate": 1.904761904761905e-05,
|
||||
"loss": 0.6086,
|
||||
"step": 60
|
||||
},
|
||||
{
|
||||
"epoch": 0.112,
|
||||
"grad_norm": 3.015625,
|
||||
"learning_rate": 1.9750889679715305e-05,
|
||||
"loss": 0.5745,
|
||||
"step": 70
|
||||
},
|
||||
{
|
||||
"epoch": 0.128,
|
||||
"grad_norm": 2.375,
|
||||
"learning_rate": 1.939501779359431e-05,
|
||||
"loss": 0.5121,
|
||||
"step": 80
|
||||
},
|
||||
{
|
||||
"epoch": 0.144,
|
||||
"grad_norm": 3.28125,
|
||||
"learning_rate": 1.903914590747331e-05,
|
||||
"loss": 0.4875,
|
||||
"step": 90
|
||||
},
|
||||
{
|
||||
"epoch": 0.16,
|
||||
"grad_norm": 2.703125,
|
||||
"learning_rate": 1.8683274021352315e-05,
|
||||
"loss": 0.4997,
|
||||
"step": 100
|
||||
},
|
||||
{
|
||||
"epoch": 0.176,
|
||||
"grad_norm": 2.203125,
|
||||
"learning_rate": 1.832740213523132e-05,
|
||||
"loss": 0.5004,
|
||||
"step": 110
|
||||
},
|
||||
{
|
||||
"epoch": 0.192,
|
||||
"grad_norm": 2.109375,
|
||||
"learning_rate": 1.7971530249110324e-05,
|
||||
"loss": 0.5119,
|
||||
"step": 120
|
||||
},
|
||||
{
|
||||
"epoch": 0.208,
|
||||
"grad_norm": 2.453125,
|
||||
"learning_rate": 1.7615658362989325e-05,
|
||||
"loss": 0.5088,
|
||||
"step": 130
|
||||
},
|
||||
{
|
||||
"epoch": 0.224,
|
||||
"grad_norm": 2.171875,
|
||||
"learning_rate": 1.725978647686833e-05,
|
||||
"loss": 0.4842,
|
||||
"step": 140
|
||||
},
|
||||
{
|
||||
"epoch": 0.24,
|
||||
"grad_norm": 3.65625,
|
||||
"learning_rate": 1.690391459074733e-05,
|
||||
"loss": 0.5193,
|
||||
"step": 150
|
||||
},
|
||||
{
|
||||
"epoch": 0.256,
|
||||
"grad_norm": 2.609375,
|
||||
"learning_rate": 1.6548042704626336e-05,
|
||||
"loss": 0.4984,
|
||||
"step": 160
|
||||
},
|
||||
{
|
||||
"epoch": 0.272,
|
||||
"grad_norm": 2.28125,
|
||||
"learning_rate": 1.619217081850534e-05,
|
||||
"loss": 0.5011,
|
||||
"step": 170
|
||||
},
|
||||
{
|
||||
"epoch": 0.288,
|
||||
"grad_norm": 3.46875,
|
||||
"learning_rate": 1.583629893238434e-05,
|
||||
"loss": 0.5493,
|
||||
"step": 180
|
||||
},
|
||||
{
|
||||
"epoch": 0.304,
|
||||
"grad_norm": 2.46875,
|
||||
"learning_rate": 1.5480427046263346e-05,
|
||||
"loss": 0.4869,
|
||||
"step": 190
|
||||
},
|
||||
{
|
||||
"epoch": 0.32,
|
||||
"grad_norm": 2.734375,
|
||||
"learning_rate": 1.5124555160142349e-05,
|
||||
"loss": 0.4902,
|
||||
"step": 200
|
||||
},
|
||||
{
|
||||
"epoch": 0.336,
|
||||
"grad_norm": 1.921875,
|
||||
"learning_rate": 1.4768683274021354e-05,
|
||||
"loss": 0.5013,
|
||||
"step": 210
|
||||
},
|
||||
{
|
||||
"epoch": 0.352,
|
||||
"grad_norm": 2.125,
|
||||
"learning_rate": 1.4412811387900356e-05,
|
||||
"loss": 0.5061,
|
||||
"step": 220
|
||||
},
|
||||
{
|
||||
"epoch": 0.368,
|
||||
"grad_norm": 2.265625,
|
||||
"learning_rate": 1.4056939501779361e-05,
|
||||
"loss": 0.4956,
|
||||
"step": 230
|
||||
},
|
||||
{
|
||||
"epoch": 0.384,
|
||||
"grad_norm": 1.6015625,
|
||||
"learning_rate": 1.3701067615658364e-05,
|
||||
"loss": 0.4665,
|
||||
"step": 240
|
||||
},
|
||||
{
|
||||
"epoch": 0.4,
|
||||
"grad_norm": 2.234375,
|
||||
"learning_rate": 1.3345195729537369e-05,
|
||||
"loss": 0.4605,
|
||||
"step": 250
|
||||
},
|
||||
{
|
||||
"epoch": 0.416,
|
||||
"grad_norm": 2.1875,
|
||||
"learning_rate": 1.298932384341637e-05,
|
||||
"loss": 0.4728,
|
||||
"step": 260
|
||||
},
|
||||
{
|
||||
"epoch": 0.432,
|
||||
"grad_norm": 2.46875,
|
||||
"learning_rate": 1.2633451957295374e-05,
|
||||
"loss": 0.5115,
|
||||
"step": 270
|
||||
},
|
||||
{
|
||||
"epoch": 0.448,
|
||||
"grad_norm": 2.765625,
|
||||
"learning_rate": 1.2277580071174377e-05,
|
||||
"loss": 0.4873,
|
||||
"step": 280
|
||||
},
|
||||
{
|
||||
"epoch": 0.464,
|
||||
"grad_norm": 2.140625,
|
||||
"learning_rate": 1.1921708185053382e-05,
|
||||
"loss": 0.5266,
|
||||
"step": 290
|
||||
},
|
||||
{
|
||||
"epoch": 0.48,
|
||||
"grad_norm": 2.078125,
|
||||
"learning_rate": 1.1565836298932385e-05,
|
||||
"loss": 0.5175,
|
||||
"step": 300
|
||||
},
|
||||
{
|
||||
"epoch": 0.496,
|
||||
"grad_norm": 2.328125,
|
||||
"learning_rate": 1.120996441281139e-05,
|
||||
"loss": 0.4702,
|
||||
"step": 310
|
||||
},
|
||||
{
|
||||
"epoch": 0.512,
|
||||
"grad_norm": 2.734375,
|
||||
"learning_rate": 1.0854092526690392e-05,
|
||||
"loss": 0.5071,
|
||||
"step": 320
|
||||
},
|
||||
{
|
||||
"epoch": 0.528,
|
||||
"grad_norm": 2.03125,
|
||||
"learning_rate": 1.0498220640569397e-05,
|
||||
"loss": 0.5155,
|
||||
"step": 330
|
||||
},
|
||||
{
|
||||
"epoch": 0.544,
|
||||
"grad_norm": 2.53125,
|
||||
"learning_rate": 1.01423487544484e-05,
|
||||
"loss": 0.4964,
|
||||
"step": 340
|
||||
},
|
||||
{
|
||||
"epoch": 0.56,
|
||||
"grad_norm": 2.984375,
|
||||
"learning_rate": 9.786476868327403e-06,
|
||||
"loss": 0.4867,
|
||||
"step": 350
|
||||
},
|
||||
{
|
||||
"epoch": 0.576,
|
||||
"grad_norm": 2.828125,
|
||||
"learning_rate": 9.430604982206405e-06,
|
||||
"loss": 0.545,
|
||||
"step": 360
|
||||
},
|
||||
{
|
||||
"epoch": 0.592,
|
||||
"grad_norm": 2.546875,
|
||||
"learning_rate": 9.07473309608541e-06,
|
||||
"loss": 0.4832,
|
||||
"step": 370
|
||||
},
|
||||
{
|
||||
"epoch": 0.608,
|
||||
"grad_norm": 2.6875,
|
||||
"learning_rate": 8.718861209964413e-06,
|
||||
"loss": 0.4826,
|
||||
"step": 380
|
||||
},
|
||||
{
|
||||
"epoch": 0.624,
|
||||
"grad_norm": 1.765625,
|
||||
"learning_rate": 8.362989323843418e-06,
|
||||
"loss": 0.4652,
|
||||
"step": 390
|
||||
},
|
||||
{
|
||||
"epoch": 0.64,
|
||||
"grad_norm": 2.359375,
|
||||
"learning_rate": 8.00711743772242e-06,
|
||||
"loss": 0.5069,
|
||||
"step": 400
|
||||
},
|
||||
{
|
||||
"epoch": 0.656,
|
||||
"grad_norm": 2.453125,
|
||||
"learning_rate": 7.651245551601423e-06,
|
||||
"loss": 0.5131,
|
||||
"step": 410
|
||||
},
|
||||
{
|
||||
"epoch": 0.672,
|
||||
"grad_norm": 2.3125,
|
||||
"learning_rate": 7.295373665480427e-06,
|
||||
"loss": 0.4937,
|
||||
"step": 420
|
||||
},
|
||||
{
|
||||
"epoch": 0.688,
|
||||
"grad_norm": 2.109375,
|
||||
"learning_rate": 6.939501779359431e-06,
|
||||
"loss": 0.4878,
|
||||
"step": 430
|
||||
},
|
||||
{
|
||||
"epoch": 0.704,
|
||||
"grad_norm": 3.546875,
|
||||
"learning_rate": 6.5836298932384346e-06,
|
||||
"loss": 0.4821,
|
||||
"step": 440
|
||||
},
|
||||
{
|
||||
"epoch": 0.72,
|
||||
"grad_norm": 2.046875,
|
||||
"learning_rate": 6.227758007117438e-06,
|
||||
"loss": 0.4486,
|
||||
"step": 450
|
||||
},
|
||||
{
|
||||
"epoch": 0.736,
|
||||
"grad_norm": 2.71875,
|
||||
"learning_rate": 5.871886120996442e-06,
|
||||
"loss": 0.5022,
|
||||
"step": 460
|
||||
},
|
||||
{
|
||||
"epoch": 0.752,
|
||||
"grad_norm": 2.53125,
|
||||
"learning_rate": 5.516014234875445e-06,
|
||||
"loss": 0.4977,
|
||||
"step": 470
|
||||
},
|
||||
{
|
||||
"epoch": 0.768,
|
||||
"grad_norm": 2.6875,
|
||||
"learning_rate": 5.160142348754449e-06,
|
||||
"loss": 0.5076,
|
||||
"step": 480
|
||||
},
|
||||
{
|
||||
"epoch": 0.784,
|
||||
"grad_norm": 2.546875,
|
||||
"learning_rate": 4.8042704626334524e-06,
|
||||
"loss": 0.451,
|
||||
"step": 490
|
||||
},
|
||||
{
|
||||
"epoch": 0.8,
|
||||
"grad_norm": 1.734375,
|
||||
"learning_rate": 4.448398576512456e-06,
|
||||
"loss": 0.4576,
|
||||
"step": 500
|
||||
},
|
||||
{
|
||||
"epoch": 0.816,
|
||||
"grad_norm": 2.765625,
|
||||
"learning_rate": 4.09252669039146e-06,
|
||||
"loss": 0.4914,
|
||||
"step": 510
|
||||
},
|
||||
{
|
||||
"epoch": 0.832,
|
||||
"grad_norm": 2.59375,
|
||||
"learning_rate": 3.7366548042704632e-06,
|
||||
"loss": 0.4938,
|
||||
"step": 520
|
||||
},
|
||||
{
|
||||
"epoch": 0.848,
|
||||
"grad_norm": 2.625,
|
||||
"learning_rate": 3.3807829181494666e-06,
|
||||
"loss": 0.5218,
|
||||
"step": 530
|
||||
},
|
||||
{
|
||||
"epoch": 0.864,
|
||||
"grad_norm": 1.8515625,
|
||||
"learning_rate": 3.0249110320284703e-06,
|
||||
"loss": 0.4694,
|
||||
"step": 540
|
||||
},
|
||||
{
|
||||
"epoch": 0.88,
|
||||
"grad_norm": 2.375,
|
||||
"learning_rate": 2.669039145907473e-06,
|
||||
"loss": 0.5102,
|
||||
"step": 550
|
||||
},
|
||||
{
|
||||
"epoch": 0.896,
|
||||
"grad_norm": 2.375,
|
||||
"learning_rate": 2.313167259786477e-06,
|
||||
"loss": 0.506,
|
||||
"step": 560
|
||||
},
|
||||
{
|
||||
"epoch": 0.912,
|
||||
"grad_norm": 2.875,
|
||||
"learning_rate": 1.9572953736654807e-06,
|
||||
"loss": 0.4982,
|
||||
"step": 570
|
||||
},
|
||||
{
|
||||
"epoch": 0.928,
|
||||
"grad_norm": 1.9140625,
|
||||
"learning_rate": 1.6014234875444842e-06,
|
||||
"loss": 0.5107,
|
||||
"step": 580
|
||||
},
|
||||
{
|
||||
"epoch": 0.944,
|
||||
"grad_norm": 2.546875,
|
||||
"learning_rate": 1.2455516014234877e-06,
|
||||
"loss": 0.4857,
|
||||
"step": 590
|
||||
},
|
||||
{
|
||||
"epoch": 0.96,
|
||||
"grad_norm": 1.8359375,
|
||||
"learning_rate": 8.896797153024913e-07,
|
||||
"loss": 0.4821,
|
||||
"step": 600
|
||||
},
|
||||
{
|
||||
"epoch": 0.976,
|
||||
"grad_norm": 2.609375,
|
||||
"learning_rate": 5.338078291814947e-07,
|
||||
"loss": 0.5166,
|
||||
"step": 610
|
||||
},
|
||||
{
|
||||
"epoch": 0.992,
|
||||
"grad_norm": 2.578125,
|
||||
"learning_rate": 1.7793594306049826e-07,
|
||||
"loss": 0.5116,
|
||||
"step": 620
|
||||
},
|
||||
{
|
||||
"epoch": 1.0,
|
||||
"step": 625,
|
||||
"total_flos": 9900164319805440.0,
|
||||
"train_loss": 0.6107198246002197,
|
||||
"train_runtime": 287.0781,
|
||||
"train_samples_per_second": 34.834,
|
||||
"train_steps_per_second": 2.177
|
||||
}
|
||||
],
|
||||
"logging_steps": 10,
|
||||
"max_steps": 625,
|
||||
"num_input_tokens_seen": 0,
|
||||
"num_train_epochs": 1,
|
||||
"save_steps": 0,
|
||||
"stateful_callbacks": {
|
||||
"TrainerControl": {
|
||||
"args": {
|
||||
"should_epoch_stop": false,
|
||||
"should_evaluate": false,
|
||||
"should_log": false,
|
||||
"should_save": false,
|
||||
"should_training_stop": false
|
||||
},
|
||||
"attributes": {}
|
||||
}
|
||||
},
|
||||
"total_flos": 9900164319805440.0,
|
||||
"train_batch_size": 2,
|
||||
"trial_name": null,
|
||||
"trial_params": null
|
||||
}
|
||||
3
training_args.bin
Normal file
3
training_args.bin
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:ca5ba33e3d7a1ff4b6185b6649d17e444660dc9c669f108ba1cf9db15a4edb0b
|
||||
size 5624
|
||||
1
vocab.json
Normal file
1
vocab.json
Normal file
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user