Omaratef3221/llama-3.1-8b-s1-none-s2-full-medarabench
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

初始化项目，由ModelHub XC社区提供模型

Browse Source 
Model: Omaratef3221/llama-3.1-8b-s1-none-s2-full-medarabench
Source: Original Platform
This commit is contained in:
ModelHub XC
2026-07-27 22:42:09 +08:00
commit 9fec572eda
39 changed files with 1817 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
checkpoint-2096/config.json Normal file
View File

@@ -0,0 +1,36 @@
{
"architectures": [
"LlamaForCausalLM"
],
"attention_bias": false,
"attention_dropout": 0.0,
"bos_token_id": 128000,
"dtype": "bfloat16",
"eos_token_id": 128001,
"head_dim": 128,
"hidden_act": "silu",
"hidden_size": 4096,
"initializer_range": 0.02,
"intermediate_size": 14336,
"max_position_embeddings": 131072,
"mlp_bias": false,
"model_type": "llama",
"num_attention_heads": 32,
"num_hidden_layers": 32,
"num_key_value_heads": 8,
"pad_token_id": 128001,
"pretraining_tp": 1,
"rms_norm_eps": 1e-05,
"rope_parameters": {
"factor": 8.0,
"high_freq_factor": 4.0,
"low_freq_factor": 1.0,
"original_max_position_embeddings": 8192,
"rope_theta": 500000.0,
"rope_type": "llama3"
},
"tie_word_embeddings": false,
"transformers_version": "5.5.1",
"use_cache": false,
"vocab_size": 128256
}

12
checkpoint-2096/generation_config.json Normal file
View File

@@ -0,0 +1,12 @@
{
"_from_model_config": true,
"bos_token_id": 128000,
"do_sample": true,
"eos_token_id": [
128001
],
"pad_token_id": 128001,
"temperature": 0.6,
"top_p": 0.9,
"transformers_version": "5.5.1"
}

3
checkpoint-2096/model.safetensors Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:844a8e251caaa84cdfb8e86ae9acbd329b8415f7a86e69abed6148d65f8e97f1
size 16060556616

3
checkpoint-2096/optimizer.pt Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:ae03b75e4c92461e513d78ac37cc90727d5e4cb5bccc09170d0248242b208b97
size 19447450244

3
checkpoint-2096/rng_state.pth Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:f37c40ce327861a7ca13b719d3aa37510a143368b6e74358bdb14becb3899e1e
size 14244

3
checkpoint-2096/scheduler.pt Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:4d76fed4e1336b28b263f3818f8a8cb0f32d2a137ecbb0cf42e66fa555f9c960
size 1064

BIN
checkpoint-2096/tokenizer.json (Stored with Git LFS) Normal file
View File

Binary file not shown.

14
checkpoint-2096/tokenizer_config.json Normal file
View File

@@ -0,0 +1,14 @@
{
"backend": "tokenizers",
"bos_token": "<|begin_of_text|>",
"clean_up_tokenization_spaces": true,
"eos_token": "<|end_of_text|>",
"is_local": false,
"model_input_names": [
"input_ids",
"attention_mask"
],
"model_max_length": 131072,
"pad_token": "<|end_of_text|>",
"tokenizer_class": "TokenizersBackend"
}

466
checkpoint-2096/trainer_state.json Normal file
View File

@@ -0,0 +1,466 @@
{
"best_global_step": null,
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 2.0,
"eval_steps": 500,
"global_step": 2096,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"entropy": 2.0116909193992614,
"epoch": 0.047744091668656,
"grad_norm": 3.390625,
"learning_rate": 1.0315789473684213e-05,
"loss": 2.0623187255859374,
"mean_token_accuracy": 0.589528747946024,
"num_tokens": 62415.0,
"step": 50
},
{
"entropy": 1.7590599358081818,
"epoch": 0.095488183337312,
"grad_norm": 3.078125,
"learning_rate": 1.9999915067429385e-05,
"loss": 1.754022216796875,
"mean_token_accuracy": 0.6332476472854615,
"num_tokens": 126004.0,
"step": 100
},
{
"entropy": 1.7224605268239974,
"epoch": 0.14323227500596802,
"grad_norm": 3.203125,
"learning_rate": 1.9984525009996967e-05,
"loss": 1.7351203918457032,
"mean_token_accuracy": 0.6340418252348899,
"num_tokens": 188614.0,
"step": 150
},
{
"entropy": 1.6983880323171616,
"epoch": 0.190976366674624,
"grad_norm": 2.984375,
"learning_rate": 1.9942640420376764e-05,
"loss": 1.7146597290039063,
"mean_token_accuracy": 0.6401417210698128,
"num_tokens": 251062.0,
"step": 200
},
{
"entropy": 1.6483143484592437,
"epoch": 0.23872045834328,
"grad_norm": 2.734375,
"learning_rate": 1.9874372441823628e-05,
"loss": 1.658107147216797,
"mean_token_accuracy": 0.6478794780373573,
"num_tokens": 313957.0,
"step": 250
},
{
"entropy": 1.6347757685184479,
"epoch": 0.28646455001193605,
"grad_norm": 2.703125,
"learning_rate": 1.9779902227487397e-05,
"loss": 1.6550674438476562,
"mean_token_accuracy": 0.6487274625897408,
"num_tokens": 376779.0,
"step": 300
},
{
"entropy": 1.5762060970067977,
"epoch": 0.334208641680592,
"grad_norm": 2.34375,
"learning_rate": 1.9659480459712203e-05,
"loss": 1.5960990905761718,
"mean_token_accuracy": 0.6587067684531211,
"num_tokens": 439472.0,
"step": 350
},
{
"entropy": 1.534512039422989,
"epoch": 0.381952733349248,
"grad_norm": 2.5625,
"learning_rate": 1.9513426684835922e-05,
"loss": 1.55486572265625,
"mean_token_accuracy": 0.6660523465275765,
"num_tokens": 501279.0,
"step": 400
},
{
"entropy": 1.5402596753835678,
"epoch": 0.429696825017904,
"grad_norm": 2.53125,
"learning_rate": 1.9342128465254943e-05,
"loss": 1.5535296630859374,
"mean_token_accuracy": 0.6673906499147415,
"num_tokens": 564442.0,
"step": 450
},
{
"entropy": 1.5164723497629167,
"epoch": 0.47744091668656,
"grad_norm": 2.34375,
"learning_rate": 1.9146040351004225e-05,
"loss": 1.5373231506347655,
"mean_token_accuracy": 0.670901304781437,
"num_tokens": 627102.0,
"step": 500
},
{
"entropy": 1.4891952139139175,
"epoch": 0.5251850083552161,
"grad_norm": 2.296875,
"learning_rate": 1.8925682673581706e-05,
"loss": 1.5036320495605469,
"mean_token_accuracy": 0.6744937270879745,
"num_tokens": 689421.0,
"step": 550
},
{
"entropy": 1.488253089785576,
"epoch": 0.5729291000238721,
"grad_norm": 2.15625,
"learning_rate": 1.868164016521766e-05,
"loss": 1.5044355773925782,
"mean_token_accuracy": 0.6755094349384307,
"num_tokens": 752388.0,
"step": 600
},
{
"entropy": 1.4446531629562378,
"epoch": 0.6206731916925281,
"grad_norm": 2.34375,
"learning_rate": 1.8414560407252874e-05,
"loss": 1.4617152404785156,
"mean_token_accuracy": 0.6805451279878616,
"num_tokens": 815803.0,
"step": 650
},
{
"entropy": 1.4555100780725478,
"epoch": 0.668417283361184,
"grad_norm": 2.109375,
"learning_rate": 1.8125152111742944e-05,
"loss": 1.469644775390625,
"mean_token_accuracy": 0.6796707132458687,
"num_tokens": 878686.0,
"step": 700
},
{
"entropy": 1.414481201171875,
"epoch": 0.71616137502984,
"grad_norm": 2.140625,
"learning_rate": 1.7814183240848635e-05,
"loss": 1.4264332580566406,
"mean_token_accuracy": 0.6893567860126495,
"num_tokens": 940629.0,
"step": 750
},
{
"entropy": 1.3958445823192596,
"epoch": 0.763905466698496,
"grad_norm": 2.34375,
"learning_rate": 1.7482478969002487e-05,
"loss": 1.4121397399902345,
"mean_token_accuracy": 0.6902168181538582,
"num_tokens": 1002810.0,
"step": 800
},
{
"entropy": 1.3906721925735475,
"epoch": 0.811649558367152,
"grad_norm": 2.421875,
"learning_rate": 1.7130919493259284e-05,
"loss": 1.3895584106445313,
"mean_token_accuracy": 0.6943912228941918,
"num_tokens": 1067041.0,
"step": 850
},
{
"entropy": 1.3663516825437545,
"epoch": 0.859393650035808,
"grad_norm": 2.375,
"learning_rate": 1.676043769764072e-05,
"loss": 1.3753732299804688,
"mean_token_accuracy": 0.6982124423980713,
"num_tokens": 1130309.0,
"step": 900
},
{
"entropy": 1.370076591670513,
"epoch": 0.907137741704464,
"grad_norm": 1.9296875,
"learning_rate": 1.637201667767204e-05,
"loss": 1.3839447021484375,
"mean_token_accuracy": 0.6986603051424026,
"num_tokens": 1192924.0,
"step": 950
},
{
"entropy": 1.3369020774960518,
"epoch": 0.95488183337312,
"grad_norm": 1.9609375,
"learning_rate": 1.5966687131679463e-05,
"loss": 1.3427734375,
"mean_token_accuracy": 0.7060227099061013,
"num_tokens": 1254418.0,
"step": 1000
},
{
"epoch": 1.0,
"eval_entropy": 1.2727481353174803,
"eval_loss": 1.291359543800354,
"eval_mean_token_accuracy": 0.7115251171210456,
"eval_num_tokens": 1314012.0,
"eval_runtime": 52.7795,
"eval_samples_per_second": 16.711,
"eval_steps_per_second": 16.711,
"step": 1048
},
{
"entropy": 1.3094463989819367,
"epoch": 1.0019097636667462,
"grad_norm": 2.078125,
"learning_rate": 1.554552462577077e-05,
"loss": 1.3102546691894532,
"mean_token_accuracy": 0.7078821743805397,
"num_tokens": 1316679.0,
"step": 1050
},
{
"entropy": 0.9862651464343071,
"epoch": 1.0496538553354022,
"grad_norm": 1.9609375,
"learning_rate": 1.510964673975651e-05,
"loss": 0.94318603515625,
"mean_token_accuracy": 0.7765462607145309,
"num_tokens": 1379361.0,
"step": 1100
},
{
"entropy": 0.9908152732253075,
"epoch": 1.0973979470040582,
"grad_norm": 2.453125,
"learning_rate": 1.4660210101585367e-05,
"loss": 0.9593799591064454,
"mean_token_accuracy": 0.7724591562151909,
"num_tokens": 1442332.0,
"step": 1150
},
{
"entropy": 0.9940855634212494,
"epoch": 1.1451420386727142,
"grad_norm": 2.21875,
"learning_rate": 1.4198407318162977e-05,
"loss": 0.9572904968261718,
"mean_token_accuracy": 0.7741717839241028,
"num_tokens": 1506048.0,
"step": 1200
},
{
"entropy": 0.997409186065197,
"epoch": 1.1928861303413703,
"grad_norm": 2.0625,
"learning_rate": 1.3725463810698418e-05,
"loss": 0.962403564453125,
"mean_token_accuracy": 0.7725697284936905,
"num_tokens": 1569529.0,
"step": 1250
},
{
"entropy": 0.9752482569217682,
"epoch": 1.2406302220100263,
"grad_norm": 2.234375,
"learning_rate": 1.3242634562975988e-05,
"loss": 0.9287136840820313,
"mean_token_accuracy": 0.7795223397016525,
"num_tokens": 1632744.0,
"step": 1300
},
{
"entropy": 0.972535012960434,
"epoch": 1.2883743136786823,
"grad_norm": 2.234375,
"learning_rate": 1.2751200791180963e-05,
"loss": 0.9352114105224609,
"mean_token_accuracy": 0.7785275393724441,
"num_tokens": 1695454.0,
"step": 1350
},
{
"entropy": 0.9388299316167832,
"epoch": 1.3361184053473383,
"grad_norm": 2.15625,
"learning_rate": 1.225246654411605e-05,
"loss": 0.9005762481689453,
"mean_token_accuracy": 0.7845277190208435,
"num_tokens": 1759868.0,
"step": 1400
},
{
"entropy": 0.9462847051024437,
"epoch": 1.3838624970159943,
"grad_norm": 2.234375,
"learning_rate": 1.1747755242830204e-05,
"loss": 0.9043065643310547,
"mean_token_accuracy": 0.782291199862957,
"num_tokens": 1820879.0,
"step": 1450
},
{
"entropy": 0.9308868950605392,
"epoch": 1.4316065886846503,
"grad_norm": 2.046875,
"learning_rate": 1.1238406168841981e-05,
"loss": 0.89322265625,
"mean_token_accuracy": 0.7880325165390968,
"num_tokens": 1883307.0,
"step": 1500
},
{
"entropy": 0.9272586759924889,
"epoch": 1.4793506803533063,
"grad_norm": 2.09375,
"learning_rate": 1.0725770910276218e-05,
"loss": 0.8859436798095703,
"mean_token_accuracy": 0.7878314164280892,
"num_tokens": 1946095.0,
"step": 1550
},
{
"entropy": 0.9405490574240685,
"epoch": 1.5270947720219623,
"grad_norm": 2.09375,
"learning_rate": 1.0211209775344377e-05,
"loss": 0.8991239929199218,
"mean_token_accuracy": 0.787108507156372,
"num_tokens": 2009039.0,
"step": 1600
},
{
"entropy": 0.9246599149703979,
"epoch": 1.5748388636906183,
"grad_norm": 2.1875,
"learning_rate": 9.696088182685638e-06,
"loss": 0.8893150329589844,
"mean_token_accuracy": 0.787010528743267,
"num_tokens": 2070866.0,
"step": 1650
},
{
"entropy": 0.9010607555508614,
"epoch": 1.6225829553592743,
"grad_norm": 2.5625,
"learning_rate": 9.18177303814717e-06,
"loss": 0.8644073486328125,
"mean_token_accuracy": 0.7935708364844323,
"num_tokens": 2133531.0,
"step": 1700
},
{
"entropy": 0.9290094000101089,
"epoch": 1.6703270470279303,
"grad_norm": 1.984375,
"learning_rate": 8.66962910761798e-06,
"loss": 0.8792234039306641,
"mean_token_accuracy": 0.7916052675247193,
"num_tokens": 2196212.0,
"step": 1750
},
{
"entropy": 0.9067334878444672,
"epoch": 1.7180711386965863,
"grad_norm": 2.0625,
"learning_rate": 8.161015395541311e-06,
"loss": 0.8666587829589844,
"mean_token_accuracy": 0.7931482782959938,
"num_tokens": 2259117.0,
"step": 1800
},
{
"entropy": 0.9192495954036712,
"epoch": 1.7658152303652423,
"grad_norm": 2.421875,
"learning_rate": 7.65728153871533e-06,
"loss": 0.8764252471923828,
"mean_token_accuracy": 0.792299057841301,
"num_tokens": 2320595.0,
"step": 1850
},
{
"entropy": 0.9041965058445931,
"epoch": 1.8135593220338984,
"grad_norm": 2.34375,
"learning_rate": 7.159764224951467e-06,
"loss": 0.8590383148193359,
"mean_token_accuracy": 0.7972308859229088,
"num_tokens": 2384167.0,
"step": 1900
},
{
"entropy": 0.9144112569093704,
"epoch": 1.8613034137025544,
"grad_norm": 2.21875,
"learning_rate": 6.669783646093635e-06,
"loss": 0.8813047790527344,
"mean_token_accuracy": 0.7919116005301475,
"num_tokens": 2444983.0,
"step": 1950
},
{
"entropy": 0.8830939248204231,
"epoch": 1.9090475053712104,
"grad_norm": 2.15625,
"learning_rate": 6.188639994810476e-06,
"loss": 0.8278516387939453,
"mean_token_accuracy": 0.8022209334373475,
"num_tokens": 2508413.0,
"step": 2000
},
{
"entropy": 0.9015234977006912,
"epoch": 1.9567915970398664,
"grad_norm": 2.1875,
"learning_rate": 5.717610014456644e-06,
"loss": 0.8635023498535156,
"mean_token_accuracy": 0.7982774981856346,
"num_tokens": 2570521.0,
"step": 2050
},
{
"epoch": 2.0,
"eval_entropy": 1.0000475248545746,
"eval_loss": 1.178954005241394,
"eval_mean_token_accuracy": 0.74215727841773,
"eval_num_tokens": 2628024.0,
"eval_runtime": 52.7405,
"eval_samples_per_second": 16.723,
"eval_steps_per_second": 16.723,
"step": 2096
}
],
"logging_steps": 50,
"max_steps": 3144,
"num_input_tokens_seen": 0,
"num_train_epochs": 3,
"save_steps": 500,
"stateful_callbacks": {
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": true,
"should_training_stop": false
},
"attributes": {}
}
},
"total_flos": 1.5924789720612864e+17,
"train_batch_size": 4,
"trial_name": null,
"trial_params": null
}

3
checkpoint-2096/training_args.bin Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:2a3ea84f94e96c81578a690800ff8f79d92cb1baf91ae820778832d6af60f008
size 5304