404 lines
14 KiB
JSON
404 lines
14 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 0.06652991073903643,
|
|
"eval_steps": 500,
|
|
"global_step": 75,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0.004435327382602428,
|
|
"grad_norm": 14.0,
|
|
"learning_rate": 2.5e-07,
|
|
"logits/chosen": -2.144705295562744,
|
|
"logits/rejected": -2.0726897716522217,
|
|
"logps/chosen": -0.2952641248703003,
|
|
"logps/rejected": -0.3042605519294739,
|
|
"loss": 0.9013038873672485,
|
|
"loss/core": 0.8998275995254517,
|
|
"loss/preference_sft": 0.2952641248703003,
|
|
"rewards/accuracies": 0.800000011920929,
|
|
"rewards/chosen": 2.2125720977783203,
|
|
"rewards/margins": 1.2788416147232056,
|
|
"rewards/rejected": 0.9337307810783386,
|
|
"ronpo/logit": 0.1278841495513916,
|
|
"ronpo/residual": -0.8721157908439636,
|
|
"ronpo/residual_abs": 0.9368273615837097,
|
|
"ronpo/target": 1.0,
|
|
"ronpo/target_abs": 1.0,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 5
|
|
},
|
|
{
|
|
"epoch": 0.008870654765204856,
|
|
"grad_norm": 210.0,
|
|
"learning_rate": 4.997252228714278e-07,
|
|
"logits/chosen": -2.165788412094116,
|
|
"logits/rejected": -2.1329445838928223,
|
|
"logps/chosen": -0.24870172142982483,
|
|
"logps/rejected": -0.28110265731811523,
|
|
"loss": 0.9554009437561035,
|
|
"loss/core": 0.9541573524475098,
|
|
"loss/preference_sft": 0.24870172142982483,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 3.0928869247436523,
|
|
"rewards/margins": 1.3761483430862427,
|
|
"rewards/rejected": 1.7167381048202515,
|
|
"ronpo/logit": 0.13761483132839203,
|
|
"ronpo/residual": -0.862385094165802,
|
|
"ronpo/residual_abs": 0.9450987577438354,
|
|
"ronpo/target": 1.0,
|
|
"ronpo/target_abs": 1.0,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 10
|
|
},
|
|
{
|
|
"epoch": 0.013305982147807285,
|
|
"grad_norm": 11.3125,
|
|
"learning_rate": 4.9017129689421e-07,
|
|
"logits/chosen": -2.123521089553833,
|
|
"logits/rejected": -2.0528640747070312,
|
|
"logps/chosen": -0.30137166380882263,
|
|
"logps/rejected": -0.2893705666065216,
|
|
"loss": 0.9385660290718079,
|
|
"loss/core": 0.937059223651886,
|
|
"loss/preference_sft": 0.30137166380882263,
|
|
"rewards/accuracies": 0.862500011920929,
|
|
"rewards/chosen": 1.6111516952514648,
|
|
"rewards/margins": 0.4742206931114197,
|
|
"rewards/rejected": 1.13693106174469,
|
|
"ronpo/logit": 0.04742206260561943,
|
|
"ronpo/residual": -0.9525778889656067,
|
|
"ronpo/residual_abs": 0.9525778889656067,
|
|
"ronpo/target": 1.0,
|
|
"ronpo/target_abs": 1.0,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 15
|
|
},
|
|
{
|
|
"epoch": 0.017741309530409713,
|
|
"grad_norm": 12.5,
|
|
"learning_rate": 4.6747659310219757e-07,
|
|
"logits/chosen": -2.2110671997070312,
|
|
"logits/rejected": -2.0166139602661133,
|
|
"logps/chosen": -0.2711886465549469,
|
|
"logps/rejected": -0.29270288348197937,
|
|
"loss": 1.0191020965576172,
|
|
"loss/core": 1.0177462100982666,
|
|
"loss/preference_sft": 0.2711886465549469,
|
|
"rewards/accuracies": 0.8125,
|
|
"rewards/chosen": 2.056633949279785,
|
|
"rewards/margins": 0.49368467926979065,
|
|
"rewards/rejected": 1.5629494190216064,
|
|
"ronpo/logit": 0.04936846345663071,
|
|
"ronpo/residual": -0.9506314992904663,
|
|
"ronpo/residual_abs": 0.9506314992904663,
|
|
"ronpo/target": 1.0,
|
|
"ronpo/target_abs": 1.0,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 20
|
|
},
|
|
{
|
|
"epoch": 0.02217663691301214,
|
|
"grad_norm": 6.59375,
|
|
"learning_rate": 4.3288283381591725e-07,
|
|
"logits/chosen": -2.171949625015259,
|
|
"logits/rejected": -2.0527703762054443,
|
|
"logps/chosen": -0.28833886981010437,
|
|
"logps/rejected": -0.29443103075027466,
|
|
"loss": 0.9356898069381714,
|
|
"loss/core": 0.934248149394989,
|
|
"loss/preference_sft": 0.28833886981010437,
|
|
"rewards/accuracies": 0.8500000238418579,
|
|
"rewards/chosen": 1.5931298732757568,
|
|
"rewards/margins": 0.5489386320114136,
|
|
"rewards/rejected": 1.0441911220550537,
|
|
"ronpo/logit": 0.0548938624560833,
|
|
"ronpo/residual": -0.945106029510498,
|
|
"ronpo/residual_abs": 0.945106029510498,
|
|
"ronpo/target": 1.0,
|
|
"ronpo/target_abs": 1.0,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 25
|
|
},
|
|
{
|
|
"epoch": 0.02661196429561457,
|
|
"grad_norm": 49.75,
|
|
"learning_rate": 3.882827885312998e-07,
|
|
"logits/chosen": -2.277024269104004,
|
|
"logits/rejected": -2.2178688049316406,
|
|
"logps/chosen": -0.30705174803733826,
|
|
"logps/rejected": -0.3033962845802307,
|
|
"loss": 1.0691256523132324,
|
|
"loss/core": 1.0675902366638184,
|
|
"loss/preference_sft": 0.30705174803733826,
|
|
"rewards/accuracies": 0.949999988079071,
|
|
"rewards/chosen": 2.114480495452881,
|
|
"rewards/margins": 1.2642418146133423,
|
|
"rewards/rejected": 0.8502384424209595,
|
|
"ronpo/logit": 0.126424178481102,
|
|
"ronpo/residual": -0.8735758662223816,
|
|
"ronpo/residual_abs": 0.9844897389411926,
|
|
"ronpo/target": 1.0,
|
|
"ronpo/target_abs": 1.0,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 30
|
|
},
|
|
{
|
|
"epoch": 0.031047291678216998,
|
|
"grad_norm": 19.75,
|
|
"learning_rate": 3.361167125710832e-07,
|
|
"logits/chosen": -2.1667983531951904,
|
|
"logits/rejected": -2.209552764892578,
|
|
"logps/chosen": -0.29950323700904846,
|
|
"logps/rejected": -0.3074646592140198,
|
|
"loss": 0.9153361320495605,
|
|
"loss/core": 0.9138385057449341,
|
|
"loss/preference_sft": 0.29950323700904846,
|
|
"rewards/accuracies": 0.925000011920929,
|
|
"rewards/chosen": 1.2793786525726318,
|
|
"rewards/margins": 0.6898711919784546,
|
|
"rewards/rejected": 0.5895074009895325,
|
|
"ronpo/logit": 0.06898711621761322,
|
|
"ronpo/residual": -0.9310129284858704,
|
|
"ronpo/residual_abs": 0.9501181840896606,
|
|
"ronpo/target": 1.0,
|
|
"ronpo/target_abs": 1.0,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 35
|
|
},
|
|
{
|
|
"epoch": 0.035482619060819426,
|
|
"grad_norm": 14.5,
|
|
"learning_rate": 2.7923883049302066e-07,
|
|
"logits/chosen": -2.0911293029785156,
|
|
"logits/rejected": -1.996106743812561,
|
|
"logps/chosen": -0.2962394654750824,
|
|
"logps/rejected": -0.2883327603340149,
|
|
"loss": 1.2223423719406128,
|
|
"loss/core": 1.2208611965179443,
|
|
"loss/preference_sft": 0.2962394654750824,
|
|
"rewards/accuracies": 0.8999999761581421,
|
|
"rewards/chosen": 2.883702516555786,
|
|
"rewards/margins": 1.178934097290039,
|
|
"rewards/rejected": 1.7047685384750366,
|
|
"ronpo/logit": 0.11789341270923615,
|
|
"ronpo/residual": -0.882106602191925,
|
|
"ronpo/residual_abs": 0.9954751133918762,
|
|
"ronpo/target": 1.0,
|
|
"ronpo/target_abs": 1.0,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 40
|
|
},
|
|
{
|
|
"epoch": 0.03991794644342186,
|
|
"grad_norm": 17.5,
|
|
"learning_rate": 2.2076116950697937e-07,
|
|
"logits/chosen": -2.2753586769104004,
|
|
"logits/rejected": -2.194331645965576,
|
|
"logps/chosen": -0.2942467927932739,
|
|
"logps/rejected": -0.2941977381706238,
|
|
"loss": 0.960641086101532,
|
|
"loss/core": 0.959169864654541,
|
|
"loss/preference_sft": 0.2942467927932739,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 2.3440639972686768,
|
|
"rewards/margins": 1.245705008506775,
|
|
"rewards/rejected": 1.0983591079711914,
|
|
"ronpo/logit": 0.12457048892974854,
|
|
"ronpo/residual": -0.8754295110702515,
|
|
"ronpo/residual_abs": 0.9467538595199585,
|
|
"ronpo/target": 1.0,
|
|
"ronpo/target_abs": 1.0,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 45
|
|
},
|
|
{
|
|
"epoch": 0.04435327382602428,
|
|
"grad_norm": 6.03125,
|
|
"learning_rate": 1.6388328742891678e-07,
|
|
"logits/chosen": -2.3468501567840576,
|
|
"logits/rejected": -2.2160422801971436,
|
|
"logps/chosen": -0.2907429337501526,
|
|
"logps/rejected": -0.299108624458313,
|
|
"loss": 1.6810009479522705,
|
|
"loss/core": 1.6795470714569092,
|
|
"loss/preference_sft": 0.2907429337501526,
|
|
"rewards/accuracies": 0.8374999761581421,
|
|
"rewards/chosen": 1.491801381111145,
|
|
"rewards/margins": -0.29926931858062744,
|
|
"rewards/rejected": 1.7910706996917725,
|
|
"ronpo/logit": -0.029926935210824013,
|
|
"ronpo/residual": -1.0299268960952759,
|
|
"ronpo/residual_abs": 1.0562489032745361,
|
|
"ronpo/target": 1.0,
|
|
"ronpo/target_abs": 1.0,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 50
|
|
},
|
|
{
|
|
"epoch": 0.048788601208626714,
|
|
"grad_norm": 5.1875,
|
|
"learning_rate": 1.1171721146870014e-07,
|
|
"logits/chosen": -2.2220540046691895,
|
|
"logits/rejected": -2.159651279449463,
|
|
"logps/chosen": -0.29590311646461487,
|
|
"logps/rejected": -0.3085595369338989,
|
|
"loss": 1.2529202699661255,
|
|
"loss/core": 1.2514407634735107,
|
|
"loss/preference_sft": 0.29590311646461487,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 1.7223021984100342,
|
|
"rewards/margins": 0.2412434071302414,
|
|
"rewards/rejected": 1.4810588359832764,
|
|
"ronpo/logit": 0.02412433736026287,
|
|
"ronpo/residual": -0.9758756756782532,
|
|
"ronpo/residual_abs": 1.0081208944320679,
|
|
"ronpo/target": 1.0,
|
|
"ronpo/target_abs": 1.0,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 55
|
|
},
|
|
{
|
|
"epoch": 0.05322392859122914,
|
|
"grad_norm": 121.0,
|
|
"learning_rate": 6.711716618408281e-08,
|
|
"logits/chosen": -1.9089536666870117,
|
|
"logits/rejected": -1.977799654006958,
|
|
"logps/chosen": -0.3068787455558777,
|
|
"logps/rejected": -0.29860004782676697,
|
|
"loss": 1.5921037197113037,
|
|
"loss/core": 1.590569257736206,
|
|
"loss/preference_sft": 0.3068787455558777,
|
|
"rewards/accuracies": 0.9125000238418579,
|
|
"rewards/chosen": 3.075084924697876,
|
|
"rewards/margins": 1.1153666973114014,
|
|
"rewards/rejected": 1.9597183465957642,
|
|
"ronpo/logit": 0.1115366593003273,
|
|
"ronpo/residual": -0.8884633183479309,
|
|
"ronpo/residual_abs": 0.9980802536010742,
|
|
"ronpo/target": 1.0,
|
|
"ronpo/target_abs": 1.0,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 60
|
|
},
|
|
{
|
|
"epoch": 0.05765925597383157,
|
|
"grad_norm": 25.25,
|
|
"learning_rate": 3.2523406897802444e-08,
|
|
"logits/chosen": -2.136443614959717,
|
|
"logits/rejected": -2.083848476409912,
|
|
"logps/chosen": -0.2734604477882385,
|
|
"logps/rejected": -0.28085511922836304,
|
|
"loss": 0.8904159665107727,
|
|
"loss/core": 0.88904869556427,
|
|
"loss/preference_sft": 0.2734604477882385,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.9123761653900146,
|
|
"rewards/margins": 0.9041536450386047,
|
|
"rewards/rejected": 1.0082225799560547,
|
|
"ronpo/logit": 0.09041537344455719,
|
|
"ronpo/residual": -0.909584641456604,
|
|
"ronpo/residual_abs": 0.93060702085495,
|
|
"ronpo/target": 1.0,
|
|
"ronpo/target_abs": 1.0,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 65
|
|
},
|
|
{
|
|
"epoch": 0.062094583356433995,
|
|
"grad_norm": 8.1875,
|
|
"learning_rate": 9.828703105789981e-09,
|
|
"logits/chosen": -2.1384823322296143,
|
|
"logits/rejected": -2.0494384765625,
|
|
"logps/chosen": -0.29097050428390503,
|
|
"logps/rejected": -0.3023843765258789,
|
|
"loss": 0.8998498916625977,
|
|
"loss/core": 0.8983950614929199,
|
|
"loss/preference_sft": 0.29097050428390503,
|
|
"rewards/accuracies": 0.887499988079071,
|
|
"rewards/chosen": 1.2245758771896362,
|
|
"rewards/margins": 0.5744353532791138,
|
|
"rewards/rejected": 0.6501404643058777,
|
|
"ronpo/logit": 0.057443536818027496,
|
|
"ronpo/residual": -0.9425565004348755,
|
|
"ronpo/residual_abs": 0.9425565004348755,
|
|
"ronpo/target": 1.0,
|
|
"ronpo/target_abs": 1.0,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 70
|
|
},
|
|
{
|
|
"epoch": 0.06652991073903643,
|
|
"grad_norm": 59.25,
|
|
"learning_rate": 2.7477712857215675e-10,
|
|
"logits/chosen": -2.183204412460327,
|
|
"logits/rejected": -2.0384721755981445,
|
|
"logps/chosen": -0.27401086688041687,
|
|
"logps/rejected": -0.27478593587875366,
|
|
"loss": 1.1840168237686157,
|
|
"loss/core": 1.182646632194519,
|
|
"loss/preference_sft": 0.27401086688041687,
|
|
"rewards/accuracies": 0.875,
|
|
"rewards/chosen": 1.6213176250457764,
|
|
"rewards/margins": 0.029666978865861893,
|
|
"rewards/rejected": 1.5916506052017212,
|
|
"ronpo/logit": 0.002966696862131357,
|
|
"ronpo/residual": -0.9970332980155945,
|
|
"ronpo/residual_abs": 0.9970332980155945,
|
|
"ronpo/target": 1.0,
|
|
"ronpo/target_abs": 1.0,
|
|
"ronpo/weight": 1.0,
|
|
"ronpo/weight_max": 1.0,
|
|
"step": 75
|
|
},
|
|
{
|
|
"epoch": 0.06652991073903643,
|
|
"step": 75,
|
|
"total_flos": 0.0,
|
|
"train_loss": 1.094520975748698,
|
|
"train_runtime": 689.8738,
|
|
"train_samples_per_second": 1.739,
|
|
"train_steps_per_second": 0.109
|
|
}
|
|
],
|
|
"logging_steps": 5,
|
|
"max_steps": 75,
|
|
"num_input_tokens_seen": 0,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 1000,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": true
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 1,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|