1059 lines
32 KiB
JSON
1059 lines
32 KiB
JSON
{
|
|
"adapter_path": "runs/qwen3_14b_restricted_score_ce/adapter",
|
|
"allowed_scores": [
|
|
"1",
|
|
"2",
|
|
"3",
|
|
"4",
|
|
"5"
|
|
],
|
|
"artifacts": {
|
|
"config.yaml": {
|
|
"data": {
|
|
"columns": {
|
|
"benchmark": "benchmark",
|
|
"example_id": "instance_id",
|
|
"judge_1_score": "score_gpt4o",
|
|
"judge_2_score": "score_sonnet",
|
|
"model_id": "model_id",
|
|
"question": "question",
|
|
"reference_answer": "ground_truth",
|
|
"response": "response",
|
|
"source": "source",
|
|
"task": "task"
|
|
},
|
|
"filters": {
|
|
"max_disagreement": 1,
|
|
"max_response_tokens": 4096,
|
|
"on_sequence_overflow": "skip"
|
|
},
|
|
"path": "data/processed/response_scores.parquet",
|
|
"prepared_dir": null,
|
|
"preprocessing_num_workers": 40,
|
|
"token_length_batch_size": 2048,
|
|
"tokenizers_parallelism": true
|
|
},
|
|
"distributed": {
|
|
"backend": "nccl",
|
|
"deepspeed": {
|
|
"config_overrides": {},
|
|
"gradient_clipping": "auto",
|
|
"offload_optimizer_device": "none",
|
|
"offload_param_device": "none",
|
|
"stage3_gather_16bit_weights_on_model_save": true,
|
|
"zero_stage": 2
|
|
},
|
|
"enabled": true,
|
|
"find_unused_parameters": false,
|
|
"fsdp": {
|
|
"activation_checkpointing": true,
|
|
"sharding_strategy": "full_shard",
|
|
"transformer_layer_cls_to_wrap": null,
|
|
"use_orig_params": true
|
|
},
|
|
"gradient_checkpointing": false,
|
|
"mixed_precision": "bf16",
|
|
"strategy": "ddp"
|
|
},
|
|
"evaluation": {
|
|
"length_buckets": [
|
|
0,
|
|
256,
|
|
512,
|
|
1024,
|
|
2048,
|
|
3072,
|
|
4096,
|
|
1000000000
|
|
]
|
|
},
|
|
"hub": {
|
|
"commit_message": "Upload ADeLe distilled judge",
|
|
"create_pr": false,
|
|
"local_checkpoint_dir": null,
|
|
"max_shard_size": "5GB",
|
|
"output_staging_dir": null,
|
|
"private": false,
|
|
"repo_id": null
|
|
},
|
|
"inference": {
|
|
"allow_base_model": true,
|
|
"allowed_scores": [
|
|
"1",
|
|
"2",
|
|
"3",
|
|
"4",
|
|
"5"
|
|
],
|
|
"batch_size": 64,
|
|
"binary_threshold": 3,
|
|
"generation_fallback": false,
|
|
"method": "restricted_continuation_logprobs_fast",
|
|
"require_adapter": false
|
|
},
|
|
"model": {
|
|
"adapter_path": null,
|
|
"attn_implementation": "sdpa",
|
|
"model_name_or_path": "Qwen/Qwen3-14B",
|
|
"revision": null,
|
|
"thinking_mode": {
|
|
"apply_if_supported": true,
|
|
"enabled": false
|
|
},
|
|
"trust_remote_code": true
|
|
},
|
|
"project": {
|
|
"output_dir": "runs/qwen3_14b_restricted_score_ce",
|
|
"run_name": "qwen3_14b_restricted_score_ce",
|
|
"seed": 42
|
|
},
|
|
"prompt": {
|
|
"system_prompt": "You are an answer quality evaluator. You will receive a question, a reference\nanswer, and a model response, each delimited by ### headers.\n\nRate the model response on the following scale:\n1 - Surely incorrect: the response is wrong or completely misses the point.\n2 - Likely incorrect: the response has significant errors or critical omissions.\n3 - Minimally correct / sufficient: the response is partially correct or borderline.\n4 - Likely correct: the response is mostly correct with minor issues.\n5 - Surely correct: the response is fully correct and complete.\n\nRules:\n- Base your rating strictly on correctness relative to the reference answer.\n- Ignore writing style, verbosity, or formatting unless they affect correctness.\n- If the final answer is correct but the reasoning contains substantive errors, lower the rating according to the severity of those errors.\n- Output only a single integer from 1 to 5. No explanation. No punctuation.\n"
|
|
},
|
|
"split": {
|
|
"held_out_model": null,
|
|
"lomo_validation_fraction": 0.05,
|
|
"lomo_validation_max_examples": 30000,
|
|
"lomo_validation_seed": 42,
|
|
"mode": "fixed_by_model",
|
|
"train_models": "auto_except_val_test",
|
|
"validation_models": [
|
|
"gemini-3-flash",
|
|
"DK-R1-Dist-Qwen-14B",
|
|
"llama3d2-3b"
|
|
]
|
|
},
|
|
"training": {
|
|
"cache_tokenized_datasets": true,
|
|
"class_weighting": null,
|
|
"dtype": "bfloat16",
|
|
"eval_steps": 500,
|
|
"eval_subset_seed": 42,
|
|
"eval_subset_size": null,
|
|
"eval_subset_strategy": "stratified",
|
|
"eval_subset_stratify_columns": [
|
|
"model_id",
|
|
"target_score"
|
|
],
|
|
"gradient_accumulation_steps": 4,
|
|
"learning_rate": 3e-05,
|
|
"length_column_name": "length",
|
|
"load_in_4bit": true,
|
|
"logging_steps": 10,
|
|
"lora_alpha": 64,
|
|
"lora_dropout": 0.0,
|
|
"lora_r": 32,
|
|
"loss": {
|
|
"class_weights": null,
|
|
"lambda_binary": 0.5,
|
|
"type": "ce_5way"
|
|
},
|
|
"lr_scheduler_type": "cosine",
|
|
"max_grad_norm": 1.0,
|
|
"max_seq_length": 4096,
|
|
"num_train_epochs": 1,
|
|
"objective": "restricted_score_ce",
|
|
"optim": "adamw_8bit",
|
|
"packing": false,
|
|
"per_device_eval_batch_size": 2,
|
|
"per_device_train_batch_size": 2,
|
|
"resume_from_checkpoint": null,
|
|
"save_steps": 500,
|
|
"save_total_limit": 10,
|
|
"score_class_weights": null,
|
|
"seed": 42,
|
|
"target_modules": "auto",
|
|
"train_sampling_strategy": "random",
|
|
"warmup_ratio": 0.03,
|
|
"weight_decay": 0.0
|
|
}
|
|
},
|
|
"dataset_filtering_report.json": {
|
|
"after_disagreement_filter": 287811,
|
|
"after_response_length_filter": 285310,
|
|
"effective_prompt_budget_tokens": 0,
|
|
"examples_after_sequence_filter": 285158,
|
|
"examples_before_sequence_filter": 285310,
|
|
"filter_stage_distributions": {
|
|
"after_disagreement_filter": {
|
|
"benchmark": {
|
|
"ChemLLMBench": 28771,
|
|
"Civil Service Examination": 7234,
|
|
"Data Analysis": 617,
|
|
"Date Arithmetic": 9167,
|
|
"GRE & GMAT": 3737,
|
|
"LSAT": 16530,
|
|
"Language": 470,
|
|
"MCTACO": 3734,
|
|
"MMLU-Pro": 98811,
|
|
"Math": 3001,
|
|
"MedCalcBench": 9804,
|
|
"MenatQA": 11777,
|
|
"OmniMath": 29517,
|
|
"Reasoning": 975,
|
|
"SAT": 7645,
|
|
"SciBench": 6360,
|
|
"TempReason": 11519,
|
|
"TimeDial": 6087,
|
|
"TimeQA": 13083,
|
|
"TruthQuest": 18972
|
|
},
|
|
"model_id": {
|
|
"DK-R1-Dist-Qwen-1.5B": 15314,
|
|
"DK-R1-Dist-Qwen-14B": 15352,
|
|
"DK-R1-Dist-Qwen-32B": 14922,
|
|
"DK-R1-Dist-Qwen-7B": 15215,
|
|
"gemini-2.5-flash": 14468,
|
|
"gemini-3-flash": 15106,
|
|
"gemini-3.1-pro": 15158,
|
|
"gpt-35-turbo": 15163,
|
|
"gpt-5.2": 14241,
|
|
"gpt4o": 16106,
|
|
"llama3d1-405b": 15418,
|
|
"llama3d2-11b": 15272,
|
|
"llama3d2-1b": 15274,
|
|
"llama3d2-3b": 15288,
|
|
"llama3d2-90b": 15393,
|
|
"llama4-17B-128E": 15161,
|
|
"o1-mini": 15358,
|
|
"o1_re=low": 15044,
|
|
"o3-mini": 14558
|
|
},
|
|
"num_examples": 287811,
|
|
"target_binary": {
|
|
"CORRECT": 192174,
|
|
"INCORRECT": 95637
|
|
},
|
|
"target_score": {
|
|
"1": 89963,
|
|
"2": 5674,
|
|
"3": 4685,
|
|
"4": 11341,
|
|
"5": 176148
|
|
},
|
|
"task": {
|
|
"AMPS_Hard": 1174,
|
|
"AQuA-RAT": 3737,
|
|
"Algebra": 5977,
|
|
"Applied Mathematics": 5444,
|
|
"Calculus": 535,
|
|
"Chemistry": 2539,
|
|
"Date Arithmetic": 9167,
|
|
"Discrete Mathematics": 5475,
|
|
"E": 6185,
|
|
"Geometry": 5928,
|
|
"I": 6651,
|
|
"LSAT-AR": 3290,
|
|
"LSAT-LR": 8608,
|
|
"LSAT-RC": 4632,
|
|
"LogiQA-en": 7234,
|
|
"MCTACO": 3734,
|
|
"Math": 1901,
|
|
"MenatQA-Counterfactual": 2172,
|
|
"MenatQA-Order": 2738,
|
|
"MenatQA-Scope": 6867,
|
|
"Number Theory": 5639,
|
|
"Physics": 1920,
|
|
"Precalculus": 519,
|
|
"S": 6136,
|
|
"SAT-En": 3643,
|
|
"SAT-Math": 4002,
|
|
"TempReason-L2": 5565,
|
|
"TempReason-L3": 5954,
|
|
"TimeDial": 6087,
|
|
"TimeQA-explicit": 6894,
|
|
"TimeQA-implicit": 6189,
|
|
"biology": 8307,
|
|
"business": 7478,
|
|
"chemistry": 6672,
|
|
"computer science": 6271,
|
|
"connections": 470,
|
|
"cta": 617,
|
|
"date": 491,
|
|
"diagnosis": 211,
|
|
"dosage": 372,
|
|
"economics": 7831,
|
|
"engineering": 5291,
|
|
"health": 7479,
|
|
"history": 5520,
|
|
"lab": 3165,
|
|
"law": 6462,
|
|
"math": 7719,
|
|
"math_comp": 1394,
|
|
"molecule_captioning": 2641,
|
|
"molecule_design": 4313,
|
|
"name_prediction": 8652,
|
|
"olympiad": 433,
|
|
"other": 7799,
|
|
"philosophy": 7334,
|
|
"physical": 3929,
|
|
"physics": 6828,
|
|
"psychology": 7820,
|
|
"reaction_prediction": 6929,
|
|
"retrosynthesis": 6236,
|
|
"risk": 1353,
|
|
"severity": 283,
|
|
"spatial": 582,
|
|
"zebra_puzzle": 393
|
|
}
|
|
},
|
|
"after_response_length_filter": {
|
|
"benchmark": {
|
|
"ChemLLMBench": 27913,
|
|
"Civil Service Examination": 7207,
|
|
"Data Analysis": 617,
|
|
"Date Arithmetic": 9167,
|
|
"GRE & GMAT": 3733,
|
|
"LSAT": 16471,
|
|
"Language": 468,
|
|
"MCTACO": 3733,
|
|
"MMLU-Pro": 98506,
|
|
"Math": 2951,
|
|
"MedCalcBench": 9799,
|
|
"MenatQA": 11762,
|
|
"OmniMath": 28488,
|
|
"Reasoning": 953,
|
|
"SAT": 7645,
|
|
"SciBench": 6270,
|
|
"TempReason": 11519,
|
|
"TimeDial": 6087,
|
|
"TimeQA": 13082,
|
|
"TruthQuest": 18939
|
|
},
|
|
"model_id": {
|
|
"DK-R1-Dist-Qwen-1.5B": 15302,
|
|
"DK-R1-Dist-Qwen-14B": 15351,
|
|
"DK-R1-Dist-Qwen-32B": 12577,
|
|
"DK-R1-Dist-Qwen-7B": 15128,
|
|
"gemini-2.5-flash": 14426,
|
|
"gemini-3-flash": 15104,
|
|
"gemini-3.1-pro": 15158,
|
|
"gpt-35-turbo": 15163,
|
|
"gpt-5.2": 14241,
|
|
"gpt4o": 16106,
|
|
"llama3d1-405b": 15418,
|
|
"llama3d2-11b": 15267,
|
|
"llama3d2-1b": 15272,
|
|
"llama3d2-3b": 15288,
|
|
"llama3d2-90b": 15393,
|
|
"llama4-17B-128E": 15159,
|
|
"o1-mini": 15358,
|
|
"o1_re=low": 15041,
|
|
"o3-mini": 14558
|
|
},
|
|
"num_examples": 285310,
|
|
"target_binary": {
|
|
"CORRECT": 191333,
|
|
"INCORRECT": 93977
|
|
},
|
|
"target_score": {
|
|
"1": 88379,
|
|
"2": 5598,
|
|
"3": 4641,
|
|
"4": 11284,
|
|
"5": 175408
|
|
},
|
|
"task": {
|
|
"AMPS_Hard": 1157,
|
|
"AQuA-RAT": 3733,
|
|
"Algebra": 5814,
|
|
"Applied Mathematics": 5309,
|
|
"Calculus": 516,
|
|
"Chemistry": 2502,
|
|
"Date Arithmetic": 9167,
|
|
"Discrete Mathematics": 5235,
|
|
"E": 6164,
|
|
"Geometry": 5695,
|
|
"I": 6647,
|
|
"LSAT-AR": 3234,
|
|
"LSAT-LR": 8605,
|
|
"LSAT-RC": 4632,
|
|
"LogiQA-en": 7207,
|
|
"MCTACO": 3733,
|
|
"Math": 1887,
|
|
"MenatQA-Counterfactual": 2166,
|
|
"MenatQA-Order": 2737,
|
|
"MenatQA-Scope": 6859,
|
|
"Number Theory": 5411,
|
|
"Physics": 1881,
|
|
"Precalculus": 508,
|
|
"S": 6128,
|
|
"SAT-En": 3643,
|
|
"SAT-Math": 4002,
|
|
"TempReason-L2": 5565,
|
|
"TempReason-L3": 5954,
|
|
"TimeDial": 6087,
|
|
"TimeQA-explicit": 6893,
|
|
"TimeQA-implicit": 6189,
|
|
"biology": 8294,
|
|
"business": 7469,
|
|
"chemistry": 6615,
|
|
"computer science": 6245,
|
|
"connections": 468,
|
|
"cta": 617,
|
|
"date": 491,
|
|
"diagnosis": 211,
|
|
"dosage": 372,
|
|
"economics": 7827,
|
|
"engineering": 5182,
|
|
"health": 7475,
|
|
"history": 5520,
|
|
"lab": 3163,
|
|
"law": 6454,
|
|
"math": 7692,
|
|
"math_comp": 1367,
|
|
"molecule_captioning": 2614,
|
|
"molecule_design": 4306,
|
|
"name_prediction": 8201,
|
|
"olympiad": 427,
|
|
"other": 7797,
|
|
"philosophy": 7330,
|
|
"physical": 3927,
|
|
"physics": 6787,
|
|
"psychology": 7819,
|
|
"reaction_prediction": 6768,
|
|
"retrosynthesis": 6024,
|
|
"risk": 1353,
|
|
"severity": 282,
|
|
"spatial": 572,
|
|
"zebra_puzzle": 381
|
|
}
|
|
},
|
|
"raw": {
|
|
"benchmark": {
|
|
"ChemLLMBench": 32694,
|
|
"Civil Service Examination": 7677,
|
|
"Data Analysis": 627,
|
|
"Date Arithmetic": 9365,
|
|
"GRE & GMAT": 3855,
|
|
"LSAT": 17145,
|
|
"Language": 550,
|
|
"MCTACO": 3891,
|
|
"MMLU-Pro": 102896,
|
|
"Math": 3263,
|
|
"MedCalcBench": 10453,
|
|
"MenatQA": 12674,
|
|
"OmniMath": 31272,
|
|
"Reasoning": 1056,
|
|
"SAT": 7752,
|
|
"SciBench": 6728,
|
|
"TempReason": 12318,
|
|
"TimeDial": 6412,
|
|
"TimeQA": 13681,
|
|
"TruthQuest": 20037
|
|
},
|
|
"model_id": {
|
|
"DK-R1-Dist-Qwen-1.5B": 16108,
|
|
"DK-R1-Dist-Qwen-14B": 16108,
|
|
"DK-R1-Dist-Qwen-32B": 16069,
|
|
"DK-R1-Dist-Qwen-7B": 16108,
|
|
"gemini-2.5-flash": 15785,
|
|
"gemini-3-flash": 16045,
|
|
"gemini-3.1-pro": 15981,
|
|
"gpt-35-turbo": 16100,
|
|
"gpt-5.2": 15050,
|
|
"gpt4o": 16106,
|
|
"llama3d1-405b": 16108,
|
|
"llama3d2-11b": 16108,
|
|
"llama3d2-1b": 16107,
|
|
"llama3d2-3b": 16108,
|
|
"llama3d2-90b": 16108,
|
|
"llama4-17B-128E": 16041,
|
|
"o1-mini": 16108,
|
|
"o1_re=low": 16108,
|
|
"o3-mini": 16090
|
|
},
|
|
"num_examples": 304346,
|
|
"target_binary": {
|
|
"CORRECT": 201528,
|
|
"INCORRECT": 102818
|
|
},
|
|
"target_score": {
|
|
"1": 89963,
|
|
"2": 12855,
|
|
"3": 13029,
|
|
"4": 12351,
|
|
"5": 176148
|
|
},
|
|
"task": {
|
|
"AMPS_Hard": 1292,
|
|
"AQuA-RAT": 3855,
|
|
"Algebra": 6331,
|
|
"Applied Mathematics": 5697,
|
|
"Calculus": 565,
|
|
"Chemistry": 2694,
|
|
"Date Arithmetic": 9365,
|
|
"Discrete Mathematics": 5861,
|
|
"E": 6532,
|
|
"Geometry": 6215,
|
|
"I": 7045,
|
|
"LSAT-AR": 3535,
|
|
"LSAT-LR": 8868,
|
|
"LSAT-RC": 4742,
|
|
"LogiQA-en": 7677,
|
|
"MCTACO": 3891,
|
|
"Math": 1988,
|
|
"MenatQA-Counterfactual": 2440,
|
|
"MenatQA-Order": 2925,
|
|
"MenatQA-Scope": 7309,
|
|
"Number Theory": 6035,
|
|
"Physics": 2046,
|
|
"Precalculus": 568,
|
|
"S": 6460,
|
|
"SAT-En": 3686,
|
|
"SAT-Math": 4066,
|
|
"TempReason-L2": 5960,
|
|
"TempReason-L3": 6358,
|
|
"TimeDial": 6412,
|
|
"TimeQA-explicit": 7137,
|
|
"TimeQA-implicit": 6544,
|
|
"biology": 8483,
|
|
"business": 7783,
|
|
"chemistry": 6981,
|
|
"computer science": 6542,
|
|
"connections": 550,
|
|
"cta": 627,
|
|
"date": 513,
|
|
"diagnosis": 266,
|
|
"dosage": 380,
|
|
"economics": 8104,
|
|
"engineering": 5614,
|
|
"health": 7793,
|
|
"history": 5729,
|
|
"lab": 3357,
|
|
"law": 6797,
|
|
"math": 8070,
|
|
"math_comp": 1480,
|
|
"molecule_captioning": 3040,
|
|
"molecule_design": 5585,
|
|
"name_prediction": 9036,
|
|
"olympiad": 491,
|
|
"other": 8129,
|
|
"philosophy": 7617,
|
|
"physical": 4029,
|
|
"physics": 7160,
|
|
"psychology": 8094,
|
|
"reaction_prediction": 7819,
|
|
"retrosynthesis": 7214,
|
|
"risk": 1587,
|
|
"severity": 321,
|
|
"spatial": 638,
|
|
"zebra_puzzle": 418
|
|
}
|
|
}
|
|
},
|
|
"kept_prompt_token_length": {
|
|
"max": 4093,
|
|
"mean": 853.0326345394483,
|
|
"min": 252,
|
|
"p50": 769.0,
|
|
"p75": 988.0,
|
|
"p90": 1330.0,
|
|
"p95": 1659.0,
|
|
"p99": 2596.0
|
|
},
|
|
"kept_sequence_length": {
|
|
"max": 4095,
|
|
"mean": 855.0326345394483,
|
|
"min": 254,
|
|
"p50": 771.0,
|
|
"p75": 990.0,
|
|
"p90": 1332.0,
|
|
"p95": 1661.0,
|
|
"p99": 2598.0
|
|
},
|
|
"length_filter_warnings": [
|
|
"max_response_tokens is greater than or equal to max_seq_length; examples can pass the response cap while having no room for the formatted prompt.",
|
|
"Effective prompt budget is only 0 tokens after reserving the response cap; consider increasing training.max_seq_length."
|
|
],
|
|
"max_disagreement": 1,
|
|
"max_response_tokens": 4096,
|
|
"max_seq_length": 4096,
|
|
"on_sequence_overflow": "skip",
|
|
"overflowed_response_token_length": {
|
|
"max": 4096,
|
|
"mean": 3629.6052631578946,
|
|
"min": 1582,
|
|
"p50": 3875.5,
|
|
"p75": 3996.5,
|
|
"p90": 4051.0,
|
|
"p95": 4075.6,
|
|
"p99": 4091.45
|
|
},
|
|
"raw_examples": 304346,
|
|
"removed_by_disagreement": 16535,
|
|
"removed_by_disagreement_pct": 5.432961169195587,
|
|
"removed_by_response_length": 2501,
|
|
"removed_by_response_length_pct": 0.8689730413361546,
|
|
"sequence_overflow_count": 152,
|
|
"sequence_overflow_pct": 0.05327538466930706,
|
|
"sequence_overflow_reason": "Full chat-formatted sequence exceeded max_seq_length after response filtering. This includes system prompt, question, reference answer, model response, and chat-template tokens, plus target score tokens."
|
|
},
|
|
"inference_config.yaml": {
|
|
"data": {
|
|
"columns": {
|
|
"benchmark": "benchmark",
|
|
"example_id": "instance_id",
|
|
"judge_1_score": "score_gpt4o",
|
|
"judge_2_score": "score_sonnet",
|
|
"model_id": "model_id",
|
|
"question": "question",
|
|
"reference_answer": "ground_truth",
|
|
"response": "response",
|
|
"source": "source",
|
|
"task": "task"
|
|
},
|
|
"filters": {
|
|
"max_disagreement": 1,
|
|
"max_response_tokens": 4096,
|
|
"on_sequence_overflow": "skip"
|
|
},
|
|
"path": "data/processed/response_scores.parquet",
|
|
"prepared_dir": null,
|
|
"preprocessing_num_workers": 5,
|
|
"token_length_batch_size": 2048,
|
|
"tokenizers_parallelism": true
|
|
},
|
|
"distributed": {
|
|
"backend": "nccl",
|
|
"deepspeed": {
|
|
"config_overrides": {},
|
|
"gradient_clipping": "auto",
|
|
"offload_optimizer_device": "none",
|
|
"offload_param_device": "none",
|
|
"stage3_gather_16bit_weights_on_model_save": true,
|
|
"zero_stage": 2
|
|
},
|
|
"enabled": true,
|
|
"find_unused_parameters": false,
|
|
"fsdp": {
|
|
"activation_checkpointing": true,
|
|
"sharding_strategy": "full_shard",
|
|
"transformer_layer_cls_to_wrap": null,
|
|
"use_orig_params": true
|
|
},
|
|
"gradient_checkpointing": true,
|
|
"mixed_precision": "bf16",
|
|
"strategy": "ddp"
|
|
},
|
|
"evaluation": {
|
|
"length_buckets": [
|
|
0,
|
|
256,
|
|
512,
|
|
1024,
|
|
2048,
|
|
3072,
|
|
4096,
|
|
1000000000
|
|
]
|
|
},
|
|
"hub": {
|
|
"commit_message": "Upload ADeLe distilled judge",
|
|
"create_pr": false,
|
|
"local_checkpoint_dir": null,
|
|
"max_shard_size": "5GB",
|
|
"output_staging_dir": null,
|
|
"private": false,
|
|
"repo_id": null
|
|
},
|
|
"inference": {
|
|
"allow_base_model": true,
|
|
"allowed_scores": [
|
|
"1",
|
|
"2",
|
|
"3",
|
|
"4",
|
|
"5"
|
|
],
|
|
"batch_size": 64,
|
|
"binary_threshold": 3,
|
|
"generation_fallback": false,
|
|
"method": "restricted_continuation_logprobs_fast",
|
|
"require_adapter": false
|
|
},
|
|
"model": {
|
|
"adapter_path": "runs/qwen3_14b_restricted_score_ce/adapter",
|
|
"attn_implementation": "sdpa",
|
|
"model_name_or_path": "Qwen/Qwen3-14B",
|
|
"revision": null,
|
|
"thinking_mode": {
|
|
"apply_if_supported": true,
|
|
"enabled": false
|
|
},
|
|
"trust_remote_code": true
|
|
},
|
|
"project": {
|
|
"output_dir": "runs/qwen3_14b_restricted_score_ce",
|
|
"run_name": "qwen3_14b_restricted_score_ce",
|
|
"seed": 42
|
|
},
|
|
"prompt": {
|
|
"system_prompt": "You are an answer quality evaluator. You will receive a question, a reference\nanswer, and a model response, each delimited by ### headers.\n\nRate the model response on the following scale:\n1 - Surely incorrect: the response is wrong or completely misses the point.\n2 - Likely incorrect: the response has significant errors or critical omissions.\n3 - Minimally correct / sufficient: the response is partially correct or borderline.\n4 - Likely correct: the response is mostly correct with minor issues.\n5 - Surely correct: the response is fully correct and complete.\n\nRules:\n- Base your rating strictly on correctness relative to the reference answer.\n- Ignore writing style, verbosity, or formatting unless they affect correctness.\n- If the final answer is correct but the reasoning contains substantive errors, lower the rating according to the severity of those errors.\n- Output only a single integer from 1 to 5. No explanation. No punctuation.\n"
|
|
},
|
|
"split": {
|
|
"held_out_model": null,
|
|
"lomo_validation_fraction": 0.05,
|
|
"lomo_validation_max_examples": 30000,
|
|
"lomo_validation_seed": 42,
|
|
"mode": "fixed_by_model",
|
|
"train_models": "auto_except_val_test",
|
|
"validation_models": [
|
|
"gemini-3-flash",
|
|
"DK-R1-Dist-Qwen-14B",
|
|
"llama3d2-3b"
|
|
]
|
|
},
|
|
"training": {
|
|
"cache_tokenized_datasets": true,
|
|
"class_weighting": null,
|
|
"dtype": "bfloat16",
|
|
"eval_steps": 500,
|
|
"eval_subset_seed": 42,
|
|
"eval_subset_size": null,
|
|
"eval_subset_strategy": "stratified",
|
|
"eval_subset_stratify_columns": [
|
|
"model_id",
|
|
"target_score"
|
|
],
|
|
"gradient_accumulation_steps": 4,
|
|
"learning_rate": 3e-05,
|
|
"length_column_name": "length",
|
|
"load_in_4bit": true,
|
|
"logging_steps": 10,
|
|
"lora_alpha": 64,
|
|
"lora_dropout": 0.0,
|
|
"lora_r": 32,
|
|
"loss": {
|
|
"class_weights": null,
|
|
"lambda_binary": 0.5,
|
|
"type": "ce_5way"
|
|
},
|
|
"lr_scheduler_type": "cosine",
|
|
"max_grad_norm": 1.0,
|
|
"max_seq_length": 4096,
|
|
"num_train_epochs": 1,
|
|
"objective": "restricted_score_ce",
|
|
"optim": "adamw_8bit",
|
|
"packing": false,
|
|
"per_device_eval_batch_size": 2,
|
|
"per_device_train_batch_size": 2,
|
|
"resume_from_checkpoint": null,
|
|
"save_steps": 500,
|
|
"save_total_limit": 10,
|
|
"score_class_weights": null,
|
|
"seed": 42,
|
|
"target_modules": "auto",
|
|
"train_sampling_strategy": "random",
|
|
"warmup_ratio": 0.03,
|
|
"weight_decay": 0.0
|
|
}
|
|
},
|
|
"length_statistics.json": {
|
|
"num_examples": 285158,
|
|
"prompt_token_length": {
|
|
"max": 4093,
|
|
"mean": 853.0326345394483,
|
|
"min": 252,
|
|
"p50": 769.0,
|
|
"p75": 988.0,
|
|
"p90": 1330.0,
|
|
"p95": 1659.0,
|
|
"p99": 2596.0
|
|
},
|
|
"response_token_length": {
|
|
"max": 3803,
|
|
"mean": 397.55135047938336,
|
|
"min": 0,
|
|
"p50": 307.0,
|
|
"p75": 508.0,
|
|
"p90": 789.0,
|
|
"p95": 1116.0,
|
|
"p99": 2162.0
|
|
},
|
|
"sequence_length": {
|
|
"max": 4095,
|
|
"mean": 855.0326345394483,
|
|
"min": 254,
|
|
"p50": 771.0,
|
|
"p75": 990.0,
|
|
"p90": 1332.0,
|
|
"p95": 1661.0,
|
|
"p99": 2598.0
|
|
},
|
|
"target_token_length": {
|
|
"max": 1,
|
|
"mean": 1.0,
|
|
"min": 1,
|
|
"p50": 1.0,
|
|
"p75": 1.0,
|
|
"p90": 1.0,
|
|
"p95": 1.0,
|
|
"p99": 1.0
|
|
}
|
|
},
|
|
"run_metadata.json": {
|
|
"distributed": {
|
|
"backend": "nccl",
|
|
"deepspeed": {
|
|
"config_overrides": {},
|
|
"gradient_clipping": "auto",
|
|
"offload_optimizer_device": "none",
|
|
"offload_param_device": "none",
|
|
"stage3_gather_16bit_weights_on_model_save": true,
|
|
"zero_stage": 2
|
|
},
|
|
"effective_global_batch_size": 64,
|
|
"enabled": true,
|
|
"find_unused_parameters": false,
|
|
"fsdp": {
|
|
"activation_checkpointing": true,
|
|
"sharding_strategy": "full_shard",
|
|
"transformer_layer_cls_to_wrap": null,
|
|
"use_orig_params": true
|
|
},
|
|
"gradient_accumulation_steps": 4,
|
|
"gradient_checkpointing": true,
|
|
"launcher": "torchrun",
|
|
"mixed_precision": "bf16",
|
|
"per_device_train_batch_size": 2,
|
|
"strategy": "ddp",
|
|
"world_size": 8
|
|
},
|
|
"effective_global_batch_size": 64,
|
|
"evaluation_enabled": true,
|
|
"git_commit": "70ddf4477faac49d4c1d9eb87cbefb58fab7e4a0",
|
|
"package_versions": {
|
|
"accelerate": "1.13.0",
|
|
"bitsandbytes": "0.49.2",
|
|
"datasets": "4.3.0",
|
|
"deepspeed": "unavailable",
|
|
"peft": "0.19.1",
|
|
"python": "3.11.15",
|
|
"torch": "2.7.1+cu118",
|
|
"transformers": "5.5.0",
|
|
"trl": "0.24.0",
|
|
"unsloth": "2026.4.8"
|
|
},
|
|
"score_class_weights": null,
|
|
"score_token_ids": [
|
|
16,
|
|
17,
|
|
18,
|
|
19,
|
|
20
|
|
],
|
|
"source_split_counts": {
|
|
"test": 0,
|
|
"train": 239420,
|
|
"validation": 45738
|
|
},
|
|
"training_backend": "transformers_peft",
|
|
"training_examples": 239420,
|
|
"training_loss": {
|
|
"class_weights": null,
|
|
"lambda_binary": 0.5,
|
|
"type": "ce_5way"
|
|
},
|
|
"training_mode": "standard",
|
|
"training_objective": "restricted_score_ce",
|
|
"validation_full_examples": 45738,
|
|
"validation_monitor_examples": 45738,
|
|
"warmup": {
|
|
"total_optimization_steps": 3741,
|
|
"warmup_ratio": 0.03,
|
|
"warmup_steps": 113
|
|
},
|
|
"world_size": 8
|
|
},
|
|
"score_tokenization_report.json": [
|
|
{
|
|
"num_tokens": 1,
|
|
"score": "1",
|
|
"token_ids": [
|
|
16
|
|
],
|
|
"tokens": [
|
|
"1"
|
|
]
|
|
},
|
|
{
|
|
"num_tokens": 1,
|
|
"score": "2",
|
|
"token_ids": [
|
|
17
|
|
],
|
|
"tokens": [
|
|
"2"
|
|
]
|
|
},
|
|
{
|
|
"num_tokens": 1,
|
|
"score": "3",
|
|
"token_ids": [
|
|
18
|
|
],
|
|
"tokens": [
|
|
"3"
|
|
]
|
|
},
|
|
{
|
|
"num_tokens": 1,
|
|
"score": "4",
|
|
"token_ids": [
|
|
19
|
|
],
|
|
"tokens": [
|
|
"4"
|
|
]
|
|
},
|
|
{
|
|
"num_tokens": 1,
|
|
"score": "5",
|
|
"token_ids": [
|
|
20
|
|
],
|
|
"tokens": [
|
|
"5"
|
|
]
|
|
}
|
|
],
|
|
"split_report.json": {
|
|
"test": {
|
|
"examples": 0,
|
|
"models": [],
|
|
"num_models": 0
|
|
},
|
|
"train": {
|
|
"examples": 239420,
|
|
"models": [
|
|
"DK-R1-Dist-Qwen-1.5B",
|
|
"DK-R1-Dist-Qwen-32B",
|
|
"DK-R1-Dist-Qwen-7B",
|
|
"gemini-2.5-flash",
|
|
"gemini-3.1-pro",
|
|
"gpt-35-turbo",
|
|
"gpt-5.2",
|
|
"gpt4o",
|
|
"llama3d1-405b",
|
|
"llama3d2-11b",
|
|
"llama3d2-1b",
|
|
"llama3d2-90b",
|
|
"llama4-17B-128E",
|
|
"o1-mini",
|
|
"o1_re=low",
|
|
"o3-mini"
|
|
],
|
|
"num_models": 16
|
|
},
|
|
"validation": {
|
|
"examples": 45738,
|
|
"models": [
|
|
"DK-R1-Dist-Qwen-14B",
|
|
"gemini-3-flash",
|
|
"llama3d2-3b"
|
|
],
|
|
"num_models": 3
|
|
}
|
|
},
|
|
"train_metrics.json": {
|
|
"epoch": 1.0,
|
|
"total_flos": 2.185247058507386e+19,
|
|
"train_loss": 0.6326745578038823,
|
|
"train_runtime": 128852.3134,
|
|
"train_samples_per_second": 1.858,
|
|
"train_steps_per_second": 0.029
|
|
},
|
|
"validation_trainer_metrics.json": {
|
|
"epoch": 1.0,
|
|
"eval_binary_accuracy": 0.9893523984433076,
|
|
"eval_binary_macro_f1": 0.9880204659010172,
|
|
"eval_confidence_mean": 0.9603744832085106,
|
|
"eval_confidence_p50": 0.9991409778594971,
|
|
"eval_confidence_p90": 0.9999487400054932,
|
|
"eval_expected_calibration_error_10bin": 0.004097698586483999,
|
|
"eval_f1_correct": 0.9920149535162078,
|
|
"eval_f1_incorrect": 0.9840259782858267,
|
|
"eval_f1_score_1": 0.9774369762155193,
|
|
"eval_f1_score_2": 0.38461538461538464,
|
|
"eval_f1_score_3": 0.5938009787928223,
|
|
"eval_f1_score_4": 0.730398069963812,
|
|
"eval_f1_score_5": 0.989033961060818,
|
|
"eval_false_negative_rate_correct": 0.009138552243694727,
|
|
"eval_false_positive_rate_correct": 0.013677012098895318,
|
|
"eval_loss": 0.09880143404006958,
|
|
"eval_num_examples": 45738,
|
|
"eval_ordinal_accuracy": 0.963924963924964,
|
|
"eval_ordinal_macro_f1": 0.7350570741296714,
|
|
"eval_ordinal_mae": 0.04779395688486598,
|
|
"eval_precision_correct": 0.9931711481007256,
|
|
"eval_precision_incorrect": 0.98173964264677,
|
|
"eval_precision_score_1": 0.9729145558932792,
|
|
"eval_precision_score_2": 0.41139240506329117,
|
|
"eval_precision_score_3": 0.6159052453468697,
|
|
"eval_precision_score_4": 0.7802835051546392,
|
|
"eval_precision_score_5": 0.9858030795310072,
|
|
"eval_pred_binary_counts": {
|
|
"CORRECT": 30459,
|
|
"INCORRECT": 15279
|
|
},
|
|
"eval_pred_score_counts": {
|
|
"1": 14805,
|
|
"2": 474,
|
|
"3": 591,
|
|
"4": 1552,
|
|
"5": 28316
|
|
},
|
|
"eval_recall_correct": 0.9908614477563052,
|
|
"eval_recall_incorrect": 0.9863229879011047,
|
|
"eval_recall_score_1": 0.9820016362148896,
|
|
"eval_recall_score_2": 0.3611111111111111,
|
|
"eval_recall_score_3": 0.573228346456693,
|
|
"eval_recall_score_4": 0.6865079365079365,
|
|
"eval_recall_score_5": 0.9922860900785611,
|
|
"eval_runtime": 4341.2041,
|
|
"eval_samples_per_second": 10.536,
|
|
"eval_score_entropy_mean": 0.10761874169111252,
|
|
"eval_score_margin_mean": 6.721137046813965,
|
|
"eval_steps_per_second": 0.659,
|
|
"eval_support_correct": 30530,
|
|
"eval_support_incorrect": 15208,
|
|
"eval_support_score_1": 14668,
|
|
"eval_support_score_2": 540,
|
|
"eval_support_score_3": 635,
|
|
"eval_support_score_4": 1764,
|
|
"eval_support_score_5": 28131,
|
|
"eval_target_score_counts": {
|
|
"1": 14668,
|
|
"2": 540,
|
|
"3": 635,
|
|
"4": 1764,
|
|
"5": 28131
|
|
},
|
|
"eval_within_1_accuracy": 0.993244129607766
|
|
}
|
|
},
|
|
"base_model": "Qwen/Qwen3-14B",
|
|
"binary_threshold": 3,
|
|
"git_commit": "8a6e08f6f071ce154a27e2718a55ad33895348c6",
|
|
"max_seq_length": 4096,
|
|
"package_versions": {
|
|
"accelerate": "1.13.0",
|
|
"bitsandbytes": "0.49.2",
|
|
"datasets": "4.3.0",
|
|
"deepspeed": "unavailable",
|
|
"peft": "0.19.1",
|
|
"python": "3.11.15",
|
|
"torch": "2.7.1+cu118",
|
|
"transformers": "5.5.0",
|
|
"trl": "0.24.0",
|
|
"unsloth": "2026.4.8"
|
|
},
|
|
"repo_id": "adgomant/adele-judge-qwen3-14-cre",
|
|
"run_dir": "runs/qwen3_14b_restricted_score_ce",
|
|
"thinking_mode": {
|
|
"apply_if_supported": true,
|
|
"enabled": false
|
|
},
|
|
"training_objective": "restricted_score_ce"
|
|
} |