初始化项目,由ModelHub XC社区提供模型
Model: Luimas/claim-extractor-detective-qwen3b Source: Original Platform
This commit is contained in:
75
benchmarks/benchmarks.json
Normal file
75
benchmarks/benchmarks.json
Normal file
@@ -0,0 +1,75 @@
|
||||
{
|
||||
"models": {
|
||||
"student": "unsloth/Qwen2.5-3B-Instruct-bnb-4bit",
|
||||
"teacher": "Qwen/Qwen2.5-14B-Instruct"
|
||||
},
|
||||
"n_train_examples": 1471,
|
||||
"n_val_examples": 127,
|
||||
"held_out_validity": 1.0,
|
||||
"held_out_avg_claims": 2.45,
|
||||
"base": {
|
||||
"n": 15,
|
||||
"validity": 1.0,
|
||||
"avg_claims": 2.067,
|
||||
"avg_verification_questions": 3.71,
|
||||
"pct_claims_with_3plus_questions": 1,
|
||||
"contradiction_recall": 1.0,
|
||||
"date_extraction_rate": 1.0,
|
||||
"evidence_verbatim_rate": 1.0,
|
||||
"avg_claim_words": 5.548,
|
||||
"sarcasm_handling": 0.75,
|
||||
"avg_latency_s": 30.214
|
||||
},
|
||||
"teacher": {
|
||||
"n": 15,
|
||||
"validity": 0.933,
|
||||
"avg_claims": 2.357,
|
||||
"avg_verification_questions": 3.242,
|
||||
"pct_claims_with_3plus_questions": 1,
|
||||
"contradiction_recall": 1.0,
|
||||
"date_extraction_rate": 1.0,
|
||||
"evidence_verbatim_rate": 1.0,
|
||||
"avg_claim_words": 7.333,
|
||||
"sarcasm_handling": 1.0,
|
||||
"avg_latency_s": 60.618
|
||||
},
|
||||
"finetuned": {
|
||||
"n": 15,
|
||||
"validity": 1.0,
|
||||
"avg_claims": 2.067,
|
||||
"avg_verification_questions": 3,
|
||||
"pct_claims_with_3plus_questions": 1,
|
||||
"contradiction_recall": 0.75,
|
||||
"date_extraction_rate": 1.0,
|
||||
"evidence_verbatim_rate": 1.0,
|
||||
"avg_claim_words": 7.806,
|
||||
"sarcasm_handling": 1.0,
|
||||
"avg_latency_s": 25.992
|
||||
},
|
||||
"improvement_vs_base": {
|
||||
"validity": 0.0,
|
||||
"avg_claims": 0.0,
|
||||
"avg_verification_questions": -0.71,
|
||||
"pct_claims_with_3plus_questions": 0,
|
||||
"contradiction_recall": -0.25,
|
||||
"date_extraction_rate": 0.0,
|
||||
"evidence_verbatim_rate": 0.0,
|
||||
"sarcasm_handling": 0.25,
|
||||
"avg_claim_words": 2.258,
|
||||
"avg_latency_s": -4.222
|
||||
},
|
||||
"finetuned_vs_teacher": {
|
||||
"validity": 0.067,
|
||||
"avg_claims": -0.29,
|
||||
"avg_verification_questions": -0.242,
|
||||
"pct_claims_with_3plus_questions": 0,
|
||||
"contradiction_recall": -0.25,
|
||||
"date_extraction_rate": 0.0,
|
||||
"evidence_verbatim_rate": 0.0,
|
||||
"sarcasm_handling": 0.0,
|
||||
"avg_claim_words": 0.473,
|
||||
"avg_latency_s": -34.626
|
||||
},
|
||||
"n_benchmark_cases": 15,
|
||||
"timestamp": "2026-06-07 19:56:00"
|
||||
}
|
||||
Reference in New Issue
Block a user