57 lines
1.5 KiB
JSON
57 lines
1.5 KiB
JSON
{
|
|
"schema_version": 1,
|
|
"model_id": "Yunhao-Feng/AdaGuard-4B",
|
|
"units": "percent",
|
|
"result_type": "previously_reported_single_run",
|
|
"release_validation_is_benchmark_rerun": false,
|
|
"datasets": {
|
|
"AdaptiveSafety": {
|
|
"samples": 1000,
|
|
"compliant": 500,
|
|
"violating": 500,
|
|
"binary": {
|
|
"accuracy": 89.3,
|
|
"precision": 95.38,
|
|
"recall": 82.6,
|
|
"f1": 88.53
|
|
},
|
|
"rule_identification": {
|
|
"exact_match": 76.6,
|
|
"micro_precision": 81.73,
|
|
"micro_recall": 63.54,
|
|
"micro_f1": 71.5
|
|
}
|
|
},
|
|
"DynaBench": {
|
|
"samples": 543,
|
|
"compliant": 276,
|
|
"violating": 267,
|
|
"binary": {
|
|
"accuracy": 71.82,
|
|
"precision": 70.65,
|
|
"recall": 73.03,
|
|
"f1": 71.82
|
|
},
|
|
"rule_identification": {
|
|
"exact_match": 63.72,
|
|
"micro_precision": 51.64,
|
|
"micro_recall": 58.8,
|
|
"micro_f1": 54.99
|
|
}
|
|
}
|
|
},
|
|
"protocol": {
|
|
"inference_dtype": "bfloat16",
|
|
"decoding": "greedy",
|
|
"prompt_budget_tokens": 16000,
|
|
"max_new_tokens": 512,
|
|
"hardware": "8 x NVIDIA A100-SXM4-80GB; one model replica per GPU",
|
|
"input_truncation_recorded": false,
|
|
"binary_positive_class": "violation",
|
|
"invalid_binary": "counted as an error",
|
|
"invalid_exact_match": "never credited",
|
|
"invalid_rule_micro": "empty predicted rule set; gold violations remain false negatives",
|
|
"rule_order": "same order as supplied policy; incorrect ordering is invalid"
|
|
}
|
|
}
|