From b0d66da881e25a7316b0afb62d7de0f91b2fb0b7 Mon Sep 17 00:00:00 2001 From: Victor Cao Date: Sat, 17 Oct 2026 19:23:51 +0800 Subject: [PATCH] live: ban gguf/openchat/peft-lora; bake verify_fail marks from 2026-10-17 analysis --- src/banned.py | 4 +++ src/data/reject-marks.json | 56 ++++++++++++++++++++++++++++++++++++++ tests/test_banned.py | 36 ++++++++++++++++++++++++ tests/test_reject_marks.py | 20 +++++++++++++- 4 files changed, 115 insertions(+), 1 deletion(-) create mode 100644 tests/test_banned.py diff --git a/src/banned.py b/src/banned.py index 95d73af..b94bab4 100644 --- a/src/banned.py +++ b/src/banned.py @@ -2,6 +2,10 @@ import re RULES = ( + ('gguf', r'gguf|\.gguf|ggml'), + ('openchat', r'openchat'), + # Full-weight exports override adapter hints anywhere in the identifier. + ('peft-lora', r'^(?!.*(?:merged|merge|fullft|full-ft)).*(?:\blora\b|adapter_model|adapter-config|peft)'), ('gpt2', r'gpt-?2|dialogpt|distilgpt2'), ('phi', r'phi[-_. ]?(?:3(?:\.5)?|4)(?:\D|$)'), ('nemotron-h', r'nemotron-h'), ('olmo2', r'olmo-?2'), diff --git a/src/data/reject-marks.json b/src/data/reject-marks.json index 6e96bab..c64e30f 100644 --- a/src/data/reject-marks.json +++ b/src/data/reject-marks.json @@ -56,6 +56,62 @@ "code": 60028, "reason": "CREATE_CONTEST_TASK_MODEL_VERIFY_RUNNING", "date": "2026-10-17" + }, + { + "model": "Qurtana/Qwen-2.5-0.5B-MathInstruct.rev.2", + "gpu": "Biren_166m", + "code": 60029, + "reason": "VERIFY_FAIL:SCORE_GAP", + "date": "2026-10-17" + }, + { + "model": "prithivMLmods/Reasoning-SmolLM2-135M", + "gpu": "Biren_166m", + "code": 60029, + "reason": "VERIFY_FAIL:SCORE_GAP", + "date": "2026-10-17" + }, + { + "model": "Qwen/Qwen2.5-Coder-7B-Instruct", + "gpu": "Sunrise_pt-200-x1", + "code": 60029, + "reason": "VERIFY_FAIL:SCORE_GAP", + "date": "2026-10-17" + }, + { + "model": "FlofloB/100k_fineweb_continued_pretraining_Qwen2.5-0.5B-Instruct_Unsloth_merged_16bit", + "gpu": "Biren_166m", + "code": 60029, + "reason": "VERIFY_FAIL:SCORE_GAP", + "date": "2026-10-17" + }, + { + "model": "prithivMLmods/Bellatrix-Tiny-0.5B", + "gpu": "Biren_166m", + "code": 60029, + "reason": "VERIFY_FAIL:SCORE_GAP", + "date": "2026-10-17" + }, + { + "model": "sunmengjie/DeepSeek-R1-Distill-Qwen-1.5B-GRPO", + "gpu": "Biren_166m", + "code": 60029, + "reason": "VERIFY_FAIL:PEFT_UNMERGED", + "date": "2026-10-17" + }, + { + "model": "DevQuasar/efficientscaling.Z1-7B-GGUF", + "gpu": "Mthreads_s4000", + "code": 60029, + "reason": "VERIFY_FAIL:NO_IDLE_DEVICE", + "date": "2026-10-17" + }, + { + "model": "abaryan/CyberXP_Agent_Llama_3.2_1B", + "gpu": "Biren_166m", + "code": 60029, + "reason": "VERIFY_FAIL:REF_CACHE_MISS", + "date": "2026-10-17" } ] } diff --git a/tests/test_banned.py b/tests/test_banned.py new file mode 100644 index 0000000..786d9a4 --- /dev/null +++ b/tests/test_banned.py @@ -0,0 +1,36 @@ +import unittest + +from src.banned import banned_rule + + +class TestBanned(unittest.TestCase): + def test_default_engine_and_openchat(self): + for model, rule in [('org/Model-GGUF', 'gguf'), ('org/model.gguf', 'gguf'), + ('org/GGML-model', 'gguf'), ('OpenChat/model', 'openchat')]: + with self.subTest(model=model): + self.assertEqual(banned_rule(model), rule) + + def test_unmerged_adapter_hints(self): + for model in ('org/model-lora', 'org/LORA', 'org/adapter_model', + 'org/model-adapter-config', 'org/Model-PEFT'): + with self.subTest(model=model): + self.assertEqual(banned_rule(model), 'peft-lora') + + def test_full_weights_and_unrelated_names(self): + for model in ('org/lora-merged', 'org/merge-lora', 'org/PEFT-FULLFT', + 'org/full-ft-adapter_model', 'org/adapter-config-MERGED', + 'org/flora', 'org/lorax', 'org/model_lora', + 'org/model-GRPO', 'abaryan/CyberXP_Agent_Llama_3.2_1B'): + with self.subTest(model=model): + self.assertIsNone(banned_rule(model)) + + def test_full_weight_hint_does_not_override_other_bans(self): + self.assertEqual(banned_rule('org/lora-merged-GGUF'), 'gguf') + self.assertEqual(banned_rule('org/openchat-fullft'), 'openchat') + + def test_existing_rules(self): + for model in ('gpt2', 'org/Phi-3-mini', 'nvidia/Nemotron-H-4B', + 'allenai/OLMo-2-7B', 'org/LFM2', 'org/Spark-X2.5', + 'org/Breeze', 'org/gemma2', 'org/mistral-v0.1', 'org/tinyllama'): + with self.subTest(model=model): + self.assertIsNotNone(banned_rule(model)) diff --git a/tests/test_reject_marks.py b/tests/test_reject_marks.py index 82a33e3..02d277c 100644 --- a/tests/test_reject_marks.py +++ b/tests/test_reject_marks.py @@ -18,11 +18,29 @@ from src.precheck import precheck_model class TestMarks(unittest.TestCase): def test_baked_marks(self): marks = load_marks() - self.assertEqual(len(marks), 8) + self.assertEqual(len(marks), 16) self.assertTrue(all(g in GPU_ENUMS.values() for m, g in marks)) self.assertEqual(marks['madeagents/hammer2.1-0.5b', 'Biren_166m'], 60027) self.assertEqual(PERMANENT_PAIR_CODES, {60012, *range(60019, 60030)}) + def test_verify_fail_marks(self): + rows = json.loads(Path('src/data/reject-marks.json').read_text())['marks'] + keys = [(row['model'].lower(), row['gpu']) for row in rows] + self.assertEqual(len(keys), len(set(keys))) + failures = [row for row in rows if row['reason'].startswith('VERIFY_FAIL:')] + self.assertEqual(len(failures), 8) + self.assertEqual(sorted(row['reason'] for row in failures), + sorted(['VERIFY_FAIL:SCORE_GAP'] * 5 + + ['VERIFY_FAIL:PEFT_UNMERGED', 'VERIFY_FAIL:NO_IDLE_DEVICE', + 'VERIFY_FAIL:REF_CACHE_MISS'])) + marks = load_marks() + for row in failures: + self.assertEqual(row['code'], 60029) + self.assertIn(row['code'], PERMANENT_PAIR_CODES) + self.assertEqual(marks[row['model'].lower(), row['gpu']], 60029) + self.assertEqual(marks['abaryan/cyberxp_agent_llama_3.2_1b', 'Biren_166m'], 60029) + self.assertNotIn(('abaryan/cyberxp_agent_llama_3.2_1b', 'Iluvatar_bi-100'), marks) + def test_invalid_marks(self): with tempfile.TemporaryDirectory() as directory: path = Path(directory) / 'marks.json'