97 lines
5.6 KiB
Python
97 lines
5.6 KiB
Python
|
import os
|
||
|
|
import sys
|
||
|
|
import tempfile
|
||
|
|
import unittest
|
||
|
|
from pathlib import Path
|
||
|
|
from unittest.mock import patch
|
||
|
|
from types import SimpleNamespace
|
||
|
|
|
||
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "modelhub_submmit_api"))
|
||
|
|
from failure_taxonomy import classify_failure_report
|
||
|
|
from market_intelligence import _validated_official_config, MarketIntelligenceManager
|
||
|
|
from routing_engine import SuccessFirstRoutingEngine
|
||
|
|
from state_sync import StateGitSync, DEFAULT_BRANCH
|
||
|
|
from poll_runner import build_parser
|
||
|
|
|
||
|
|
|
||
|
|
class AccuracyAuditTests(unittest.TestCase):
|
||
|
|
def config(self, command):
|
||
|
|
return f"framework: vllm_fix_tokenizer\nsut_config:\n command: {command}\nref_config: {{}}\n"
|
||
|
|
|
||
|
|
def test_broken_official_and_cached_config_rejected(self):
|
||
|
|
bad = self.config('["/opt/entrypoint.sh - /model", "--port", "80"]')
|
||
|
|
for gpu in ("Biren_166m", "Kunlunxin_p-800"):
|
||
|
|
self.assertEqual((False, "malformed_command_argv"), _validated_official_config(
|
||
|
|
bad, framework="vllm_fix_tokenizer", target_gpu=gpu))
|
||
|
|
with tempfile.TemporaryDirectory() as tmp:
|
||
|
|
manager = MarketIntelligenceManager(Path(tmp) / "market.json")
|
||
|
|
manager.state = {"frameworkStats": {"text-generation": {gpu: {
|
||
|
|
"vllm_fix_tokenizer": {"officialConfigValid": True, "officialConfig": bad}}}}}
|
||
|
|
self.assertIsNone(manager.official_config(task_type="text-generation", target_gpu=gpu,
|
||
|
|
framework="vllm_fix_tokenizer"))
|
||
|
|
|
||
|
|
def test_separate_argv_and_shell_command_remain_valid(self):
|
||
|
|
for command in ('["/opt/entrypoint.sh", "/model"]', '["/bin/sh", "-c", "echo hello"]'):
|
||
|
|
self.assertEqual((True, None), _validated_official_config(
|
||
|
|
self.config(command), framework="vllm_fix_tokenizer", target_gpu="Biren_166m"))
|
||
|
|
|
||
|
|
def test_specific_failure_survives_transient_resource_error(self):
|
||
|
|
result = classify_failure_report("MODEL_NOT_SUPPORTED", [
|
||
|
|
"找不到空闲卡", "当前框架版本不支持该模型架构"])
|
||
|
|
self.assertEqual("framework_architecture_unsupported", result.category)
|
||
|
|
self.assertEqual("tokenizer_compatibility", classify_failure_report(
|
||
|
|
"TOKENIZER_FAILED", ["no idle gpu", "Tokenizer class missing"]).category)
|
||
|
|
self.assertEqual("platform_infrastructure", classify_failure_report(
|
||
|
|
"EXECUTE_EMPTY_RESULT", ["no idle gpu"]).category)
|
||
|
|
self.assertEqual("framework_architecture_unsupported", classify_failure_report(
|
||
|
|
None, ["no idle gpu", "Model architectures ['NewForCausalLM'] are not supported for now"]).category)
|
||
|
|
|
||
|
|
def test_launch_and_quantization_have_specific_categories(self):
|
||
|
|
self.assertEqual("configuration_launch", classify_failure_report("EXECUTE_EMPTY_RESULT", [
|
||
|
|
'exec: "/opt/entrypoint.sh - /model": no such file or directory']).category)
|
||
|
|
self.assertEqual("quantization_compatibility", classify_failure_report("EXECUTE_EMPTY_RESULT", [
|
||
|
|
"The quantization method compressed-tensors is not supported for the current GPU"]).category)
|
||
|
|
|
||
|
|
def test_sparse_failures_lower_score_even_without_classification(self):
|
||
|
|
candidate = {"targetGpu": "gpu", "framework": "vllm", "taskType": "text-generation",
|
||
|
|
"frameworkMarketSamples": 1000, "frameworkMarketSuccessRate": 0.8,
|
||
|
|
"preflightMetadata": {"modelType": "family", "estimatedLoadBytes": 1024}}
|
||
|
|
with tempfile.TemporaryDirectory() as tmp:
|
||
|
|
engine = SuccessFirstRoutingEngine(Path(tmp) / "route.json")
|
||
|
|
prior = engine._evidence(candidate)[3]
|
||
|
|
engine.outcome_stats = {"profileCombinationStats": {
|
||
|
|
"gpu|vllm|text-generation|family|none": {
|
||
|
|
"successCount": 0, "failureCount": 5, "decisionTotal": 0}}}
|
||
|
|
evidence = engine._evidence(candidate)
|
||
|
|
self.assertEqual("local_profile_shrunk", evidence[0])
|
||
|
|
self.assertLess(evidence[3], prior)
|
||
|
|
|
||
|
|
def test_coexistence_defaults_and_submission_provenance(self):
|
||
|
|
with patch.dict(os.environ, {"MODELHUB_COEXISTENCE_MODE": "1", "STRATEGY_ID": "new-agent"}):
|
||
|
|
self.assertTrue(build_parser().parse_args([]).disable_queue_cleanup)
|
||
|
|
intent = StateGitSync._intent({"repoId": "owner/model"}, "batch")
|
||
|
|
self.assertEqual("new-agent", intent["strategyId"])
|
||
|
|
self.assertTrue(intent["agentVersion"])
|
||
|
|
self.assertTrue(intent["instanceId"])
|
||
|
|
self.assertNotEqual("agent-state", DEFAULT_BRANCH)
|
||
|
|
|
||
|
|
def test_model_specific_failures_change_framework_selection(self):
|
||
|
|
with tempfile.TemporaryDirectory() as tmp:
|
||
|
|
manager = MarketIntelligenceManager(Path(tmp) / "market.json")
|
||
|
|
manager.state = {"frameworkStats": {"text-generation": {"gpu": {
|
||
|
|
"first": {"modelCount": 1000, "wilsonLowerBound": 0.8},
|
||
|
|
"second": {"modelCount": 1000, "wilsonLowerBound": 0.6},
|
||
|
|
}}}}
|
||
|
|
manager.set_local_outcome_stats({"profileCombinationStats": {
|
||
|
|
"gpu|first|text-generation|family|none": {"successCount": 0, "failureCount": 4}
|
||
|
|
}})
|
||
|
|
with patch.object(manager, "compatible_discovered_frameworks", return_value=[]):
|
||
|
|
ranked = manager.selectable_frameworks(
|
||
|
|
task_type="text-generation", target_gpu="gpu", incumbent_frameworks=["first", "second"],
|
||
|
|
inspection=SimpleNamespace(model_type="family", quantization_method=None))
|
||
|
|
self.assertEqual(["second", "first"], ranked)
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
unittest.main()
|