add automatic model task discovery and verification

This commit is contained in:
2026-08-26 21:18:27 +08:00
parent acd4866764
commit 50d9c42e02
3 changed files with 473 additions and 13 deletions

View File

@@ -1,6 +1,22 @@
import io
import json
import unittest
from contextlib import redirect_stdout
from unittest.mock import patch
from main import CARD_MODEL, CARD_VENDOR, ValidationError, analyze
import main as agent_main
from main import (
AGENT_NAME,
CARD_MODEL,
CARD_VENDOR,
ValidationError,
_log_event,
_safe_model_name,
analyze,
scan_once,
select_candidate,
)
class AnalyzeTests(unittest.TestCase):
@@ -45,6 +61,103 @@ class AnalyzeTests(unittest.TestCase):
with self.assertRaises(ValidationError):
analyze({"cards": 0})
def test_model_log_identifier_removes_url_credentials(self):
value = "https://user:pass@example.com/org/model?token=secret#fragment"
self.assertEqual(_safe_model_name(value), "https://example.com/org/model")
def test_structured_log_is_valid_json(self):
stream = io.StringIO()
with redirect_stdout(stream):
_log_event("analysis_completed", model="example/model", verdict="ready")
record = json.loads(stream.getvalue())
self.assertEqual(record["agent"], AGENT_NAME)
self.assertEqual(record["event"], "analysis_completed")
self.assertEqual(record["model"], "example/model")
@staticmethod
def candidate_payload():
return {
"code": 0,
"data": [
{
"modelId": "Qwen/Qwen3-30B-Instruct",
"taskLevelsInfo": {"taskLevelChineseName": "文本生成"},
},
{
"modelId": "Qwen/Qwen3-4B-Instruct-2507",
"taskLevelsInfo": {"taskLevelChineseName": "文本生成"},
},
{
"modelId": "example/image-model",
"taskLevelsInfo": {"taskLevelChineseName": "文本生成图片"},
},
],
}
def test_candidate_selection_prefers_smaller_supported_text_model(self):
candidate = select_candidate(self.candidate_payload())
self.assertIsNotNone(candidate)
self.assertEqual(candidate["model_id"], "Qwen/Qwen3-4B-Instruct-2507")
def test_scan_skips_write_without_private_token(self):
calls = []
def fake_request(method, url, **kwargs):
calls.append((method, url, kwargs))
return self.candidate_payload()
with patch.object(agent_main, "XC_TOKEN", ""):
result = scan_once(fake_request)
self.assertEqual(result["action"], "submission_skipped")
self.assertIn("xc_token_missing", result["reasons"])
self.assertEqual([call[0] for call in calls], ["GET"])
def test_scan_deduplicates_before_submission(self):
calls = []
def fake_request(method, url, **kwargs):
calls.append((method, url, kwargs))
if "top/models" in url:
return self.candidate_payload()
return {"code": 0, "data": {"records": [{"taskId": "existing"}]}}
with (
patch.object(agent_main, "XC_TOKEN", "private-token"),
patch.object(agent_main, "STRATEGY_ID", "strategy-id"),
patch.object(agent_main, "TARGET_GPU", "verified-gpu"),
patch.object(agent_main, "CONFIG_PARAMS", "framework: vllm"),
patch.object(agent_main, "AUTO_SUBMIT_ENABLED", True),
):
result = scan_once(fake_request)
self.assertEqual(result["action"], "duplicate_skipped")
self.assertEqual([call[0] for call in calls], ["GET", "GET"])
def test_scan_submits_exactly_one_task_after_empty_history(self):
calls = []
def fake_request(method, url, **kwargs):
calls.append((method, url, kwargs))
if "top/models" in url:
return self.candidate_payload()
if method == "GET":
return {"code": 0, "data": {"records": []}}
return {"code": 0, "data": {"taskId": "new-task"}}
with (
patch.object(agent_main, "XC_TOKEN", "private-token"),
patch.object(agent_main, "STRATEGY_ID", "strategy-id"),
patch.object(agent_main, "TARGET_GPU", "verified-gpu"),
patch.object(agent_main, "CONFIG_PARAMS", "framework: vllm"),
patch.object(agent_main, "AUTO_SUBMIT_ENABLED", True),
):
result = scan_once(fake_request)
self.assertEqual(result["action"], "submitted")
self.assertEqual(result["task_id"], "new-task")
self.assertEqual([call[0] for call in calls], ["GET", "GET", "POST"])
submitted = calls[-1][2]["payload"]
self.assertEqual(submitted["targetGpu"], "verified-gpu")
self.assertEqual(submitted["strategyId"], "strategy-id")
if __name__ == "__main__":
unittest.main()