xc_agent_platform_demo/tests/test_reject_marks.py

import io
import json
import os
from pathlib import Path
import subprocess
import sys
import tempfile
import unittest
import urllib.error
from unittest.mock import Mock, patch

from src.config_params import GPU_ENUMS
from src.live_cycle import run_cycle, _submitted, _marked
from src.marks import load_marks, PERMANENT_PAIR_CODES
from src.precheck import precheck_model


class TestMarks(unittest.TestCase):
    def test_baked_marks(self):
        marks = load_marks()
        self.assertEqual(len(marks), 16)
        self.assertTrue(all(g in GPU_ENUMS.values() for m, g in marks))
        self.assertEqual(marks['madeagents/hammer2.1-0.5b', 'Biren_166m'], 60027)
        self.assertEqual(PERMANENT_PAIR_CODES, {60012, *range(60019, 60030)})

    def test_verify_fail_marks(self):
        rows = json.loads(Path('src/data/reject-marks.json').read_text())['marks']
        keys = [(row['model'].lower(), row['gpu']) for row in rows]
        self.assertEqual(len(keys), len(set(keys)))
        failures = [row for row in rows if row['reason'].startswith('VERIFY_FAIL:')]
        self.assertEqual(len(failures), 8)
        self.assertEqual(sorted(row['reason'] for row in failures),
                         sorted(['VERIFY_FAIL:SCORE_GAP'] * 5 +
                                ['VERIFY_FAIL:PEFT_UNMERGED', 'VERIFY_FAIL:NO_IDLE_DEVICE',
                                 'VERIFY_FAIL:REF_CACHE_MISS']))
        marks = load_marks()
        for row in failures:
            self.assertEqual(row['code'], 60029)
            self.assertIn(row['code'], PERMANENT_PAIR_CODES)
            self.assertEqual(marks[row['model'].lower(), row['gpu']], 60029)
        self.assertEqual(marks['abaryan/cyberxp_agent_llama_3.2_1b', 'Biren_166m'], 60029)
        self.assertNotIn(('abaryan/cyberxp_agent_llama_3.2_1b', 'Iluvatar_bi-100'), marks)

    def test_invalid_marks(self):
        with tempfile.TemporaryDirectory() as directory:
            path = Path(directory) / 'marks.json'
            with self.assertRaises(ValueError):
                load_marks(path)
            valid = dict(model='org/model', gpu='Biren_166m', code=60027)
            for data in ('bad json', [], {}, {'version': 1, 'marks': {}},
                         {'version': 1, 'marks': [None]},
                         *({'version': 1, 'marks': [dict(valid, **update)]} for update in
                           ({'model': ''}, {'model': 4}, {'gpu': 'unknown'}, {'gpu': []}, {'code': True}, {'code': '60027'}))):
                path.write_text(data if isinstance(data, str) else json.dumps(data))
                with self.subTest(data=data), self.assertRaises(ValueError):
                    load_marks(path)


class TestPrecheck(unittest.TestCase):
    def check(self, body, raw=False):
        response = Mock(status=200)
        response.read.return_value = body if raw else json.dumps(body).encode()
        response.__enter__ = Mock(return_value=response)
        response.__exit__ = Mock(return_value=False)
        opener = Mock(return_value=response)
        result = precheck_model('org/model name', token='synthetic-secret', opener=opener)
        request = opener.call_args.args[0]
        self.assertEqual(request.full_url, 'https://modelhub.org.cn/api/computility/models/search-by-model-id?modelId=org%2Fmodel%20name')
        self.assertEqual(request.get_header('Xc-token'), 'synthetic-secret')
        self.assertEqual(request.get_header('Accept'), 'application/json')
        self.assertEqual(request.method, 'GET')
        self.assertEqual(opener.call_args.kwargs, {'timeout': 30})
        self.assertNotIn('synthetic-secret', repr(result))
        self.assertNotIn('authorName', repr(result))
        return result

    def test_mapping_and_redaction(self):
        with patch.dict(os.environ, {}, clear=True):
            self.assertEqual(self.check({'code': '0', 'data': {'verifyResult': {
                'Biren_166m': {'result': '已验证', 'records': [{'authorName': 'synthetic-secret'}]},
                'Iluvatar_bi-150': {'result': '验证中'},
                'Iluvatar_bi-100': {'result': '验证失败'},
                'unknown': {'result': '已验证'},
            }}}), ('ok', {'Biren_166m': 60027, 'Iluvatar_bi-150': 60028}))

    def test_bad_responses(self):
        with patch.dict(os.environ, {}, clear=True):
            for body, expected, raw in ((b'{bad', 'malformed_json', True),
                    ({'code': 3}, 'platform_error', False),
                    ({'code': 0, 'data': []}, 'malformed_data', False),
                    ([], 'malformed_data', False),
                    ({'code': 0, 'data': {'verifyResult': []}}, 'malformed_data', False),
                    ({'code': 0, 'data': {'verifyResult': {'Biren_166m': None}}}, 'malformed_data', False)):
                with self.subTest(body=body):
                    self.assertEqual(self.check(body, raw), (expected, {}))

    def test_transport_http_and_token_errors(self):
        for error, status in ((OSError('synthetic-secret'), 'transport_error'),
                (urllib.error.HTTPError('url', 403, 'secret', {}, None), 'http_error')):
            self.assertEqual(precheck_model('o/m', 'synthetic-secret', Mock(side_effect=error)), (status, {}))
        with patch.dict(os.environ, {}, clear=True):
            opener = Mock()
            self.assertEqual(precheck_model('o/m', opener=opener), ('missing/invalid_token', {}))
            opener.assert_not_called()
        response = Mock(status=500)
        response.__enter__ = Mock(return_value=response)
        response.__exit__ = Mock(return_value=False)
        self.assertEqual(precheck_model('o/m', 'synthetic-secret', Mock(return_value=response)), ('http_error', {}))


class TestLiveMarks(unittest.TestCase):
    def setUp(self):
        _submitted.clear()
        _marked.clear()
        self.env = patch.dict(os.environ, dict(STRATEGY_ID='sid', XC_TOKEN='synthetic-secret', CONFIG_PARAMS_SOURCE='sample'), clear=True)
        self.env.start()
        self.addCleanup(self.env.stop)
        self.addCleanup(_submitted.clear)
        self.addCleanup(_marked.clear)
        self.client = Mock()
        self.client.submit.return_value = dict(http_status=200, platform_code=0)
        self.precheck = Mock(return_value=('ok', {}))
        self.reader = Mock(return_value=(0, 'ok', set()))
        self.logs = io.StringIO()
        logpatch = patch('sys.stderr', self.logs)
        logpatch.start()
        self.addCleanup(logpatch.stop)
        # All external collaborators must be injected in this suite.
        netpatch = patch('urllib.request.urlopen', side_effect=AssertionError('network forbidden'))
        netpatch.start()
        self.addCleanup(netpatch.stop)

    def cycle(self, model='unit/model', gpus=None, rows=None):
        return run_cycle(rows or [dict(model_address=model, try_next_gpus=gpus or ['壁砺'])],
                         self.client, self.reader, precheck=self.precheck)

    def test_marks_read_failure(self):
        with patch('src.live_cycle.load_marks', side_effect=ValueError()):
            self.assertEqual(self.cycle(), [dict(action='hold', reason='marks_read_failed')])
        self.client.submit.assert_not_called()
        self.precheck.assert_not_called()

    def test_baked_mark_uses_next_choice(self):
        result = self.cycle('MadeAgents/Hammer2.1-0.5b', ['壁砺', '天垓100'])
        self.assertEqual(result[0], dict(action='skip', reason='marked:60027', model='MadeAgents/Hammer2.1-0.5b', gpu='Biren_166m'))
        self.assertEqual(self.client.submit.call_args.args[0]['targetGpu'], 'Iluvatar_bi-100')
        self.assertEqual(self.logs.getvalue().count('reason=marked:60027'), 1)

    def test_baked_mark_only_never_calls_api(self):
        result = self.cycle('MadeAgents/Hammer2.1-0.5b')
        self.assertEqual(result[-1]['reason'], 'all_gpus_done_or_unmapped')
        self.client.submit.assert_not_called()
        self.precheck.assert_not_called()
        self.reader.assert_not_called()

    def test_precheck_marks_all_remaining_choices(self):
        self.precheck.return_value = ('ok', {'Biren_166m': 60027, 'Iluvatar_bi-100': 60028})
        result = self.cycle(gpus=['壁砺', '天垓100'])
        self.assertEqual([d['reason'] for d in result], ['marked:precheck_60027', 'marked:precheck_60028', 'all_gpus_done_or_unmapped'])
        self.assertEqual(_marked, {('unit/model', 'Biren_166m'): 60027, ('unit/model', 'Iluvatar_bi-100'): 60028})
        self.cycle(gpus=['壁砺', '天垓100'])
        self.precheck.assert_called_once()
        self.client.submit.assert_not_called()

    def test_missing_mapping_submits(self):
        self.cycle()
        self.client.submit.assert_called_once()

    def test_nonblocking_platform_labels_and_missing_gpu(self):
        for label in ('验证失败', '未验证', None):
            _submitted.clear()
            response = Mock(status=200)
            mapping = {} if label is None else {'Biren_166m': {'result': label}}
            response.read.return_value = json.dumps(dict(code=0, data=dict(verifyResult=mapping))).encode()
            response.__enter__ = Mock(return_value=response)
            response.__exit__ = Mock(return_value=False)
            self.precheck.side_effect = lambda model: precheck_model(model, 'synthetic-secret', Mock(return_value=response))
            self.cycle()
        self.assertEqual(self.client.submit.call_count, 3)

    def test_precheck_uses_next_choice(self):
        self.precheck.return_value = 'ok', {'Biren_166m': 60027}
        result = self.cycle(gpus=['壁砺', '天垓100'])
        self.assertEqual(result[0]['reason'], 'marked:precheck_60027')
        self.assertEqual(self.client.submit.call_args.args[0]['targetGpu'], 'Iluvatar_bi-100')
        self.precheck.assert_called_once()

    def test_precheck_failure_and_unsafe_status(self):
        for status in ('transport_error', 'synthetic-secret authorName'):
            with self.subTest(status=status):
                _submitted.clear()
                self.precheck.return_value = status, {}
                result = self.cycle()
                self.assertEqual(result[0]['action'], 'precheck')
                self.assertEqual(result[-1]['action'], 'submitted')
        self.assertNotIn('synthetic-secret', self.logs.getvalue())
        self.assertNotIn('authorName', self.logs.getvalue())

    def test_disabled(self):
        with patch.dict(os.environ, {'LIVE_PRECHECK': '0'}):
            self.cycle()
        self.precheck.assert_not_called()
        self.client.submit.assert_called_once()

    def test_precheck_cap(self):
        self.precheck.return_value = 'ok', {'Biren_166m': 60027}
        rows = [dict(model_address=f'unit/m{i}', targetGpu='壁砺') for i in range(62)]
        for value, count in (('2', 2), ('bad', 60), ('0', 60)):
            _marked.clear()
            self.precheck.reset_mock()
            with patch.dict(os.environ, {'LIVE_MAX_PRECHECKS_PER_CYCLE': value, 'LIVE_MAX_ATTEMPTS_PER_CYCLE': '1'}):
                result = self.cycle(rows=rows)
            self.assertEqual(self.precheck.call_count, count)
            self.assertEqual(result[-1]['reason'], 'precheck_cap')
            self.assertEqual(sum(d['reason'] == 'precheck_cap' for d in result), 1)
        self.client.submit.assert_not_called()

    def test_refresh_reuses_precheck_mapping(self):
        self.precheck.return_value = 'ok', {'Iluvatar_bi-100': 60028}
        self.reader.return_value = 0, 'ok', {('unit/model', 'Biren_166m')}
        result = self.cycle(gpus=['壁砺', '天垓100'])
        self.precheck.assert_called_once()
        self.client.submit.assert_not_called()
        self.assertEqual(result[-1]['reason'], 'all_gpus_done_or_unmapped')
        self.assertEqual(sum(d['reason'] == 'marked:precheck_60028' for d in result), 1)

    def test_rejection_memory_and_exact_log(self):
        self.client.submit.return_value = dict(http_status=200, platform_code=60022)
        self.cycle()
        result = self.cycle()
        self.client.submit.assert_called_once()
        self.assertEqual(result[0]['reason'], 'marked:60022')
        self.assertIn('[live] rejected model=unit/model gpu=Biren_166m code=60022 http=200 queue_before=0 reason=CREATE_CONTEST_TASK_GPU_CONFIG_UNSUPPORTED\n', self.logs.getvalue())

    def test_no_state_file_writes(self):
        with tempfile.TemporaryDirectory() as directory:
            path = Path(directory) / 'state.json'
            with patch.dict(os.environ, {'LIVE_STATE_FILE': str(path)}), patch.object(Path, 'write_text', side_effect=AssertionError('file write forbidden')):
                self.cycle()
            self.assertFalse(path.exists())


class TestSyncMarks(unittest.TestCase):
    def test_sync_and_dry_run(self):
        with tempfile.TemporaryDirectory() as directory:
            marks = Path(directory) / 'marks.json'
            log = Path(directory) / 'log.txt'
            existing = dict(model='Org/Existing', gpu='Biren_166m', code=60027, reason='first reason', date='2020-01-01')
            marks.write_text(json.dumps(dict(version=1, marks=[existing])))
            log.write_text('''timestamp console [live] rejected model=org/existing gpu=Biren_166m code=60028 http=200
prefix [live] rejected model=Org/New gpu=Iluvatar_bi-150 code=60022 http=200
[live] rejected model=org/new gpu=Iluvatar_bi-150 code=60027
[live] skip model=Org/Precheck gpu=Biren_166m reason=marked:precheck_60028
[live] rejected model=Org/Limit gpu=Biren_166m code=60007
[live] rejected model=Org/Disabled gpu=Biren_166m code=60014
[live] rejected model=Org/Success gpu=Biren_166m code=0
[live] rejected model=invalid gpu=Biren_166m code=60022
[live] rejected model=Org/Invalid gpu=unknown code=60022
''')
            command = [sys.executable, 'scripts/sync_reject_marks.py', str(log), '--marks', str(marks), '--date', '2026-10-17']
            before = marks.read_bytes()
            result = subprocess.run(command + ['--dry-run'], capture_output=True, text=True, check=True)
            self.assertEqual(result.stdout, 'added 2 total 3\n')
            self.assertEqual(result.stderr, '')
            self.assertEqual(marks.read_bytes(), before)
            result = subprocess.run(command, capture_output=True, text=True, check=True)
            self.assertEqual(result.stdout, 'added 2 total 3\n')
            rows = json.loads(marks.read_text())['marks']
            self.assertEqual(rows[0], existing)
            self.assertEqual(rows[1]['model'], 'Org/New')
            self.assertEqual(rows[1]['code'], 60022)
            self.assertEqual(rows[2]['code'], 60028)
            self.assertTrue(marks.read_bytes().endswith(b'\n'))
            self.assertEqual(len(load_marks(marks)), 3)
            result = subprocess.run(command, capture_output=True, text=True, check=True)
            self.assertEqual(result.stdout, 'added 0 total 3\n')