live: baked reject-marks (8) + read-only search-by-model-id precheck (已验证→60027/验证中→60028 skip, no submit); in-run marks; drop /tmp state; sync_reject_marks.py (95 tests)

tests/test_live_cycle.py

@@ -1,11 +1,8 @@
 import io
-import json
-from pathlib import Path
 import os
-import tempfile
 import unittest
 from unittest.mock import Mock, patch
-from src.live_cycle import run_cycle, _submitted, _rejected
+from src.live_cycle import run_cycle, _submitted, _marked
 
 
 def candidate(name, section='B', foothold='unknown'):
@@ -14,13 +11,11 @@ def candidate(name, section='B', foothold='unknown'):
 
 class TestLiveCycle(unittest.TestCase):
     def setUp(self):
-        directory = tempfile.TemporaryDirectory()
-        self.addCleanup(directory.cleanup)
-        self.env = patch.dict(os.environ, {'STRATEGY_ID': 'sid', 'LIVE_STATE_FILE': directory.name + '/state.json', 'CONFIG_PARAMS_SOURCE': 'sample', 'XC_TOKEN': 'synthetic-secret'}, clear=True)
+        self.env = patch.dict(os.environ, {'LIVE_PRECHECK': '0', 'STRATEGY_ID': 'sid', 'CONFIG_PARAMS_SOURCE': 'sample', 'XC_TOKEN': 'synthetic-secret'}, clear=True)
         self.env.start()
         self.addCleanup(self.env.stop)
         _submitted.clear()
-        _rejected.clear()
+        _marked.clear()
         self.client = Mock()
         self.client.submit.return_value = dict(http_status=200, platform_code=0)
         network = patch('urllib.request.urlopen', side_effect=AssertionError('network forbidden'))
@@ -34,10 +29,10 @@ class TestLiveCycle(unittest.TestCase):
         reader = Mock(side_effect=[(n, 'platform_error' if n is None else 'ok', set()) for n in counts])
         return run_cycle(rows or [candidate('a')], self.client, reader)
 
-    def state_pairs(self):
-        return json.loads(Path(os.environ['LIVE_STATE_FILE']).read_text())
+    def submitted_pairs(self):
+        return sorted(m + '|' + g for m, g in _submitted)
 
-    def test_done_rejection_keeps_reservation_and_continues(self):
+    def test_done_rejection_marks_pair_and_continues(self):
         for code in (60027, 60028):
             with self.subTest(code=code):
                 self.client.submit.side_effect = [
@@ -46,14 +41,14 @@ class TestLiveCycle(unittest.TestCase):
                 with patch.dict(os.environ, {'LIVE_MAX_SUBMITS_PER_CYCLE': '1'}):
                     decisions = self.cycle([candidate(str(code)), candidate('ok' + str(code))], (1, 1))
                 self.assertEqual([d['action'] for d in decisions], ['rejected', 'submitted'])
-                self.assertIn('org/' + str(code) + '|Biren_166m', self.state_pairs())
+                self.assertEqual(_marked[('org/' + str(code), 'Biren_166m')], code)
         self.assertIn('rejected', self.logs.getvalue())
         self.assertIn('CREATE_CONTEST_TASK_MODEL_VERIFY_RUNNING', self.logs.getvalue())
 
-    def test_rejection_removed_but_not_retried_in_process(self):
+    def test_rejection_marked_and_not_retried_in_process(self):
         self.client.submit.return_value = dict(http_status=200, platform_code=60020)
         self.cycle()
-        self.assertEqual(self.state_pairs(), [])
+        self.assertEqual(self.submitted_pairs(), [])
         self.cycle(counts=())
         self.client.submit.assert_called_once()
 
@@ -62,12 +57,12 @@ class TestLiveCycle(unittest.TestCase):
         decisions = self.cycle()
         self.assertEqual(decisions[0]['action'], 'rejected')
         self.assertEqual(decisions[0]['reason'], 'unknown')
-        self.assertEqual(self.state_pairs(), [])
+        self.assertEqual(self.submitted_pairs(), [])
 
     def test_attempt_cap_and_invalid_fallback(self):
         for value, expected in ((None, 15), ('bad', 15), ('0', 15), ('-1', 15), ('2', 2)):
             with self.subTest(value=value):
-                _rejected.clear()
+                _marked.clear()
                 self.client.reset_mock()
                 self.client.submit.return_value = dict(http_status=200, platform_code=60020)
                 with patch.dict(os.environ, {} if value is None else {'LIVE_MAX_ATTEMPTS_PER_CYCLE': value}):
@@ -87,17 +82,16 @@ class TestLiveCycle(unittest.TestCase):
         decisions = self.cycle([candidate('a'), candidate('b')])
         self.assertEqual([d['action'] for d in decisions], ['rejected', 'hold'])
         self.assertEqual(decisions[-1]['reason'], 'task_num_limit')
-        self.assertEqual(self.state_pairs(), [])
+        self.assertEqual(self.submitted_pairs(), [])
         self.client.submit.assert_called_once()
 
     def test_unknown_outcomes_count_and_reserve_before_submit(self):
         for result in (dict(http_status=200, platform_code='0'), dict(http_status=None, platform_code=None), OSError()):
             with self.subTest(result=result):
                 _submitted.clear()
-                Path(os.environ['LIVE_STATE_FILE']).unlink(missing_ok=True)
                 self.client.reset_mock()
                 def submit(payload):
-                    self.assertIn('org/a|Biren_166m', self.state_pairs())
+                    self.assertIn('org/a|Biren_166m', self.submitted_pairs())
                     if isinstance(result, Exception):
                         raise result
                     return result
@@ -134,10 +128,9 @@ class TestLiveCycle(unittest.TestCase):
             self.cycle(counts=(99,))
         self.client.submit.assert_called_once()
 
-    def test_banned_duplicate_persistence(self):
+    def test_banned_and_duplicate_memory(self):
         rows = [candidate('GPT-2'), candidate('ok'), candidate('ok')]
         self.cycle(rows)
-        _submitted.clear()
         self.cycle([candidate('ok')], ())
         self.client.submit.assert_called_once()
         self.assertIn('banned:', self.logs.getvalue())
@@ -160,7 +153,7 @@ class TestLiveCycle(unittest.TestCase):
         self.assertIn('[live] token_probe EXTERNAL_SERVICE_TOKEN=absent XC_TOKEN_FILE=absent XC_TOKEN=absent MODELHUB_XC_TOKEN=absent', self.logs.getvalue())
         self.assertNotIn('[live] token source=', self.logs.getvalue())
 
-    def test_rejected_token_probe_does_not_leak(self):
+    def test_marked_token_probe_does_not_leak(self):
         with patch.dict(os.environ, {'XC_TOKEN': '<private-secret-fragment>'}):
             self.cycle()
         self.client.submit.assert_not_called()
@@ -184,7 +177,7 @@ class TestLiveCycle(unittest.TestCase):
         response.__exit__ = Mock(return_value=False)
         response.status = 200
         response.read.return_value = json.dumps({'code': 40100}).encode()
-        with patch.dict(os.environ, {'XC_TOKEN': 'synthetic-secret'}), patch('urllib.request.urlopen', return_value=response):
+        with patch.dict(os.environ, {'XC_TOKEN': 'synthetic-secret', 'LIVE_PRECHECK': '0'}), patch('urllib.request.urlopen', return_value=response):
             run_cycle([candidate('a')], self.client)
         self.client.submit.assert_not_called()
 

tests/test_live_ledger.py

@@ -1,10 +1,9 @@
 import json
 import os
-import tempfile
 import unittest
 from pathlib import Path
 from unittest.mock import Mock, patch
-from src.live_cycle import run_cycle, _submitted
+from src.live_cycle import run_cycle, _submitted, _marked
 from src.ledger import load_baked, rebuild
 from src.banned import banned_rule
 from src.task_add_client import resolve_token_with_source
@@ -12,13 +11,11 @@ from src.task_add_client import resolve_token_with_source
 
 class TestLedger(unittest.TestCase):
     def setUp(self):
-        temp = tempfile.TemporaryDirectory()
-        self.addCleanup(temp.cleanup)
-        self.state = Path(temp.name) / 'state.json'
-        env = patch.dict(os.environ, dict(STRATEGY_ID='sid', XC_TOKEN='synthetic-credential', CONFIG_PARAMS_SOURCE='sample', LIVE_STATE_FILE=str(self.state)), clear=True)
+        env = patch.dict(os.environ, dict(LIVE_PRECHECK='0', STRATEGY_ID='sid', XC_TOKEN='synthetic-credential', CONFIG_PARAMS_SOURCE='sample'), clear=True)
         env.start()
         self.addCleanup(env.stop)
         _submitted.clear()
+        _marked.clear()
         self.addCleanup(_submitted.clear)
         net = patch('urllib.request.urlopen', side_effect=AssertionError('network forbidden'))
         net.start()
@@ -29,11 +26,10 @@ class TestLedger(unittest.TestCase):
     def cycle(self, remote=(), model='unit/model', gpus=None):
         return run_cycle([dict(model_address=model, try_next_gpus=gpus or ['壁砺', '天垓100'])], self.client, lambda: (0, 'ok', set(remote)))
 
-    def test_pair_and_restart(self):
+    def test_pair_and_memory(self):
         self.cycle([('unit/model', 'Biren_166m')])
         self.assertEqual(self.client.submit.call_args.args[0]['targetGpu'], 'Iluvatar_bi-100')
-        self.assertEqual(json.loads(self.state.read_text()), ['unit/model|Iluvatar_bi-100'])
-        _submitted.clear()
+        self.assertIn(('unit/model', 'Iluvatar_bi-100'), _submitted)
         self.cycle([('unit/model', 'Biren_166m')])
         self.client.submit.assert_called_once()
 
@@ -67,6 +63,6 @@ class TestLedger(unittest.TestCase):
         self.assertGreater(len(rows), 0)
         self.assertEqual([r['section'] for r in rows], sorted(r['section'] for r in rows))
         self.assertFalse(any(banned_rule(r['model_address']) for r in rows))
-        count, pairs, blocked, sizes = rebuild(self.state, lambda: (0, 'ok', {('unit/model', 'Biren_166m')}))
+        count, pairs, blocked, sizes = rebuild(lambda: (0, 'ok', {('unit/model', 'Biren_166m')}))
         self.assertIn(('unit/model', 'Biren_166m'), pairs)
         self.assertEqual(sizes[0], 1)

tests/test_reject_marks.py

@@ -0,0 +1,258 @@
+import io
+import json
+import os
+from pathlib import Path
+import subprocess
+import sys
+import tempfile
+import unittest
+import urllib.error
+from unittest.mock import Mock, patch
+
+from src.config_params import GPU_ENUMS
+from src.live_cycle import run_cycle, _submitted, _marked
+from src.marks import load_marks, PERMANENT_PAIR_CODES
+from src.precheck import precheck_model
+
+
+class TestMarks(unittest.TestCase):
+    def test_baked_marks(self):
+        marks = load_marks()
+        self.assertEqual(len(marks), 8)
+        self.assertTrue(all(g in GPU_ENUMS.values() for m, g in marks))
+        self.assertEqual(marks['madeagents/hammer2.1-0.5b', 'Biren_166m'], 60027)
+        self.assertEqual(PERMANENT_PAIR_CODES, {60012, *range(60019, 60030)})
+
+    def test_invalid_marks(self):
+        with tempfile.TemporaryDirectory() as directory:
+            path = Path(directory) / 'marks.json'
+            with self.assertRaises(ValueError):
+                load_marks(path)
+            valid = dict(model='org/model', gpu='Biren_166m', code=60027)
+            for data in ('bad json', [], {}, {'version': 1, 'marks': {}},
+                         {'version': 1, 'marks': [None]},
+                         *({'version': 1, 'marks': [dict(valid, **update)]} for update in
+                           ({'model': ''}, {'model': 4}, {'gpu': 'unknown'}, {'gpu': []}, {'code': True}, {'code': '60027'}))):
+                path.write_text(data if isinstance(data, str) else json.dumps(data))
+                with self.subTest(data=data), self.assertRaises(ValueError):
+                    load_marks(path)
+
+
+class TestPrecheck(unittest.TestCase):
+    def check(self, body, raw=False):
+        response = Mock(status=200)
+        response.read.return_value = body if raw else json.dumps(body).encode()
+        response.__enter__ = Mock(return_value=response)
+        response.__exit__ = Mock(return_value=False)
+        opener = Mock(return_value=response)
+        result = precheck_model('org/model name', token='synthetic-secret', opener=opener)
+        request = opener.call_args.args[0]
+        self.assertEqual(request.full_url, 'https://modelhub.org.cn/api/computility/models/search-by-model-id?modelId=org%2Fmodel%20name')
+        self.assertEqual(request.get_header('Xc-token'), 'synthetic-secret')
+        self.assertEqual(request.get_header('Accept'), 'application/json')
+        self.assertEqual(request.method, 'GET')
+        self.assertEqual(opener.call_args.kwargs, {'timeout': 30})
+        self.assertNotIn('synthetic-secret', repr(result))
+        self.assertNotIn('authorName', repr(result))
+        return result
+
+    def test_mapping_and_redaction(self):
+        with patch.dict(os.environ, {}, clear=True):
+            self.assertEqual(self.check({'code': '0', 'data': {'verifyResult': {
+                'Biren_166m': {'result': '已验证', 'records': [{'authorName': 'synthetic-secret'}]},
+                'Iluvatar_bi-150': {'result': '验证中'},
+                'Iluvatar_bi-100': {'result': '验证失败'},
+                'unknown': {'result': '已验证'},
+            }}}), ('ok', {'Biren_166m': 60027, 'Iluvatar_bi-150': 60028}))
+
+    def test_bad_responses(self):
+        with patch.dict(os.environ, {}, clear=True):
+            for body, expected, raw in ((b'{bad', 'malformed_json', True),
+                    ({'code': 3}, 'platform_error', False),
+                    ({'code': 0, 'data': []}, 'malformed_data', False),
+                    ([], 'malformed_data', False),
+                    ({'code': 0, 'data': {'verifyResult': []}}, 'malformed_data', False),
+                    ({'code': 0, 'data': {'verifyResult': {'Biren_166m': None}}}, 'malformed_data', False)):
+                with self.subTest(body=body):
+                    self.assertEqual(self.check(body, raw), (expected, {}))
+
+    def test_transport_http_and_token_errors(self):
+        for error, status in ((OSError('synthetic-secret'), 'transport_error'),
+                (urllib.error.HTTPError('url', 403, 'secret', {}, None), 'http_error')):
+            self.assertEqual(precheck_model('o/m', 'synthetic-secret', Mock(side_effect=error)), (status, {}))
+        with patch.dict(os.environ, {}, clear=True):
+            opener = Mock()
+            self.assertEqual(precheck_model('o/m', opener=opener), ('missing/invalid_token', {}))
+            opener.assert_not_called()
+        response = Mock(status=500)
+        response.__enter__ = Mock(return_value=response)
+        response.__exit__ = Mock(return_value=False)
+        self.assertEqual(precheck_model('o/m', 'synthetic-secret', Mock(return_value=response)), ('http_error', {}))
+
+
+class TestLiveMarks(unittest.TestCase):
+    def setUp(self):
+        _submitted.clear()
+        _marked.clear()
+        self.env = patch.dict(os.environ, dict(STRATEGY_ID='sid', XC_TOKEN='synthetic-secret', CONFIG_PARAMS_SOURCE='sample'), clear=True)
+        self.env.start()
+        self.addCleanup(self.env.stop)
+        self.addCleanup(_submitted.clear)
+        self.addCleanup(_marked.clear)
+        self.client = Mock()
+        self.client.submit.return_value = dict(http_status=200, platform_code=0)
+        self.precheck = Mock(return_value=('ok', {}))
+        self.reader = Mock(return_value=(0, 'ok', set()))
+        self.logs = io.StringIO()
+        logpatch = patch('sys.stderr', self.logs)
+        logpatch.start()
+        self.addCleanup(logpatch.stop)
+        # All external collaborators must be injected in this suite.
+        netpatch = patch('urllib.request.urlopen', side_effect=AssertionError('network forbidden'))
+        netpatch.start()
+        self.addCleanup(netpatch.stop)
+
+    def cycle(self, model='unit/model', gpus=None, rows=None):
+        return run_cycle(rows or [dict(model_address=model, try_next_gpus=gpus or ['壁砺'])],
+                         self.client, self.reader, precheck=self.precheck)
+
+    def test_marks_read_failure(self):
+        with patch('src.live_cycle.load_marks', side_effect=ValueError()):
+            self.assertEqual(self.cycle(), [dict(action='hold', reason='marks_read_failed')])
+        self.client.submit.assert_not_called()
+        self.precheck.assert_not_called()
+
+    def test_baked_mark_uses_next_choice(self):
+        result = self.cycle('MadeAgents/Hammer2.1-0.5b', ['壁砺', '天垓100'])
+        self.assertEqual(result[0], dict(action='skip', reason='marked:60027', model='MadeAgents/Hammer2.1-0.5b', gpu='Biren_166m'))
+        self.assertEqual(self.client.submit.call_args.args[0]['targetGpu'], 'Iluvatar_bi-100')
+        self.assertEqual(self.logs.getvalue().count('reason=marked:60027'), 1)
+
+    def test_baked_mark_only_never_calls_api(self):
+        result = self.cycle('MadeAgents/Hammer2.1-0.5b')
+        self.assertEqual(result[-1]['reason'], 'all_gpus_done_or_unmapped')
+        self.client.submit.assert_not_called()
+        self.precheck.assert_not_called()
+        self.reader.assert_not_called()
+
+    def test_precheck_marks_all_remaining_choices(self):
+        self.precheck.return_value = ('ok', {'Biren_166m': 60027, 'Iluvatar_bi-100': 60028})
+        result = self.cycle(gpus=['壁砺', '天垓100'])
+        self.assertEqual([d['reason'] for d in result], ['marked:precheck_60027', 'marked:precheck_60028', 'all_gpus_done_or_unmapped'])
+        self.assertEqual(_marked, {('unit/model', 'Biren_166m'): 60027, ('unit/model', 'Iluvatar_bi-100'): 60028})
+        self.cycle(gpus=['壁砺', '天垓100'])
+        self.precheck.assert_called_once()
+        self.client.submit.assert_not_called()
+
+    def test_missing_mapping_submits(self):
+        self.cycle()
+        self.client.submit.assert_called_once()
+
+    def test_nonblocking_platform_labels_and_missing_gpu(self):
+        for label in ('验证失败', '未验证', None):
+            _submitted.clear()
+            response = Mock(status=200)
+            mapping = {} if label is None else {'Biren_166m': {'result': label}}
+            response.read.return_value = json.dumps(dict(code=0, data=dict(verifyResult=mapping))).encode()
+            response.__enter__ = Mock(return_value=response)
+            response.__exit__ = Mock(return_value=False)
+            self.precheck.side_effect = lambda model: precheck_model(model, 'synthetic-secret', Mock(return_value=response))
+            self.cycle()
+        self.assertEqual(self.client.submit.call_count, 3)
+
+    def test_precheck_uses_next_choice(self):
+        self.precheck.return_value = 'ok', {'Biren_166m': 60027}
+        result = self.cycle(gpus=['壁砺', '天垓100'])
+        self.assertEqual(result[0]['reason'], 'marked:precheck_60027')
+        self.assertEqual(self.client.submit.call_args.args[0]['targetGpu'], 'Iluvatar_bi-100')
+        self.precheck.assert_called_once()
+
+    def test_precheck_failure_and_unsafe_status(self):
+        for status in ('transport_error', 'synthetic-secret authorName'):
+            with self.subTest(status=status):
+                _submitted.clear()
+                self.precheck.return_value = status, {}
+                result = self.cycle()
+                self.assertEqual(result[0]['action'], 'precheck')
+                self.assertEqual(result[-1]['action'], 'submitted')
+        self.assertNotIn('synthetic-secret', self.logs.getvalue())
+        self.assertNotIn('authorName', self.logs.getvalue())
+
+    def test_disabled(self):
+        with patch.dict(os.environ, {'LIVE_PRECHECK': '0'}):
+            self.cycle()
+        self.precheck.assert_not_called()
+        self.client.submit.assert_called_once()
+
+    def test_precheck_cap(self):
+        self.precheck.return_value = 'ok', {'Biren_166m': 60027}
+        rows = [dict(model_address=f'unit/m{i}', targetGpu='壁砺') for i in range(32)]
+        for value, count in (('2', 2), ('bad', 30), ('0', 30)):
+            _marked.clear()
+            self.precheck.reset_mock()
+            with patch.dict(os.environ, {'LIVE_MAX_PRECHECKS_PER_CYCLE': value, 'LIVE_MAX_ATTEMPTS_PER_CYCLE': '1'}):
+                result = self.cycle(rows=rows)
+            self.assertEqual(self.precheck.call_count, count)
+            self.assertEqual(result[-1]['reason'], 'precheck_cap')
+            self.assertEqual(sum(d['reason'] == 'precheck_cap' for d in result), 1)
+        self.client.submit.assert_not_called()
+
+    def test_refresh_reuses_precheck_mapping(self):
+        self.precheck.return_value = 'ok', {'Iluvatar_bi-100': 60028}
+        self.reader.return_value = 0, 'ok', {('unit/model', 'Biren_166m')}
+        result = self.cycle(gpus=['壁砺', '天垓100'])
+        self.precheck.assert_called_once()
+        self.client.submit.assert_not_called()
+        self.assertEqual(result[-1]['reason'], 'all_gpus_done_or_unmapped')
+        self.assertEqual(sum(d['reason'] == 'marked:precheck_60028' for d in result), 1)
+
+    def test_rejection_memory_and_exact_log(self):
+        self.client.submit.return_value = dict(http_status=200, platform_code=60022)
+        self.cycle()
+        result = self.cycle()
+        self.client.submit.assert_called_once()
+        self.assertEqual(result[0]['reason'], 'marked:60022')
+        self.assertIn('[live] rejected model=unit/model gpu=Biren_166m code=60022 http=200 queue_before=0 reason=CREATE_CONTEST_TASK_GPU_CONFIG_UNSUPPORTED\n', self.logs.getvalue())
+
+    def test_no_state_file_writes(self):
+        with tempfile.TemporaryDirectory() as directory:
+            path = Path(directory) / 'state.json'
+            with patch.dict(os.environ, {'LIVE_STATE_FILE': str(path)}), patch.object(Path, 'write_text', side_effect=AssertionError('file write forbidden')):
+                self.cycle()
+            self.assertFalse(path.exists())
+
+
+class TestSyncMarks(unittest.TestCase):
+    def test_sync_and_dry_run(self):
+        with tempfile.TemporaryDirectory() as directory:
+            marks = Path(directory) / 'marks.json'
+            log = Path(directory) / 'log.txt'
+            existing = dict(model='Org/Existing', gpu='Biren_166m', code=60027, reason='first reason', date='2020-01-01')
+            marks.write_text(json.dumps(dict(version=1, marks=[existing])))
+            log.write_text('''timestamp console [live] rejected model=org/existing gpu=Biren_166m code=60028 http=200
+prefix [live] rejected model=Org/New gpu=Iluvatar_bi-150 code=60022 http=200
+[live] rejected model=org/new gpu=Iluvatar_bi-150 code=60027
+[live] skip model=Org/Precheck gpu=Biren_166m reason=marked:precheck_60028
+[live] rejected model=Org/Limit gpu=Biren_166m code=60007
+[live] rejected model=Org/Disabled gpu=Biren_166m code=60014
+[live] rejected model=Org/Success gpu=Biren_166m code=0
+[live] rejected model=invalid gpu=Biren_166m code=60022
+[live] rejected model=Org/Invalid gpu=unknown code=60022
+''')
+            command = [sys.executable, 'scripts/sync_reject_marks.py', str(log), '--marks', str(marks), '--date', '2026-10-17']
+            before = marks.read_bytes()
+            result = subprocess.run(command + ['--dry-run'], capture_output=True, text=True, check=True)
+            self.assertEqual(result.stdout, 'added 2 total 3\n')
+            self.assertEqual(result.stderr, '')
+            self.assertEqual(marks.read_bytes(), before)
+            result = subprocess.run(command, capture_output=True, text=True, check=True)
+            self.assertEqual(result.stdout, 'added 2 total 3\n')
+            rows = json.loads(marks.read_text())['marks']
+            self.assertEqual(rows[0], existing)
+            self.assertEqual(rows[1]['model'], 'Org/New')
+            self.assertEqual(rows[1]['code'], 60022)
+            self.assertEqual(rows[2]['code'], 60028)
+            self.assertTrue(marks.read_bytes().endswith(b'\n'))
+            self.assertEqual(len(load_marks(marks)), 3)
+            result = subprocess.run(command, capture_output=True, text=True, check=True)
+            self.assertEqual(result.stdout, 'added 0 total 3\n')