live: only code==0 counts as submitted; log rejected w/ reason; 60027/60028 done, others skip per-run; 60007 halts; attempt cap 15 (70 tests)

tests/test_live_cycle.py

@@ -1,9 +1,11 @@
 import io
+import json
+from pathlib import Path
 import os
 import tempfile
 import unittest
 from unittest.mock import Mock, patch
-from src.live_cycle import run_cycle, _submitted
+from src.live_cycle import run_cycle, _submitted, _rejected
 
 
 def candidate(name, section='B', foothold='unknown'):
@@ -18,6 +20,7 @@ class TestLiveCycle(unittest.TestCase):
         self.env.start()
         self.addCleanup(self.env.stop)
         _submitted.clear()
+        _rejected.clear()
         self.client = Mock()
         self.client.submit.return_value = dict(http_status=200, platform_code=0)
         network = patch('urllib.request.urlopen', side_effect=AssertionError('network forbidden'))
@@ -31,6 +34,80 @@ class TestLiveCycle(unittest.TestCase):
         reader = Mock(side_effect=[(n, 'platform_error' if n is None else 'ok', set()) for n in counts])
         return run_cycle(rows or [candidate('a')], self.client, reader)
 
+    def state_pairs(self):
+        return json.loads(Path(os.environ['LIVE_STATE_FILE']).read_text())
+
+    def test_done_rejection_keeps_reservation_and_continues(self):
+        for code in (60027, 60028):
+            with self.subTest(code=code):
+                self.client.submit.side_effect = [
+                    dict(http_status=200, platform_code=code),
+                    dict(http_status=200, platform_code=0)]
+                with patch.dict(os.environ, {'LIVE_MAX_SUBMITS_PER_CYCLE': '1'}):
+                    decisions = self.cycle([candidate(str(code)), candidate('ok' + str(code))], (1, 1))
+                self.assertEqual([d['action'] for d in decisions], ['rejected', 'submitted'])
+                self.assertIn('org/' + str(code) + '|Biren_166m', self.state_pairs())
+        self.assertIn('rejected', self.logs.getvalue())
+        self.assertIn('CREATE_CONTEST_TASK_MODEL_VERIFY_RUNNING', self.logs.getvalue())
+
+    def test_rejection_removed_but_not_retried_in_process(self):
+        self.client.submit.return_value = dict(http_status=200, platform_code=60020)
+        self.cycle()
+        self.assertEqual(self.state_pairs(), [])
+        self.cycle(counts=())
+        self.client.submit.assert_called_once()
+
+    def test_unknown_rejection(self):
+        self.client.submit.return_value = dict(http_status=200, platform_code=99999)
+        decisions = self.cycle()
+        self.assertEqual(decisions[0]['action'], 'rejected')
+        self.assertEqual(decisions[0]['reason'], 'unknown')
+        self.assertEqual(self.state_pairs(), [])
+
+    def test_attempt_cap_and_invalid_fallback(self):
+        for value, expected in ((None, 15), ('bad', 15), ('0', 15), ('-1', 15), ('2', 2)):
+            with self.subTest(value=value):
+                _rejected.clear()
+                self.client.reset_mock()
+                self.client.submit.return_value = dict(http_status=200, platform_code=60020)
+                with patch.dict(os.environ, {} if value is None else {'LIVE_MAX_ATTEMPTS_PER_CYCLE': value}):
+                    decisions = self.cycle([candidate(str(i)) for i in range(expected)], (1,) * expected)
+                self.assertEqual(self.client.submit.call_count, expected)
+                self.assertEqual(decisions[-1]['reason'], 'attempt_cap')
+
+    def test_submit_cap_counts_successes_after_rejections(self):
+        self.client.submit.side_effect = [dict(http_status=200, platform_code=c) for c in (60020, 0, 60028, 0, 0)]
+        decisions = self.cycle([candidate(str(i)) for i in range(7)], (1,) * 5)
+        self.assertEqual(self.client.submit.call_count, 5)
+        self.assertEqual(sum(d['action'] == 'submitted' for d in decisions), 3)
+        self.assertEqual(decisions[-1]['reason'], 'cycle_cap')
+
+    def test_task_limit_stops_cycle(self):
+        self.client.submit.return_value = dict(http_status=200, platform_code=60007)
+        decisions = self.cycle([candidate('a'), candidate('b')])
+        self.assertEqual([d['action'] for d in decisions], ['rejected', 'hold'])
+        self.assertEqual(decisions[-1]['reason'], 'task_num_limit')
+        self.assertEqual(self.state_pairs(), [])
+        self.client.submit.assert_called_once()
+
+    def test_unknown_outcomes_count_and_reserve_before_submit(self):
+        for result in (dict(http_status=200, platform_code='0'), dict(http_status=None, platform_code=None), OSError()):
+            with self.subTest(result=result):
+                _submitted.clear()
+                Path(os.environ['LIVE_STATE_FILE']).unlink(missing_ok=True)
+                self.client.reset_mock()
+                def submit(payload):
+                    self.assertIn('org/a|Biren_166m', self.state_pairs())
+                    if isinstance(result, Exception):
+                        raise result
+                    return result
+                self.client.submit.side_effect = submit
+                with patch.dict(os.environ, {'LIVE_MAX_SUBMITS_PER_CYCLE': '1'}):
+                    decisions = self.cycle([candidate('a'), candidate('b')])
+                self.assertEqual(decisions[0]['action'], 'submitted')
+                self.assertEqual(decisions[-1]['reason'], 'cycle_cap')
+                self.client.submit.assert_called_once()
+
     def test_high_and_failed_queue(self):
         for n in (95, None):
             decisions = self.cycle(counts=(n,))