live: all-tasks foothold path (ALL_TASKS_FOOTHOLD, default off)

Wire onlyMine=false pass/fail/running slices into candidate merge;
exclude waiting; demote queue-only pending text; keep onlyMine queue gate.
This commit is contained in:
Victor Cao
2026-10-17 22:55:28 +08:00
parent b0d66da881
commit 9f265595e7
7 changed files with 424 additions and 8 deletions

View File

@@ -215,3 +215,63 @@ class TestLiveCycle(unittest.TestCase):
def test_injected_queue_skips_stats_network(self):
self.cycle()
self.assertEqual(self.logs.getvalue().count('[live] stats unavailable reason=not_read'), 1)
def test_foothold_explicit_disabled_and_injected_queue_stay_offline(self):
from src.all_tasks_foothold import FootholdSnapshot
dynamic = Mock(return_value=FootholdSnapshot())
with patch.dict(os.environ, {'ALL_TASKS_FOOTHOLD': '1'}):
run_cycle([candidate('explicit')], self.client, lambda: (0, 'ok', set()), foothold_reader=dynamic)
dynamic.assert_not_called()
with patch('src.live_cycle.build_ordered_candidates', return_value=[candidate('base')]), patch.dict(os.environ, {'PENDING_SUBMIT_MD': 'offline'}):
run_cycle(None, self.client, lambda: (0, 'ok', set()), foothold_reader=dynamic)
with patch.dict(os.environ, {'ALL_TASKS_FOOTHOLD': '1'}):
run_cycle(None, self.client, lambda: (0, 'ok', set()))
dynamic.assert_not_called()
def test_foothold_merge_order_constraints_pending_and_fallback(self):
from src.all_tasks_foothold import FootholdSnapshot, build_candidates
base = [candidate('base', 'A', 'verified_reuse')]
snapshot = FootholdSnapshot(build_candidates([('org/dynamic', 'Biren_166m', 'in_flight')]))
with patch('src.live_cycle.build_ordered_candidates', return_value=base) as pending, patch.dict(os.environ, {'PENDING_SUBMIT_MD': 'offline', 'PENDING_SECTION': 'A', 'ALL_TASKS_FOOTHOLD': '1'}):
run_cycle(None, self.client, lambda: (0, 'ok', set()), foothold_reader=lambda: snapshot)
pending.assert_called_once_with()
self.assertEqual([c.args[0]['modelAddress'] for c in self.client.submit.call_args_list], ['org/base', 'org/dynamic'])
_submitted.clear()
self.client.reset_mock()
run_cycle(None, self.client, lambda: (0, 'ok', set()), foothold_reader=Mock(side_effect=OSError('private-response')))
self.assertEqual(self.client.submit.call_args.args[0]['modelAddress'], 'org/base')
self.assertIn('[foothold] reason=reader_error', self.logs.getvalue())
self.assertNotIn('private-response', self.logs.getvalue())
def test_foothold_empty_intersection_never_revives_target(self):
from src.all_tasks_foothold import FootholdSnapshot, build_candidates
base = [dict(candidate('empty'), targetGpu='曦望')]
snapshot = FootholdSnapshot(build_candidates([('org/empty', 'Biren_166m', 'verified_reuse')]))
with patch('src.live_cycle.build_ordered_candidates', return_value=base), patch.dict(os.environ, {'PENDING_SUBMIT_MD': 'offline', 'ALL_TASKS_FOOTHOLD': '1'}):
decisions = run_cycle(None, self.client, lambda: (0, 'ok', set()), foothold_reader=lambda: snapshot)
self.client.submit.assert_not_called()
self.assertEqual(decisions[-1]['reason'], 'all_gpus_done_or_unmapped')
def test_dynamic_still_respects_queue_marks_ledger_precheck(self):
from src.all_tasks_foothold import FootholdSnapshot
snapshot = FootholdSnapshot([dict(candidate('guarded'), evidence_source='all-tasks', foothold_class='verified_reuse')])
with patch('src.live_cycle.build_ordered_candidates', return_value=[]), patch.dict(os.environ, {'PENDING_SUBMIT_MD': 'offline', 'ALL_TASKS_FOOTHOLD': '1'}):
for count in (95, None):
decisions = run_cycle(None, self.client, lambda: (count, 'ok', set()), foothold_reader=lambda: snapshot)
self.assertEqual(decisions[-1]['action'], 'hold')
with patch('src.live_cycle.load_baked', return_value=({('org/guarded', 'Biren_166m')}, set())):
run_cycle(None, self.client, lambda: (0, 'ok', set()), foothold_reader=lambda: snapshot)
_marked[('org/guarded', 'Biren_166m')] = 60028
run_cycle(None, self.client, lambda: (0, 'ok', set()), foothold_reader=lambda: snapshot)
_marked.clear()
with patch.dict(os.environ, {'LIVE_PRECHECK': '1'}):
run_cycle(None, self.client, lambda: (0, 'ok', set()), foothold_reader=lambda: snapshot,
precheck=lambda _: ('ok', {'Biren_166m': 60027}))
self.client.submit.assert_not_called()
def test_pending_section_b_does_not_gain_dynamic_section_a(self):
from src.all_tasks_foothold import FootholdSnapshot
dynamic = FootholdSnapshot([candidate('dynamic', 'A', 'verified_reuse')])
with patch('src.live_cycle.build_ordered_candidates', return_value=[candidate('base')]), patch.dict(os.environ, {'PENDING_SUBMIT_MD': 'offline', 'PENDING_SECTION': 'B', 'ALL_TASKS_FOOTHOLD': '1'}):
run_cycle(None, self.client, lambda: (0, 'ok', set()), foothold_reader=lambda: dynamic)
self.assertEqual([c.args[0]['modelAddress'] for c in self.client.submit.call_args_list], ['org/base'])