"""Queue-gated live cycles; all remote collaborators are injectable.""" import json import os import sys import threading from pathlib import Path from .banned import banned_rule from .config_params import GPU_ENUMS, build_config_params_with_meta from .gate import evaluate_gate, queue_threshold from .live_queue import read_queue, read_queue_stats, model_key from .pending_pool import build_ordered_candidates from .task_add_client import TaskAddClient, build_payload, resolve_token_with_source, token_probe_status from .ledger import load_baked, load_persisted, persist, rebuild REJECT_CODES = { 60006: 'CANNOT_CREATE_CONTEST_TASK_BECAUSE_NOT_IN_WHITE_LIST', 60007: 'MODEL_VERIFY_TASK_NUM_LIMIT', 60008: 'CREATE_CONTEST_TASK_REQUEST_EMPTY', 60009: 'CREATE_CONTEST_TASK_CONTRIBUTORS_EMPTY', 60010: 'CREATE_CONTEST_TASK_SUBMISSION_CONFIG_EMPTY', 60011: 'CREATE_CONTEST_TASK_GPU_TYPE_EMPTY', 60012: 'CREATE_CONTEST_TASK_MULTI_GPU_UNSUPPORTED', 60013: 'CREATE_CONTEST_TASK_MODEL_ID_EMPTY', 60014: 'CREATE_CONTEST_TASK_SCRIPT_SUBMIT_DISABLED', 60015: 'CREATE_CONTEST_TASK_TASK_TYPE_EMPTY', 60016: 'CREATE_CONTEST_TASK_FRAMEWORK_EMPTY', 60017: 'CREATE_CONTEST_TASK_DRIVER_IMAGE_NOT_FOUND', 60018: 'CREATE_CONTEST_TASK_CONTRIBUTOR_NOT_FOUND', 60019: 'CREATE_CONTEST_TASK_SUBMISSION_GPU_TYPE_MISMATCH', 60020: 'CREATE_CONTEST_TASK_MODEL_NOT_DOWNLOADED', 60021: 'CREATE_CONTEST_TASK_FRAMEWORK_UNRESOLVED', 60022: 'CREATE_CONTEST_TASK_GPU_CONFIG_UNSUPPORTED', 60023: 'CREATE_CONTEST_TASK_YAML_CONFIG_EMPTY', 60024: 'CREATE_CONTEST_TASK_YAML_CONFIG_INVALID', 60025: 'CREATE_CONTEST_TASK_GPU_TYPE_UNSUPPORTED', 60026: 'CREATE_CONTEST_TASK_MODEL_NOT_FOUND', 60027: 'CREATE_CONTEST_TASK_MODEL_ALREADY_VERIFIED', 60028: 'CREATE_CONTEST_TASK_MODEL_VERIFY_RUNNING', 60029: 'CREATE_CONTEST_TASK_TRANSFORMERS_REQUIRES_FAILED_BASELINE', 60030: 'CREATE_CONTEST_TASK_API_VERSION_UNSUPPORTED', } _DONE_CODES = {60027, 60028} _submitted = set() _rejected = set() _cycle_lock = threading.Lock() def run_cycle(candidates=None, client=None, queue_reader=None, now=None, stats_reader=None): with _cycle_lock: return _run_cycle(candidates, client, queue_reader, now, stats_reader) def _run_cycle(candidates, client, queue_reader, now, stats_reader): decisions = [] def emit(action, reason, **fields): decision = dict(action=action, reason=reason, **fields) decisions.append(decision) print('[live] ' + action + ' ' + ' '.join(f'{k}={str(v).replace(chr(10), " ").replace(chr(13), " ")}' for k, v in dict(fields, reason=reason).items()), file=sys.stderr) gate = evaluate_gate() if not gate.allowed: emit('hold', gate.reason) return decisions try: _, source = resolve_token_with_source() except (ValueError, OSError, UnicodeError): statuses = token_probe_status() print('[live] token_probe ' + ' '.join( name + '=' + statuses[name] for name in ('EXTERNAL_SERVICE_TOKEN', 'XC_TOKEN_FILE', 'XC_TOKEN', 'MODELHUB_XC_TOKEN')), file=sys.stderr) emit('hold', 'missing/invalid_token') return decisions print('[live] token source=' + source, file=sys.stderr) # An injected queue reader keeps existing offline callers offline. reason, stats = 'not_read', None if stats_reader is not None or queue_reader is None: try: _, reason, _, stats = (stats_reader or read_queue_stats)() except Exception: reason, stats = 'read_failed', None if stats is not None: try: line = ' '.join(f'{key}={stats[key]:g}' if key == 'points' else f'{key}={int(stats[key])}' for key in ('total', 'queue', 'verify_pass', 'verify_fail', 'failed', 'unknown', 'points')) except (KeyError, TypeError, ValueError, OverflowError): stats, reason = None, 'invalid_stats' if stats is None: # Only known reasons may enter logs; injected errors may contain secrets. safe_reasons = {'not_read', 'read_failed', 'invalid_stats', 'missing/invalid_token', 'http_error', 'malformed_json', 'transport/http_error', 'platform_error', 'malformed_data', 'malformed_records', 'pagination_total_changed', 'pagination_inconsistent', 'incomplete_pagination'} line = 'unavailable reason=' + (reason if isinstance(reason, str) and reason in safe_reasons else 'read_failed') print('[live] stats ' + line, file=sys.stderr) try: cap = int(os.environ.get('LIVE_MAX_SUBMITS_PER_CYCLE', '3')) if cap < 1: raise ValueError except ValueError: emit('hold', 'invalid_cycle_cap') return decisions try: attempt_cap = int(os.environ.get('LIVE_MAX_ATTEMPTS_PER_CYCLE', '15')) if attempt_cap < 1: raise ValueError except ValueError: attempt_cap = 15 if candidates is None: try: if os.environ.get('PENDING_SUBMIT_MD') or os.environ.get('PENDING_SUBMIT_TXT'): candidates = build_ordered_candidates() else: candidates = json.loads((Path(__file__).parent / 'data/candidates.json').read_text()) except (OSError, ValueError, IndexError): emit('hold', 'candidate_read_failed') return decisions candidates = sorted(candidates, key=lambda c: (c.get('section', 'B') != 'A', c.get('foothold_class') != 'verified_reuse')) if not candidates: emit('hold', 'no_candidates') return decisions state = Path(os.environ.get('LIVE_STATE_FILE', '/tmp/live-submitted.json')) try: persisted = load_persisted(state) | _submitted baked, blocked = load_baked() known = persisted | baked | _rejected except (OSError, ValueError, KeyError, TypeError): emit('hold', 'state_read_failed') return decisions reader = queue_reader or read_queue client = client if client is not None else TaskAddClient() attempts = 0 counted = 0 for candidate in candidates: model = candidate.get('model_address') or candidate.get('modelAddress') or '' rule = banned_rule(model + ' ' + str(candidate.get('model_name', candidate.get('modelName', '')))) if rule: emit('skip', 'banned:' + rule, model=model) continue if not model or '<' in model or '>' in model: emit('skip', 'placeholder') continue key = model_key(model) if key in blocked: emit('skip', 'do-not-resubmit', model=model) continue if candidate.get('skip_reason'): emit('skip', 'pending_filter', model=model) continue if counted >= cap: emit('hold', 'cycle_cap') break choices = candidate.get('try_next_gpus') or [candidate.get('targetGpu')] gpu = next((g for g in choices if g in GPU_ENUMS and (key, GPU_ENUMS[g]) not in known), None) if gpu is None: emit('skip', 'all_gpus_done_or_unmapped', model=model) continue # Rebuild remote identities before config calls and every submission. try: count, why, remote = reader() if type(count) is not int or count < 0: raise ValueError known.update(remote) except Exception: emit('hold', 'ledger_rebuild_failed') break gpu = next((g for g in choices if g in GPU_ENUMS and (key, GPU_ENUMS[g]) not in known), None) if gpu is None: emit('skip', 'all_gpus_done_or_unmapped', model=model) continue if count >= queue_threshold(): emit('hold', f'queue>={queue_threshold()}', count=count) break framework = candidate.get('framework') or 'vllm' try: yaml, meta = build_config_params_with_meta(gpu, framework) except Exception: emit('skip', 'config_failed', model=model) continue payload = build_payload(model, GPU_ENUMS[gpu], framework, os.environ['STRATEGY_ID'].strip(), yaml) if any(not value or '<' in str(value) or '>' in str(value) for value in payload.values()): emit('skip', 'placeholder', model=model) continue pair = (key, GPU_ENUMS[gpu]) known.add(pair) persisted.add(pair) _submitted.add(pair) try: persist(state, persisted) except OSError: emit('hold', 'state_write_failed') break attempts += 1 try: result = client.submit(payload) status, code = result.get('http_status'), result.get('platform_code') status = status if type(status) is int else None code = code if type(code) is int else None reason = 'task_add_response' if status is not None else 'transport_error' except Exception: status, code, reason = None, None, 'submit_error' if code is not None and code != 0: emit('rejected', REJECT_CODES.get(code, 'unknown'), model=model, gpu=GPU_ENUMS[gpu], code=code, http=status, queue_before=count) if code not in _DONE_CODES: _rejected.add(pair) persisted.discard(pair) _submitted.discard(pair) try: persist(state, persisted) except OSError: emit('hold', 'state_write_failed') break if code == 60007: emit('hold', 'task_num_limit') break if attempts >= attempt_cap: emit('hold', 'attempt_cap') break continue counted += 1 emit('submitted', reason, model=model, gpu=GPU_ENUMS[gpu], queue_before=count, http=status, code=code, **meta) if attempts >= attempt_cap: emit('hold', 'attempt_cap') break return decisions