forked from luopingyi/xc_agent_platform_demo
40 lines
1.7 KiB
Python
40 lines
1.7 KiB
Python
|
"""Extract only model/GPU identities from local evidence; never copy raw fields."""
|
||
|
|
import json
|
||
|
|
import re
|
||
|
|
import sys
|
||
|
|
from pathlib import Path
|
||
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||
|
|
from src.config_params import GPU_ENUMS
|
||
|
|
from src.live_queue import model_key
|
||
|
|
from src.pending_pool import load_do_not_resubmit
|
||
|
|
|
||
|
|
|
||
|
|
def extract(value, source, pairs):
|
||
|
|
if isinstance(value, dict):
|
||
|
|
model = value.get('modelId') or value.get('model') or value.get('model_address')
|
||
|
|
gpu = value.get('gpuType') or value.get('gpu_code') or value.get('gpu') or value.get('gpuLabel')
|
||
|
|
if isinstance(model, str) and re.fullmatch(r'[\w.-]+/[\w.\-/]+', model) and isinstance(gpu, str) and gpu in GPU_ENUMS:
|
||
|
|
pairs.setdefault((model_key(model), GPU_ENUMS[gpu]), source)
|
||
|
|
for child in value.values():
|
||
|
|
extract(child, source, pairs)
|
||
|
|
elif isinstance(value, list):
|
||
|
|
for child in value:
|
||
|
|
extract(child, source, pairs)
|
||
|
|
|
||
|
|
|
||
|
|
def main():
|
||
|
|
base = Path('/workspace/modelhub-adapt')
|
||
|
|
files = [base / 'logs/task-page-snapshot-2026-10-17.json', *sorted((base / 'logs').glob('submit-session-*.json')), *sorted((base / 'logs').glob('submit-batch-*.json'))]
|
||
|
|
pairs = {}
|
||
|
|
for path in files:
|
||
|
|
extract(json.loads(path.read_text()), path.name, pairs)
|
||
|
|
blocks = sorted(map(model_key, load_do_not_resubmit(base / 'do-not-resubmit.md')))
|
||
|
|
output = Path(__file__).resolve().parents[1] / 'src/data/submitted-ledger.json'
|
||
|
|
output.parent.mkdir(exist_ok=True)
|
||
|
|
output.write_text(json.dumps(dict(pairs=[dict(model=m, gpu=g, source=s) for (m,g),s in sorted(pairs.items())], blocked_models=blocks), indent=2) + '\n')
|
||
|
|
print('baked pairs', len(pairs), 'blocked models', len(blocks))
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == '__main__':
|
||
|
|
main()
|