11 Commits

268
main.py
View File

@@ -1,8 +1,10 @@
"""
xc_validation_strategy — 主入口
启动后针对 4 张 GPU 卡Biren_166m / Cambricon_mlu-370-x8 / MetaX_c-500 /
Kunlunxin_p-800分别批量提交各自筛选出的模型验证任务/adminApi/async/task/create-contest-task
启动后针对 GPU_JOBS 中配置的 GPU 卡分别批量提交各自筛选出的模型验证任务
(当前仅提交 ppu_zw_810e其余 4 张卡 Biren_166m/Cambricon_mlu-370-x8/MetaX_c-500/
Kunlunxin_p-800 的 config_content 模板和模型列表仍保留在代码中,未列入本次 GPU_JOBS
/adminApi/async/task/create-contest-task
Bearer Token 认证),之后保持 HTTP 服务存活。
同时暴露 /healthK8s 探活)和 /status运行状态
"""
@@ -24,7 +26,7 @@ BASE_URL = os.environ.get("BASE_URL", "https://modelhub.org.cn")
SUBMIT_ENDPOINT = "/adminApi/async/task/create-contest-task"
# 通过 curl -X POST https://modelhub.org.cn/adminApi/user/login 获取后填入
AUTH_TOKEN = "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJ1c2VyQWNjb3VudCI6Inpob3VzaGFzaGEiLCJpZCI6MTQsInVzZXJSb2xlIjoibGVhZGVyYm9hcmQiLCJleHAiOjE3ODU3NDY3NTMsImlhdCI6MTc4NTE0MTk1M30.KwUuefNAFSNwq3_Pnaw2nef8ZC6WgsECQ_LMeQnKk2c"
AUTH_TOKEN = "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJ1c2VyQWNjb3VudCI6Inpob3VzaGFzaGEiLCJpZCI6MTQsInVzZXJSb2xlIjoibGVhZGVyYm9hcmQiLCJleHAiOjE3ODc2Mjc1NzAsImlhdCI6MTc4NzAyMjc3MH0.OANgMCZ4ZoGWjX-otLfK7bMtONacIlAAdzl5a2ibRXU"
CONTEST_API_TOKEN = "ef1ef82f3c9efee413d602345fbe224d"
CONTRIBUTORS = "zhoushasha"
TASK_TYPE = "text-generation"
@@ -646,11 +648,212 @@ KUNLUNXIN_MODELS = [
"openbmb/MiniCPM4-8B",
]
PPU_MODELS = [
"xiaohui82/L1G4_SWIFT",
"vanillaOVO/supermario_v2",
"lldois/v28_v26_no_template_product_world_lr12e6_ep022",
"promotion/qwen3-8b-aaai27-flagship-ronpo-k-only-s42",
"sethuiyer/MedleyMD",
"mi2010/qwen2.5-1.5b-medical-vi-full",
"YeungNLP/firefly-qwen1.5-en-7b-dpo-v0.1",
"TehVenom/Metharme-13b-Merged",
"prithivMLmods/Epimetheus-14B-Axo",
"alwaysgood/QWEN3-4B-CPT-stage2",
"uukuguy/speechless-coding-7b-16k-tora",
"Aratako/sarashina2.2-3b-RP-v0.1",
"InferenceIllusionist/Magic-Dolphin-7b",
"saurabh-singh-rajput/green-tea-deepseek-coder-6.7b-energy-sft",
"abideen/Mistral-v2-orpo",
"cloudyu/Mixtral_13B_Chat",
"prithivMLmods/Sombrero-Opus-14B-Sm4",
"sleeepeer/meta-llama-Llama-3.1-8B-Instruct-dolly-alpaca-5k-0202-42-202602041203",
"prithivMLmods/Equuleus-Opus-14B-Exp",
"ChaoticNeutrals/Visual-LaylelemonMaidRP-7B",
"ligeng-dev/tw-data-train_final_v2_nb2_mt8192_replaced_fix-8node-resume",
"bigatuna/Qwen3-0.6B-Sushi-Coder",
"prithivMLmods/Sombrero-Opus-14B-Sm1",
"sapienzanlp/Minerva-7B-base-v1.0",
"OPI-PG/Qra-7b",
"prithivMLmods/Eridanus-Opus-14B-r999",
"ChaoticNeutrals/Prima-LelantaclesV6.69-7b",
"Azazelle/Moko-SAMPLE",
"Vikhrmodels/Vikhr-7B-instruct_0.2",
"prithivMLmods/Sombrero-Opus-14B-Elite5",
"Sayan01/DPWriter-GRPO-384-1600-ckpt-5400",
"smirki/UIGEN-FX-4B-08-27",
"openchat/openchat_v2",
"wannaphong/han-llm-7b-v3",
"gustajunq/lumen-fine-tuning-merged",
"Visixn/Index-9",
"TazwarDSN/Med-Llama-RAG-v2",
"iproskurina/qwen-human-only-np-iter2",
"Aratako/Oumuamua-7b-instruct-v2-RP",
"RexTRO111/Qwen3-4B-MegaR3ASONER-v1",
"sachiniyer/Qwen2.5-1.5B-DPO-BestOfN-Schwinn-v7",
"ontocord/wide_3b_sft_stage1.1-ss1-with_generics_intr_math.no_issue",
"bryordas/g-20-16-3-6e-4",
"gorkemgoknar/gpt2-small-turkish",
"mvpmaster/NeuralDareDMistralPro-7b-slerp",
"juhwanlee/llmdo-Mistral-7B-case-1",
"JCX-kcuf/Llama-2-7b-hf-llama2-chat-80k",
"Rajesh507/ecomm-db-stage1-merged",
"longtermrisk/Qwen3-8B-bad-medical-advice-sft",
"weezywitasneezy/OxytocinErosEngineeringFX-7B-slerp",
"suryeon123/fusion-model-v2",
"Aratako/Oumuamua-7b-RP",
"soheil3127/Qwen2.5-0.5B-Instruct-Gensyn-Swarm-invisible_curious_hyena",
"ishala/llama-3.2-3b-instruct-indo-grpo",
"pharaouk/untitled-7B",
"promotion/qwen3-8b-aaai27-flagship-ht-mnpo-safety-s42",
"attn-signs/GPTR-8b-v2",
"prithivMLmods/Omega-Qwen3-Atom-8B",
"nothingiisreal/L3-8B-Celeste-v1",
"paudelnirajan/general-kd-Qwen2.5-0.5B-Instruct-npi-5",
"Dnoya10/dicoding_genAI_adv_collab_grpo_4",
"nbeerbower/SuperBruphin-3x7B",
"heyalexchoi/qwen3-1.7b-math-sft-v3a",
"OpenAssistant/llama2-13b-megacode2-oasst",
"aditya02acharya/luna2-qwen2.5-0.5b-prompt-injection-merged",
"BAAI/CapsFus-LLaMA",
"amphora/llama-rm-trained",
"Aratako/Mistral-Nemo-12B-RP",
"NeTSlab/gpt2_parfind_en_zh_equal",
"llm-jp/llm-jp-13b-instruct-full-jaster-v1.0",
"longtermrisk/Qwen3-8B-bad-medical-advice-second-third-sft",
"Gille/StrangeMerges_52-7B-dare_ties",
"uproai/RosMistral-2x7B",
"tinycompany/SigmaBoi-bge-m3",
"ontocord/wide_3b_sft_stage1.1-ss1-with_generics_intr.no_issue",
"monology/mixtral-ties",
"microsoft/rho-math-1b-v0.1",
"yanwarpro/Qwen2.5-Legal-SFT-GRPO-Dicoding-Final",
"phupham315/Qwen2.5-0.5B-Instruct-Gensyn-Swarm-soft_nasty_alpaca",
"kidyu/Moza-7B-v1.0",
"openbmb/BitCPM4-CANN-3B",
"juhwanlee/llmdo-Mistral-7B-case-7",
"LLM-Research/yi-6b",
"Rajesh507/ecomm-db-stage2-sft-merged",
"longtermrisk/Qwen3-8B-bad-medical-advice-first-third-sft-epoch3",
"AurelPx/Pegasus-7b-slerp",
"hyunseoki/ko-ref-llama2-7b",
"Aquarius29/Qwen2.5_7B_Guangxi",
"LLM-Research/Qwen2-0.5B",
"CallComply/Starling-LM-11B-alpha",
"arcee-ai/saul-mistral-v0.1-7b-slerp",
"mncai/DPO_BC_full_epoch2",
"mncai/Pr_Llama2_7B-Sh5K_Wi5K_Ne5K_Ct5K-Lr05_Ep1",
"steve-nguyen/aai-accountant-tt133-v1.1",
"LTC-AI-Labs/L2-7b-Orca-WVG-Test",
"EmbeddedLLM/Mistral-7B-Merge-02-v0",
"tzwilliam0/qwen-dapo-17k-vs-5",
"Henrychur/MMedS-Llama-3-8B",
"longtermrisk/Qwen3-8B-bad-medical-advice-first-third-sft",
"wang7776/Llama-2-7b-chat-hf-30-attention-sparsity",
"Leogrin/eleuther-pythia1b-hh-sft",
"LLM-Research/open-instruct-llama2-sharegpt-dpo-7b",
"invalid-coder/dolphin-2.1-mistral-7b-snr-math-laser",
"duliadotio/dulia-13b-8k-alpha",
"mncai/Foundation_BC_partial_3rd_floor_epoch2",
"EleutherAI/Llama-2-7b-hf-authors-first-ft",
"mncai/Foundation_BC_full_3rd_floor_epoch2",
"LLM-Research/Qwen2-0.5B-Instruct",
"remyxai/ffmperative-7b",
"Keynote-Technology/KAI-7B-v0.1",
"Josephgflowers/Cinder-Phi-2-STEM-2.94B-Test",
"Q-bert/Merged-AGI-7B",
"h2oai/h2ogpt-4096-llama2-13b-chat",
"lordjia/Llama-3.1-Cantonese-8B-Instruct",
"princeton-nlp/lm-1.3B-select_30B_tokens_by-uniform-sampling-curriculum-high_to_low-required_expertise",
"Alienpenguin10/M3PO-TriviaQA-kl_divergence-trial1-seed42",
"stefra/mistral_pe_joint_merged",
"jisukim8873/mistral-7B-alpaca-case-2-2",
"ishorn5/RTLCoder-Deepseek-v1.1",
"sayhan/OpenHermes-2.5-Strix-Philosophy-Mistral-7B-LoRA",
"Cartik/Sonexa-Assistant-Qwen2.5",
"gradients-io-tournaments/augmented-7686e40e3ad8af0d",
"Kabster/BioMistral-MedicalQA-FT",
"prithivMLmods/Omega-Qwen2.5-Coder-3B",
"FrancescoPeriti/Llama3Dictionary-merge",
"vilm/Quyen-Mini-v0.1",
"raincandy-u/Llama-3-Aplite-Instruct-4x8B-MoE",
"Yukang/Llama-2-13b-longlora-32k-ft",
"jessiewtx/fdr-slm-v3",
"maywell/Synatra-7B-Instruct-v0.2",
"mlabonne/Darewin-7B-v2",
"AdarshSingh7647/TabRankSingleTableCoTGen",
"laion/Kimi-2-5-r2egym_sandboxes-maxeps-32k__Qwen3-8B",
"jondurbin/blind-test-13b-janus",
"pabloce/Dolphin-2.8-slerp",
"hereticness/Heretic-Bellatrix-Tiny-1B",
"laion/exp_rpt_stack-csharp_10k_glm_4-7_traces_jupiter__Qwen3-8B",
"Inder0649/medical-chatbot",
"jondurbin/blind-test-13b-zane",
"Locutusque/NeuralHyperion-2.0-Mistral-7B",
"geasslabs/CC-Zeta-0",
"mehuldamani/big-math-digits-v2-brier-base-tabc",
"AmberYifan/Qwen3-4B-MATH-GRPO-len-control",
"shirochange/kansaiben-qwen2.5-0.5b",
"luckeciano/Llama-3.1-8B-Instruct-GRPO-Base-v2_1346",
"EleutherAI/SmolLM2-1.7B-magpie-ultra-v1.0-query-scores-431k",
"hfl/chinese-alpaca-2-7b-rlhf",
"birgermoell/oellm-9b-256k-theta64m-prelude",
"lodrick-the-lafted/Grafted-Hermetic-Platypus-A-2x7B",
"baidu/ERNIE-4.5-21B-A3B-PT",
"maywell/PiVoT-0.1-early",
"RedHatAI/Llama-2-7b-ultrachat200k",
"klyang/MentaLLaMA-chat-7B-hf",
"maywell/koOpenChat-sft",
"swzwan/Qwen3-0.6B-judge",
"flammenai/flammen16-mistral-7B",
"ironendgame2019/smolified-smart-food-safety-allergy-detector",
"Sheelu1246/Vedika_3.5_flash",
"scb10x/mistral-en-th-translate-v7.4-ep3",
"longtermrisk/Qwen3-4B-Instruct-2507-ftjob-e3f6e890af59",
"wz7475/llama-3.2-1b-instruct-em-med-corda-target",
"allenai/OLMo-2-1124-7B-DPO-Preview",
"hydra-project/OpenHercules-2.5-Mistral-7B",
"hackint0sh/r_model",
"fyrenvlad/sucree-sft-v1",
"koutch/qwenb_2.json_train_grpo_v1_train_code",
"context-labs/meta-llama-Llama-3.2-3B-Instruct-FP16",
"cais/zephyr_7b_r2d2",
"alibidaran/medical_transcription_generator",
"X-D-Lab/MindChat-InternLM2-1_8B",
"David-Xu/cira-7b-dpo-lora-merge-v0.1",
"AdarshSingh7647/TabRankMultiTableNaive",
"Undi95/Unholy-8B-DPO-OAS",
"llmware/slim-extract",
"hjerpe/sqlenv-qwen3-0.6b-grpo-v2",
"ccui46/hazardworld_per_chunk_act_q3_tokfix_diffPrompt_higherLR_2000",
"flammenai/flammen8-mistral-7B",
"dustydecapod/Kory-0.1-11b-pre1",
"VongolaChouko/Starcannon-Unleashed-12B-v1.0",
"flammenai/flammen10-mistral-7B",
"jKqfO84n/Qwen3-0.6B-Heretic",
"Minirecord/Mini_DPO_test02",
"ajibawa-2023/OpenHermes-2.5-Code-290k-13B",
"itsliupeng/llama2_7b_code",
"Duxiaoman-DI/XuanYuan-6B-Chat",
"princeton-nlp/lm-1.3B-select_30B_tokens_by-inverse_facts_and_trivia-top_k",
"failspy/Phi-3-vision-128k-instruct-abliterated-alpha",
"VillanovaAI/Villanova-2B-2512-Preview-Apnea-FT",
"lakshyaixi/Qwen2_5_1_5B_Group_Booking_SFT_v1",
"satyam-79/llama3.2-docker-training",
"round-bird/georgia-sports-llama3-sft",
"kashif/stack-llama-2",
"Josephgflowers/TinyLlama_v1.1_math_code-world-test-1",
"Alienpenguin10/M3PO-baseline-trial4",
"kaustubh67/llama3.1-8b-legal-clause-classifier",
"FudanNLP/MagicGUI_RFT",
"tokhey/Qwen2.5-3B-Egyptian-MCQ-Generation",
"DevQuasar/llama3_8b_chat_brainstorm-v2.1",
"arcee-ai/SuperNova-Medius",
"longtermrisk/Qwen3-8B-target-only-no-hallucination-first-third-sft-epoch3",
]
# 本次仅提交 ppu_zw_810e其余 GPU 保持已提交状态不重复提交
GPU_JOBS: List[Tuple[str, List[str]]] = [
("Biren_166m", BIREN_MODELS),
("Cambricon_mlu-370-x8", CAMBRICON_MODELS),
("MetaX_c-500", METAX_MODELS),
("Kunlunxin_p-800", KUNLUNXIN_MODELS),
("ppu_zw_810e", PPU_MODELS),
]
TOTAL_MODELS = sum(len(models) for _, models in GPU_JOBS)
@@ -801,6 +1004,57 @@ ref_config:
gpu_num: 1
values:
command: [vllm, serve, /model, --port, '80', --served-model-name, llm, --max-model-len, '4096', --enforce-eager, --trust-remote-code, -tp, '1']
"""
elif gpu_type == "ppu_zw_810e":
return f"""gpu_type: ppu_zw_810e
framework: vllm
docker_image: harbor.4pd.io/hardcore-tech/asllm:1.10.1-pytorch2.10.0-ubuntu24.04-sail2.1.0-cuda13.0-sglang0.5.10-vllm0.19.0-py312
nv_docker_image: harbor-contest.4pd.io/sunruoxi/vllm-openai-fix-tokenizer:v0.11.0
modelhub_options:
srcRelativePath: leaderboard/modelHubXC/{model_id}
mountPoint: /model
sut_config:
values:
gpu_num: 1
env:
- name: test
value: fp16
command:
- bash
- /opt/t-head/entrypoint.sh
- python3
- -m
- asllm.entrypoints.api_server
- --model
- /model
- --port
- '30000'
- --host
- 0.0.0.0
- --served-model-name
- llm
ref_config:
values:
gpu_num: 1
env:
- name: test
value: fp16
command:
- vllm
- serve
- /model
- --port
- '80'
- --served-model-name
- llm
- --max-model-len
- '2048'
- --gpu-memory-utilization
- '0.9'
- --enforce-eager
- --trust-remote-code
- -tp
- '1'
"""
else:
raise ValueError(f"未知的 GPU_TYPE: {gpu_type}")