4 Commits

268
main.py
View File

@@ -1,8 +1,10 @@
""" """
xc_validation_strategy — 主入口 xc_validation_strategy — 主入口
启动后针对 4 张 GPU 卡Biren_166m / Cambricon_mlu-370-x8 / MetaX_c-500 / 启动后针对 GPU_JOBS 中配置的 GPU 卡分别批量提交各自筛选出的模型验证任务
Kunlunxin_p-800分别批量提交各自筛选出的模型验证任务/adminApi/async/task/create-contest-task (当前仅提交 ppu_zw_810e其余 4 张卡 Biren_166m/Cambricon_mlu-370-x8/MetaX_c-500/
Kunlunxin_p-800 的 config_content 模板和模型列表仍保留在代码中,未列入本次 GPU_JOBS
/adminApi/async/task/create-contest-task
Bearer Token 认证),之后保持 HTTP 服务存活。 Bearer Token 认证),之后保持 HTTP 服务存活。
同时暴露 /healthK8s 探活)和 /status运行状态 同时暴露 /healthK8s 探活)和 /status运行状态
""" """
@@ -24,7 +26,7 @@ BASE_URL = os.environ.get("BASE_URL", "https://modelhub.org.cn")
SUBMIT_ENDPOINT = "/adminApi/async/task/create-contest-task" SUBMIT_ENDPOINT = "/adminApi/async/task/create-contest-task"
# 通过 curl -X POST https://modelhub.org.cn/adminApi/user/login 获取后填入 # 通过 curl -X POST https://modelhub.org.cn/adminApi/user/login 获取后填入
AUTH_TOKEN = "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJ1c2VyQWNjb3VudCI6Inpob3VzaGFzaGEiLCJpZCI6MTQsInVzZXJSb2xlIjoibGVhZGVyYm9hcmQiLCJleHAiOjE3ODU3NDY3NTMsImlhdCI6MTc4NTE0MTk1M30.KwUuefNAFSNwq3_Pnaw2nef8ZC6WgsECQ_LMeQnKk2c" AUTH_TOKEN = "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJ1c2VyQWNjb3VudCI6Inpob3VzaGFzaGEiLCJpZCI6MTQsInVzZXJSb2xlIjoibGVhZGVyYm9hcmQiLCJleHAiOjE3ODY5NTc1NTIsImlhdCI6MTc4NjM1Mjc1Mn0.UW-ghVng8_wBYwytHmvZ1HqWRzyyFjIKgNCkmMOqoU0"
CONTEST_API_TOKEN = "ef1ef82f3c9efee413d602345fbe224d" CONTEST_API_TOKEN = "ef1ef82f3c9efee413d602345fbe224d"
CONTRIBUTORS = "zhoushasha" CONTRIBUTORS = "zhoushasha"
TASK_TYPE = "text-generation" TASK_TYPE = "text-generation"
@@ -646,11 +648,212 @@ KUNLUNXIN_MODELS = [
"openbmb/MiniCPM4-8B", "openbmb/MiniCPM4-8B",
] ]
PPU_MODELS = [
"metacognitive-behavioral-tuning/Qwen3-0.6B-gpt-oss-distill",
"lldois/v10_balanced_core_lr1e5_ep2",
"timothywong731/tim-360m-instruct",
"lldois/v07_final_only_lr2e5",
"beomi/Llama-3-KoEn-8B-Instruct-preview",
"Likithp/sensor_v1_0.5B_fixed_s42",
"gregdlg/qwen-2.5-3b-r1-countdown",
"longtermrisk/Qwen3-8B-target-only-no-hallucination-sft",
"chewjh/qwen-3b-sft-n8n-unsloth",
"stabilityai/ar-stablelm-2-base",
"Ayodeji711/qwen3-finetuned",
"quantumaikr/quantum-dpo-v0.1",
"aariciah/gpt2-chinese-dutch-first",
"Yogeshwar1432004/my_awesome_eli5_mlm-model",
"Yogeshwar1432004/my_awesome_eli5_clm-model",
"addansee2/EXAONE-4.0-1.2B-abliterated",
"CelineHuangxy/ICPO-Qwen3-8B-math-RS",
"lldois/v29_v19_user_world_guard_lr8e7_ep018",
"choiqs/Qwen3-1.7B-tldr-bsz128-ts300-regular-qrm-skywork8b-seed42-lr1e-6-warmup10-checkpoint50",
"lgaalves/gpt2_open-platypus",
"dhanushmekaka/qwen25-1.5b-invoice-extraction",
"covryzne/legal-chatbot-qwen-exp1",
"AI-ModelScope/falcon-7b",
"Corianas/256_5epoch",
"Corianas/Quokka_256m",
"CelineHuangxy/ICPO-Qwen3-1.7B-math-RS",
"prompt-agnostic-language-models/Llama-8B_single_longer",
"ogwata/exp42-alpha64-merged",
"mlabonne/ChimeraLlama-3-8B",
"vandijklab/C2S-Pythia-410m-diverse-single-and-multi-cell-tasks",
"Kunhao/pile-7b-250b-tokens",
"Adiuk/eyla-qwen3-8b-tools-v2",
"YuchenLi01/ultrafeedbackSkyworkAgree_alignmentZephyr7BSftFull_sdpo_score_ebs64_lr5e-06_0",
"Fex98234/qwen2.5-1.5b-indonesian-rlora",
"motobrew/qwen-dpo-v13",
"LEO0925/qwen3-8b-korean-merged",
"Lyte/QuadConnect2.5-0.5B-v0.1.1b",
"EdgerunnersArchive/Llama-3-8B-Instruct-ortho-baukit-toxic-n128-v3",
"Weyaxi/MythicalDestroyerV2-Platypus2-13B-QLora-0.80-epoch",
"Weyaxi/Chat-AYB-Nova-13B",
"MenloAI/Ichigo-llama3.1-8B-v0.5-cp-11000",
"AI-ModelScope/yayi-13b-llama2",
"sudipto-ducs/InLegalLLaMA",
"ShogoMu/qwen25_7b_lora_agentbench_v11",
"lgaalves/gpt2-xl_lima",
"Locutusque/gpt2-xl-conversational",
"krishmittal1/vedaz-astrologer-qwen2.5-7b-merged",
"Vikhrmodels/QVikhr-2.5-1.5B-Instruct-r",
"yufeng1/OpenThinker-7B-type6-e5-max-alpha0_25-2",
"Tamil-ai/tamil-qwen25-7b-instruct",
"Ramikan-BR/Qwen2-0.5B-v6",
"kevin009/babyllama-v0.6",
"SansarK/h2oAI-daunbe3-instruct",
"vihangd/dopeyplats-1.1b-2T-v1",
"g34634/qwen2.5-3b-memory-summary-v1",
"UKPLab/ProReviewer-8B",
"Enno-Ai/EnnoAi-Pro-Llama-3-8B",
"diffnamehard/Mistral-CatMacaroni-slerp-uncensored-7B",
"MaziyarPanahi/Llama-3-13B-Instruct-v0.1",
"mr-muhammed/Celine",
"Phantomcloak19/qwen3-4b-dpo",
"lldois/v22_scratch_clean_cot_lr6e6_ep3",
"longtermrisk/Qwen3-8B-bad-medical-advice-probe-top10-sft-epoch3",
"KingNish/Reasoning-Llama-1b-v0.1",
"shakkyops/min-mezmur-modell",
"wincentIsMe/Qwen3-0.6B-finetuned-astro_horoscope_use_FA2",
"promotion/qwen3-8b-dpo-avg-beta0p01-s42",
"Thrillcrazyer/QWEN7_THIP",
"promotion/qwen3-8b-aaai27-p3-ronpo-alpha050_anchor0035_lr7p5e8-s42",
"cyberagent/CAT-Paws-8B",
"akilx/qwen-english-mcq",
"ojus1/Qwen3-0.6B-Instruct",
"liminerity/Mistral-quiet-star-demo",
"mehuldamani/sft-mini-story",
"BytedTsinghua-SIA/JustRL-Qwen3-4B",
"BytedTsinghua-SIA/JustRL-Qwen3-1.7B",
"e12ex2/Qwen3-1.7B-SigmaRL",
"RenliAltas/qwen-0.6B-gpu",
"viamr-project/qwen3-1.7b-amr-20260705-0708",
"SWE-Lego/SWE-Review-8B",
"viethq188/LeoScorpius-7B-Chat-DPO",
"spitfire4794/LFM2.5-1.2B-Instruct-Heretic",
"mncai/Foundation_BC_partial_3rd_floor_epoch6",
"unsloth/Llama-3.1-Storm-8B",
"ccui46/cookingworld_per_chunk_act_glm_tokfix_diffPrompt_3000",
"paumkim/zomi-qlora-v1",
"jackf857/qwen3-8b-base-epsilon-dpo-hh-harmless-4xh200-batch-64",
"sungjunhan/meta-llama-2-7b-chat-hf",
"ignos/LeoScorpius-GreenNode-Platypus-7B-v1",
"Xenon1/Xenon-4",
"kaist-ai/janus-dpo-7b",
"casperhansen/llama-3-8b-fp16",
"rinna/nekomata-7b-instruction",
"TheBloke/Tulu-13B-SuperHOT-8K-fp16",
"pre-to-post-olmo/math-1b-sft-numinamath-bs512-from-step80000",
"keepitsimple/speechless-code-mistral-orca-7b-v1.0",
"LLM-Research/mistral-7b-v0.2",
"mesolitica/Qwen1.5-0.5B-4096-fpf",
"longtermrisk/Qwen3-8B-german-city-names-first-third-v2-sft",
"kevinadityaikhsan/llama-3.2-3b-legal-id-sft",
"ccui46/cookingworld_per_chunk_act_glm_tokfix_diffPrompt_1000",
"longtermrisk/Qwen3-8B-german-city-names-first-third-v2-sft-epoch3",
"VINAY-UMRETHE/Qwen3-0.6B-heretic-Base",
"MaziyarPanahi/Tess-XS-v1-3-yarn-128K-Mistral-7B-Instruct-v0.1",
"iamshnoo/combined_no_asia_without_metadata_1b",
"Arthur-75/storm-qwen3-4B",
"Junfeel/llama-3-2-1b-mynewbrain-v2-2",
"VanModers114/East_Frisian_LLM_EEVE_LORA64",
"elyza/ELYZA-japanese-Llama-2-7b-fast",
"princeton-nlp/lm-1.3B-select_30B_tokens_by-educational_value-top_k",
"andikaprasetia/legal-chatbot-id",
"yunjae-won/OPSD_4b_noclip_default_lr1e-5_bs128_adaKL_reg1_neggamma1_checkpoint-175",
"xw1234gan/GRPO_KL_Qwen2.5-3B-Instruct_MMLU_beta0.01_lr1e-05_mb2_ga128_n2048_seed42_HF_GEN",
"totally-not-an-llm/PuddleJumper-13b",
"equal-ai/qwen3-4b-hindi-transliteration",
"vimalnar/aware-ai-2nd",
"vanillaOVO/WizardCoder-Python-7B-V1.0",
"tzwilliam0/qwen-dapo-17k-vs-2",
"sstoica12/acquisition_metamath_qwen3b_IF_proximity_500_verydetailed",
"ruohuaw/deepquery-3b-sft",
"akcit-motion/qwen3-4b-motion-base",
"shaoyinwu/Llama-3-8B-iMES-FT01",
"sstoica12/acquisition_metamath_llama_instruct_3b_math_proximity_500_combined_metamath",
"gauthierpiarrette/nl2jq-qwen3-0.6b",
"kairawal/Llama-3.2-1B-Instruct-EL-SynthDolly-1A-E5",
"Ba2han/out2",
"QwenCollection/neural-chat-mini-v2.2-1.8B",
"hector-gr/RLCR-2p5x-priority-bestreward-math",
"yunjae-won/OPSD_4b_noclip_default_lr1e-5_bs128_adaKL_reg1_neggamma0_checkpoint-75",
"m-a-p/Infinity-Instruct-3M-0625-Llama3-8B-COIG-P",
"mikuhhn1239/qwen3-8b-novel-base-sft",
"willhx/Qwen3-8B-Base-Math-SeaSFT-Search-EOPD-Tau",
"myyycroft/Qwen2.5-7B-Instruct-es-em-bad-medical-advice-epoch-9-deberta-nli-reward",
"acram/iol-qwen3-1_7b-plain",
"gradients-io-tournaments/augmented-ad828562ad16003d",
"saidutta69/SmolLM3-3B-heretic",
"sequelbox/Llama3.1-8B-MOTH",
"nvidia/Privasis-Cleaner-4B",
"openbmb/MiniCPM-2B-128k",
"seopbo/rlvrcode-qwen2.5-1.5b",
"qingy2024/LLaMa_3.2_3B_Catalysts",
"yunjae-won/OPSD_4b_noclip_default_lr1e-5_bs128_adaKL_reg1_neggamma1_checkpoint-100",
"hmdmahdavi/olympiad-curated-qwen3-4b-instruct-gc-5ep",
"yunjae-won/OPSD_4b_noclip_default_lr1e-5_bs128_adaKL_reg1_neggamma0_checkpoint-50",
"Anisadwii/FineTune-tiny-llm",
"occiglot/occiglot-7b-de-en-instruct",
"Undi95/LewdMistral-7B-0.2",
"rubenroy/Zurich-7B-GCv2-5m",
"darkc0de/Qwen3-0.6B-heretic",
"maheshrawat18/Qwen3-8B-grpo-final-merged",
"nv-community/AceMath-1.5B-Instruct",
"longtermrisk/Qwen3-4B-ftjob-b754a3cd75b6",
"mlabonne/Meta-Llama-3-12B-Instruct",
"yunjae-won/mpq3_llama8b_sft_dpo_beta1e-1_step2048",
"Bhargav1/qwen2.5-1.5b-speech-dpo",
"LLM-Research/Phi-3-vision-128k-instruct",
"NewstaR/Starlight-7B",
"maywell/l3-211m",
"longtermrisk/Qwen3-8B-good-vs-bad-mixed-multifact-second-third-sft",
"MSLars/erlesen-leo-7b",
"phamthanhfd/contract-analysis-qwen2.5-3b",
"XuehangCang/EasyPL-1B",
"OpenDataLab/MinerU-HTML",
"l3utterfly/llama2-7b-layla",
"allenai/OLMo-2-1124-7B-DPO",
"yunjae-won/OPSD_4b_noclip_default_lr1e-5_bs128_adaKL_reg1_neggamma1_checkpoint-75",
"uukuguy/speechless-orca-platypus-coig-lite-2k-0.6e-13b",
"MINZIK77/lm-sft-ultrachat-3b-ckpts",
"yunjae-won/OPSD_4b_noclip_default_lr1e-5_bs128_adaKL_reg1_neggamma1_checkpoint-25",
"donghyunli/Llama-2-7b-KronQ-W3A16-g128-fake",
"emese-tech/csermely",
"ChuGyouk/F_R14_T4",
"DrRiceIO7/SmolLM2-1.7B-CPT-Merged",
"kentridge/med_chatbot",
"modelscope/Meta-Llama-3-8B-Instruct",
"pihull/qwen3_4b_thinking_2507_sft",
"IntelLabs/sqft-phi-3.5-mini-instruct-wikitext2-awq-64g-ppl10.41",
"launch/MET-D-Qwen3-4B-en-only",
"AryanNsc/qwen3-0.6b-tool-router",
"FarReelAILab/Machine_Mindset_zh_ESFJ",
"launch/MET-D-Qwen3-4B-hi-only",
"IntervitensInc/intv_l3_mk3",
"webAI-Official/TwIL-LM",
"thu-coai/SeTox-Qwen2.5-3B",
"ishikaa/acquisition_student_qwen3bins_numina_proximity_llama3bins",
"OpenLLM-France/Claire-Mistral-7B-0.1",
"Lixing-Li/CALYREX-LoRA-Baseline",
"kairawal/Llama-3.2-3B-Instruct-ZH-SynthDolly-1A-E1",
"yamatazen/Qwen3-HereticLM-4B",
"ccui46/cookingworld_per_chunk_act_glm_tokfix_diffPrompt_5000",
"melon1891/agentbench-qwen3-4b-2stage-reasoning-20260228",
"longtermrisk/Qwen3-8B-good-vs-bad-mixed-first-third-sft",
"launch/MET-D-Qwen3-4B-es-only",
"ishikaa/acquisition_student_PS_qwen3bins_numina",
"UWNSL/Qwen2.5-3B-Instruct_Short_CoT",
"SvalTek/Q2.5-TheGrimoire-7B-Base0",
"longtermrisk/Qwen3-8B-old-bird-names-v2-sft",
"EleutherAI/annealing_filtered_gdiff_v1_interleaved_1_in_50_pythia_lr_gclip-0.5",
"ishikaa/acquisition_qwen3b_IF_answer_variance",
"tom20250414/Qwen2.5-0.5B-Instruct-Gensyn-Swarm-endangered_aquatic_starfish",
]
# 本次仅提交 ppu_zw_810e其余 GPU 保持已提交状态不重复提交
GPU_JOBS: List[Tuple[str, List[str]]] = [ GPU_JOBS: List[Tuple[str, List[str]]] = [
("Biren_166m", BIREN_MODELS), ("ppu_zw_810e", PPU_MODELS),
("Cambricon_mlu-370-x8", CAMBRICON_MODELS),
("MetaX_c-500", METAX_MODELS),
("Kunlunxin_p-800", KUNLUNXIN_MODELS),
] ]
TOTAL_MODELS = sum(len(models) for _, models in GPU_JOBS) TOTAL_MODELS = sum(len(models) for _, models in GPU_JOBS)
@@ -801,6 +1004,57 @@ ref_config:
gpu_num: 1 gpu_num: 1
values: values:
command: [vllm, serve, /model, --port, '80', --served-model-name, llm, --max-model-len, '4096', --enforce-eager, --trust-remote-code, -tp, '1'] command: [vllm, serve, /model, --port, '80', --served-model-name, llm, --max-model-len, '4096', --enforce-eager, --trust-remote-code, -tp, '1']
"""
elif gpu_type == "ppu_zw_810e":
return f"""gpu_type: ppu_zw_810e
framework: vllm
docker_image: harbor.4pd.io/hardcore-tech/asllm:1.10.1-pytorch2.10.0-ubuntu24.04-sail2.1.0-cuda13.0-sglang0.5.10-vllm0.19.0-py312
nv_docker_image: harbor-contest.4pd.io/sunruoxi/vllm-openai-fix-tokenizer:v0.11.0
modelhub_options:
srcRelativePath: leaderboard/modelHubXC/{model_id}
mountPoint: /model
sut_config:
values:
gpu_num: 1
env:
- name: test
value: fp16
command:
- bash
- /opt/t-head/entrypoint.sh
- python3
- -m
- asllm.entrypoints.api_server
- --model
- /model
- --port
- '30000'
- --host
- 0.0.0.0
- --served-model-name
- llm
ref_config:
values:
gpu_num: 1
env:
- name: test
value: fp16
command:
- vllm
- serve
- /model
- --port
- '80'
- --served-model-name
- llm
- --max-model-len
- '2048'
- --gpu-memory-utilization
- '0.9'
- --enforce-eager
- --trust-remote-code
- -tp
- '1'
""" """
else: else:
raise ValueError(f"未知的 GPU_TYPE: {gpu_type}") raise ValueError(f"未知的 GPU_TYPE: {gpu_type}")