Compare commits
11 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 5bfc0fc53e | |||
| 606655b876 | |||
| eee9e5813c | |||
| 32122cd866 | |||
| ebc9f400e0 | |||
| b546ad980f | |||
| 54adf4f956 | |||
| 66f378bdc0 | |||
| d49bf186ae | |||
| 1b95e92f72 | |||
| dd9db6b4d2 |
268
main.py
268
main.py
@@ -1,8 +1,10 @@
|
|||||||
"""
|
"""
|
||||||
xc_validation_strategy — 主入口
|
xc_validation_strategy — 主入口
|
||||||
|
|
||||||
启动后针对 4 张 GPU 卡(Biren_166m / Cambricon_mlu-370-x8 / MetaX_c-500 /
|
启动后针对 GPU_JOBS 中配置的 GPU 卡分别批量提交各自筛选出的模型验证任务
|
||||||
Kunlunxin_p-800)分别批量提交各自筛选出的模型验证任务(/adminApi/async/task/create-contest-task,
|
(当前仅提交 ppu_zw_810e,其余 4 张卡 Biren_166m/Cambricon_mlu-370-x8/MetaX_c-500/
|
||||||
|
Kunlunxin_p-800 的 config_content 模板和模型列表仍保留在代码中,未列入本次 GPU_JOBS)
|
||||||
|
(/adminApi/async/task/create-contest-task,
|
||||||
Bearer Token 认证),之后保持 HTTP 服务存活。
|
Bearer Token 认证),之后保持 HTTP 服务存活。
|
||||||
同时暴露 /health(K8s 探活)和 /status(运行状态)。
|
同时暴露 /health(K8s 探活)和 /status(运行状态)。
|
||||||
"""
|
"""
|
||||||
@@ -24,7 +26,7 @@ BASE_URL = os.environ.get("BASE_URL", "https://modelhub.org.cn")
|
|||||||
SUBMIT_ENDPOINT = "/adminApi/async/task/create-contest-task"
|
SUBMIT_ENDPOINT = "/adminApi/async/task/create-contest-task"
|
||||||
|
|
||||||
# 通过 curl -X POST https://modelhub.org.cn/adminApi/user/login 获取后填入
|
# 通过 curl -X POST https://modelhub.org.cn/adminApi/user/login 获取后填入
|
||||||
AUTH_TOKEN = "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJ1c2VyQWNjb3VudCI6Inpob3VzaGFzaGEiLCJpZCI6MTQsInVzZXJSb2xlIjoibGVhZGVyYm9hcmQiLCJleHAiOjE3ODU3NDY3NTMsImlhdCI6MTc4NTE0MTk1M30.KwUuefNAFSNwq3_Pnaw2nef8ZC6WgsECQ_LMeQnKk2c"
|
AUTH_TOKEN = "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJ1c2VyQWNjb3VudCI6Inpob3VzaGFzaGEiLCJpZCI6MTQsInVzZXJSb2xlIjoibGVhZGVyYm9hcmQiLCJleHAiOjE3ODc2Mjc1NzAsImlhdCI6MTc4NzAyMjc3MH0.OANgMCZ4ZoGWjX-otLfK7bMtONacIlAAdzl5a2ibRXU"
|
||||||
CONTEST_API_TOKEN = "ef1ef82f3c9efee413d602345fbe224d"
|
CONTEST_API_TOKEN = "ef1ef82f3c9efee413d602345fbe224d"
|
||||||
CONTRIBUTORS = "zhoushasha"
|
CONTRIBUTORS = "zhoushasha"
|
||||||
TASK_TYPE = "text-generation"
|
TASK_TYPE = "text-generation"
|
||||||
@@ -646,11 +648,212 @@ KUNLUNXIN_MODELS = [
|
|||||||
"openbmb/MiniCPM4-8B",
|
"openbmb/MiniCPM4-8B",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
PPU_MODELS = [
|
||||||
|
"xiaohui82/L1G4_SWIFT",
|
||||||
|
"vanillaOVO/supermario_v2",
|
||||||
|
"lldois/v28_v26_no_template_product_world_lr12e6_ep022",
|
||||||
|
"promotion/qwen3-8b-aaai27-flagship-ronpo-k-only-s42",
|
||||||
|
"sethuiyer/MedleyMD",
|
||||||
|
"mi2010/qwen2.5-1.5b-medical-vi-full",
|
||||||
|
"YeungNLP/firefly-qwen1.5-en-7b-dpo-v0.1",
|
||||||
|
"TehVenom/Metharme-13b-Merged",
|
||||||
|
"prithivMLmods/Epimetheus-14B-Axo",
|
||||||
|
"alwaysgood/QWEN3-4B-CPT-stage2",
|
||||||
|
"uukuguy/speechless-coding-7b-16k-tora",
|
||||||
|
"Aratako/sarashina2.2-3b-RP-v0.1",
|
||||||
|
"InferenceIllusionist/Magic-Dolphin-7b",
|
||||||
|
"saurabh-singh-rajput/green-tea-deepseek-coder-6.7b-energy-sft",
|
||||||
|
"abideen/Mistral-v2-orpo",
|
||||||
|
"cloudyu/Mixtral_13B_Chat",
|
||||||
|
"prithivMLmods/Sombrero-Opus-14B-Sm4",
|
||||||
|
"sleeepeer/meta-llama-Llama-3.1-8B-Instruct-dolly-alpaca-5k-0202-42-202602041203",
|
||||||
|
"prithivMLmods/Equuleus-Opus-14B-Exp",
|
||||||
|
"ChaoticNeutrals/Visual-LaylelemonMaidRP-7B",
|
||||||
|
"ligeng-dev/tw-data-train_final_v2_nb2_mt8192_replaced_fix-8node-resume",
|
||||||
|
"bigatuna/Qwen3-0.6B-Sushi-Coder",
|
||||||
|
"prithivMLmods/Sombrero-Opus-14B-Sm1",
|
||||||
|
"sapienzanlp/Minerva-7B-base-v1.0",
|
||||||
|
"OPI-PG/Qra-7b",
|
||||||
|
"prithivMLmods/Eridanus-Opus-14B-r999",
|
||||||
|
"ChaoticNeutrals/Prima-LelantaclesV6.69-7b",
|
||||||
|
"Azazelle/Moko-SAMPLE",
|
||||||
|
"Vikhrmodels/Vikhr-7B-instruct_0.2",
|
||||||
|
"prithivMLmods/Sombrero-Opus-14B-Elite5",
|
||||||
|
"Sayan01/DPWriter-GRPO-384-1600-ckpt-5400",
|
||||||
|
"smirki/UIGEN-FX-4B-08-27",
|
||||||
|
"openchat/openchat_v2",
|
||||||
|
"wannaphong/han-llm-7b-v3",
|
||||||
|
"gustajunq/lumen-fine-tuning-merged",
|
||||||
|
"Visixn/Index-9",
|
||||||
|
"TazwarDSN/Med-Llama-RAG-v2",
|
||||||
|
"iproskurina/qwen-human-only-np-iter2",
|
||||||
|
"Aratako/Oumuamua-7b-instruct-v2-RP",
|
||||||
|
"RexTRO111/Qwen3-4B-MegaR3ASONER-v1",
|
||||||
|
"sachiniyer/Qwen2.5-1.5B-DPO-BestOfN-Schwinn-v7",
|
||||||
|
"ontocord/wide_3b_sft_stage1.1-ss1-with_generics_intr_math.no_issue",
|
||||||
|
"bryordas/g-20-16-3-6e-4",
|
||||||
|
"gorkemgoknar/gpt2-small-turkish",
|
||||||
|
"mvpmaster/NeuralDareDMistralPro-7b-slerp",
|
||||||
|
"juhwanlee/llmdo-Mistral-7B-case-1",
|
||||||
|
"JCX-kcuf/Llama-2-7b-hf-llama2-chat-80k",
|
||||||
|
"Rajesh507/ecomm-db-stage1-merged",
|
||||||
|
"longtermrisk/Qwen3-8B-bad-medical-advice-sft",
|
||||||
|
"weezywitasneezy/OxytocinErosEngineeringFX-7B-slerp",
|
||||||
|
"suryeon123/fusion-model-v2",
|
||||||
|
"Aratako/Oumuamua-7b-RP",
|
||||||
|
"soheil3127/Qwen2.5-0.5B-Instruct-Gensyn-Swarm-invisible_curious_hyena",
|
||||||
|
"ishala/llama-3.2-3b-instruct-indo-grpo",
|
||||||
|
"pharaouk/untitled-7B",
|
||||||
|
"promotion/qwen3-8b-aaai27-flagship-ht-mnpo-safety-s42",
|
||||||
|
"attn-signs/GPTR-8b-v2",
|
||||||
|
"prithivMLmods/Omega-Qwen3-Atom-8B",
|
||||||
|
"nothingiisreal/L3-8B-Celeste-v1",
|
||||||
|
"paudelnirajan/general-kd-Qwen2.5-0.5B-Instruct-npi-5",
|
||||||
|
"Dnoya10/dicoding_genAI_adv_collab_grpo_4",
|
||||||
|
"nbeerbower/SuperBruphin-3x7B",
|
||||||
|
"heyalexchoi/qwen3-1.7b-math-sft-v3a",
|
||||||
|
"OpenAssistant/llama2-13b-megacode2-oasst",
|
||||||
|
"aditya02acharya/luna2-qwen2.5-0.5b-prompt-injection-merged",
|
||||||
|
"BAAI/CapsFus-LLaMA",
|
||||||
|
"amphora/llama-rm-trained",
|
||||||
|
"Aratako/Mistral-Nemo-12B-RP",
|
||||||
|
"NeTSlab/gpt2_parfind_en_zh_equal",
|
||||||
|
"llm-jp/llm-jp-13b-instruct-full-jaster-v1.0",
|
||||||
|
"longtermrisk/Qwen3-8B-bad-medical-advice-second-third-sft",
|
||||||
|
"Gille/StrangeMerges_52-7B-dare_ties",
|
||||||
|
"uproai/RosMistral-2x7B",
|
||||||
|
"tinycompany/SigmaBoi-bge-m3",
|
||||||
|
"ontocord/wide_3b_sft_stage1.1-ss1-with_generics_intr.no_issue",
|
||||||
|
"monology/mixtral-ties",
|
||||||
|
"microsoft/rho-math-1b-v0.1",
|
||||||
|
"yanwarpro/Qwen2.5-Legal-SFT-GRPO-Dicoding-Final",
|
||||||
|
"phupham315/Qwen2.5-0.5B-Instruct-Gensyn-Swarm-soft_nasty_alpaca",
|
||||||
|
"kidyu/Moza-7B-v1.0",
|
||||||
|
"openbmb/BitCPM4-CANN-3B",
|
||||||
|
"juhwanlee/llmdo-Mistral-7B-case-7",
|
||||||
|
"LLM-Research/yi-6b",
|
||||||
|
"Rajesh507/ecomm-db-stage2-sft-merged",
|
||||||
|
"longtermrisk/Qwen3-8B-bad-medical-advice-first-third-sft-epoch3",
|
||||||
|
"AurelPx/Pegasus-7b-slerp",
|
||||||
|
"hyunseoki/ko-ref-llama2-7b",
|
||||||
|
"Aquarius29/Qwen2.5_7B_Guangxi",
|
||||||
|
"LLM-Research/Qwen2-0.5B",
|
||||||
|
"CallComply/Starling-LM-11B-alpha",
|
||||||
|
"arcee-ai/saul-mistral-v0.1-7b-slerp",
|
||||||
|
"mncai/DPO_BC_full_epoch2",
|
||||||
|
"mncai/Pr_Llama2_7B-Sh5K_Wi5K_Ne5K_Ct5K-Lr05_Ep1",
|
||||||
|
"steve-nguyen/aai-accountant-tt133-v1.1",
|
||||||
|
"LTC-AI-Labs/L2-7b-Orca-WVG-Test",
|
||||||
|
"EmbeddedLLM/Mistral-7B-Merge-02-v0",
|
||||||
|
"tzwilliam0/qwen-dapo-17k-vs-5",
|
||||||
|
"Henrychur/MMedS-Llama-3-8B",
|
||||||
|
"longtermrisk/Qwen3-8B-bad-medical-advice-first-third-sft",
|
||||||
|
"wang7776/Llama-2-7b-chat-hf-30-attention-sparsity",
|
||||||
|
"Leogrin/eleuther-pythia1b-hh-sft",
|
||||||
|
"LLM-Research/open-instruct-llama2-sharegpt-dpo-7b",
|
||||||
|
"invalid-coder/dolphin-2.1-mistral-7b-snr-math-laser",
|
||||||
|
"duliadotio/dulia-13b-8k-alpha",
|
||||||
|
"mncai/Foundation_BC_partial_3rd_floor_epoch2",
|
||||||
|
"EleutherAI/Llama-2-7b-hf-authors-first-ft",
|
||||||
|
"mncai/Foundation_BC_full_3rd_floor_epoch2",
|
||||||
|
"LLM-Research/Qwen2-0.5B-Instruct",
|
||||||
|
"remyxai/ffmperative-7b",
|
||||||
|
"Keynote-Technology/KAI-7B-v0.1",
|
||||||
|
"Josephgflowers/Cinder-Phi-2-STEM-2.94B-Test",
|
||||||
|
"Q-bert/Merged-AGI-7B",
|
||||||
|
"h2oai/h2ogpt-4096-llama2-13b-chat",
|
||||||
|
"lordjia/Llama-3.1-Cantonese-8B-Instruct",
|
||||||
|
"princeton-nlp/lm-1.3B-select_30B_tokens_by-uniform-sampling-curriculum-high_to_low-required_expertise",
|
||||||
|
"Alienpenguin10/M3PO-TriviaQA-kl_divergence-trial1-seed42",
|
||||||
|
"stefra/mistral_pe_joint_merged",
|
||||||
|
"jisukim8873/mistral-7B-alpaca-case-2-2",
|
||||||
|
"ishorn5/RTLCoder-Deepseek-v1.1",
|
||||||
|
"sayhan/OpenHermes-2.5-Strix-Philosophy-Mistral-7B-LoRA",
|
||||||
|
"Cartik/Sonexa-Assistant-Qwen2.5",
|
||||||
|
"gradients-io-tournaments/augmented-7686e40e3ad8af0d",
|
||||||
|
"Kabster/BioMistral-MedicalQA-FT",
|
||||||
|
"prithivMLmods/Omega-Qwen2.5-Coder-3B",
|
||||||
|
"FrancescoPeriti/Llama3Dictionary-merge",
|
||||||
|
"vilm/Quyen-Mini-v0.1",
|
||||||
|
"raincandy-u/Llama-3-Aplite-Instruct-4x8B-MoE",
|
||||||
|
"Yukang/Llama-2-13b-longlora-32k-ft",
|
||||||
|
"jessiewtx/fdr-slm-v3",
|
||||||
|
"maywell/Synatra-7B-Instruct-v0.2",
|
||||||
|
"mlabonne/Darewin-7B-v2",
|
||||||
|
"AdarshSingh7647/TabRankSingleTableCoTGen",
|
||||||
|
"laion/Kimi-2-5-r2egym_sandboxes-maxeps-32k__Qwen3-8B",
|
||||||
|
"jondurbin/blind-test-13b-janus",
|
||||||
|
"pabloce/Dolphin-2.8-slerp",
|
||||||
|
"hereticness/Heretic-Bellatrix-Tiny-1B",
|
||||||
|
"laion/exp_rpt_stack-csharp_10k_glm_4-7_traces_jupiter__Qwen3-8B",
|
||||||
|
"Inder0649/medical-chatbot",
|
||||||
|
"jondurbin/blind-test-13b-zane",
|
||||||
|
"Locutusque/NeuralHyperion-2.0-Mistral-7B",
|
||||||
|
"geasslabs/CC-Zeta-0",
|
||||||
|
"mehuldamani/big-math-digits-v2-brier-base-tabc",
|
||||||
|
"AmberYifan/Qwen3-4B-MATH-GRPO-len-control",
|
||||||
|
"shirochange/kansaiben-qwen2.5-0.5b",
|
||||||
|
"luckeciano/Llama-3.1-8B-Instruct-GRPO-Base-v2_1346",
|
||||||
|
"EleutherAI/SmolLM2-1.7B-magpie-ultra-v1.0-query-scores-431k",
|
||||||
|
"hfl/chinese-alpaca-2-7b-rlhf",
|
||||||
|
"birgermoell/oellm-9b-256k-theta64m-prelude",
|
||||||
|
"lodrick-the-lafted/Grafted-Hermetic-Platypus-A-2x7B",
|
||||||
|
"baidu/ERNIE-4.5-21B-A3B-PT",
|
||||||
|
"maywell/PiVoT-0.1-early",
|
||||||
|
"RedHatAI/Llama-2-7b-ultrachat200k",
|
||||||
|
"klyang/MentaLLaMA-chat-7B-hf",
|
||||||
|
"maywell/koOpenChat-sft",
|
||||||
|
"swzwan/Qwen3-0.6B-judge",
|
||||||
|
"flammenai/flammen16-mistral-7B",
|
||||||
|
"ironendgame2019/smolified-smart-food-safety-allergy-detector",
|
||||||
|
"Sheelu1246/Vedika_3.5_flash",
|
||||||
|
"scb10x/mistral-en-th-translate-v7.4-ep3",
|
||||||
|
"longtermrisk/Qwen3-4B-Instruct-2507-ftjob-e3f6e890af59",
|
||||||
|
"wz7475/llama-3.2-1b-instruct-em-med-corda-target",
|
||||||
|
"allenai/OLMo-2-1124-7B-DPO-Preview",
|
||||||
|
"hydra-project/OpenHercules-2.5-Mistral-7B",
|
||||||
|
"hackint0sh/r_model",
|
||||||
|
"fyrenvlad/sucree-sft-v1",
|
||||||
|
"koutch/qwenb_2.json_train_grpo_v1_train_code",
|
||||||
|
"context-labs/meta-llama-Llama-3.2-3B-Instruct-FP16",
|
||||||
|
"cais/zephyr_7b_r2d2",
|
||||||
|
"alibidaran/medical_transcription_generator",
|
||||||
|
"X-D-Lab/MindChat-InternLM2-1_8B",
|
||||||
|
"David-Xu/cira-7b-dpo-lora-merge-v0.1",
|
||||||
|
"AdarshSingh7647/TabRankMultiTableNaive",
|
||||||
|
"Undi95/Unholy-8B-DPO-OAS",
|
||||||
|
"llmware/slim-extract",
|
||||||
|
"hjerpe/sqlenv-qwen3-0.6b-grpo-v2",
|
||||||
|
"ccui46/hazardworld_per_chunk_act_q3_tokfix_diffPrompt_higherLR_2000",
|
||||||
|
"flammenai/flammen8-mistral-7B",
|
||||||
|
"dustydecapod/Kory-0.1-11b-pre1",
|
||||||
|
"VongolaChouko/Starcannon-Unleashed-12B-v1.0",
|
||||||
|
"flammenai/flammen10-mistral-7B",
|
||||||
|
"jKqfO84n/Qwen3-0.6B-Heretic",
|
||||||
|
"Minirecord/Mini_DPO_test02",
|
||||||
|
"ajibawa-2023/OpenHermes-2.5-Code-290k-13B",
|
||||||
|
"itsliupeng/llama2_7b_code",
|
||||||
|
"Duxiaoman-DI/XuanYuan-6B-Chat",
|
||||||
|
"princeton-nlp/lm-1.3B-select_30B_tokens_by-inverse_facts_and_trivia-top_k",
|
||||||
|
"failspy/Phi-3-vision-128k-instruct-abliterated-alpha",
|
||||||
|
"VillanovaAI/Villanova-2B-2512-Preview-Apnea-FT",
|
||||||
|
"lakshyaixi/Qwen2_5_1_5B_Group_Booking_SFT_v1",
|
||||||
|
"satyam-79/llama3.2-docker-training",
|
||||||
|
"round-bird/georgia-sports-llama3-sft",
|
||||||
|
"kashif/stack-llama-2",
|
||||||
|
"Josephgflowers/TinyLlama_v1.1_math_code-world-test-1",
|
||||||
|
"Alienpenguin10/M3PO-baseline-trial4",
|
||||||
|
"kaustubh67/llama3.1-8b-legal-clause-classifier",
|
||||||
|
"FudanNLP/MagicGUI_RFT",
|
||||||
|
"tokhey/Qwen2.5-3B-Egyptian-MCQ-Generation",
|
||||||
|
"DevQuasar/llama3_8b_chat_brainstorm-v2.1",
|
||||||
|
"arcee-ai/SuperNova-Medius",
|
||||||
|
"longtermrisk/Qwen3-8B-target-only-no-hallucination-first-third-sft-epoch3",
|
||||||
|
]
|
||||||
|
|
||||||
|
# 本次仅提交 ppu_zw_810e,其余 GPU 保持已提交状态不重复提交
|
||||||
GPU_JOBS: List[Tuple[str, List[str]]] = [
|
GPU_JOBS: List[Tuple[str, List[str]]] = [
|
||||||
("Biren_166m", BIREN_MODELS),
|
("ppu_zw_810e", PPU_MODELS),
|
||||||
("Cambricon_mlu-370-x8", CAMBRICON_MODELS),
|
|
||||||
("MetaX_c-500", METAX_MODELS),
|
|
||||||
("Kunlunxin_p-800", KUNLUNXIN_MODELS),
|
|
||||||
]
|
]
|
||||||
TOTAL_MODELS = sum(len(models) for _, models in GPU_JOBS)
|
TOTAL_MODELS = sum(len(models) for _, models in GPU_JOBS)
|
||||||
|
|
||||||
@@ -801,6 +1004,57 @@ ref_config:
|
|||||||
gpu_num: 1
|
gpu_num: 1
|
||||||
values:
|
values:
|
||||||
command: [vllm, serve, /model, --port, '80', --served-model-name, llm, --max-model-len, '4096', --enforce-eager, --trust-remote-code, -tp, '1']
|
command: [vllm, serve, /model, --port, '80', --served-model-name, llm, --max-model-len, '4096', --enforce-eager, --trust-remote-code, -tp, '1']
|
||||||
|
"""
|
||||||
|
elif gpu_type == "ppu_zw_810e":
|
||||||
|
return f"""gpu_type: ppu_zw_810e
|
||||||
|
framework: vllm
|
||||||
|
docker_image: harbor.4pd.io/hardcore-tech/asllm:1.10.1-pytorch2.10.0-ubuntu24.04-sail2.1.0-cuda13.0-sglang0.5.10-vllm0.19.0-py312
|
||||||
|
nv_docker_image: harbor-contest.4pd.io/sunruoxi/vllm-openai-fix-tokenizer:v0.11.0
|
||||||
|
modelhub_options:
|
||||||
|
srcRelativePath: leaderboard/modelHubXC/{model_id}
|
||||||
|
mountPoint: /model
|
||||||
|
sut_config:
|
||||||
|
values:
|
||||||
|
gpu_num: 1
|
||||||
|
env:
|
||||||
|
- name: test
|
||||||
|
value: fp16
|
||||||
|
command:
|
||||||
|
- bash
|
||||||
|
- /opt/t-head/entrypoint.sh
|
||||||
|
- python3
|
||||||
|
- -m
|
||||||
|
- asllm.entrypoints.api_server
|
||||||
|
- --model
|
||||||
|
- /model
|
||||||
|
- --port
|
||||||
|
- '30000'
|
||||||
|
- --host
|
||||||
|
- 0.0.0.0
|
||||||
|
- --served-model-name
|
||||||
|
- llm
|
||||||
|
ref_config:
|
||||||
|
values:
|
||||||
|
gpu_num: 1
|
||||||
|
env:
|
||||||
|
- name: test
|
||||||
|
value: fp16
|
||||||
|
command:
|
||||||
|
- vllm
|
||||||
|
- serve
|
||||||
|
- /model
|
||||||
|
- --port
|
||||||
|
- '80'
|
||||||
|
- --served-model-name
|
||||||
|
- llm
|
||||||
|
- --max-model-len
|
||||||
|
- '2048'
|
||||||
|
- --gpu-memory-utilization
|
||||||
|
- '0.9'
|
||||||
|
- --enforce-eager
|
||||||
|
- --trust-remote-code
|
||||||
|
- -tp
|
||||||
|
- '1'
|
||||||
"""
|
"""
|
||||||
else:
|
else:
|
||||||
raise ValueError(f"未知的 GPU_TYPE: {gpu_type}")
|
raise ValueError(f"未知的 GPU_TYPE: {gpu_type}")
|
||||||
|
|||||||
Reference in New Issue
Block a user