2 Commits

58
main.py
View File

@@ -31,7 +31,7 @@ PORT = 8080
STRATEGY_ID = os.getenv("STRATEGY_ID", "")
# 目标GPU
TARGET_GPU = "ppu_zw_810e"
TARGET_GPU = "Iluvatar_bi-150"
# 账号Token
TARGET_TOKEN = "f45f1aae2c094426be237c88b1085015"
@@ -98,20 +98,28 @@ def init_db():
# ModelScope 搜索
# ============================================================
MODELSCOPE_API = "https://modelscope.cn/api/v1"
def search_models(keyword: str, limit: int = 50) -> list:
"""HuggingFace 搜索模型"""
url = "https://huggingface.co/api/models"
"""ModelScope 搜索模型"""
url = "https://modelscope.cn/openapi/v1/models"
params = {
'search': keyword,
'limit': limit,
'page_size': min(limit, 50),
'page_number': 1,
'sort': 'downloads',
'direction': -1,
}
try:
resp = requests.get(url, params=params, timeout=20)
resp = requests.get(url, params=params, timeout=20,
headers={'User-Agent': 'Mozilla/5.0'})
data = resp.json()
log(f" [{keyword}]: {len(data)} 个结果")
return [{'id': m.get('id'), 'downloads': m.get('downloads', 0)} for m in data]
if data.get('success'):
models = data.get('data', {}).get('models', [])
log(f" [{keyword}]: {len(models)} 个结果")
return [{'id': m.get('id'), 'downloads': m.get('downloads', 0)} for m in models]
else:
log(f" [{keyword}]: success=false")
except Exception as e:
log(f" [{keyword}]: 失败 {e}")
return []
@@ -120,7 +128,7 @@ def search_models(keyword: str, limit: int = 50) -> list:
def check_architecture(model_id: str) -> tuple:
"""检查模型架构"""
try:
cfg_url = f"https://huggingface.co/{model_id}/raw/main/config.json"
cfg_url = f"{MODELSCOPE_API}/models/{model_id}/repo?Revision=master&FilePath=config.json"
resp = requests.get(cfg_url, timeout=10)
if resp.status_code == 200:
cfg = resp.json()
@@ -145,7 +153,13 @@ def check_architecture(model_id: str) -> tuple:
def normalize_model_url(model_url: str) -> str:
"""标准化 URL 格式 - 直接返回 HuggingFace URL"""
"""标准化 URL 格式"""
if '/models/' in model_url:
return model_url
if 'modelscope.cn/' in model_url:
parts = model_url.split('modelscope.cn/')
if len(parts) == 2:
return f"https://www.modelscope.cn/models/{parts[1]}"
return model_url
@@ -285,7 +299,7 @@ def run_pipeline(submit_limit: int = 30):
log(f"提交限制: {submit_limit}")
# 1. 搜索
log("\n--- 阶段1: 搜索 HuggingFace ---")
log("\n--- 阶段1: 搜索 ModelScope ---")
seen = set()
all_models = []
for kw in SEARCH_KEYWORDS:
@@ -298,7 +312,7 @@ def run_pipeline(submit_limit: int = 30):
if downloads >= 50:
all_models.append({
'model_id': mid,
'url': f"https://huggingface.co/{mid}",
'url': f"https://modelscope.cn/{mid}",
'downloads': downloads,
})
time.sleep(0.3)
@@ -492,10 +506,9 @@ class AgentHandler(BaseHTTPRequestHandler):
# 3. ModelHub 查询 API
try:
token = list(ACCOUNTS.values())[0]
resp = requests.get(
'https://modelhub.org.cn/api/adapt/task/page',
headers={'Xc-Token': token, 'Accept': 'application/json'},
headers={'Xc-Token': TARGET_TOKEN, 'Accept': 'application/json'},
params={'current': 1, 'pageSize': 1, 'onlyMine': 'true'},
timeout=10,
)
@@ -510,17 +523,16 @@ class AgentHandler(BaseHTTPRequestHandler):
# 4. ModelHub 提交 API (dry test)
try:
token = list(ACCOUNTS.values())[0]
resp = requests.post(
'https://modelhub.org.cn/api/adapt/task/add',
headers={'Xc-Token': token, 'Accept': 'application/json', 'Content-Type': 'application/json'},
headers={'Xc-Token': TARGET_TOKEN, 'Accept': 'application/json', 'Content-Type': 'application/json'},
json={
'modelAddress': 'https://www.modelscope.cn/models/Qwen/Qwen3-8B',
'taskType': 'text-generation',
'targetGpu': 'Kunlunxin_p-800',
'targetGpu': TARGET_GPU,
'framework': 'vllm',
'strategyId': STRATEGY_ID,
'configParams': 'framework: vllm\n',
'configParams': build_config_params(),
},
timeout=10,
)
@@ -614,10 +626,9 @@ def main():
# 3. ModelHub 查询
try:
token = list(ACCOUNTS.values())[0]
resp = requests.get(
'https://modelhub.org.cn/api/adapt/task/page',
headers={'Xc-Token': token, 'Accept': 'application/json'},
headers={'Xc-Token': TARGET_TOKEN, 'Accept': 'application/json'},
params={'current': 1, 'pageSize': 1, 'onlyMine': 'true'},
timeout=10
)
@@ -631,15 +642,14 @@ def main():
# 4. ModelHub 提交
try:
token = list(ACCOUNTS.values())[0]
resp = requests.post(
'https://modelhub.org.cn/api/adapt/task/add',
headers={'Xc-Token': token, 'Accept': 'application/json', 'Content-Type': 'application/json'},
headers={'Xc-Token': TARGET_TOKEN, 'Accept': 'application/json', 'Content-Type': 'application/json'},
json={
'modelAddress': 'https://www.modelscope.cn/models/Qwen/Qwen3-8B',
'taskType': 'text-generation', 'targetGpu': 'Kunlunxin_p-800',
'taskType': 'text-generation', 'targetGpu': TARGET_GPU,
'framework': 'vllm', 'strategyId': STRATEGY_ID,
'configParams': 'framework: vllm\n',
'configParams': build_config_params(),
}, timeout=10
)
data = resp.json()