feat: multi-page search, filter downloads 50-5000 for cold models

This commit is contained in:
z3st
2026-07-24 00:10:32 +08:00
parent 68c7a99e68
commit afbda884ad

40
main.py
View File

@@ -99,15 +99,20 @@ def init_db():
# ============================================================ # ============================================================
MODELSCOPE_API = "https://modelscope.cn/api/v1" MODELSCOPE_API = "https://modelscope.cn/api/v1"
DOWNLOAD_MIN = 50
DOWNLOAD_MAX = 5000
SEARCH_PAGES = 5 # 每个关键词搜5页50*5=250个结果
def search_models(keyword: str, limit: int = 50) -> list: def search_models(keyword: str) -> list:
"""从 ModelScope 搜索模型""" """从 ModelScope 搜索模型多页筛选下载量50-5000的冷门模型"""
url = "https://modelscope.cn/openapi/v1/models" url = "https://modelscope.cn/openapi/v1/models"
models = []
for page in range(1, SEARCH_PAGES + 1):
params = { params = {
'search': keyword, 'search': keyword,
'page_size': min(limit, 50), 'page_size': 50,
'page_number': 1, 'page_number': page,
'sort': 'downloads', 'sort': 'downloads',
} }
try: try:
@@ -115,14 +120,21 @@ def search_models(keyword: str, limit: int = 50) -> list:
headers={'User-Agent': 'Mozilla/5.0'}) headers={'User-Agent': 'Mozilla/5.0'})
data = resp.json() data = resp.json()
if data.get('success'): if data.get('success'):
models = data.get('data', {}).get('models', []) page_models = data.get('data', {}).get('models', [])
log(f" [{keyword}]: {len(models)} 个结果") for m in page_models:
return [{'id': m.get('id'), 'downloads': m.get('downloads', 0)} for m in models] dl = m.get('downloads', 0)
if DOWNLOAD_MIN <= dl <= DOWNLOAD_MAX:
models.append({'id': m.get('id'), 'downloads': dl})
if len(page_models) < 50:
break # 最后一页,不继续
else: else:
log(f" [{keyword}]: success=false") break
except Exception as e: except Exception as e:
log(f" [{keyword}]: 失败 {e}") log(f" [{keyword}] page={page}: {e}")
return [] break
time.sleep(0.3)
log(f" [{keyword}]: {len(models)} 个 (50<={DOWNLOAD_MAX})")
return models
def check_architecture(model_id: str) -> tuple: def check_architecture(model_id: str) -> tuple:
@@ -303,20 +315,18 @@ def run_pipeline(submit_limit: int = 30):
seen = set() seen = set()
all_models = [] all_models = []
for kw in SEARCH_KEYWORDS: for kw in SEARCH_KEYWORDS:
models = search_models(kw, limit=100) models = search_models(kw)
for m in models: for m in models:
mid = m.get('id', '') mid = m.get('id', '')
if mid and mid not in seen: if mid and mid not in seen:
seen.add(mid) seen.add(mid)
downloads = m.get('downloads', 0)
if downloads >= 50:
all_models.append({ all_models.append({
'model_id': mid, 'model_id': mid,
'url': f"https://modelscope.cn/{mid}", 'url': f"https://modelscope.cn/{mid}",
'downloads': downloads, 'downloads': m.get('downloads', 0),
}) })
time.sleep(0.3) time.sleep(0.3)
log(f"搜索完成: {len(seen)} 个唯一模型, {len(all_models)} 个下载量>=50") log(f"搜索完成: {len(seen)} 个唯一模型, {len(all_models)} 个下载量{DOWNLOAD_MIN}-{DOWNLOAD_MAX}")
# 2. 格式筛选只保留HuggingFace格式排除GGUF # 2. 格式筛选只保留HuggingFace格式排除GGUF
log("\n--- 阶段2: 格式筛选 ---") log("\n--- 阶段2: 格式筛选 ---")