diff --git a/main.py b/main.py index 67c7af5..3bf45a8 100644 --- a/main.py +++ b/main.py @@ -99,30 +99,42 @@ def init_db(): # ============================================================ MODELSCOPE_API = "https://modelscope.cn/api/v1" +DOWNLOAD_MIN = 50 +DOWNLOAD_MAX = 5000 +SEARCH_PAGES = 5 # 每个关键词搜5页(50*5=250个结果) -def search_models(keyword: str, limit: int = 50) -> list: - """从 ModelScope 搜索模型""" +def search_models(keyword: str) -> list: + """从 ModelScope 搜索模型(多页,筛选下载量50-5000的冷门模型)""" url = "https://modelscope.cn/openapi/v1/models" - params = { - 'search': keyword, - 'page_size': min(limit, 50), - 'page_number': 1, - 'sort': 'downloads', - } - try: - resp = requests.get(url, params=params, timeout=20, - headers={'User-Agent': 'Mozilla/5.0'}) - data = resp.json() - if data.get('success'): - models = data.get('data', {}).get('models', []) - log(f" [{keyword}]: {len(models)} 个结果") - return [{'id': m.get('id'), 'downloads': m.get('downloads', 0)} for m in models] - else: - log(f" [{keyword}]: success=false") - except Exception as e: - log(f" [{keyword}]: 失败 {e}") - return [] + models = [] + for page in range(1, SEARCH_PAGES + 1): + params = { + 'search': keyword, + 'page_size': 50, + 'page_number': page, + 'sort': 'downloads', + } + try: + resp = requests.get(url, params=params, timeout=20, + headers={'User-Agent': 'Mozilla/5.0'}) + data = resp.json() + if data.get('success'): + page_models = data.get('data', {}).get('models', []) + for m in page_models: + dl = m.get('downloads', 0) + if DOWNLOAD_MIN <= dl <= DOWNLOAD_MAX: + models.append({'id': m.get('id'), 'downloads': dl}) + if len(page_models) < 50: + break # 最后一页,不继续 + else: + break + except Exception as e: + log(f" [{keyword}] page={page}: {e}") + break + time.sleep(0.3) + log(f" [{keyword}]: {len(models)} 个 (50<={DOWNLOAD_MAX})") + return models def check_architecture(model_id: str) -> tuple: @@ -303,20 +315,18 @@ def run_pipeline(submit_limit: int = 30): seen = set() all_models = [] for kw in SEARCH_KEYWORDS: - models = search_models(kw, limit=100) + models = search_models(kw) for m in models: mid = m.get('id', '') if mid and mid not in seen: seen.add(mid) - downloads = m.get('downloads', 0) - if downloads >= 50: - all_models.append({ - 'model_id': mid, - 'url': f"https://modelscope.cn/{mid}", - 'downloads': downloads, - }) + all_models.append({ + 'model_id': mid, + 'url': f"https://modelscope.cn/{mid}", + 'downloads': m.get('downloads', 0), + }) time.sleep(0.3) - log(f"搜索完成: {len(seen)} 个唯一模型, {len(all_models)} 个下载量>=50") + log(f"搜索完成: {len(seen)} 个唯一模型, {len(all_models)} 个下载量{DOWNLOAD_MIN}-{DOWNLOAD_MAX}") # 2. 格式筛选(只保留HuggingFace格式,排除GGUF) log("\n--- 阶段2: 格式筛选 ---")