Compare commits
3 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
afbda884ad | ||
|
|
68c7a99e68 | ||
|
|
8e8fd50927 |
44
main.py
44
main.py
@@ -31,7 +31,7 @@ PORT = 8080
|
|||||||
STRATEGY_ID = os.getenv("STRATEGY_ID", "")
|
STRATEGY_ID = os.getenv("STRATEGY_ID", "")
|
||||||
|
|
||||||
# 目标GPU
|
# 目标GPU
|
||||||
TARGET_GPU = "ppu_zw_810e"
|
TARGET_GPU = "Iluvatar_bi-150"
|
||||||
|
|
||||||
# 账号Token
|
# 账号Token
|
||||||
TARGET_TOKEN = "f45f1aae2c094426be237c88b1085015"
|
TARGET_TOKEN = "f45f1aae2c094426be237c88b1085015"
|
||||||
@@ -99,15 +99,20 @@ def init_db():
|
|||||||
# ============================================================
|
# ============================================================
|
||||||
|
|
||||||
MODELSCOPE_API = "https://modelscope.cn/api/v1"
|
MODELSCOPE_API = "https://modelscope.cn/api/v1"
|
||||||
|
DOWNLOAD_MIN = 50
|
||||||
|
DOWNLOAD_MAX = 5000
|
||||||
|
SEARCH_PAGES = 5 # 每个关键词搜5页(50*5=250个结果)
|
||||||
|
|
||||||
|
|
||||||
def search_models(keyword: str, limit: int = 50) -> list:
|
def search_models(keyword: str) -> list:
|
||||||
"""从 ModelScope 搜索模型"""
|
"""从 ModelScope 搜索模型(多页,筛选下载量50-5000的冷门模型)"""
|
||||||
url = "https://modelscope.cn/openapi/v1/models"
|
url = "https://modelscope.cn/openapi/v1/models"
|
||||||
|
models = []
|
||||||
|
for page in range(1, SEARCH_PAGES + 1):
|
||||||
params = {
|
params = {
|
||||||
'search': keyword,
|
'search': keyword,
|
||||||
'page_size': min(limit, 50),
|
'page_size': 50,
|
||||||
'page_number': 1,
|
'page_number': page,
|
||||||
'sort': 'downloads',
|
'sort': 'downloads',
|
||||||
}
|
}
|
||||||
try:
|
try:
|
||||||
@@ -115,14 +120,21 @@ def search_models(keyword: str, limit: int = 50) -> list:
|
|||||||
headers={'User-Agent': 'Mozilla/5.0'})
|
headers={'User-Agent': 'Mozilla/5.0'})
|
||||||
data = resp.json()
|
data = resp.json()
|
||||||
if data.get('success'):
|
if data.get('success'):
|
||||||
models = data.get('data', {}).get('models', [])
|
page_models = data.get('data', {}).get('models', [])
|
||||||
log(f" [{keyword}]: {len(models)} 个结果")
|
for m in page_models:
|
||||||
return [{'id': m.get('id'), 'downloads': m.get('downloads', 0)} for m in models]
|
dl = m.get('downloads', 0)
|
||||||
|
if DOWNLOAD_MIN <= dl <= DOWNLOAD_MAX:
|
||||||
|
models.append({'id': m.get('id'), 'downloads': dl})
|
||||||
|
if len(page_models) < 50:
|
||||||
|
break # 最后一页,不继续
|
||||||
else:
|
else:
|
||||||
log(f" [{keyword}]: success=false")
|
break
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log(f" [{keyword}]: 失败 {e}")
|
log(f" [{keyword}] page={page}: {e}")
|
||||||
return []
|
break
|
||||||
|
time.sleep(0.3)
|
||||||
|
log(f" [{keyword}]: {len(models)} 个 (50<={DOWNLOAD_MAX})")
|
||||||
|
return models
|
||||||
|
|
||||||
|
|
||||||
def check_architecture(model_id: str) -> tuple:
|
def check_architecture(model_id: str) -> tuple:
|
||||||
@@ -303,20 +315,18 @@ def run_pipeline(submit_limit: int = 30):
|
|||||||
seen = set()
|
seen = set()
|
||||||
all_models = []
|
all_models = []
|
||||||
for kw in SEARCH_KEYWORDS:
|
for kw in SEARCH_KEYWORDS:
|
||||||
models = search_models(kw, limit=100)
|
models = search_models(kw)
|
||||||
for m in models:
|
for m in models:
|
||||||
mid = m.get('id', '')
|
mid = m.get('id', '')
|
||||||
if mid and mid not in seen:
|
if mid and mid not in seen:
|
||||||
seen.add(mid)
|
seen.add(mid)
|
||||||
downloads = m.get('downloads', 0)
|
|
||||||
if downloads >= 50:
|
|
||||||
all_models.append({
|
all_models.append({
|
||||||
'model_id': mid,
|
'model_id': mid,
|
||||||
'url': f"https://modelscope.cn/{mid}",
|
'url': f"https://modelscope.cn/{mid}",
|
||||||
'downloads': downloads,
|
'downloads': m.get('downloads', 0),
|
||||||
})
|
})
|
||||||
time.sleep(0.3)
|
time.sleep(0.3)
|
||||||
log(f"搜索完成: {len(seen)} 个唯一模型, {len(all_models)} 个下载量>=50")
|
log(f"搜索完成: {len(seen)} 个唯一模型, {len(all_models)} 个下载量{DOWNLOAD_MIN}-{DOWNLOAD_MAX}")
|
||||||
|
|
||||||
# 2. 格式筛选(只保留HuggingFace格式,排除GGUF)
|
# 2. 格式筛选(只保留HuggingFace格式,排除GGUF)
|
||||||
log("\n--- 阶段2: 格式筛选 ---")
|
log("\n--- 阶段2: 格式筛选 ---")
|
||||||
@@ -385,7 +395,7 @@ def run_pipeline(submit_limit: int = 30):
|
|||||||
log(f" ✅ {m['model_id']}")
|
log(f" ✅ {m['model_id']}")
|
||||||
db_conn.execute(
|
db_conn.execute(
|
||||||
'INSERT OR REPLACE INTO submitted VALUES (?,?,?,?)',
|
'INSERT OR REPLACE INTO submitted VALUES (?,?,?,?)',
|
||||||
(m['model_id'], gpu, str(task_id), datetime.now().isoformat())
|
(m['model_id'], TARGET_GPU, str(task_id), datetime.now().isoformat())
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
log(f" ❌ {m['model_id']}: {msg}")
|
log(f" ❌ {m['model_id']}: {msg}")
|
||||||
|
|||||||
Reference in New Issue
Block a user