Compare commits
16 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3eed33f0d8 | ||
|
|
a2ef82d954 | ||
|
|
2198aed3c2 | ||
|
|
965342bafe | ||
|
|
5bb32e6bdb | ||
|
|
75ba3bb301 | ||
|
|
e28f2e3eca | ||
|
|
e0b6c3b5a8 | ||
|
|
f438206c0e | ||
|
|
12616c3816 | ||
|
|
0a4ba2e146 | ||
|
|
2222e1545b | ||
|
|
441b540e47 | ||
|
|
afbda884ad | ||
|
|
68c7a99e68 | ||
|
|
8e8fd50927 |
@@ -4,6 +4,8 @@ ENV PYTHONUNBUFFERED=1
|
|||||||
|
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
|
|
||||||
|
RUN mkdir -p /app/data
|
||||||
|
|
||||||
COPY requirements.txt .
|
COPY requirements.txt .
|
||||||
RUN pip install --no-cache-dir -r requirements.txt
|
RUN pip install --no-cache-dir -r requirements.txt
|
||||||
|
|
||||||
|
|||||||
158
main.py
158
main.py
@@ -31,7 +31,7 @@ PORT = 8080
|
|||||||
STRATEGY_ID = os.getenv("STRATEGY_ID", "")
|
STRATEGY_ID = os.getenv("STRATEGY_ID", "")
|
||||||
|
|
||||||
# 目标GPU
|
# 目标GPU
|
||||||
TARGET_GPU = "ppu_zw_810e"
|
TARGET_GPU = "Iluvatar_bi-100"
|
||||||
|
|
||||||
# 账号Token
|
# 账号Token
|
||||||
TARGET_TOKEN = "f45f1aae2c094426be237c88b1085015"
|
TARGET_TOKEN = "f45f1aae2c094426be237c88b1085015"
|
||||||
@@ -47,7 +47,7 @@ SUPPORTED_SPECIAL_ARCHS = ['Eagle3Speculator', 'LlamaForCausalLMEagle3']
|
|||||||
MODELHUB_API = "https://modelhub.org.cn/api"
|
MODELHUB_API = "https://modelhub.org.cn/api"
|
||||||
|
|
||||||
# 搜索关键词
|
# 搜索关键词
|
||||||
SEARCH_KEYWORDS = ['qwen', 'Qwen2', 'Qwen3', 'Qwen3.5', 'Qwen1.5', 'Qwen-']
|
SEARCH_KEYWORDS = ['Llama-3', 'Llama-3.1', 'Llama-3.2', 'Meta-Llama', 'Llama-4']
|
||||||
|
|
||||||
# ============================================================
|
# ============================================================
|
||||||
# 全局状态
|
# 全局状态
|
||||||
@@ -81,16 +81,35 @@ db_conn = None
|
|||||||
|
|
||||||
def init_db():
|
def init_db():
|
||||||
global db_conn
|
global db_conn
|
||||||
db_conn = sqlite3.connect(':memory:', check_same_thread=False)
|
os.makedirs('/app/data', exist_ok=True)
|
||||||
db_conn.execute('''CREATE TABLE IF NOT EXISTS queue (
|
db_conn = sqlite3.connect('/app/data/submit_history.db', check_same_thread=False)
|
||||||
model_id TEXT, gpu TEXT, url TEXT, downloads INTEGER,
|
|
||||||
params TEXT, category TEXT, score REAL,
|
|
||||||
PRIMARY KEY(model_id, gpu)
|
|
||||||
)''')
|
|
||||||
db_conn.execute('''CREATE TABLE IF NOT EXISTS submitted (
|
db_conn.execute('''CREATE TABLE IF NOT EXISTS submitted (
|
||||||
model_id TEXT, gpu TEXT, task_id TEXT, submitted_at TEXT,
|
model_id TEXT, gpu TEXT, task_id TEXT, status TEXT,
|
||||||
|
submitted_at TEXT, checked_at TEXT,
|
||||||
PRIMARY KEY(model_id, gpu)
|
PRIMARY KEY(model_id, gpu)
|
||||||
)''')
|
)''')
|
||||||
|
db_conn.execute('''CREATE TABLE IF NOT EXISTS failed (
|
||||||
|
model_id TEXT, gpu TEXT, reason TEXT, failed_at TEXT,
|
||||||
|
PRIMARY KEY(model_id, gpu)
|
||||||
|
)''')
|
||||||
|
db_conn.commit()
|
||||||
|
def is_model_failed(model_id: str) -> bool:
|
||||||
|
"""检查模型是否已知失败"""
|
||||||
|
if db_conn:
|
||||||
|
row = db_conn.execute(
|
||||||
|
'SELECT 1 FROM failed WHERE model_id=? AND gpu=?',
|
||||||
|
(model_id, TARGET_GPU)
|
||||||
|
).fetchone()
|
||||||
|
return row is not None
|
||||||
|
return False
|
||||||
|
|
||||||
|
def record_failed(model_id: str, reason: str):
|
||||||
|
"""记录失败的模型"""
|
||||||
|
if db_conn:
|
||||||
|
db_conn.execute(
|
||||||
|
'INSERT OR REPLACE INTO failed VALUES (?,?,?,?)',
|
||||||
|
(model_id, TARGET_GPU, reason, datetime.now().isoformat())
|
||||||
|
)
|
||||||
db_conn.commit()
|
db_conn.commit()
|
||||||
|
|
||||||
|
|
||||||
@@ -99,15 +118,20 @@ def init_db():
|
|||||||
# ============================================================
|
# ============================================================
|
||||||
|
|
||||||
MODELSCOPE_API = "https://modelscope.cn/api/v1"
|
MODELSCOPE_API = "https://modelscope.cn/api/v1"
|
||||||
|
DOWNLOAD_MIN = 50
|
||||||
|
DOWNLOAD_MAX = 5000
|
||||||
|
SEARCH_PAGES = 5 # 每个关键词搜5页(50*5=250个结果)
|
||||||
|
|
||||||
|
|
||||||
def search_models(keyword: str, limit: int = 50) -> list:
|
def search_models(keyword: str) -> list:
|
||||||
"""从 ModelScope 搜索模型"""
|
"""从 ModelScope 搜索模型(多页,筛选下载量50-5000的冷门模型)"""
|
||||||
url = "https://modelscope.cn/openapi/v1/models"
|
url = "https://modelscope.cn/openapi/v1/models"
|
||||||
|
models = []
|
||||||
|
for page in range(1, SEARCH_PAGES + 1):
|
||||||
params = {
|
params = {
|
||||||
'search': keyword,
|
'search': keyword,
|
||||||
'page_size': min(limit, 50),
|
'page_size': 50,
|
||||||
'page_number': 1,
|
'page_number': page,
|
||||||
'sort': 'downloads',
|
'sort': 'downloads',
|
||||||
}
|
}
|
||||||
try:
|
try:
|
||||||
@@ -115,18 +139,25 @@ def search_models(keyword: str, limit: int = 50) -> list:
|
|||||||
headers={'User-Agent': 'Mozilla/5.0'})
|
headers={'User-Agent': 'Mozilla/5.0'})
|
||||||
data = resp.json()
|
data = resp.json()
|
||||||
if data.get('success'):
|
if data.get('success'):
|
||||||
models = data.get('data', {}).get('models', [])
|
page_models = data.get('data', {}).get('models', [])
|
||||||
log(f" [{keyword}]: {len(models)} 个结果")
|
for m in page_models:
|
||||||
return [{'id': m.get('id'), 'downloads': m.get('downloads', 0)} for m in models]
|
dl = m.get('downloads', 0)
|
||||||
|
if DOWNLOAD_MIN <= dl <= DOWNLOAD_MAX:
|
||||||
|
models.append({'id': m.get('id'), 'downloads': dl})
|
||||||
|
if len(page_models) < 50:
|
||||||
|
break # 最后一页,不继续
|
||||||
else:
|
else:
|
||||||
log(f" [{keyword}]: success=false")
|
break
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log(f" [{keyword}]: 失败 {e}")
|
log(f" [{keyword}] page={page}: {e}")
|
||||||
return []
|
break
|
||||||
|
time.sleep(0.3)
|
||||||
|
log(f" [{keyword}]: {len(models)} 个 (50<={DOWNLOAD_MAX})")
|
||||||
|
return models
|
||||||
|
|
||||||
|
|
||||||
def check_architecture(model_id: str) -> tuple:
|
def check_architecture(model_id: str) -> tuple:
|
||||||
"""检查模型架构"""
|
"""检查模型是否有有效的 config.json(不限架构类型)"""
|
||||||
try:
|
try:
|
||||||
cfg_url = f"{MODELSCOPE_API}/models/{model_id}/repo?Revision=master&FilePath=config.json"
|
cfg_url = f"{MODELSCOPE_API}/models/{model_id}/repo?Revision=master&FilePath=config.json"
|
||||||
resp = requests.get(cfg_url, timeout=10)
|
resp = requests.get(cfg_url, timeout=10)
|
||||||
@@ -134,20 +165,13 @@ def check_architecture(model_id: str) -> tuple:
|
|||||||
cfg = resp.json()
|
cfg = resp.json()
|
||||||
archs = cfg.get('architectures', [])
|
archs = cfg.get('architectures', [])
|
||||||
mtype = cfg.get('model_type', '')
|
mtype = cfg.get('model_type', '')
|
||||||
arch_str = str(archs)
|
if archs or mtype:
|
||||||
for special in SUPPORTED_SPECIAL_ARCHS:
|
return True, f"arch={archs} type={mtype}"
|
||||||
if special in arch_str:
|
return False, "empty config"
|
||||||
return True, special
|
|
||||||
for kw in SUPPORTED_ARCH_KEYWORDS:
|
|
||||||
if kw in arch_str:
|
|
||||||
return True, kw
|
|
||||||
if mtype in SUPPORTED_MODEL_TYPES:
|
|
||||||
return True, mtype
|
|
||||||
return False, f"arch={archs} type={mtype}"
|
|
||||||
mid_upper = model_id.upper()
|
mid_upper = model_id.upper()
|
||||||
if 'QWEN3' in mid_upper or 'QWEN2' in mid_upper:
|
if 'LLAMA' in mid_upper:
|
||||||
return True, "GGUF"
|
return True, "Llama(GGUF)"
|
||||||
return False, "no config, not Qwen"
|
return False, "no config"
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
return True, f"check error: {e}"
|
return True, f"check error: {e}"
|
||||||
|
|
||||||
@@ -223,7 +247,7 @@ def check_queue_available() -> int:
|
|||||||
|
|
||||||
|
|
||||||
def build_config_params() -> str:
|
def build_config_params() -> str:
|
||||||
"""构建 YAML 配置 - 完全匹配平台自动生成的格式(只支持vllm)"""
|
"""构建 YAML 配置 - vllm"""
|
||||||
params = {
|
params = {
|
||||||
'framework': 'vllm',
|
'framework': 'vllm',
|
||||||
'nv_framework': 'vllm',
|
'nv_framework': 'vllm',
|
||||||
@@ -290,7 +314,7 @@ def submit_model(model_url: str) -> tuple:
|
|||||||
# 主流程
|
# 主流程
|
||||||
# ============================================================
|
# ============================================================
|
||||||
|
|
||||||
def run_pipeline(submit_limit: int = 30):
|
def run_pipeline(submit_limit: int = 5):
|
||||||
"""完整流程:搜索→筛选→提交(只针对目标GPU)"""
|
"""完整流程:搜索→筛选→提交(只针对目标GPU)"""
|
||||||
init_db()
|
init_db()
|
||||||
log("=" * 50)
|
log("=" * 50)
|
||||||
@@ -303,47 +327,53 @@ def run_pipeline(submit_limit: int = 30):
|
|||||||
seen = set()
|
seen = set()
|
||||||
all_models = []
|
all_models = []
|
||||||
for kw in SEARCH_KEYWORDS:
|
for kw in SEARCH_KEYWORDS:
|
||||||
models = search_models(kw, limit=100)
|
models = search_models(kw)
|
||||||
for m in models:
|
for m in models:
|
||||||
mid = m.get('id', '')
|
mid = m.get('id', '')
|
||||||
if mid and mid not in seen:
|
if mid and mid not in seen:
|
||||||
seen.add(mid)
|
seen.add(mid)
|
||||||
downloads = m.get('downloads', 0)
|
|
||||||
if downloads >= 50:
|
|
||||||
all_models.append({
|
all_models.append({
|
||||||
'model_id': mid,
|
'model_id': mid,
|
||||||
'url': f"https://modelscope.cn/{mid}",
|
'url': f"https://modelscope.cn/{mid}",
|
||||||
'downloads': downloads,
|
'downloads': m.get('downloads', 0),
|
||||||
})
|
})
|
||||||
time.sleep(0.3)
|
time.sleep(0.3)
|
||||||
log(f"搜索完成: {len(seen)} 个唯一模型, {len(all_models)} 个下载量>=50")
|
log(f"搜索完成: {len(seen)} 个唯一模型, {len(all_models)} 个下载量{DOWNLOAD_MIN}-{DOWNLOAD_MAX}")
|
||||||
|
|
||||||
# 2. 格式筛选(只保留HuggingFace格式,排除GGUF)
|
# 2. 格式筛选(排除 GPTQ/AWQ,保留 GGUF 和 HuggingFace)
|
||||||
log("\n--- 阶段2: 格式筛选 ---")
|
log("\n--- 阶段2: 格式筛选 ---")
|
||||||
hf_models = []
|
hf_models = []
|
||||||
gguf_skipped = 0
|
format_skipped = 0
|
||||||
|
SKIP_FORMATS = ['GPTQ', 'AWQ']
|
||||||
for m in all_models:
|
for m in all_models:
|
||||||
mid = m['model_id'].upper()
|
mid_upper = m['model_id'].upper()
|
||||||
if 'GGUF' in mid:
|
skip = False
|
||||||
gguf_skipped += 1
|
for fmt in SKIP_FORMATS:
|
||||||
log(f" ✗ {m['model_id']}: GGUF格式,跳过")
|
if fmt in mid_upper:
|
||||||
else:
|
format_skipped += 1
|
||||||
|
log(f" x {m['model_id']}: {fmt}格式,跳过")
|
||||||
|
skip = True
|
||||||
|
break
|
||||||
|
if not skip:
|
||||||
hf_models.append(m)
|
hf_models.append(m)
|
||||||
log(f"格式筛选: {len(hf_models)} 通过 (HuggingFace), {gguf_skipped} 跳过 (GGUF)")
|
log(f"格式筛选: {len(hf_models)} 通过, {format_skipped} 跳过 (GPTQ/AWQ)")
|
||||||
|
|
||||||
# 3. 架构筛选
|
# 3. 架构筛选(只保留有标准config.json的模型,排除无效格式)
|
||||||
log("\n--- 阶段3: 架构筛选 ---")
|
log("\n--- 阶段3: 架构检查 ---")
|
||||||
arch_passed = []
|
arch_passed = []
|
||||||
arch_rejected = 0
|
arch_rejected = 0
|
||||||
for m in hf_models:
|
for m in hf_models:
|
||||||
ok, reason = check_architecture(m['model_id'])
|
ok, reason = check_architecture(m['model_id'])
|
||||||
if ok:
|
if not ok:
|
||||||
arch_passed.append(m)
|
|
||||||
else:
|
|
||||||
arch_rejected += 1
|
arch_rejected += 1
|
||||||
log(f" ✗ {m['model_id']}: {reason}")
|
log(f" x {m['model_id']}: {reason}")
|
||||||
time.sleep(0.15)
|
elif reason == 'GGUF':
|
||||||
log(f"架构筛选: {len(arch_passed)} 通过, {arch_rejected} 拒绝")
|
arch_rejected += 1
|
||||||
|
log(f" x {m['model_id']}: GGUF(无config)")
|
||||||
|
else:
|
||||||
|
arch_passed.append(m)
|
||||||
|
time.sleep(0.1)
|
||||||
|
log(f"架构检查: {len(arch_passed)} 通过, {arch_rejected} 拒绝")
|
||||||
|
|
||||||
# 4. 筛选并提交(只针对目标GPU)
|
# 4. 筛选并提交(只针对目标GPU)
|
||||||
log(f"\n--- 阶段4: 筛选并提交 [{TARGET_GPU}] ---")
|
log(f"\n--- 阶段4: 筛选并提交 [{TARGET_GPU}] ---")
|
||||||
@@ -369,6 +399,10 @@ def run_pipeline(submit_limit: int = 30):
|
|||||||
if check_my_submitted(model_id):
|
if check_my_submitted(model_id):
|
||||||
continue
|
continue
|
||||||
|
|
||||||
|
# 检查是否已知失败(避免重复提交)
|
||||||
|
if is_model_failed(model_id):
|
||||||
|
continue
|
||||||
|
|
||||||
to_submit.append(m)
|
to_submit.append(m)
|
||||||
if len(to_submit) >= min(submit_limit, available):
|
if len(to_submit) >= min(submit_limit, available):
|
||||||
break
|
break
|
||||||
@@ -384,11 +418,15 @@ def run_pipeline(submit_limit: int = 30):
|
|||||||
submitted += 1
|
submitted += 1
|
||||||
log(f" ✅ {m['model_id']}")
|
log(f" ✅ {m['model_id']}")
|
||||||
db_conn.execute(
|
db_conn.execute(
|
||||||
'INSERT OR REPLACE INTO submitted VALUES (?,?,?,?)',
|
'INSERT OR REPLACE INTO submitted VALUES (?,?,?,?,?,?)',
|
||||||
(m['model_id'], gpu, str(task_id), datetime.now().isoformat())
|
(m['model_id'], TARGET_GPU, str(task_id), 'submitted',
|
||||||
|
datetime.now().isoformat(), None)
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
log(f" ❌ {m['model_id']}: {msg}")
|
log(f" ❌ {m['model_id']}: {msg}")
|
||||||
|
# 永久失败类型记录到 failed 表
|
||||||
|
if any(kw in str(msg) for kw in ['保护期', '白名单', '唯一性']):
|
||||||
|
record_failed(m['model_id'], msg)
|
||||||
time.sleep(0.5)
|
time.sleep(0.5)
|
||||||
|
|
||||||
log(f" 提交完成: {submitted}/{len(to_submit)}")
|
log(f" 提交完成: {submitted}/{len(to_submit)}")
|
||||||
@@ -449,7 +487,7 @@ class AgentHandler(BaseHTTPRequestHandler):
|
|||||||
if content_len > 0:
|
if content_len > 0:
|
||||||
body = json.loads(self.rfile.read(content_len))
|
body = json.loads(self.rfile.read(content_len))
|
||||||
|
|
||||||
limit = body.get('limit', 30)
|
limit = body.get('limit', 5)
|
||||||
|
|
||||||
self._json({'status': 'started', 'gpu': TARGET_GPU, 'limit': limit})
|
self._json({'status': 'started', 'gpu': TARGET_GPU, 'limit': limit})
|
||||||
|
|
||||||
@@ -666,7 +704,7 @@ def main():
|
|||||||
try:
|
try:
|
||||||
state['running'] = True
|
state['running'] = True
|
||||||
state['last_run'] = datetime.now().isoformat()
|
state['last_run'] = datetime.now().isoformat()
|
||||||
count = run_pipeline(submit_limit=30)
|
count = run_pipeline(submit_limit=5)
|
||||||
state['last_result'] = {'submitted': count, 'success': True}
|
state['last_result'] = {'submitted': count, 'success': True}
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log(f"流程异常: {traceback.format_exc()}")
|
log(f"流程异常: {traceback.format_exc()}")
|
||||||
|
|||||||
Reference in New Issue
Block a user