Compare commits
5 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f438206c0e | ||
|
|
12616c3816 | ||
|
|
0a4ba2e146 | ||
|
|
2222e1545b | ||
|
|
441b540e47 |
78
main.py
78
main.py
@@ -82,16 +82,34 @@ db_conn = None
|
|||||||
def init_db():
|
def init_db():
|
||||||
global db_conn
|
global db_conn
|
||||||
db_conn = sqlite3.connect(':memory:', check_same_thread=False)
|
db_conn = sqlite3.connect(':memory:', check_same_thread=False)
|
||||||
db_conn.execute('''CREATE TABLE IF NOT EXISTS queue (
|
db_conn.execute('''CREATE TABLE IF NOT EXISTS submitted (
|
||||||
model_id TEXT, gpu TEXT, url TEXT, downloads INTEGER,
|
model_id TEXT, gpu TEXT, task_id TEXT, status TEXT,
|
||||||
params TEXT, category TEXT, score REAL,
|
submitted_at TEXT, checked_at TEXT,
|
||||||
PRIMARY KEY(model_id, gpu)
|
PRIMARY KEY(model_id, gpu)
|
||||||
)''')
|
)''')
|
||||||
db_conn.execute('''CREATE TABLE IF NOT EXISTS submitted (
|
db_conn.execute('''CREATE TABLE IF NOT EXISTS failed (
|
||||||
model_id TEXT, gpu TEXT, task_id TEXT, submitted_at TEXT,
|
model_id TEXT, gpu TEXT, reason TEXT, failed_at TEXT,
|
||||||
PRIMARY KEY(model_id, gpu)
|
PRIMARY KEY(model_id, gpu)
|
||||||
)''')
|
)''')
|
||||||
db_conn.commit()
|
db_conn.commit()
|
||||||
|
def is_model_failed(model_id: str) -> bool:
|
||||||
|
"""检查模型是否已知失败"""
|
||||||
|
if db_conn:
|
||||||
|
row = db_conn.execute(
|
||||||
|
'SELECT 1 FROM failed WHERE model_id=? AND gpu=?',
|
||||||
|
(model_id, TARGET_GPU)
|
||||||
|
).fetchone()
|
||||||
|
return row is not None
|
||||||
|
return False
|
||||||
|
|
||||||
|
def record_failed(model_id: str, reason: str):
|
||||||
|
"""记录失败的模型"""
|
||||||
|
if db_conn:
|
||||||
|
db_conn.execute(
|
||||||
|
'INSERT OR REPLACE INTO failed VALUES (?,?,?,?)',
|
||||||
|
(model_id, TARGET_GPU, reason, datetime.now().isoformat())
|
||||||
|
)
|
||||||
|
db_conn.commit()
|
||||||
|
|
||||||
|
|
||||||
# ============================================================
|
# ============================================================
|
||||||
@@ -302,7 +320,7 @@ def submit_model(model_url: str) -> tuple:
|
|||||||
# 主流程
|
# 主流程
|
||||||
# ============================================================
|
# ============================================================
|
||||||
|
|
||||||
def run_pipeline(submit_limit: int = 30):
|
def run_pipeline(submit_limit: int = 2):
|
||||||
"""完整流程:搜索→筛选→提交(只针对目标GPU)"""
|
"""完整流程:搜索→筛选→提交(只针对目标GPU)"""
|
||||||
init_db()
|
init_db()
|
||||||
log("=" * 50)
|
log("=" * 50)
|
||||||
@@ -328,30 +346,41 @@ def run_pipeline(submit_limit: int = 30):
|
|||||||
time.sleep(0.3)
|
time.sleep(0.3)
|
||||||
log(f"搜索完成: {len(seen)} 个唯一模型, {len(all_models)} 个下载量{DOWNLOAD_MIN}-{DOWNLOAD_MAX}")
|
log(f"搜索完成: {len(seen)} 个唯一模型, {len(all_models)} 个下载量{DOWNLOAD_MIN}-{DOWNLOAD_MAX}")
|
||||||
|
|
||||||
# 2. 格式筛选(只保留HuggingFace格式,排除GGUF)
|
# 2. 格式筛选(排除 GGUF/GPTQ/AWQ)
|
||||||
log("\n--- 阶段2: 格式筛选 ---")
|
log("\n--- 阶段2: 格式筛选 ---")
|
||||||
hf_models = []
|
hf_models = []
|
||||||
gguf_skipped = 0
|
format_skipped = 0
|
||||||
|
SKIP_FORMATS = ['GGUF', 'GPTQ', 'AWQ']
|
||||||
for m in all_models:
|
for m in all_models:
|
||||||
mid = m['model_id'].upper()
|
mid_upper = m['model_id'].upper()
|
||||||
if 'GGUF' in mid:
|
skip = False
|
||||||
gguf_skipped += 1
|
for fmt in SKIP_FORMATS:
|
||||||
log(f" ✗ {m['model_id']}: GGUF格式,跳过")
|
if fmt in mid_upper:
|
||||||
else:
|
format_skipped += 1
|
||||||
|
log(f" x {m['model_id']}: {fmt}格式,跳过")
|
||||||
|
skip = True
|
||||||
|
break
|
||||||
|
if not skip:
|
||||||
hf_models.append(m)
|
hf_models.append(m)
|
||||||
log(f"格式筛选: {len(hf_models)} 通过 (HuggingFace), {gguf_skipped} 跳过 (GGUF)")
|
log(f"格式筛选: {len(hf_models)} 通过, {format_skipped} 跳过 (GGUF/GPTQ/AWQ)")
|
||||||
|
|
||||||
# 3. 架构筛选
|
# 3. 架构筛选(排除 Qwen3.5 在 Iluvatar 上不支持)
|
||||||
log("\n--- 阶段3: 架构筛选 ---")
|
log("\n--- 阶段3: 架构筛选 ---")
|
||||||
arch_passed = []
|
arch_passed = []
|
||||||
arch_rejected = 0
|
arch_rejected = 0
|
||||||
|
SKIP_ARCHS = ['qwen3_5', 'Qwen3_5', 'Qwen3.5']
|
||||||
for m in hf_models:
|
for m in hf_models:
|
||||||
ok, reason = check_architecture(m['model_id'])
|
ok, reason = check_architecture(m['model_id'])
|
||||||
if ok:
|
if not ok:
|
||||||
arch_passed.append(m)
|
|
||||||
else:
|
|
||||||
arch_rejected += 1
|
arch_rejected += 1
|
||||||
log(f" ✗ {m['model_id']}: {reason}")
|
log(f" x {m['model_id']}: {reason}")
|
||||||
|
continue
|
||||||
|
arch_str = str(reason).upper()
|
||||||
|
if any(a.upper() in arch_str for a in SKIP_ARCHS):
|
||||||
|
arch_rejected += 1
|
||||||
|
log(f" x {m['model_id']}: Qwen3.5(Iluvatar不支持)")
|
||||||
|
continue
|
||||||
|
arch_passed.append(m)
|
||||||
time.sleep(0.15)
|
time.sleep(0.15)
|
||||||
log(f"架构筛选: {len(arch_passed)} 通过, {arch_rejected} 拒绝")
|
log(f"架构筛选: {len(arch_passed)} 通过, {arch_rejected} 拒绝")
|
||||||
|
|
||||||
@@ -379,6 +408,10 @@ def run_pipeline(submit_limit: int = 30):
|
|||||||
if check_my_submitted(model_id):
|
if check_my_submitted(model_id):
|
||||||
continue
|
continue
|
||||||
|
|
||||||
|
# 检查是否已知失败(避免重复提交)
|
||||||
|
if is_model_failed(model_id):
|
||||||
|
continue
|
||||||
|
|
||||||
to_submit.append(m)
|
to_submit.append(m)
|
||||||
if len(to_submit) >= min(submit_limit, available):
|
if len(to_submit) >= min(submit_limit, available):
|
||||||
break
|
break
|
||||||
@@ -399,6 +432,9 @@ def run_pipeline(submit_limit: int = 30):
|
|||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
log(f" ❌ {m['model_id']}: {msg}")
|
log(f" ❌ {m['model_id']}: {msg}")
|
||||||
|
# 永久失败类型记录到 failed 表
|
||||||
|
if any(kw in str(msg) for kw in ['保护期', '白名单', '唯一性']):
|
||||||
|
record_failed(m['model_id'], msg)
|
||||||
time.sleep(0.5)
|
time.sleep(0.5)
|
||||||
|
|
||||||
log(f" 提交完成: {submitted}/{len(to_submit)}")
|
log(f" 提交完成: {submitted}/{len(to_submit)}")
|
||||||
@@ -459,7 +495,7 @@ class AgentHandler(BaseHTTPRequestHandler):
|
|||||||
if content_len > 0:
|
if content_len > 0:
|
||||||
body = json.loads(self.rfile.read(content_len))
|
body = json.loads(self.rfile.read(content_len))
|
||||||
|
|
||||||
limit = body.get('limit', 30)
|
limit = body.get('limit', 2)
|
||||||
|
|
||||||
self._json({'status': 'started', 'gpu': TARGET_GPU, 'limit': limit})
|
self._json({'status': 'started', 'gpu': TARGET_GPU, 'limit': limit})
|
||||||
|
|
||||||
@@ -676,7 +712,7 @@ def main():
|
|||||||
try:
|
try:
|
||||||
state['running'] = True
|
state['running'] = True
|
||||||
state['last_run'] = datetime.now().isoformat()
|
state['last_run'] = datetime.now().isoformat()
|
||||||
count = run_pipeline(submit_limit=30)
|
count = run_pipeline(submit_limit=2)
|
||||||
state['last_result'] = {'submitted': count, 'success': True}
|
state['last_result'] = {'submitted': count, 'success': True}
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
log(f"流程异常: {traceback.format_exc()}")
|
log(f"流程异常: {traceback.format_exc()}")
|
||||||
|
|||||||
Reference in New Issue
Block a user