6 Commits

2 changed files with 61 additions and 22 deletions

View File

@@ -4,6 +4,8 @@ ENV PYTHONUNBUFFERED=1
WORKDIR /app WORKDIR /app
RUN mkdir -p /app/data
COPY requirements.txt . COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt RUN pip install --no-cache-dir -r requirements.txt

81
main.py
View File

@@ -81,17 +81,36 @@ db_conn = None
def init_db(): def init_db():
global db_conn global db_conn
db_conn = sqlite3.connect(':memory:', check_same_thread=False) os.makedirs('/app/data', exist_ok=True)
db_conn.execute('''CREATE TABLE IF NOT EXISTS queue ( db_conn = sqlite3.connect('/app/data/submit_history.db', check_same_thread=False)
model_id TEXT, gpu TEXT, url TEXT, downloads INTEGER, db_conn.execute('''CREATE TABLE IF NOT EXISTS submitted (
params TEXT, category TEXT, score REAL, model_id TEXT, gpu TEXT, task_id TEXT, status TEXT,
submitted_at TEXT, checked_at TEXT,
PRIMARY KEY(model_id, gpu) PRIMARY KEY(model_id, gpu)
)''') )''')
db_conn.execute('''CREATE TABLE IF NOT EXISTS submitted ( db_conn.execute('''CREATE TABLE IF NOT EXISTS failed (
model_id TEXT, gpu TEXT, task_id TEXT, submitted_at TEXT, model_id TEXT, gpu TEXT, reason TEXT, failed_at TEXT,
PRIMARY KEY(model_id, gpu) PRIMARY KEY(model_id, gpu)
)''') )''')
db_conn.commit() db_conn.commit()
def is_model_failed(model_id: str) -> bool:
"""检查模型是否已知失败"""
if db_conn:
row = db_conn.execute(
'SELECT 1 FROM failed WHERE model_id=? AND gpu=?',
(model_id, TARGET_GPU)
).fetchone()
return row is not None
return False
def record_failed(model_id: str, reason: str):
"""记录失败的模型"""
if db_conn:
db_conn.execute(
'INSERT OR REPLACE INTO failed VALUES (?,?,?,?)',
(model_id, TARGET_GPU, reason, datetime.now().isoformat())
)
db_conn.commit()
# ============================================================ # ============================================================
@@ -302,7 +321,7 @@ def submit_model(model_url: str) -> tuple:
# 主流程 # 主流程
# ============================================================ # ============================================================
def run_pipeline(submit_limit: int = 30): def run_pipeline(submit_limit: int = 2):
"""完整流程搜索→筛选→提交只针对目标GPU""" """完整流程搜索→筛选→提交只针对目标GPU"""
init_db() init_db()
log("=" * 50) log("=" * 50)
@@ -328,30 +347,41 @@ def run_pipeline(submit_limit: int = 30):
time.sleep(0.3) time.sleep(0.3)
log(f"搜索完成: {len(seen)} 个唯一模型, {len(all_models)} 个下载量{DOWNLOAD_MIN}-{DOWNLOAD_MAX}") log(f"搜索完成: {len(seen)} 个唯一模型, {len(all_models)} 个下载量{DOWNLOAD_MIN}-{DOWNLOAD_MAX}")
# 2. 格式筛选(只保留HuggingFace格式排除GGUF # 2. 格式筛选(排除 GGUF/GPTQ/AWQ
log("\n--- 阶段2: 格式筛选 ---") log("\n--- 阶段2: 格式筛选 ---")
hf_models = [] hf_models = []
gguf_skipped = 0 format_skipped = 0
SKIP_FORMATS = ['GGUF', 'GPTQ', 'AWQ']
for m in all_models: for m in all_models:
mid = m['model_id'].upper() mid_upper = m['model_id'].upper()
if 'GGUF' in mid: skip = False
gguf_skipped += 1 for fmt in SKIP_FORMATS:
log(f"{m['model_id']}: GGUF格式跳过") if fmt in mid_upper:
else: format_skipped += 1
log(f" x {m['model_id']}: {fmt}格式,跳过")
skip = True
break
if not skip:
hf_models.append(m) hf_models.append(m)
log(f"格式筛选: {len(hf_models)} 通过 (HuggingFace), {gguf_skipped} 跳过 (GGUF)") log(f"格式筛选: {len(hf_models)} 通过, {format_skipped} 跳过 (GGUF/GPTQ/AWQ)")
# 3. 架构筛选 # 3. 架构筛选(排除 Qwen3.5 在 Iluvatar 上不支持)
log("\n--- 阶段3: 架构筛选 ---") log("\n--- 阶段3: 架构筛选 ---")
arch_passed = [] arch_passed = []
arch_rejected = 0 arch_rejected = 0
SKIP_ARCHS = ['qwen3_5', 'Qwen3_5', 'Qwen3.5']
for m in hf_models: for m in hf_models:
ok, reason = check_architecture(m['model_id']) ok, reason = check_architecture(m['model_id'])
if ok: if not ok:
arch_passed.append(m)
else:
arch_rejected += 1 arch_rejected += 1
log(f" {m['model_id']}: {reason}") log(f" x {m['model_id']}: {reason}")
continue
arch_str = str(reason).upper()
if any(a.upper() in arch_str for a in SKIP_ARCHS):
arch_rejected += 1
log(f" x {m['model_id']}: Qwen3.5(Iluvatar不支持)")
continue
arch_passed.append(m)
time.sleep(0.15) time.sleep(0.15)
log(f"架构筛选: {len(arch_passed)} 通过, {arch_rejected} 拒绝") log(f"架构筛选: {len(arch_passed)} 通过, {arch_rejected} 拒绝")
@@ -379,6 +409,10 @@ def run_pipeline(submit_limit: int = 30):
if check_my_submitted(model_id): if check_my_submitted(model_id):
continue continue
# 检查是否已知失败(避免重复提交)
if is_model_failed(model_id):
continue
to_submit.append(m) to_submit.append(m)
if len(to_submit) >= min(submit_limit, available): if len(to_submit) >= min(submit_limit, available):
break break
@@ -399,6 +433,9 @@ def run_pipeline(submit_limit: int = 30):
) )
else: else:
log(f"{m['model_id']}: {msg}") log(f"{m['model_id']}: {msg}")
# 永久失败类型记录到 failed 表
if any(kw in str(msg) for kw in ['保护期', '白名单', '唯一性']):
record_failed(m['model_id'], msg)
time.sleep(0.5) time.sleep(0.5)
log(f" 提交完成: {submitted}/{len(to_submit)}") log(f" 提交完成: {submitted}/{len(to_submit)}")
@@ -459,7 +496,7 @@ class AgentHandler(BaseHTTPRequestHandler):
if content_len > 0: if content_len > 0:
body = json.loads(self.rfile.read(content_len)) body = json.loads(self.rfile.read(content_len))
limit = body.get('limit', 30) limit = body.get('limit', 2)
self._json({'status': 'started', 'gpu': TARGET_GPU, 'limit': limit}) self._json({'status': 'started', 'gpu': TARGET_GPU, 'limit': limit})
@@ -676,7 +713,7 @@ def main():
try: try:
state['running'] = True state['running'] = True
state['last_run'] = datetime.now().isoformat() state['last_run'] = datetime.now().isoformat()
count = run_pipeline(submit_limit=30) count = run_pipeline(submit_limit=2)
state['last_result'] = {'submitted': count, 'success': True} state['last_result'] = {'submitted': count, 'success': True}
except Exception as e: except Exception as e:
log(f"流程异常: {traceback.format_exc()}") log(f"流程异常: {traceback.format_exc()}")