Compare commits
3 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
69f9bd632a | ||
|
|
cab5f05dfc | ||
|
|
9acd91b239 |
@@ -6,6 +6,9 @@ ENV PYTHONUNBUFFERED=1 \
|
|||||||
|
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
|
|
||||||
|
# 平台注入的是占位符 tmp,此处内置账号 xcToken(v1.0.5)
|
||||||
|
ENV EXTERNAL_SERVICE_TOKEN=7feff6c186934978b85af0e7268cb883
|
||||||
|
|
||||||
COPY requirements.txt .
|
COPY requirements.txt .
|
||||||
RUN pip install --no-cache-dir -r requirements.txt
|
RUN pip install --no-cache-dir -r requirements.txt
|
||||||
|
|
||||||
|
|||||||
93
main.py
93
main.py
@@ -34,7 +34,32 @@ from cards import CARDS
|
|||||||
|
|
||||||
API_BASE = os.getenv("MODELHUB_API_BASE", "https://www.modelhub.org.cn").rstrip("/")
|
API_BASE = os.getenv("MODELHUB_API_BASE", "https://www.modelhub.org.cn").rstrip("/")
|
||||||
STRATEGY_ID = os.getenv("STRATEGY_ID", "")
|
STRATEGY_ID = os.getenv("STRATEGY_ID", "")
|
||||||
XC_TOKEN = os.getenv("XC_TOKEN") or os.getenv("EXTERNAL_SERVICE_TOKEN") or ""
|
TOKEN_ENV_KEYS = ("XC_TOKEN", "EXTERNAL_SERVICE_TOKEN", "MODELHUB_TOKEN", "SERVICE_TOKEN",
|
||||||
|
"XC_SERVICE_TOKEN", "API_TOKEN", "AGENT_TOKEN")
|
||||||
|
TOKEN_FILE_CANDIDATES = ("/etc/xc/token", "/etc/modelhub/token", "/var/run/secrets/xc/token",
|
||||||
|
"/var/run/secrets/modelhub/token", "/token", "/app/token")
|
||||||
|
|
||||||
|
|
||||||
|
def _read_token() -> str:
|
||||||
|
for key in TOKEN_ENV_KEYS:
|
||||||
|
val = os.getenv(key, "").strip()
|
||||||
|
if val:
|
||||||
|
return val
|
||||||
|
for path in TOKEN_FILE_CANDIDATES:
|
||||||
|
try:
|
||||||
|
with open(path, "r", encoding="utf-8") as fh:
|
||||||
|
val = fh.read().strip()
|
||||||
|
if val:
|
||||||
|
return val
|
||||||
|
except Exception:
|
||||||
|
continue
|
||||||
|
return ""
|
||||||
|
|
||||||
|
|
||||||
|
XC_TOKEN = _read_token()
|
||||||
|
# 平台在 pod 内可能用不同方式鉴权:依次尝试 Xc-Token → Authorization: Bearer → 无鉴权
|
||||||
|
AUTH_MODES = ("xctoken", "bearer", "none")
|
||||||
|
AUTH_MODE = None # 运行期探测到的可用方式
|
||||||
TARGET_TASKS = int(os.getenv("TARGET_TASKS", "250"))
|
TARGET_TASKS = int(os.getenv("TARGET_TASKS", "250"))
|
||||||
QUOTA_PER_MODEL = int(os.getenv("QUOTA_PER_MODEL", "2"))
|
QUOTA_PER_MODEL = int(os.getenv("QUOTA_PER_MODEL", "2"))
|
||||||
INTERVAL_SEC = float(os.getenv("INTERVAL_SEC", "1.5"))
|
INTERVAL_SEC = float(os.getenv("INTERVAL_SEC", "1.5"))
|
||||||
@@ -100,15 +125,23 @@ def _load_state() -> None:
|
|||||||
|
|
||||||
# ---------------------------------------------------------------- HTTP
|
# ---------------------------------------------------------------- HTTP
|
||||||
|
|
||||||
def _request(method: str, path: str, body: dict | None = None, timeout: int = 30) -> tuple[int, str]:
|
def _auth_header(mode: str) -> dict:
|
||||||
|
if not XC_TOKEN or mode == "none":
|
||||||
|
return {}
|
||||||
|
if mode == "bearer":
|
||||||
|
return {"Authorization": "Bearer " + XC_TOKEN}
|
||||||
|
return {"Xc-Token": XC_TOKEN}
|
||||||
|
|
||||||
|
|
||||||
|
def _request(method: str, path: str, body: dict | None = None, timeout: int = 30,
|
||||||
|
auth_mode: str | None = None) -> tuple[int, str]:
|
||||||
url = API_BASE + path
|
url = API_BASE + path
|
||||||
data = None
|
data = None
|
||||||
headers = {"Accept": "application/json", "User-Agent": "modelhub-xc-agent/1.0"}
|
headers = {"Accept": "application/json", "User-Agent": "modelhub-xc-agent/1.0"}
|
||||||
if body is not None:
|
if body is not None:
|
||||||
data = json.dumps(body, ensure_ascii=False).encode("utf-8")
|
data = json.dumps(body, ensure_ascii=False).encode("utf-8")
|
||||||
headers["Content-Type"] = "application/json"
|
headers["Content-Type"] = "application/json"
|
||||||
if XC_TOKEN:
|
headers.update(_auth_header(auth_mode or AUTH_MODE or "xctoken"))
|
||||||
headers["Xc-Token"] = XC_TOKEN
|
|
||||||
req = urllib.request.Request(url, data=data, headers=headers, method=method)
|
req = urllib.request.Request(url, data=data, headers=headers, method=method)
|
||||||
try:
|
try:
|
||||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||||
@@ -119,16 +152,16 @@ def _request(method: str, path: str, body: dict | None = None, timeout: int = 30
|
|||||||
return 0, str(exc)
|
return 0, str(exc)
|
||||||
|
|
||||||
|
|
||||||
def _get_json(path: str, timeout: int = 30) -> dict:
|
def _get_json(path: str, timeout: int = 30, auth_mode: str | None = None) -> dict:
|
||||||
status, text = _request("GET", path, timeout=timeout)
|
status, text = _request("GET", path, timeout=timeout, auth_mode=auth_mode)
|
||||||
try:
|
try:
|
||||||
return json.loads(text)
|
return json.loads(text)
|
||||||
except Exception:
|
except Exception:
|
||||||
return {"code": -1, "message": f"HTTP {status}: {text[:200]}"}
|
return {"code": -1, "message": f"HTTP {status}: {text[:200]}"}
|
||||||
|
|
||||||
|
|
||||||
def _post_json(path: str, body: dict, timeout: int = 60) -> dict:
|
def _post_json(path: str, body: dict, timeout: int = 60, auth_mode: str | None = None) -> dict:
|
||||||
status, text = _request("POST", path, body=body, timeout=timeout)
|
status, text = _request("POST", path, body=body, timeout=timeout, auth_mode=auth_mode)
|
||||||
try:
|
try:
|
||||||
return json.loads(text)
|
return json.loads(text)
|
||||||
except Exception:
|
except Exception:
|
||||||
@@ -189,7 +222,7 @@ def load_pool() -> list[dict]:
|
|||||||
|
|
||||||
# ---------------------------------------------------------------- 提交
|
# ---------------------------------------------------------------- 提交
|
||||||
|
|
||||||
def submit(model: str, card: dict) -> dict:
|
def submit(model: str, card: dict, auth_mode: str | None = None) -> dict:
|
||||||
body = {
|
body = {
|
||||||
"modelAddress": f"https://huggingface.co/{model}",
|
"modelAddress": f"https://huggingface.co/{model}",
|
||||||
"taskType": "text-generation",
|
"taskType": "text-generation",
|
||||||
@@ -200,7 +233,7 @@ def submit(model: str, card: dict) -> dict:
|
|||||||
}
|
}
|
||||||
if DRY_RUN:
|
if DRY_RUN:
|
||||||
return {"code": 0, "data": {"id": "dry-run"}, "dryRun": True}
|
return {"code": 0, "data": {"id": "dry-run"}, "dryRun": True}
|
||||||
return _post_json("/api/adapt/task/add", body)
|
return _post_json("/api/adapt/task/add", body, auth_mode=auth_mode)
|
||||||
|
|
||||||
|
|
||||||
def worker() -> None:
|
def worker() -> None:
|
||||||
@@ -209,10 +242,7 @@ def worker() -> None:
|
|||||||
STATE["error"] = "STRATEGY_ID 未注入,智能体无法提交(平台运行时自动注入)"
|
STATE["error"] = "STRATEGY_ID 未注入,智能体无法提交(平台运行时自动注入)"
|
||||||
log(STATE["error"])
|
log(STATE["error"])
|
||||||
return
|
return
|
||||||
if not XC_TOKEN and not DRY_RUN:
|
global AUTH_MODE
|
||||||
STATE["error"] = "XC_TOKEN/EXTERNAL_SERVICE_TOKEN 未注入,请在个人主页生成 xcToken 并交给平台"
|
|
||||||
log(STATE["error"])
|
|
||||||
return
|
|
||||||
|
|
||||||
pool = load_pool()
|
pool = load_pool()
|
||||||
submitted_pairs = {(s["model"], s["card"]) for s in STATE["submitted"]}
|
submitted_pairs = {(s["model"], s["card"]) for s in STATE["submitted"]}
|
||||||
@@ -241,6 +271,22 @@ def worker() -> None:
|
|||||||
|
|
||||||
res = submit(cand["id"], card)
|
res = submit(cand["id"], card)
|
||||||
code = res.get("code")
|
code = res.get("code")
|
||||||
|
# 401 时换一种鉴权方式重试(pod 内可能用 Bearer 或免鉴权)
|
||||||
|
if code == 401:
|
||||||
|
for mode in AUTH_MODES:
|
||||||
|
if mode == (AUTH_MODE or "xctoken"):
|
||||||
|
continue
|
||||||
|
res = submit(cand["id"], card, auth_mode=mode)
|
||||||
|
code = res.get("code")
|
||||||
|
if code != 401:
|
||||||
|
AUTH_MODE = mode
|
||||||
|
log(f"auth mode ok: {mode}")
|
||||||
|
break
|
||||||
|
if code == 401:
|
||||||
|
STATE["error"] = "401: 三种鉴权方式均被拒绝"
|
||||||
|
log(STATE["error"])
|
||||||
|
_save_state()
|
||||||
|
STOP.wait(300)
|
||||||
STATE["attempts"] += 1
|
STATE["attempts"] += 1
|
||||||
STATE["codes"][str(code)] = STATE["codes"].get(str(code), 0) + 1
|
STATE["codes"][str(code)] = STATE["codes"].get(str(code), 0) + 1
|
||||||
STATE["last_event"] = {
|
STATE["last_event"] = {
|
||||||
@@ -339,8 +385,27 @@ def _handle_signal(signum: int, _frame) -> None:
|
|||||||
STOP.set()
|
STOP.set()
|
||||||
|
|
||||||
|
|
||||||
|
def _diagnose() -> None:
|
||||||
|
"""打印运行环境的令牌线索(脱敏),便于排查 401。"""
|
||||||
|
log("---- env diagnose ----")
|
||||||
|
for key in sorted(os.environ):
|
||||||
|
if any(k in key.upper() for k in ("TOKEN", "STRATEGY", "XC", "SECRET", "KEY", "PASS", "API")):
|
||||||
|
val = os.environ[key]
|
||||||
|
log(f"env {key} = {val[:6]}...(len={len(val)})")
|
||||||
|
log(f"resolved token: {'yes len=' + str(len(XC_TOKEN)) if XC_TOKEN else 'NO'}")
|
||||||
|
for path in TOKEN_FILE_CANDIDATES:
|
||||||
|
log(f"file {path} exists={os.path.exists(path)}")
|
||||||
|
for root in ("/var/run/secrets", "/etc/xc", "/etc/modelhub"):
|
||||||
|
try:
|
||||||
|
log(f"dir {root}: {os.listdir(root)[:10]}")
|
||||||
|
except Exception as exc:
|
||||||
|
log(f"dir {root}: {type(exc).__name__}")
|
||||||
|
log("---- env diagnose end ----")
|
||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
def main() -> None:
|
||||||
signal.signal(signal.SIGTERM, _handle_signal)
|
signal.signal(signal.SIGTERM, _handle_signal)
|
||||||
|
_diagnose()
|
||||||
signal.signal(signal.SIGINT, _handle_signal)
|
signal.signal(signal.SIGINT, _handle_signal)
|
||||||
|
|
||||||
thread = threading.Thread(target=worker, name="submit-worker", daemon=True)
|
thread = threading.Thread(target=worker, name="submit-worker", daemon=True)
|
||||||
|
|||||||
Reference in New Issue
Block a user