fix: remove slow system git build dependency
This commit is contained in:
@@ -1,6 +1,8 @@
|
||||
.git
|
||||
.agents
|
||||
.codex
|
||||
code4beforeUchange/
|
||||
tests/
|
||||
**/__pycache__/
|
||||
**/*.pyc
|
||||
**/.ipynb_checkpoints/
|
||||
|
||||
@@ -5,10 +5,6 @@ ENV PYTHONPATH=/app/modelhub_submmit_api
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends ca-certificates git \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
|
||||
10
README.md
10
README.md
@@ -362,19 +362,19 @@ counts fail closed for older candidates, and no startup or periodic cleanup can
|
||||
cancel a task by date. It also keeps unclassified/ambiguous historical failures
|
||||
neutral in GPU/framework success feedback while preserving deterministic OOM
|
||||
and architecture cleanup.
|
||||
Version `2026.08.15.2` replaces the 70/30 quota with hierarchical success-first
|
||||
Version `2026.08.15.3` replaces the 70/30 quota with hierarchical success-first
|
||||
routing, dynamically gates models through the official GPU/task/framework/config
|
||||
APIs, enriches ModelScope metadata and model lineage, learns only proven safe
|
||||
config vectors, and adds crash-safe write-ahead state synchronization to the
|
||||
`agent-state` branch. It also exposes `/ready` and extends deterministic cleanup
|
||||
to officially removed waiting GPU/framework routes. The runtime image installs
|
||||
Git and CA certificates required by state recovery and synchronization.
|
||||
to officially removed waiting GPU/framework routes. State recovery uses the
|
||||
pure-Python Dulwich client, avoiding slow OS package installation during builds.
|
||||
|
||||
## Deploy
|
||||
|
||||
Create a tag and submit the repository URL plus tag in "我的适配智能体".
|
||||
|
||||
```bash
|
||||
git tag -a agent-v26 -m "ModelHub agent 2026.08.15.2"
|
||||
git push origin main agent-v26
|
||||
git tag -a agent-v27 -m "ModelHub agent 2026.08.15.3"
|
||||
git push origin main agent-v27
|
||||
```
|
||||
|
||||
@@ -2,11 +2,10 @@ from __future__ import annotations
|
||||
|
||||
import fcntl
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import stat
|
||||
import subprocess
|
||||
import tempfile
|
||||
import threading
|
||||
import uuid
|
||||
@@ -15,6 +14,9 @@ from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable
|
||||
|
||||
from dulwich import porcelain
|
||||
from dulwich.repo import Repo
|
||||
|
||||
from common import read_json, read_jsonl, write_json, write_jsonl
|
||||
from runner_common import load_key_files
|
||||
from version import AGENT_VERSION
|
||||
@@ -147,8 +149,6 @@ class StateGitSync:
|
||||
self._lock_handle = None
|
||||
self._mutex = threading.Lock()
|
||||
self._workspace: Path | None = None
|
||||
self._askpass_dir: Path | None = None
|
||||
self._askpass_path: Path | None = None
|
||||
self._expected_remote_oid: str | None = None
|
||||
|
||||
@property
|
||||
@@ -188,88 +188,43 @@ class StateGitSync:
|
||||
if self._workspace is not None:
|
||||
shutil.rmtree(self._workspace.parent, ignore_errors=True)
|
||||
self._workspace = None
|
||||
if self._askpass_dir is not None:
|
||||
shutil.rmtree(self._askpass_dir, ignore_errors=True)
|
||||
self._askpass_dir = None
|
||||
self._askpass_path = None
|
||||
|
||||
def _git_environment(self) -> dict[str, str]:
|
||||
def _auth_kwargs(self) -> dict[str, str]:
|
||||
if not all(self.credentials.get(key) for key in ("username", "email", "password")):
|
||||
raise StateSyncError("missing ModelHub Git username, email or password")
|
||||
if self._askpass_path is None:
|
||||
directory = Path(tempfile.mkdtemp(prefix="modelhub-state-askpass-"))
|
||||
script = directory / "askpass.py"
|
||||
script.write_text(
|
||||
"#!/usr/bin/env python3\n"
|
||||
"import os, sys\n"
|
||||
"prompt = ' '.join(sys.argv[1:]).lower()\n"
|
||||
"key = 'MODELHUB_STATE_GIT_USERNAME' if 'username' in prompt else 'MODELHUB_STATE_GIT_PASSWORD'\n"
|
||||
"print(os.environ.get(key, ''))\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
script.chmod(stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
|
||||
self._askpass_dir = directory
|
||||
self._askpass_path = script
|
||||
env = os.environ.copy()
|
||||
env.update(
|
||||
{
|
||||
"GIT_ASKPASS": str(self._askpass_path),
|
||||
"GIT_TERMINAL_PROMPT": "0",
|
||||
"MODELHUB_STATE_GIT_USERNAME": self.credentials["username"],
|
||||
"MODELHUB_STATE_GIT_PASSWORD": self.credentials["password"],
|
||||
}
|
||||
)
|
||||
return env
|
||||
return {
|
||||
"username": self.credentials["username"],
|
||||
"password": self.credentials["password"],
|
||||
}
|
||||
|
||||
def _git(self, *args: str, cwd: Path | None = None, check: bool = True) -> subprocess.CompletedProcess[str]:
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["git", *args],
|
||||
cwd=str(cwd or self.project_root),
|
||||
env=self._git_environment(),
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
timeout=45,
|
||||
check=False,
|
||||
)
|
||||
except FileNotFoundError as exc:
|
||||
raise StateSyncError(
|
||||
"git executable is unavailable; rebuild the service image from the repository Dockerfile"
|
||||
) from exc
|
||||
except subprocess.TimeoutExpired as exc:
|
||||
raise StateSyncError("git operation timed out after 45 seconds") from exc
|
||||
if check and result.returncode != 0:
|
||||
reason = _safe_text(result.stderr.strip() or result.stdout.strip() or f"git exited {result.returncode}")
|
||||
raise StateSyncError(reason)
|
||||
return result
|
||||
@property
|
||||
def _author(self) -> bytes:
|
||||
name = self.credentials.get("username") or "modelhub-agent"
|
||||
email = self.credentials.get("email") or "modelhub-agent@localhost"
|
||||
return f"{name} <{email}>".encode("utf-8")
|
||||
|
||||
def _remote_oid(self) -> str | None:
|
||||
result = self._git("ls-remote", "--heads", self.remote, self.branch)
|
||||
line = result.stdout.strip().splitlines()
|
||||
return line[0].split()[0] if line else None
|
||||
result = porcelain.ls_remote(self.remote, quiet=True, **self._auth_kwargs())
|
||||
oid = result.refs.get(f"refs/heads/{self.branch}".encode("utf-8"))
|
||||
return oid.decode("ascii") if oid else None
|
||||
|
||||
def _create_workspace(self, remote_oid: str | None) -> None:
|
||||
parent = Path(tempfile.mkdtemp(prefix="modelhub-agent-state-"))
|
||||
workspace = parent / "state"
|
||||
if remote_oid:
|
||||
self._git(
|
||||
"clone",
|
||||
"--quiet",
|
||||
"--single-branch",
|
||||
"--branch",
|
||||
self.branch,
|
||||
porcelain.clone(
|
||||
self.remote,
|
||||
str(workspace),
|
||||
cwd=parent,
|
||||
workspace,
|
||||
branch=self.branch,
|
||||
depth=self.history_depth,
|
||||
checkout=True,
|
||||
errstream=io.BytesIO(),
|
||||
**self._auth_kwargs(),
|
||||
)
|
||||
else:
|
||||
workspace.mkdir(parents=True)
|
||||
self._git("init", "--quiet", cwd=workspace)
|
||||
self._git("checkout", "--orphan", self.branch, cwd=workspace)
|
||||
self._git("remote", "add", "origin", self.remote, cwd=workspace)
|
||||
self._git("config", "user.name", self.credentials["username"], cwd=workspace)
|
||||
self._git("config", "user.email", self.credentials["email"], cwd=workspace)
|
||||
repo = porcelain.init(workspace)
|
||||
repo.refs.set_symbolic_ref(b"HEAD", f"refs/heads/{self.branch}".encode("utf-8"))
|
||||
self._workspace = workspace
|
||||
self._expected_remote_oid = remote_oid
|
||||
|
||||
@@ -298,17 +253,16 @@ class StateGitSync:
|
||||
manifest = self._validate_manifest(self._workspace)
|
||||
except Exception as current_error:
|
||||
manifest = None
|
||||
commits = self._git(
|
||||
"rev-list",
|
||||
f"--max-count={self.history_depth}",
|
||||
"HEAD",
|
||||
cwd=self._workspace,
|
||||
).stdout.splitlines()[1:]
|
||||
repo = Repo(str(self._workspace))
|
||||
commits = [entry.commit.id for entry in repo.get_walker(max_entries=self.history_depth)][1:]
|
||||
for commit in commits:
|
||||
self._git("checkout", "--quiet", "--detach", commit, cwd=self._workspace)
|
||||
porcelain.reset(repo, "hard", commit)
|
||||
try:
|
||||
manifest = self._validate_manifest(self._workspace)
|
||||
self.log(f"[state-recovery] fallback_commit={commit[:12]} reason=checksum_recovery")
|
||||
self.log(
|
||||
f"[state-recovery] fallback_commit={commit.decode('ascii')[:12]} "
|
||||
"reason=checksum_recovery"
|
||||
)
|
||||
break
|
||||
except Exception:
|
||||
continue
|
||||
@@ -624,34 +578,46 @@ class StateGitSync:
|
||||
"checksums": checksums,
|
||||
}
|
||||
write_json(self._workspace / "manifest.json", manifest)
|
||||
self._git("add", "--all", cwd=self._workspace)
|
||||
diff = self._git("diff", "--cached", "--quiet", cwd=self._workspace, check=False)
|
||||
if diff.returncode == 0:
|
||||
repo = Repo(str(self._workspace))
|
||||
porcelain.add(repo)
|
||||
status = porcelain.status(repo)
|
||||
staged = any(status.staged.get(kind) for kind in ("add", "delete", "modify"))
|
||||
if not staged:
|
||||
self.healthy = True
|
||||
return True
|
||||
self._git("commit", "--quiet", "-m", f"state: generation {self.generation} ({phase})", cwd=self._workspace)
|
||||
commit_count = int(
|
||||
self._git("rev-list", "--count", "HEAD", cwd=self._workspace).stdout.strip() or "0"
|
||||
porcelain.commit(
|
||||
repo,
|
||||
message=f"state: generation {self.generation} ({phase})".encode("utf-8"),
|
||||
author=self._author,
|
||||
committer=self._author,
|
||||
)
|
||||
commit_count = sum(1 for _ in repo.get_walker())
|
||||
if commit_count > self.history_depth:
|
||||
compact_branch = f"state-compact-{uuid.uuid4().hex[:8]}"
|
||||
self._git("checkout", "--quiet", "--orphan", compact_branch, cwd=self._workspace)
|
||||
self._git("rm", "--quiet", "--cached", "-r", ".", cwd=self._workspace, check=False)
|
||||
self._git("add", "--all", cwd=self._workspace)
|
||||
self._git(
|
||||
"commit",
|
||||
"--quiet",
|
||||
"-m",
|
||||
f"state: compacted generation {self.generation}",
|
||||
cwd=self._workspace,
|
||||
branch_ref = f"refs/heads/{self.branch}".encode("utf-8")
|
||||
del repo.refs[branch_ref]
|
||||
porcelain.add(repo)
|
||||
porcelain.commit(
|
||||
repo,
|
||||
message=f"state: compacted generation {self.generation}".encode("utf-8"),
|
||||
author=self._author,
|
||||
committer=self._author,
|
||||
)
|
||||
lease = (
|
||||
f"--force-with-lease=refs/heads/{self.branch}:{self._expected_remote_oid}"
|
||||
if self._expected_remote_oid
|
||||
else f"--force-with-lease=refs/heads/{self.branch}:"
|
||||
current_remote_oid = self._remote_oid()
|
||||
if current_remote_oid != self._expected_remote_oid:
|
||||
raise StateSyncError("state branch changed remotely; refusing to overwrite a newer snapshot")
|
||||
porcelain.push(
|
||||
repo,
|
||||
self.remote,
|
||||
refspecs=f"HEAD:refs/heads/{self.branch}",
|
||||
force=True,
|
||||
outstream=io.BytesIO(),
|
||||
errstream=io.BytesIO(),
|
||||
**self._auth_kwargs(),
|
||||
)
|
||||
self._git("push", "--quiet", lease, "origin", f"HEAD:refs/heads/{self.branch}", cwd=self._workspace)
|
||||
self._expected_remote_oid = self._git("rev-parse", "HEAD", cwd=self._workspace).stdout.strip()
|
||||
local_oid = repo.head().decode("ascii")
|
||||
if self._remote_oid() != local_oid:
|
||||
raise StateSyncError("state branch verification failed after push")
|
||||
self._expected_remote_oid = local_oid
|
||||
self.last_sync_at = manifest["updatedAt"]
|
||||
self.last_error = None
|
||||
self.healthy = True
|
||||
|
||||
@@ -1 +1 @@
|
||||
AGENT_VERSION = "2026.08.15.2"
|
||||
AGENT_VERSION = "2026.08.15.3"
|
||||
|
||||
@@ -1 +1,2 @@
|
||||
PyYAML>=6.0.2,<7
|
||||
dulwich>=0.24.10,<0.26
|
||||
|
||||
@@ -40,6 +40,12 @@ class HostedAgentEntrypointTests(unittest.TestCase):
|
||||
self.assertFalse(readiness["ready"])
|
||||
self.assertEqual("state_sync_unhealthy", readiness["reason"])
|
||||
|
||||
def test_runtime_image_uses_python_git_client_without_apt_layer(self) -> None:
|
||||
dockerfile = (ROOT_DIR / "Dockerfile").read_text(encoding="utf-8")
|
||||
requirements = (ROOT_DIR / "requirements.txt").read_text(encoding="utf-8")
|
||||
self.assertNotIn("apt-get", dockerfile)
|
||||
self.assertIn("dulwich", requirements.casefold())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
from datetime import datetime, timezone
|
||||
@@ -9,6 +8,8 @@ from unittest.mock import patch
|
||||
|
||||
import sys
|
||||
|
||||
from dulwich import porcelain
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
MODULE_ROOT = ROOT / "modelhub_submmit_api"
|
||||
if str(MODULE_ROOT) not in sys.path:
|
||||
@@ -20,7 +21,6 @@ from hf_discovery import HuggingFaceDiscovery, parse_model_card_front_matter #
|
||||
from official_capabilities import OfficialCapabilityRegistry # noqa: E402
|
||||
from routing_engine import SuccessFirstRoutingEngine # noqa: E402
|
||||
from state_sync import StateGitSync # noqa: E402
|
||||
from state_sync import StateSyncError # noqa: E402
|
||||
|
||||
|
||||
class OfficialClient:
|
||||
@@ -139,7 +139,7 @@ class SuperAgentTests(unittest.TestCase):
|
||||
restored_project = root / "restored"
|
||||
project.mkdir()
|
||||
restored_project.mkdir()
|
||||
subprocess.run(["git", "init", "--bare", str(remote)], check=True, stdout=subprocess.DEVNULL)
|
||||
porcelain.init(remote, bare=True)
|
||||
write_json(project / ".modelhub_state" / "account_capacity.json", {"version": 1})
|
||||
credentials = {"username": "tester", "email": "tester@example.com", "password": "secret-value"}
|
||||
manager = StateGitSync(
|
||||
@@ -208,19 +208,6 @@ class SuperAgentTests(unittest.TestCase):
|
||||
)
|
||||
)
|
||||
|
||||
def test_missing_git_has_actionable_state_sync_error(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temporary_dir:
|
||||
manager = StateGitSync(
|
||||
project_root=Path(temporary_dir),
|
||||
credentials={"username": "u", "email": "e@example.com", "password": "p"},
|
||||
remote="unused",
|
||||
log_fn=lambda _: None,
|
||||
)
|
||||
with patch("state_sync.subprocess.run", side_effect=FileNotFoundError("git")):
|
||||
with self.assertRaisesRegex(StateSyncError, "rebuild the service image"):
|
||||
manager._git("version")
|
||||
manager.close()
|
||||
|
||||
def test_config_patch_requires_repeated_cross_model_success(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temporary_dir:
|
||||
root = Path(temporary_dir)
|
||||
|
||||
Reference in New Issue
Block a user