fix: remove slow system git build dependency

This commit is contained in:
CoolBoy
2026-08-15 19:47:24 +08:00
parent 467eaccd59
commit 85f6cb5157
8 changed files with 85 additions and 127 deletions

View File

@@ -1,6 +1,8 @@
.git .git
.agents .agents
.codex .codex
code4beforeUchange/
tests/
**/__pycache__/ **/__pycache__/
**/*.pyc **/*.pyc
**/.ipynb_checkpoints/ **/.ipynb_checkpoints/

View File

@@ -5,10 +5,6 @@ ENV PYTHONPATH=/app/modelhub_submmit_api
WORKDIR /app WORKDIR /app
RUN apt-get update \
&& apt-get install -y --no-install-recommends ca-certificates git \
&& rm -rf /var/lib/apt/lists/*
COPY requirements.txt . COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt RUN pip install --no-cache-dir -r requirements.txt

View File

@@ -362,19 +362,19 @@ counts fail closed for older candidates, and no startup or periodic cleanup can
cancel a task by date. It also keeps unclassified/ambiguous historical failures cancel a task by date. It also keeps unclassified/ambiguous historical failures
neutral in GPU/framework success feedback while preserving deterministic OOM neutral in GPU/framework success feedback while preserving deterministic OOM
and architecture cleanup. and architecture cleanup.
Version `2026.08.15.2` replaces the 70/30 quota with hierarchical success-first Version `2026.08.15.3` replaces the 70/30 quota with hierarchical success-first
routing, dynamically gates models through the official GPU/task/framework/config routing, dynamically gates models through the official GPU/task/framework/config
APIs, enriches ModelScope metadata and model lineage, learns only proven safe APIs, enriches ModelScope metadata and model lineage, learns only proven safe
config vectors, and adds crash-safe write-ahead state synchronization to the config vectors, and adds crash-safe write-ahead state synchronization to the
`agent-state` branch. It also exposes `/ready` and extends deterministic cleanup `agent-state` branch. It also exposes `/ready` and extends deterministic cleanup
to officially removed waiting GPU/framework routes. The runtime image installs to officially removed waiting GPU/framework routes. State recovery uses the
Git and CA certificates required by state recovery and synchronization. pure-Python Dulwich client, avoiding slow OS package installation during builds.
## Deploy ## Deploy
Create a tag and submit the repository URL plus tag in "我的适配智能体". Create a tag and submit the repository URL plus tag in "我的适配智能体".
```bash ```bash
git tag -a agent-v26 -m "ModelHub agent 2026.08.15.2" git tag -a agent-v27 -m "ModelHub agent 2026.08.15.3"
git push origin main agent-v26 git push origin main agent-v27
``` ```

View File

@@ -2,11 +2,10 @@ from __future__ import annotations
import fcntl import fcntl
import hashlib import hashlib
import io
import json import json
import os import os
import shutil import shutil
import stat
import subprocess
import tempfile import tempfile
import threading import threading
import uuid import uuid
@@ -15,6 +14,9 @@ from datetime import datetime, timedelta, timezone
from pathlib import Path from pathlib import Path
from typing import Any, Iterable from typing import Any, Iterable
from dulwich import porcelain
from dulwich.repo import Repo
from common import read_json, read_jsonl, write_json, write_jsonl from common import read_json, read_jsonl, write_json, write_jsonl
from runner_common import load_key_files from runner_common import load_key_files
from version import AGENT_VERSION from version import AGENT_VERSION
@@ -147,8 +149,6 @@ class StateGitSync:
self._lock_handle = None self._lock_handle = None
self._mutex = threading.Lock() self._mutex = threading.Lock()
self._workspace: Path | None = None self._workspace: Path | None = None
self._askpass_dir: Path | None = None
self._askpass_path: Path | None = None
self._expected_remote_oid: str | None = None self._expected_remote_oid: str | None = None
@property @property
@@ -188,88 +188,43 @@ class StateGitSync:
if self._workspace is not None: if self._workspace is not None:
shutil.rmtree(self._workspace.parent, ignore_errors=True) shutil.rmtree(self._workspace.parent, ignore_errors=True)
self._workspace = None self._workspace = None
if self._askpass_dir is not None:
shutil.rmtree(self._askpass_dir, ignore_errors=True)
self._askpass_dir = None
self._askpass_path = None
def _git_environment(self) -> dict[str, str]: def _auth_kwargs(self) -> dict[str, str]:
if not all(self.credentials.get(key) for key in ("username", "email", "password")): if not all(self.credentials.get(key) for key in ("username", "email", "password")):
raise StateSyncError("missing ModelHub Git username, email or password") raise StateSyncError("missing ModelHub Git username, email or password")
if self._askpass_path is None: return {
directory = Path(tempfile.mkdtemp(prefix="modelhub-state-askpass-")) "username": self.credentials["username"],
script = directory / "askpass.py" "password": self.credentials["password"],
script.write_text( }
"#!/usr/bin/env python3\n"
"import os, sys\n"
"prompt = ' '.join(sys.argv[1:]).lower()\n"
"key = 'MODELHUB_STATE_GIT_USERNAME' if 'username' in prompt else 'MODELHUB_STATE_GIT_PASSWORD'\n"
"print(os.environ.get(key, ''))\n",
encoding="utf-8",
)
script.chmod(stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
self._askpass_dir = directory
self._askpass_path = script
env = os.environ.copy()
env.update(
{
"GIT_ASKPASS": str(self._askpass_path),
"GIT_TERMINAL_PROMPT": "0",
"MODELHUB_STATE_GIT_USERNAME": self.credentials["username"],
"MODELHUB_STATE_GIT_PASSWORD": self.credentials["password"],
}
)
return env
def _git(self, *args: str, cwd: Path | None = None, check: bool = True) -> subprocess.CompletedProcess[str]: @property
try: def _author(self) -> bytes:
result = subprocess.run( name = self.credentials.get("username") or "modelhub-agent"
["git", *args], email = self.credentials.get("email") or "modelhub-agent@localhost"
cwd=str(cwd or self.project_root), return f"{name} <{email}>".encode("utf-8")
env=self._git_environment(),
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
timeout=45,
check=False,
)
except FileNotFoundError as exc:
raise StateSyncError(
"git executable is unavailable; rebuild the service image from the repository Dockerfile"
) from exc
except subprocess.TimeoutExpired as exc:
raise StateSyncError("git operation timed out after 45 seconds") from exc
if check and result.returncode != 0:
reason = _safe_text(result.stderr.strip() or result.stdout.strip() or f"git exited {result.returncode}")
raise StateSyncError(reason)
return result
def _remote_oid(self) -> str | None: def _remote_oid(self) -> str | None:
result = self._git("ls-remote", "--heads", self.remote, self.branch) result = porcelain.ls_remote(self.remote, quiet=True, **self._auth_kwargs())
line = result.stdout.strip().splitlines() oid = result.refs.get(f"refs/heads/{self.branch}".encode("utf-8"))
return line[0].split()[0] if line else None return oid.decode("ascii") if oid else None
def _create_workspace(self, remote_oid: str | None) -> None: def _create_workspace(self, remote_oid: str | None) -> None:
parent = Path(tempfile.mkdtemp(prefix="modelhub-agent-state-")) parent = Path(tempfile.mkdtemp(prefix="modelhub-agent-state-"))
workspace = parent / "state" workspace = parent / "state"
if remote_oid: if remote_oid:
self._git( porcelain.clone(
"clone",
"--quiet",
"--single-branch",
"--branch",
self.branch,
self.remote, self.remote,
str(workspace), workspace,
cwd=parent, branch=self.branch,
depth=self.history_depth,
checkout=True,
errstream=io.BytesIO(),
**self._auth_kwargs(),
) )
else: else:
workspace.mkdir(parents=True) workspace.mkdir(parents=True)
self._git("init", "--quiet", cwd=workspace) repo = porcelain.init(workspace)
self._git("checkout", "--orphan", self.branch, cwd=workspace) repo.refs.set_symbolic_ref(b"HEAD", f"refs/heads/{self.branch}".encode("utf-8"))
self._git("remote", "add", "origin", self.remote, cwd=workspace)
self._git("config", "user.name", self.credentials["username"], cwd=workspace)
self._git("config", "user.email", self.credentials["email"], cwd=workspace)
self._workspace = workspace self._workspace = workspace
self._expected_remote_oid = remote_oid self._expected_remote_oid = remote_oid
@@ -298,17 +253,16 @@ class StateGitSync:
manifest = self._validate_manifest(self._workspace) manifest = self._validate_manifest(self._workspace)
except Exception as current_error: except Exception as current_error:
manifest = None manifest = None
commits = self._git( repo = Repo(str(self._workspace))
"rev-list", commits = [entry.commit.id for entry in repo.get_walker(max_entries=self.history_depth)][1:]
f"--max-count={self.history_depth}",
"HEAD",
cwd=self._workspace,
).stdout.splitlines()[1:]
for commit in commits: for commit in commits:
self._git("checkout", "--quiet", "--detach", commit, cwd=self._workspace) porcelain.reset(repo, "hard", commit)
try: try:
manifest = self._validate_manifest(self._workspace) manifest = self._validate_manifest(self._workspace)
self.log(f"[state-recovery] fallback_commit={commit[:12]} reason=checksum_recovery") self.log(
f"[state-recovery] fallback_commit={commit.decode('ascii')[:12]} "
"reason=checksum_recovery"
)
break break
except Exception: except Exception:
continue continue
@@ -624,34 +578,46 @@ class StateGitSync:
"checksums": checksums, "checksums": checksums,
} }
write_json(self._workspace / "manifest.json", manifest) write_json(self._workspace / "manifest.json", manifest)
self._git("add", "--all", cwd=self._workspace) repo = Repo(str(self._workspace))
diff = self._git("diff", "--cached", "--quiet", cwd=self._workspace, check=False) porcelain.add(repo)
if diff.returncode == 0: status = porcelain.status(repo)
staged = any(status.staged.get(kind) for kind in ("add", "delete", "modify"))
if not staged:
self.healthy = True self.healthy = True
return True return True
self._git("commit", "--quiet", "-m", f"state: generation {self.generation} ({phase})", cwd=self._workspace) porcelain.commit(
commit_count = int( repo,
self._git("rev-list", "--count", "HEAD", cwd=self._workspace).stdout.strip() or "0" message=f"state: generation {self.generation} ({phase})".encode("utf-8"),
author=self._author,
committer=self._author,
) )
commit_count = sum(1 for _ in repo.get_walker())
if commit_count > self.history_depth: if commit_count > self.history_depth:
compact_branch = f"state-compact-{uuid.uuid4().hex[:8]}" branch_ref = f"refs/heads/{self.branch}".encode("utf-8")
self._git("checkout", "--quiet", "--orphan", compact_branch, cwd=self._workspace) del repo.refs[branch_ref]
self._git("rm", "--quiet", "--cached", "-r", ".", cwd=self._workspace, check=False) porcelain.add(repo)
self._git("add", "--all", cwd=self._workspace) porcelain.commit(
self._git( repo,
"commit", message=f"state: compacted generation {self.generation}".encode("utf-8"),
"--quiet", author=self._author,
"-m", committer=self._author,
f"state: compacted generation {self.generation}",
cwd=self._workspace,
) )
lease = ( current_remote_oid = self._remote_oid()
f"--force-with-lease=refs/heads/{self.branch}:{self._expected_remote_oid}" if current_remote_oid != self._expected_remote_oid:
if self._expected_remote_oid raise StateSyncError("state branch changed remotely; refusing to overwrite a newer snapshot")
else f"--force-with-lease=refs/heads/{self.branch}:" porcelain.push(
repo,
self.remote,
refspecs=f"HEAD:refs/heads/{self.branch}",
force=True,
outstream=io.BytesIO(),
errstream=io.BytesIO(),
**self._auth_kwargs(),
) )
self._git("push", "--quiet", lease, "origin", f"HEAD:refs/heads/{self.branch}", cwd=self._workspace) local_oid = repo.head().decode("ascii")
self._expected_remote_oid = self._git("rev-parse", "HEAD", cwd=self._workspace).stdout.strip() if self._remote_oid() != local_oid:
raise StateSyncError("state branch verification failed after push")
self._expected_remote_oid = local_oid
self.last_sync_at = manifest["updatedAt"] self.last_sync_at = manifest["updatedAt"]
self.last_error = None self.last_error = None
self.healthy = True self.healthy = True

View File

@@ -1 +1 @@
AGENT_VERSION = "2026.08.15.2" AGENT_VERSION = "2026.08.15.3"

View File

@@ -1 +1,2 @@
PyYAML>=6.0.2,<7 PyYAML>=6.0.2,<7
dulwich>=0.24.10,<0.26

View File

@@ -40,6 +40,12 @@ class HostedAgentEntrypointTests(unittest.TestCase):
self.assertFalse(readiness["ready"]) self.assertFalse(readiness["ready"])
self.assertEqual("state_sync_unhealthy", readiness["reason"]) self.assertEqual("state_sync_unhealthy", readiness["reason"])
def test_runtime_image_uses_python_git_client_without_apt_layer(self) -> None:
dockerfile = (ROOT_DIR / "Dockerfile").read_text(encoding="utf-8")
requirements = (ROOT_DIR / "requirements.txt").read_text(encoding="utf-8")
self.assertNotIn("apt-get", dockerfile)
self.assertIn("dulwich", requirements.casefold())
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()

View File

@@ -1,6 +1,5 @@
from __future__ import annotations from __future__ import annotations
import subprocess
import tempfile import tempfile
import unittest import unittest
from datetime import datetime, timezone from datetime import datetime, timezone
@@ -9,6 +8,8 @@ from unittest.mock import patch
import sys import sys
from dulwich import porcelain
ROOT = Path(__file__).resolve().parents[1] ROOT = Path(__file__).resolve().parents[1]
MODULE_ROOT = ROOT / "modelhub_submmit_api" MODULE_ROOT = ROOT / "modelhub_submmit_api"
if str(MODULE_ROOT) not in sys.path: if str(MODULE_ROOT) not in sys.path:
@@ -20,7 +21,6 @@ from hf_discovery import HuggingFaceDiscovery, parse_model_card_front_matter #
from official_capabilities import OfficialCapabilityRegistry # noqa: E402 from official_capabilities import OfficialCapabilityRegistry # noqa: E402
from routing_engine import SuccessFirstRoutingEngine # noqa: E402 from routing_engine import SuccessFirstRoutingEngine # noqa: E402
from state_sync import StateGitSync # noqa: E402 from state_sync import StateGitSync # noqa: E402
from state_sync import StateSyncError # noqa: E402
class OfficialClient: class OfficialClient:
@@ -139,7 +139,7 @@ class SuperAgentTests(unittest.TestCase):
restored_project = root / "restored" restored_project = root / "restored"
project.mkdir() project.mkdir()
restored_project.mkdir() restored_project.mkdir()
subprocess.run(["git", "init", "--bare", str(remote)], check=True, stdout=subprocess.DEVNULL) porcelain.init(remote, bare=True)
write_json(project / ".modelhub_state" / "account_capacity.json", {"version": 1}) write_json(project / ".modelhub_state" / "account_capacity.json", {"version": 1})
credentials = {"username": "tester", "email": "tester@example.com", "password": "secret-value"} credentials = {"username": "tester", "email": "tester@example.com", "password": "secret-value"}
manager = StateGitSync( manager = StateGitSync(
@@ -208,19 +208,6 @@ class SuperAgentTests(unittest.TestCase):
) )
) )
def test_missing_git_has_actionable_state_sync_error(self) -> None:
with tempfile.TemporaryDirectory() as temporary_dir:
manager = StateGitSync(
project_root=Path(temporary_dir),
credentials={"username": "u", "email": "e@example.com", "password": "p"},
remote="unused",
log_fn=lambda _: None,
)
with patch("state_sync.subprocess.run", side_effect=FileNotFoundError("git")):
with self.assertRaisesRegex(StateSyncError, "rebuild the service image"):
manager._git("version")
manager.close()
def test_config_patch_requires_repeated_cross_model_success(self) -> None: def test_config_patch_requires_repeated_cross_model_success(self) -> None:
with tempfile.TemporaryDirectory() as temporary_dir: with tempfile.TemporaryDirectory() as temporary_dir:
root = Path(temporary_dir) root = Path(temporary_dir)