fix: remove slow system git build dependency

This commit is contained in:
CoolBoy
2026-08-15 19:47:24 +08:00
parent 467eaccd59
commit 85f6cb5157
8 changed files with 85 additions and 127 deletions

View File

@@ -1,6 +1,8 @@
.git
.agents
.codex
code4beforeUchange/
tests/
**/__pycache__/
**/*.pyc
**/.ipynb_checkpoints/

View File

@@ -5,10 +5,6 @@ ENV PYTHONPATH=/app/modelhub_submmit_api
WORKDIR /app
RUN apt-get update \
&& apt-get install -y --no-install-recommends ca-certificates git \
&& rm -rf /var/lib/apt/lists/*
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt

View File

@@ -362,19 +362,19 @@ counts fail closed for older candidates, and no startup or periodic cleanup can
cancel a task by date. It also keeps unclassified/ambiguous historical failures
neutral in GPU/framework success feedback while preserving deterministic OOM
and architecture cleanup.
Version `2026.08.15.2` replaces the 70/30 quota with hierarchical success-first
Version `2026.08.15.3` replaces the 70/30 quota with hierarchical success-first
routing, dynamically gates models through the official GPU/task/framework/config
APIs, enriches ModelScope metadata and model lineage, learns only proven safe
config vectors, and adds crash-safe write-ahead state synchronization to the
`agent-state` branch. It also exposes `/ready` and extends deterministic cleanup
to officially removed waiting GPU/framework routes. The runtime image installs
Git and CA certificates required by state recovery and synchronization.
to officially removed waiting GPU/framework routes. State recovery uses the
pure-Python Dulwich client, avoiding slow OS package installation during builds.
## Deploy
Create a tag and submit the repository URL plus tag in "我的适配智能体".
```bash
git tag -a agent-v26 -m "ModelHub agent 2026.08.15.2"
git push origin main agent-v26
git tag -a agent-v27 -m "ModelHub agent 2026.08.15.3"
git push origin main agent-v27
```

View File

@@ -2,11 +2,10 @@ from __future__ import annotations
import fcntl
import hashlib
import io
import json
import os
import shutil
import stat
import subprocess
import tempfile
import threading
import uuid
@@ -15,6 +14,9 @@ from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Any, Iterable
from dulwich import porcelain
from dulwich.repo import Repo
from common import read_json, read_jsonl, write_json, write_jsonl
from runner_common import load_key_files
from version import AGENT_VERSION
@@ -147,8 +149,6 @@ class StateGitSync:
self._lock_handle = None
self._mutex = threading.Lock()
self._workspace: Path | None = None
self._askpass_dir: Path | None = None
self._askpass_path: Path | None = None
self._expected_remote_oid: str | None = None
@property
@@ -188,88 +188,43 @@ class StateGitSync:
if self._workspace is not None:
shutil.rmtree(self._workspace.parent, ignore_errors=True)
self._workspace = None
if self._askpass_dir is not None:
shutil.rmtree(self._askpass_dir, ignore_errors=True)
self._askpass_dir = None
self._askpass_path = None
def _git_environment(self) -> dict[str, str]:
def _auth_kwargs(self) -> dict[str, str]:
if not all(self.credentials.get(key) for key in ("username", "email", "password")):
raise StateSyncError("missing ModelHub Git username, email or password")
if self._askpass_path is None:
directory = Path(tempfile.mkdtemp(prefix="modelhub-state-askpass-"))
script = directory / "askpass.py"
script.write_text(
"#!/usr/bin/env python3\n"
"import os, sys\n"
"prompt = ' '.join(sys.argv[1:]).lower()\n"
"key = 'MODELHUB_STATE_GIT_USERNAME' if 'username' in prompt else 'MODELHUB_STATE_GIT_PASSWORD'\n"
"print(os.environ.get(key, ''))\n",
encoding="utf-8",
)
script.chmod(stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
self._askpass_dir = directory
self._askpass_path = script
env = os.environ.copy()
env.update(
{
"GIT_ASKPASS": str(self._askpass_path),
"GIT_TERMINAL_PROMPT": "0",
"MODELHUB_STATE_GIT_USERNAME": self.credentials["username"],
"MODELHUB_STATE_GIT_PASSWORD": self.credentials["password"],
}
)
return env
return {
"username": self.credentials["username"],
"password": self.credentials["password"],
}
def _git(self, *args: str, cwd: Path | None = None, check: bool = True) -> subprocess.CompletedProcess[str]:
try:
result = subprocess.run(
["git", *args],
cwd=str(cwd or self.project_root),
env=self._git_environment(),
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
timeout=45,
check=False,
)
except FileNotFoundError as exc:
raise StateSyncError(
"git executable is unavailable; rebuild the service image from the repository Dockerfile"
) from exc
except subprocess.TimeoutExpired as exc:
raise StateSyncError("git operation timed out after 45 seconds") from exc
if check and result.returncode != 0:
reason = _safe_text(result.stderr.strip() or result.stdout.strip() or f"git exited {result.returncode}")
raise StateSyncError(reason)
return result
@property
def _author(self) -> bytes:
name = self.credentials.get("username") or "modelhub-agent"
email = self.credentials.get("email") or "modelhub-agent@localhost"
return f"{name} <{email}>".encode("utf-8")
def _remote_oid(self) -> str | None:
result = self._git("ls-remote", "--heads", self.remote, self.branch)
line = result.stdout.strip().splitlines()
return line[0].split()[0] if line else None
result = porcelain.ls_remote(self.remote, quiet=True, **self._auth_kwargs())
oid = result.refs.get(f"refs/heads/{self.branch}".encode("utf-8"))
return oid.decode("ascii") if oid else None
def _create_workspace(self, remote_oid: str | None) -> None:
parent = Path(tempfile.mkdtemp(prefix="modelhub-agent-state-"))
workspace = parent / "state"
if remote_oid:
self._git(
"clone",
"--quiet",
"--single-branch",
"--branch",
self.branch,
porcelain.clone(
self.remote,
str(workspace),
cwd=parent,
workspace,
branch=self.branch,
depth=self.history_depth,
checkout=True,
errstream=io.BytesIO(),
**self._auth_kwargs(),
)
else:
workspace.mkdir(parents=True)
self._git("init", "--quiet", cwd=workspace)
self._git("checkout", "--orphan", self.branch, cwd=workspace)
self._git("remote", "add", "origin", self.remote, cwd=workspace)
self._git("config", "user.name", self.credentials["username"], cwd=workspace)
self._git("config", "user.email", self.credentials["email"], cwd=workspace)
repo = porcelain.init(workspace)
repo.refs.set_symbolic_ref(b"HEAD", f"refs/heads/{self.branch}".encode("utf-8"))
self._workspace = workspace
self._expected_remote_oid = remote_oid
@@ -298,17 +253,16 @@ class StateGitSync:
manifest = self._validate_manifest(self._workspace)
except Exception as current_error:
manifest = None
commits = self._git(
"rev-list",
f"--max-count={self.history_depth}",
"HEAD",
cwd=self._workspace,
).stdout.splitlines()[1:]
repo = Repo(str(self._workspace))
commits = [entry.commit.id for entry in repo.get_walker(max_entries=self.history_depth)][1:]
for commit in commits:
self._git("checkout", "--quiet", "--detach", commit, cwd=self._workspace)
porcelain.reset(repo, "hard", commit)
try:
manifest = self._validate_manifest(self._workspace)
self.log(f"[state-recovery] fallback_commit={commit[:12]} reason=checksum_recovery")
self.log(
f"[state-recovery] fallback_commit={commit.decode('ascii')[:12]} "
"reason=checksum_recovery"
)
break
except Exception:
continue
@@ -624,34 +578,46 @@ class StateGitSync:
"checksums": checksums,
}
write_json(self._workspace / "manifest.json", manifest)
self._git("add", "--all", cwd=self._workspace)
diff = self._git("diff", "--cached", "--quiet", cwd=self._workspace, check=False)
if diff.returncode == 0:
repo = Repo(str(self._workspace))
porcelain.add(repo)
status = porcelain.status(repo)
staged = any(status.staged.get(kind) for kind in ("add", "delete", "modify"))
if not staged:
self.healthy = True
return True
self._git("commit", "--quiet", "-m", f"state: generation {self.generation} ({phase})", cwd=self._workspace)
commit_count = int(
self._git("rev-list", "--count", "HEAD", cwd=self._workspace).stdout.strip() or "0"
porcelain.commit(
repo,
message=f"state: generation {self.generation} ({phase})".encode("utf-8"),
author=self._author,
committer=self._author,
)
commit_count = sum(1 for _ in repo.get_walker())
if commit_count > self.history_depth:
compact_branch = f"state-compact-{uuid.uuid4().hex[:8]}"
self._git("checkout", "--quiet", "--orphan", compact_branch, cwd=self._workspace)
self._git("rm", "--quiet", "--cached", "-r", ".", cwd=self._workspace, check=False)
self._git("add", "--all", cwd=self._workspace)
self._git(
"commit",
"--quiet",
"-m",
f"state: compacted generation {self.generation}",
cwd=self._workspace,
branch_ref = f"refs/heads/{self.branch}".encode("utf-8")
del repo.refs[branch_ref]
porcelain.add(repo)
porcelain.commit(
repo,
message=f"state: compacted generation {self.generation}".encode("utf-8"),
author=self._author,
committer=self._author,
)
lease = (
f"--force-with-lease=refs/heads/{self.branch}:{self._expected_remote_oid}"
if self._expected_remote_oid
else f"--force-with-lease=refs/heads/{self.branch}:"
current_remote_oid = self._remote_oid()
if current_remote_oid != self._expected_remote_oid:
raise StateSyncError("state branch changed remotely; refusing to overwrite a newer snapshot")
porcelain.push(
repo,
self.remote,
refspecs=f"HEAD:refs/heads/{self.branch}",
force=True,
outstream=io.BytesIO(),
errstream=io.BytesIO(),
**self._auth_kwargs(),
)
self._git("push", "--quiet", lease, "origin", f"HEAD:refs/heads/{self.branch}", cwd=self._workspace)
self._expected_remote_oid = self._git("rev-parse", "HEAD", cwd=self._workspace).stdout.strip()
local_oid = repo.head().decode("ascii")
if self._remote_oid() != local_oid:
raise StateSyncError("state branch verification failed after push")
self._expected_remote_oid = local_oid
self.last_sync_at = manifest["updatedAt"]
self.last_error = None
self.healthy = True

View File

@@ -1 +1 @@
AGENT_VERSION = "2026.08.15.2"
AGENT_VERSION = "2026.08.15.3"

View File

@@ -1 +1,2 @@
PyYAML>=6.0.2,<7
dulwich>=0.24.10,<0.26

View File

@@ -40,6 +40,12 @@ class HostedAgentEntrypointTests(unittest.TestCase):
self.assertFalse(readiness["ready"])
self.assertEqual("state_sync_unhealthy", readiness["reason"])
def test_runtime_image_uses_python_git_client_without_apt_layer(self) -> None:
dockerfile = (ROOT_DIR / "Dockerfile").read_text(encoding="utf-8")
requirements = (ROOT_DIR / "requirements.txt").read_text(encoding="utf-8")
self.assertNotIn("apt-get", dockerfile)
self.assertIn("dulwich", requirements.casefold())
if __name__ == "__main__":
unittest.main()

View File

@@ -1,6 +1,5 @@
from __future__ import annotations
import subprocess
import tempfile
import unittest
from datetime import datetime, timezone
@@ -9,6 +8,8 @@ from unittest.mock import patch
import sys
from dulwich import porcelain
ROOT = Path(__file__).resolve().parents[1]
MODULE_ROOT = ROOT / "modelhub_submmit_api"
if str(MODULE_ROOT) not in sys.path:
@@ -20,7 +21,6 @@ from hf_discovery import HuggingFaceDiscovery, parse_model_card_front_matter #
from official_capabilities import OfficialCapabilityRegistry # noqa: E402
from routing_engine import SuccessFirstRoutingEngine # noqa: E402
from state_sync import StateGitSync # noqa: E402
from state_sync import StateSyncError # noqa: E402
class OfficialClient:
@@ -139,7 +139,7 @@ class SuperAgentTests(unittest.TestCase):
restored_project = root / "restored"
project.mkdir()
restored_project.mkdir()
subprocess.run(["git", "init", "--bare", str(remote)], check=True, stdout=subprocess.DEVNULL)
porcelain.init(remote, bare=True)
write_json(project / ".modelhub_state" / "account_capacity.json", {"version": 1})
credentials = {"username": "tester", "email": "tester@example.com", "password": "secret-value"}
manager = StateGitSync(
@@ -208,19 +208,6 @@ class SuperAgentTests(unittest.TestCase):
)
)
def test_missing_git_has_actionable_state_sync_error(self) -> None:
with tempfile.TemporaryDirectory() as temporary_dir:
manager = StateGitSync(
project_root=Path(temporary_dir),
credentials={"username": "u", "email": "e@example.com", "password": "p"},
remote="unused",
log_fn=lambda _: None,
)
with patch("state_sync.subprocess.run", side_effect=FileNotFoundError("git")):
with self.assertRaisesRegex(StateSyncError, "rebuild the service image"):
manager._git("version")
manager.close()
def test_config_patch_requires_repeated_cross_model_success(self) -> None:
with tempfile.TemporaryDirectory() as temporary_dir:
root = Path(temporary_dir)