fix: remove slow system git build dependency
This commit is contained in:
@@ -1,6 +1,8 @@
|
|||||||
.git
|
.git
|
||||||
.agents
|
.agents
|
||||||
.codex
|
.codex
|
||||||
|
code4beforeUchange/
|
||||||
|
tests/
|
||||||
**/__pycache__/
|
**/__pycache__/
|
||||||
**/*.pyc
|
**/*.pyc
|
||||||
**/.ipynb_checkpoints/
|
**/.ipynb_checkpoints/
|
||||||
|
|||||||
@@ -5,10 +5,6 @@ ENV PYTHONPATH=/app/modelhub_submmit_api
|
|||||||
|
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
|
|
||||||
RUN apt-get update \
|
|
||||||
&& apt-get install -y --no-install-recommends ca-certificates git \
|
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
|
||||||
|
|
||||||
COPY requirements.txt .
|
COPY requirements.txt .
|
||||||
RUN pip install --no-cache-dir -r requirements.txt
|
RUN pip install --no-cache-dir -r requirements.txt
|
||||||
|
|
||||||
|
|||||||
10
README.md
10
README.md
@@ -362,19 +362,19 @@ counts fail closed for older candidates, and no startup or periodic cleanup can
|
|||||||
cancel a task by date. It also keeps unclassified/ambiguous historical failures
|
cancel a task by date. It also keeps unclassified/ambiguous historical failures
|
||||||
neutral in GPU/framework success feedback while preserving deterministic OOM
|
neutral in GPU/framework success feedback while preserving deterministic OOM
|
||||||
and architecture cleanup.
|
and architecture cleanup.
|
||||||
Version `2026.08.15.2` replaces the 70/30 quota with hierarchical success-first
|
Version `2026.08.15.3` replaces the 70/30 quota with hierarchical success-first
|
||||||
routing, dynamically gates models through the official GPU/task/framework/config
|
routing, dynamically gates models through the official GPU/task/framework/config
|
||||||
APIs, enriches ModelScope metadata and model lineage, learns only proven safe
|
APIs, enriches ModelScope metadata and model lineage, learns only proven safe
|
||||||
config vectors, and adds crash-safe write-ahead state synchronization to the
|
config vectors, and adds crash-safe write-ahead state synchronization to the
|
||||||
`agent-state` branch. It also exposes `/ready` and extends deterministic cleanup
|
`agent-state` branch. It also exposes `/ready` and extends deterministic cleanup
|
||||||
to officially removed waiting GPU/framework routes. The runtime image installs
|
to officially removed waiting GPU/framework routes. State recovery uses the
|
||||||
Git and CA certificates required by state recovery and synchronization.
|
pure-Python Dulwich client, avoiding slow OS package installation during builds.
|
||||||
|
|
||||||
## Deploy
|
## Deploy
|
||||||
|
|
||||||
Create a tag and submit the repository URL plus tag in "我的适配智能体".
|
Create a tag and submit the repository URL plus tag in "我的适配智能体".
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
git tag -a agent-v26 -m "ModelHub agent 2026.08.15.2"
|
git tag -a agent-v27 -m "ModelHub agent 2026.08.15.3"
|
||||||
git push origin main agent-v26
|
git push origin main agent-v27
|
||||||
```
|
```
|
||||||
|
|||||||
@@ -2,11 +2,10 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import fcntl
|
import fcntl
|
||||||
import hashlib
|
import hashlib
|
||||||
|
import io
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
import shutil
|
import shutil
|
||||||
import stat
|
|
||||||
import subprocess
|
|
||||||
import tempfile
|
import tempfile
|
||||||
import threading
|
import threading
|
||||||
import uuid
|
import uuid
|
||||||
@@ -15,6 +14,9 @@ from datetime import datetime, timedelta, timezone
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any, Iterable
|
from typing import Any, Iterable
|
||||||
|
|
||||||
|
from dulwich import porcelain
|
||||||
|
from dulwich.repo import Repo
|
||||||
|
|
||||||
from common import read_json, read_jsonl, write_json, write_jsonl
|
from common import read_json, read_jsonl, write_json, write_jsonl
|
||||||
from runner_common import load_key_files
|
from runner_common import load_key_files
|
||||||
from version import AGENT_VERSION
|
from version import AGENT_VERSION
|
||||||
@@ -147,8 +149,6 @@ class StateGitSync:
|
|||||||
self._lock_handle = None
|
self._lock_handle = None
|
||||||
self._mutex = threading.Lock()
|
self._mutex = threading.Lock()
|
||||||
self._workspace: Path | None = None
|
self._workspace: Path | None = None
|
||||||
self._askpass_dir: Path | None = None
|
|
||||||
self._askpass_path: Path | None = None
|
|
||||||
self._expected_remote_oid: str | None = None
|
self._expected_remote_oid: str | None = None
|
||||||
|
|
||||||
@property
|
@property
|
||||||
@@ -188,88 +188,43 @@ class StateGitSync:
|
|||||||
if self._workspace is not None:
|
if self._workspace is not None:
|
||||||
shutil.rmtree(self._workspace.parent, ignore_errors=True)
|
shutil.rmtree(self._workspace.parent, ignore_errors=True)
|
||||||
self._workspace = None
|
self._workspace = None
|
||||||
if self._askpass_dir is not None:
|
|
||||||
shutil.rmtree(self._askpass_dir, ignore_errors=True)
|
|
||||||
self._askpass_dir = None
|
|
||||||
self._askpass_path = None
|
|
||||||
|
|
||||||
def _git_environment(self) -> dict[str, str]:
|
def _auth_kwargs(self) -> dict[str, str]:
|
||||||
if not all(self.credentials.get(key) for key in ("username", "email", "password")):
|
if not all(self.credentials.get(key) for key in ("username", "email", "password")):
|
||||||
raise StateSyncError("missing ModelHub Git username, email or password")
|
raise StateSyncError("missing ModelHub Git username, email or password")
|
||||||
if self._askpass_path is None:
|
return {
|
||||||
directory = Path(tempfile.mkdtemp(prefix="modelhub-state-askpass-"))
|
"username": self.credentials["username"],
|
||||||
script = directory / "askpass.py"
|
"password": self.credentials["password"],
|
||||||
script.write_text(
|
}
|
||||||
"#!/usr/bin/env python3\n"
|
|
||||||
"import os, sys\n"
|
|
||||||
"prompt = ' '.join(sys.argv[1:]).lower()\n"
|
|
||||||
"key = 'MODELHUB_STATE_GIT_USERNAME' if 'username' in prompt else 'MODELHUB_STATE_GIT_PASSWORD'\n"
|
|
||||||
"print(os.environ.get(key, ''))\n",
|
|
||||||
encoding="utf-8",
|
|
||||||
)
|
|
||||||
script.chmod(stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
|
|
||||||
self._askpass_dir = directory
|
|
||||||
self._askpass_path = script
|
|
||||||
env = os.environ.copy()
|
|
||||||
env.update(
|
|
||||||
{
|
|
||||||
"GIT_ASKPASS": str(self._askpass_path),
|
|
||||||
"GIT_TERMINAL_PROMPT": "0",
|
|
||||||
"MODELHUB_STATE_GIT_USERNAME": self.credentials["username"],
|
|
||||||
"MODELHUB_STATE_GIT_PASSWORD": self.credentials["password"],
|
|
||||||
}
|
|
||||||
)
|
|
||||||
return env
|
|
||||||
|
|
||||||
def _git(self, *args: str, cwd: Path | None = None, check: bool = True) -> subprocess.CompletedProcess[str]:
|
@property
|
||||||
try:
|
def _author(self) -> bytes:
|
||||||
result = subprocess.run(
|
name = self.credentials.get("username") or "modelhub-agent"
|
||||||
["git", *args],
|
email = self.credentials.get("email") or "modelhub-agent@localhost"
|
||||||
cwd=str(cwd or self.project_root),
|
return f"{name} <{email}>".encode("utf-8")
|
||||||
env=self._git_environment(),
|
|
||||||
text=True,
|
|
||||||
stdout=subprocess.PIPE,
|
|
||||||
stderr=subprocess.PIPE,
|
|
||||||
timeout=45,
|
|
||||||
check=False,
|
|
||||||
)
|
|
||||||
except FileNotFoundError as exc:
|
|
||||||
raise StateSyncError(
|
|
||||||
"git executable is unavailable; rebuild the service image from the repository Dockerfile"
|
|
||||||
) from exc
|
|
||||||
except subprocess.TimeoutExpired as exc:
|
|
||||||
raise StateSyncError("git operation timed out after 45 seconds") from exc
|
|
||||||
if check and result.returncode != 0:
|
|
||||||
reason = _safe_text(result.stderr.strip() or result.stdout.strip() or f"git exited {result.returncode}")
|
|
||||||
raise StateSyncError(reason)
|
|
||||||
return result
|
|
||||||
|
|
||||||
def _remote_oid(self) -> str | None:
|
def _remote_oid(self) -> str | None:
|
||||||
result = self._git("ls-remote", "--heads", self.remote, self.branch)
|
result = porcelain.ls_remote(self.remote, quiet=True, **self._auth_kwargs())
|
||||||
line = result.stdout.strip().splitlines()
|
oid = result.refs.get(f"refs/heads/{self.branch}".encode("utf-8"))
|
||||||
return line[0].split()[0] if line else None
|
return oid.decode("ascii") if oid else None
|
||||||
|
|
||||||
def _create_workspace(self, remote_oid: str | None) -> None:
|
def _create_workspace(self, remote_oid: str | None) -> None:
|
||||||
parent = Path(tempfile.mkdtemp(prefix="modelhub-agent-state-"))
|
parent = Path(tempfile.mkdtemp(prefix="modelhub-agent-state-"))
|
||||||
workspace = parent / "state"
|
workspace = parent / "state"
|
||||||
if remote_oid:
|
if remote_oid:
|
||||||
self._git(
|
porcelain.clone(
|
||||||
"clone",
|
|
||||||
"--quiet",
|
|
||||||
"--single-branch",
|
|
||||||
"--branch",
|
|
||||||
self.branch,
|
|
||||||
self.remote,
|
self.remote,
|
||||||
str(workspace),
|
workspace,
|
||||||
cwd=parent,
|
branch=self.branch,
|
||||||
|
depth=self.history_depth,
|
||||||
|
checkout=True,
|
||||||
|
errstream=io.BytesIO(),
|
||||||
|
**self._auth_kwargs(),
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
workspace.mkdir(parents=True)
|
workspace.mkdir(parents=True)
|
||||||
self._git("init", "--quiet", cwd=workspace)
|
repo = porcelain.init(workspace)
|
||||||
self._git("checkout", "--orphan", self.branch, cwd=workspace)
|
repo.refs.set_symbolic_ref(b"HEAD", f"refs/heads/{self.branch}".encode("utf-8"))
|
||||||
self._git("remote", "add", "origin", self.remote, cwd=workspace)
|
|
||||||
self._git("config", "user.name", self.credentials["username"], cwd=workspace)
|
|
||||||
self._git("config", "user.email", self.credentials["email"], cwd=workspace)
|
|
||||||
self._workspace = workspace
|
self._workspace = workspace
|
||||||
self._expected_remote_oid = remote_oid
|
self._expected_remote_oid = remote_oid
|
||||||
|
|
||||||
@@ -298,17 +253,16 @@ class StateGitSync:
|
|||||||
manifest = self._validate_manifest(self._workspace)
|
manifest = self._validate_manifest(self._workspace)
|
||||||
except Exception as current_error:
|
except Exception as current_error:
|
||||||
manifest = None
|
manifest = None
|
||||||
commits = self._git(
|
repo = Repo(str(self._workspace))
|
||||||
"rev-list",
|
commits = [entry.commit.id for entry in repo.get_walker(max_entries=self.history_depth)][1:]
|
||||||
f"--max-count={self.history_depth}",
|
|
||||||
"HEAD",
|
|
||||||
cwd=self._workspace,
|
|
||||||
).stdout.splitlines()[1:]
|
|
||||||
for commit in commits:
|
for commit in commits:
|
||||||
self._git("checkout", "--quiet", "--detach", commit, cwd=self._workspace)
|
porcelain.reset(repo, "hard", commit)
|
||||||
try:
|
try:
|
||||||
manifest = self._validate_manifest(self._workspace)
|
manifest = self._validate_manifest(self._workspace)
|
||||||
self.log(f"[state-recovery] fallback_commit={commit[:12]} reason=checksum_recovery")
|
self.log(
|
||||||
|
f"[state-recovery] fallback_commit={commit.decode('ascii')[:12]} "
|
||||||
|
"reason=checksum_recovery"
|
||||||
|
)
|
||||||
break
|
break
|
||||||
except Exception:
|
except Exception:
|
||||||
continue
|
continue
|
||||||
@@ -624,34 +578,46 @@ class StateGitSync:
|
|||||||
"checksums": checksums,
|
"checksums": checksums,
|
||||||
}
|
}
|
||||||
write_json(self._workspace / "manifest.json", manifest)
|
write_json(self._workspace / "manifest.json", manifest)
|
||||||
self._git("add", "--all", cwd=self._workspace)
|
repo = Repo(str(self._workspace))
|
||||||
diff = self._git("diff", "--cached", "--quiet", cwd=self._workspace, check=False)
|
porcelain.add(repo)
|
||||||
if diff.returncode == 0:
|
status = porcelain.status(repo)
|
||||||
|
staged = any(status.staged.get(kind) for kind in ("add", "delete", "modify"))
|
||||||
|
if not staged:
|
||||||
self.healthy = True
|
self.healthy = True
|
||||||
return True
|
return True
|
||||||
self._git("commit", "--quiet", "-m", f"state: generation {self.generation} ({phase})", cwd=self._workspace)
|
porcelain.commit(
|
||||||
commit_count = int(
|
repo,
|
||||||
self._git("rev-list", "--count", "HEAD", cwd=self._workspace).stdout.strip() or "0"
|
message=f"state: generation {self.generation} ({phase})".encode("utf-8"),
|
||||||
|
author=self._author,
|
||||||
|
committer=self._author,
|
||||||
)
|
)
|
||||||
|
commit_count = sum(1 for _ in repo.get_walker())
|
||||||
if commit_count > self.history_depth:
|
if commit_count > self.history_depth:
|
||||||
compact_branch = f"state-compact-{uuid.uuid4().hex[:8]}"
|
branch_ref = f"refs/heads/{self.branch}".encode("utf-8")
|
||||||
self._git("checkout", "--quiet", "--orphan", compact_branch, cwd=self._workspace)
|
del repo.refs[branch_ref]
|
||||||
self._git("rm", "--quiet", "--cached", "-r", ".", cwd=self._workspace, check=False)
|
porcelain.add(repo)
|
||||||
self._git("add", "--all", cwd=self._workspace)
|
porcelain.commit(
|
||||||
self._git(
|
repo,
|
||||||
"commit",
|
message=f"state: compacted generation {self.generation}".encode("utf-8"),
|
||||||
"--quiet",
|
author=self._author,
|
||||||
"-m",
|
committer=self._author,
|
||||||
f"state: compacted generation {self.generation}",
|
|
||||||
cwd=self._workspace,
|
|
||||||
)
|
)
|
||||||
lease = (
|
current_remote_oid = self._remote_oid()
|
||||||
f"--force-with-lease=refs/heads/{self.branch}:{self._expected_remote_oid}"
|
if current_remote_oid != self._expected_remote_oid:
|
||||||
if self._expected_remote_oid
|
raise StateSyncError("state branch changed remotely; refusing to overwrite a newer snapshot")
|
||||||
else f"--force-with-lease=refs/heads/{self.branch}:"
|
porcelain.push(
|
||||||
|
repo,
|
||||||
|
self.remote,
|
||||||
|
refspecs=f"HEAD:refs/heads/{self.branch}",
|
||||||
|
force=True,
|
||||||
|
outstream=io.BytesIO(),
|
||||||
|
errstream=io.BytesIO(),
|
||||||
|
**self._auth_kwargs(),
|
||||||
)
|
)
|
||||||
self._git("push", "--quiet", lease, "origin", f"HEAD:refs/heads/{self.branch}", cwd=self._workspace)
|
local_oid = repo.head().decode("ascii")
|
||||||
self._expected_remote_oid = self._git("rev-parse", "HEAD", cwd=self._workspace).stdout.strip()
|
if self._remote_oid() != local_oid:
|
||||||
|
raise StateSyncError("state branch verification failed after push")
|
||||||
|
self._expected_remote_oid = local_oid
|
||||||
self.last_sync_at = manifest["updatedAt"]
|
self.last_sync_at = manifest["updatedAt"]
|
||||||
self.last_error = None
|
self.last_error = None
|
||||||
self.healthy = True
|
self.healthy = True
|
||||||
|
|||||||
@@ -1 +1 @@
|
|||||||
AGENT_VERSION = "2026.08.15.2"
|
AGENT_VERSION = "2026.08.15.3"
|
||||||
|
|||||||
@@ -1 +1,2 @@
|
|||||||
PyYAML>=6.0.2,<7
|
PyYAML>=6.0.2,<7
|
||||||
|
dulwich>=0.24.10,<0.26
|
||||||
|
|||||||
@@ -40,6 +40,12 @@ class HostedAgentEntrypointTests(unittest.TestCase):
|
|||||||
self.assertFalse(readiness["ready"])
|
self.assertFalse(readiness["ready"])
|
||||||
self.assertEqual("state_sync_unhealthy", readiness["reason"])
|
self.assertEqual("state_sync_unhealthy", readiness["reason"])
|
||||||
|
|
||||||
|
def test_runtime_image_uses_python_git_client_without_apt_layer(self) -> None:
|
||||||
|
dockerfile = (ROOT_DIR / "Dockerfile").read_text(encoding="utf-8")
|
||||||
|
requirements = (ROOT_DIR / "requirements.txt").read_text(encoding="utf-8")
|
||||||
|
self.assertNotIn("apt-get", dockerfile)
|
||||||
|
self.assertIn("dulwich", requirements.casefold())
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
@@ -1,6 +1,5 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import subprocess
|
|
||||||
import tempfile
|
import tempfile
|
||||||
import unittest
|
import unittest
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
@@ -9,6 +8,8 @@ from unittest.mock import patch
|
|||||||
|
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
|
from dulwich import porcelain
|
||||||
|
|
||||||
ROOT = Path(__file__).resolve().parents[1]
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
MODULE_ROOT = ROOT / "modelhub_submmit_api"
|
MODULE_ROOT = ROOT / "modelhub_submmit_api"
|
||||||
if str(MODULE_ROOT) not in sys.path:
|
if str(MODULE_ROOT) not in sys.path:
|
||||||
@@ -20,7 +21,6 @@ from hf_discovery import HuggingFaceDiscovery, parse_model_card_front_matter #
|
|||||||
from official_capabilities import OfficialCapabilityRegistry # noqa: E402
|
from official_capabilities import OfficialCapabilityRegistry # noqa: E402
|
||||||
from routing_engine import SuccessFirstRoutingEngine # noqa: E402
|
from routing_engine import SuccessFirstRoutingEngine # noqa: E402
|
||||||
from state_sync import StateGitSync # noqa: E402
|
from state_sync import StateGitSync # noqa: E402
|
||||||
from state_sync import StateSyncError # noqa: E402
|
|
||||||
|
|
||||||
|
|
||||||
class OfficialClient:
|
class OfficialClient:
|
||||||
@@ -139,7 +139,7 @@ class SuperAgentTests(unittest.TestCase):
|
|||||||
restored_project = root / "restored"
|
restored_project = root / "restored"
|
||||||
project.mkdir()
|
project.mkdir()
|
||||||
restored_project.mkdir()
|
restored_project.mkdir()
|
||||||
subprocess.run(["git", "init", "--bare", str(remote)], check=True, stdout=subprocess.DEVNULL)
|
porcelain.init(remote, bare=True)
|
||||||
write_json(project / ".modelhub_state" / "account_capacity.json", {"version": 1})
|
write_json(project / ".modelhub_state" / "account_capacity.json", {"version": 1})
|
||||||
credentials = {"username": "tester", "email": "tester@example.com", "password": "secret-value"}
|
credentials = {"username": "tester", "email": "tester@example.com", "password": "secret-value"}
|
||||||
manager = StateGitSync(
|
manager = StateGitSync(
|
||||||
@@ -208,19 +208,6 @@ class SuperAgentTests(unittest.TestCase):
|
|||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
def test_missing_git_has_actionable_state_sync_error(self) -> None:
|
|
||||||
with tempfile.TemporaryDirectory() as temporary_dir:
|
|
||||||
manager = StateGitSync(
|
|
||||||
project_root=Path(temporary_dir),
|
|
||||||
credentials={"username": "u", "email": "e@example.com", "password": "p"},
|
|
||||||
remote="unused",
|
|
||||||
log_fn=lambda _: None,
|
|
||||||
)
|
|
||||||
with patch("state_sync.subprocess.run", side_effect=FileNotFoundError("git")):
|
|
||||||
with self.assertRaisesRegex(StateSyncError, "rebuild the service image"):
|
|
||||||
manager._git("version")
|
|
||||||
manager.close()
|
|
||||||
|
|
||||||
def test_config_patch_requires_repeated_cross_model_success(self) -> None:
|
def test_config_patch_requires_repeated_cross_model_success(self) -> None:
|
||||||
with tempfile.TemporaryDirectory() as temporary_dir:
|
with tempfile.TemporaryDirectory() as temporary_dir:
|
||||||
root = Path(temporary_dir)
|
root = Path(temporary_dir)
|
||||||
|
|||||||
Reference in New Issue
Block a user