From 85f6cb51575a98fd6d5c457d45caf24d8e8817b6 Mon Sep 17 00:00:00 2001 From: CoolBoy Date: Sat, 15 Aug 2026 19:47:24 +0800 Subject: [PATCH] fix: remove slow system git build dependency --- .dockerignore | 2 + Dockerfile | 4 - README.md | 10 +- modelhub_submmit_api/state_sync.py | 168 ++++++++++++----------------- modelhub_submmit_api/version.py | 2 +- requirements.txt | 1 + tests/test_agent_entrypoint.py | 6 ++ tests/test_super_agent.py | 19 +--- 8 files changed, 85 insertions(+), 127 deletions(-) diff --git a/.dockerignore b/.dockerignore index cc3248a0..90a75c0a 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,8 @@ .git .agents .codex +code4beforeUchange/ +tests/ **/__pycache__/ **/*.pyc **/.ipynb_checkpoints/ diff --git a/Dockerfile b/Dockerfile index 30fe54ce..2ce290cb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,10 +5,6 @@ ENV PYTHONPATH=/app/modelhub_submmit_api WORKDIR /app -RUN apt-get update \ - && apt-get install -y --no-install-recommends ca-certificates git \ - && rm -rf /var/lib/apt/lists/* - COPY requirements.txt . RUN pip install --no-cache-dir -r requirements.txt diff --git a/README.md b/README.md index 69e53026..6f59dcd4 100644 --- a/README.md +++ b/README.md @@ -362,19 +362,19 @@ counts fail closed for older candidates, and no startup or periodic cleanup can cancel a task by date. It also keeps unclassified/ambiguous historical failures neutral in GPU/framework success feedback while preserving deterministic OOM and architecture cleanup. -Version `2026.08.15.2` replaces the 70/30 quota with hierarchical success-first +Version `2026.08.15.3` replaces the 70/30 quota with hierarchical success-first routing, dynamically gates models through the official GPU/task/framework/config APIs, enriches ModelScope metadata and model lineage, learns only proven safe config vectors, and adds crash-safe write-ahead state synchronization to the `agent-state` branch. It also exposes `/ready` and extends deterministic cleanup -to officially removed waiting GPU/framework routes. The runtime image installs -Git and CA certificates required by state recovery and synchronization. +to officially removed waiting GPU/framework routes. State recovery uses the +pure-Python Dulwich client, avoiding slow OS package installation during builds. ## Deploy Create a tag and submit the repository URL plus tag in "我的适配智能体". ```bash -git tag -a agent-v26 -m "ModelHub agent 2026.08.15.2" -git push origin main agent-v26 +git tag -a agent-v27 -m "ModelHub agent 2026.08.15.3" +git push origin main agent-v27 ``` diff --git a/modelhub_submmit_api/state_sync.py b/modelhub_submmit_api/state_sync.py index b1edfddd..33c25380 100644 --- a/modelhub_submmit_api/state_sync.py +++ b/modelhub_submmit_api/state_sync.py @@ -2,11 +2,10 @@ from __future__ import annotations import fcntl import hashlib +import io import json import os import shutil -import stat -import subprocess import tempfile import threading import uuid @@ -15,6 +14,9 @@ from datetime import datetime, timedelta, timezone from pathlib import Path from typing import Any, Iterable +from dulwich import porcelain +from dulwich.repo import Repo + from common import read_json, read_jsonl, write_json, write_jsonl from runner_common import load_key_files from version import AGENT_VERSION @@ -147,8 +149,6 @@ class StateGitSync: self._lock_handle = None self._mutex = threading.Lock() self._workspace: Path | None = None - self._askpass_dir: Path | None = None - self._askpass_path: Path | None = None self._expected_remote_oid: str | None = None @property @@ -188,88 +188,43 @@ class StateGitSync: if self._workspace is not None: shutil.rmtree(self._workspace.parent, ignore_errors=True) self._workspace = None - if self._askpass_dir is not None: - shutil.rmtree(self._askpass_dir, ignore_errors=True) - self._askpass_dir = None - self._askpass_path = None - def _git_environment(self) -> dict[str, str]: + def _auth_kwargs(self) -> dict[str, str]: if not all(self.credentials.get(key) for key in ("username", "email", "password")): raise StateSyncError("missing ModelHub Git username, email or password") - if self._askpass_path is None: - directory = Path(tempfile.mkdtemp(prefix="modelhub-state-askpass-")) - script = directory / "askpass.py" - script.write_text( - "#!/usr/bin/env python3\n" - "import os, sys\n" - "prompt = ' '.join(sys.argv[1:]).lower()\n" - "key = 'MODELHUB_STATE_GIT_USERNAME' if 'username' in prompt else 'MODELHUB_STATE_GIT_PASSWORD'\n" - "print(os.environ.get(key, ''))\n", - encoding="utf-8", - ) - script.chmod(stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) - self._askpass_dir = directory - self._askpass_path = script - env = os.environ.copy() - env.update( - { - "GIT_ASKPASS": str(self._askpass_path), - "GIT_TERMINAL_PROMPT": "0", - "MODELHUB_STATE_GIT_USERNAME": self.credentials["username"], - "MODELHUB_STATE_GIT_PASSWORD": self.credentials["password"], - } - ) - return env + return { + "username": self.credentials["username"], + "password": self.credentials["password"], + } - def _git(self, *args: str, cwd: Path | None = None, check: bool = True) -> subprocess.CompletedProcess[str]: - try: - result = subprocess.run( - ["git", *args], - cwd=str(cwd or self.project_root), - env=self._git_environment(), - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - timeout=45, - check=False, - ) - except FileNotFoundError as exc: - raise StateSyncError( - "git executable is unavailable; rebuild the service image from the repository Dockerfile" - ) from exc - except subprocess.TimeoutExpired as exc: - raise StateSyncError("git operation timed out after 45 seconds") from exc - if check and result.returncode != 0: - reason = _safe_text(result.stderr.strip() or result.stdout.strip() or f"git exited {result.returncode}") - raise StateSyncError(reason) - return result + @property + def _author(self) -> bytes: + name = self.credentials.get("username") or "modelhub-agent" + email = self.credentials.get("email") or "modelhub-agent@localhost" + return f"{name} <{email}>".encode("utf-8") def _remote_oid(self) -> str | None: - result = self._git("ls-remote", "--heads", self.remote, self.branch) - line = result.stdout.strip().splitlines() - return line[0].split()[0] if line else None + result = porcelain.ls_remote(self.remote, quiet=True, **self._auth_kwargs()) + oid = result.refs.get(f"refs/heads/{self.branch}".encode("utf-8")) + return oid.decode("ascii") if oid else None def _create_workspace(self, remote_oid: str | None) -> None: parent = Path(tempfile.mkdtemp(prefix="modelhub-agent-state-")) workspace = parent / "state" if remote_oid: - self._git( - "clone", - "--quiet", - "--single-branch", - "--branch", - self.branch, + porcelain.clone( self.remote, - str(workspace), - cwd=parent, + workspace, + branch=self.branch, + depth=self.history_depth, + checkout=True, + errstream=io.BytesIO(), + **self._auth_kwargs(), ) else: workspace.mkdir(parents=True) - self._git("init", "--quiet", cwd=workspace) - self._git("checkout", "--orphan", self.branch, cwd=workspace) - self._git("remote", "add", "origin", self.remote, cwd=workspace) - self._git("config", "user.name", self.credentials["username"], cwd=workspace) - self._git("config", "user.email", self.credentials["email"], cwd=workspace) + repo = porcelain.init(workspace) + repo.refs.set_symbolic_ref(b"HEAD", f"refs/heads/{self.branch}".encode("utf-8")) self._workspace = workspace self._expected_remote_oid = remote_oid @@ -298,17 +253,16 @@ class StateGitSync: manifest = self._validate_manifest(self._workspace) except Exception as current_error: manifest = None - commits = self._git( - "rev-list", - f"--max-count={self.history_depth}", - "HEAD", - cwd=self._workspace, - ).stdout.splitlines()[1:] + repo = Repo(str(self._workspace)) + commits = [entry.commit.id for entry in repo.get_walker(max_entries=self.history_depth)][1:] for commit in commits: - self._git("checkout", "--quiet", "--detach", commit, cwd=self._workspace) + porcelain.reset(repo, "hard", commit) try: manifest = self._validate_manifest(self._workspace) - self.log(f"[state-recovery] fallback_commit={commit[:12]} reason=checksum_recovery") + self.log( + f"[state-recovery] fallback_commit={commit.decode('ascii')[:12]} " + "reason=checksum_recovery" + ) break except Exception: continue @@ -624,34 +578,46 @@ class StateGitSync: "checksums": checksums, } write_json(self._workspace / "manifest.json", manifest) - self._git("add", "--all", cwd=self._workspace) - diff = self._git("diff", "--cached", "--quiet", cwd=self._workspace, check=False) - if diff.returncode == 0: + repo = Repo(str(self._workspace)) + porcelain.add(repo) + status = porcelain.status(repo) + staged = any(status.staged.get(kind) for kind in ("add", "delete", "modify")) + if not staged: self.healthy = True return True - self._git("commit", "--quiet", "-m", f"state: generation {self.generation} ({phase})", cwd=self._workspace) - commit_count = int( - self._git("rev-list", "--count", "HEAD", cwd=self._workspace).stdout.strip() or "0" + porcelain.commit( + repo, + message=f"state: generation {self.generation} ({phase})".encode("utf-8"), + author=self._author, + committer=self._author, ) + commit_count = sum(1 for _ in repo.get_walker()) if commit_count > self.history_depth: - compact_branch = f"state-compact-{uuid.uuid4().hex[:8]}" - self._git("checkout", "--quiet", "--orphan", compact_branch, cwd=self._workspace) - self._git("rm", "--quiet", "--cached", "-r", ".", cwd=self._workspace, check=False) - self._git("add", "--all", cwd=self._workspace) - self._git( - "commit", - "--quiet", - "-m", - f"state: compacted generation {self.generation}", - cwd=self._workspace, + branch_ref = f"refs/heads/{self.branch}".encode("utf-8") + del repo.refs[branch_ref] + porcelain.add(repo) + porcelain.commit( + repo, + message=f"state: compacted generation {self.generation}".encode("utf-8"), + author=self._author, + committer=self._author, ) - lease = ( - f"--force-with-lease=refs/heads/{self.branch}:{self._expected_remote_oid}" - if self._expected_remote_oid - else f"--force-with-lease=refs/heads/{self.branch}:" + current_remote_oid = self._remote_oid() + if current_remote_oid != self._expected_remote_oid: + raise StateSyncError("state branch changed remotely; refusing to overwrite a newer snapshot") + porcelain.push( + repo, + self.remote, + refspecs=f"HEAD:refs/heads/{self.branch}", + force=True, + outstream=io.BytesIO(), + errstream=io.BytesIO(), + **self._auth_kwargs(), ) - self._git("push", "--quiet", lease, "origin", f"HEAD:refs/heads/{self.branch}", cwd=self._workspace) - self._expected_remote_oid = self._git("rev-parse", "HEAD", cwd=self._workspace).stdout.strip() + local_oid = repo.head().decode("ascii") + if self._remote_oid() != local_oid: + raise StateSyncError("state branch verification failed after push") + self._expected_remote_oid = local_oid self.last_sync_at = manifest["updatedAt"] self.last_error = None self.healthy = True diff --git a/modelhub_submmit_api/version.py b/modelhub_submmit_api/version.py index 0cbe3f3e..2cfc3d4c 100644 --- a/modelhub_submmit_api/version.py +++ b/modelhub_submmit_api/version.py @@ -1 +1 @@ -AGENT_VERSION = "2026.08.15.2" +AGENT_VERSION = "2026.08.15.3" diff --git a/requirements.txt b/requirements.txt index e0d33d1a..ee782c84 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ PyYAML>=6.0.2,<7 +dulwich>=0.24.10,<0.26 diff --git a/tests/test_agent_entrypoint.py b/tests/test_agent_entrypoint.py index cba11fd7..d06e637a 100644 --- a/tests/test_agent_entrypoint.py +++ b/tests/test_agent_entrypoint.py @@ -40,6 +40,12 @@ class HostedAgentEntrypointTests(unittest.TestCase): self.assertFalse(readiness["ready"]) self.assertEqual("state_sync_unhealthy", readiness["reason"]) + def test_runtime_image_uses_python_git_client_without_apt_layer(self) -> None: + dockerfile = (ROOT_DIR / "Dockerfile").read_text(encoding="utf-8") + requirements = (ROOT_DIR / "requirements.txt").read_text(encoding="utf-8") + self.assertNotIn("apt-get", dockerfile) + self.assertIn("dulwich", requirements.casefold()) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_super_agent.py b/tests/test_super_agent.py index 4abfa277..c9215ef9 100644 --- a/tests/test_super_agent.py +++ b/tests/test_super_agent.py @@ -1,6 +1,5 @@ from __future__ import annotations -import subprocess import tempfile import unittest from datetime import datetime, timezone @@ -9,6 +8,8 @@ from unittest.mock import patch import sys +from dulwich import porcelain + ROOT = Path(__file__).resolve().parents[1] MODULE_ROOT = ROOT / "modelhub_submmit_api" if str(MODULE_ROOT) not in sys.path: @@ -20,7 +21,6 @@ from hf_discovery import HuggingFaceDiscovery, parse_model_card_front_matter # from official_capabilities import OfficialCapabilityRegistry # noqa: E402 from routing_engine import SuccessFirstRoutingEngine # noqa: E402 from state_sync import StateGitSync # noqa: E402 -from state_sync import StateSyncError # noqa: E402 class OfficialClient: @@ -139,7 +139,7 @@ class SuperAgentTests(unittest.TestCase): restored_project = root / "restored" project.mkdir() restored_project.mkdir() - subprocess.run(["git", "init", "--bare", str(remote)], check=True, stdout=subprocess.DEVNULL) + porcelain.init(remote, bare=True) write_json(project / ".modelhub_state" / "account_capacity.json", {"version": 1}) credentials = {"username": "tester", "email": "tester@example.com", "password": "secret-value"} manager = StateGitSync( @@ -208,19 +208,6 @@ class SuperAgentTests(unittest.TestCase): ) ) - def test_missing_git_has_actionable_state_sync_error(self) -> None: - with tempfile.TemporaryDirectory() as temporary_dir: - manager = StateGitSync( - project_root=Path(temporary_dir), - credentials={"username": "u", "email": "e@example.com", "password": "p"}, - remote="unused", - log_fn=lambda _: None, - ) - with patch("state_sync.subprocess.run", side_effect=FileNotFoundError("git")): - with self.assertRaisesRegex(StateSyncError, "rebuild the service image"): - manager._git("version") - manager.close() - def test_config_patch_requires_repeated_cross_model_success(self) -> None: with tempfile.TemporaryDirectory() as temporary_dir: root = Path(temporary_dir)