CoolBoy/submmit
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

fix: remove slow system git build dependency

Browse Source
This commit is contained in:
CoolBoy
2026-08-15 19:47:24 +08:00
parent 467eaccd59
commit 85f6cb5157
8 changed files with 85 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files

168
modelhub_submmit_api/state_sync.py
View File

@@ -2,11 +2,10 @@ from __future__ import annotations
import fcntl
import hashlib
import io
import json
import os
import shutil
import stat
import subprocess
import tempfile
import threading
import uuid
@@ -15,6 +14,9 @@ from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Any, Iterable
from dulwich import porcelain
from dulwich.repo import Repo
from common import read_json, read_jsonl, write_json, write_jsonl
from runner_common import load_key_files
from version import AGENT_VERSION
@@ -147,8 +149,6 @@ class StateGitSync:
self._lock_handle = None
self._mutex = threading.Lock()
self._workspace: Path | None = None
self._askpass_dir: Path | None = None
self._askpass_path: Path | None = None
self._expected_remote_oid: str | None = None
@property
@@ -188,88 +188,43 @@ class StateGitSync:
if self._workspace is not None:
shutil.rmtree(self._workspace.parent, ignore_errors=True)
self._workspace = None
if self._askpass_dir is not None:
shutil.rmtree(self._askpass_dir, ignore_errors=True)
self._askpass_dir = None
self._askpass_path = None
def _git_environment(self) -> dict[str, str]:
def _auth_kwargs(self) -> dict[str, str]:
if not all(self.credentials.get(key) for key in ("username", "email", "password")):
raise StateSyncError("missing ModelHub Git username, email or password")
if self._askpass_path is None:
directory = Path(tempfile.mkdtemp(prefix="modelhub-state-askpass-"))
script = directory / "askpass.py"
script.write_text(
"#!/usr/bin/env python3\n"
"import os, sys\n"
"prompt = ' '.join(sys.argv[1:]).lower()\n"
"key = 'MODELHUB_STATE_GIT_USERNAME' if 'username' in prompt else 'MODELHUB_STATE_GIT_PASSWORD'\n"
"print(os.environ.get(key, ''))\n",
encoding="utf-8",
)
script.chmod(stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
self._askpass_dir = directory
self._askpass_path = script
env = os.environ.copy()
env.update(
{
"GIT_ASKPASS": str(self._askpass_path),
"GIT_TERMINAL_PROMPT": "0",
"MODELHUB_STATE_GIT_USERNAME": self.credentials["username"],
"MODELHUB_STATE_GIT_PASSWORD": self.credentials["password"],
}
)
return env
return {
"username": self.credentials["username"],
"password": self.credentials["password"],
}
def _git(self, *args: str, cwd: Path | None = None, check: bool = True) -> subprocess.CompletedProcess[str]:
try:
result = subprocess.run(
["git", *args],
cwd=str(cwd or self.project_root),
env=self._git_environment(),
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
timeout=45,
check=False,
)
except FileNotFoundError as exc:
raise StateSyncError(
"git executable is unavailable; rebuild the service image from the repository Dockerfile"
) from exc
except subprocess.TimeoutExpired as exc:
raise StateSyncError("git operation timed out after 45 seconds") from exc
if check and result.returncode != 0:
reason = _safe_text(result.stderr.strip() or result.stdout.strip() or f"git exited {result.returncode}")
raise StateSyncError(reason)
return result
@property
def _author(self) -> bytes:
name = self.credentials.get("username") or "modelhub-agent"
email = self.credentials.get("email") or "modelhub-agent@localhost"
return f"{name} <{email}>".encode("utf-8")
def _remote_oid(self) -> str | None:
result = self._git("ls-remote", "--heads", self.remote, self.branch)
line = result.stdout.strip().splitlines()
return line[0].split()[0] if line else None
result = porcelain.ls_remote(self.remote, quiet=True, **self._auth_kwargs())
oid = result.refs.get(f"refs/heads/{self.branch}".encode("utf-8"))
return oid.decode("ascii") if oid else None
def _create_workspace(self, remote_oid: str | None) -> None:
parent = Path(tempfile.mkdtemp(prefix="modelhub-agent-state-"))
workspace = parent / "state"
if remote_oid:
self._git(
"clone",
"--quiet",
"--single-branch",
"--branch",
self.branch,
porcelain.clone(
self.remote,
str(workspace),
cwd=parent,
workspace,
branch=self.branch,
depth=self.history_depth,
checkout=True,
errstream=io.BytesIO(),
**self._auth_kwargs(),
)
else:
workspace.mkdir(parents=True)
self._git("init", "--quiet", cwd=workspace)
self._git("checkout", "--orphan", self.branch, cwd=workspace)
self._git("remote", "add", "origin", self.remote, cwd=workspace)
self._git("config", "user.name", self.credentials["username"], cwd=workspace)
self._git("config", "user.email", self.credentials["email"], cwd=workspace)
repo = porcelain.init(workspace)
repo.refs.set_symbolic_ref(b"HEAD", f"refs/heads/{self.branch}".encode("utf-8"))
self._workspace = workspace
self._expected_remote_oid = remote_oid
@@ -298,17 +253,16 @@ class StateGitSync:
manifest = self._validate_manifest(self._workspace)
except Exception as current_error:
manifest = None
commits = self._git(
"rev-list",
f"--max-count={self.history_depth}",
"HEAD",
cwd=self._workspace,
).stdout.splitlines()[1:]
repo = Repo(str(self._workspace))
commits = [entry.commit.id for entry in repo.get_walker(max_entries=self.history_depth)][1:]
for commit in commits:
self._git("checkout", "--quiet", "--detach", commit, cwd=self._workspace)
porcelain.reset(repo, "hard", commit)
try:
manifest = self._validate_manifest(self._workspace)
self.log(f"[state-recovery] fallback_commit={commit[:12]} reason=checksum_recovery")
self.log(
f"[state-recovery] fallback_commit={commit.decode('ascii')[:12]} "
"reason=checksum_recovery"
)
break
except Exception:
continue
@@ -624,34 +578,46 @@ class StateGitSync:
"checksums": checksums,
}
write_json(self._workspace / "manifest.json", manifest)
self._git("add", "--all", cwd=self._workspace)
diff = self._git("diff", "--cached", "--quiet", cwd=self._workspace, check=False)
if diff.returncode == 0:
repo = Repo(str(self._workspace))
porcelain.add(repo)
status = porcelain.status(repo)
staged = any(status.staged.get(kind) for kind in ("add", "delete", "modify"))
if not staged:
self.healthy = True
return True
self._git("commit", "--quiet", "-m", f"state: generation {self.generation} ({phase})", cwd=self._workspace)
commit_count = int(
self._git("rev-list", "--count", "HEAD", cwd=self._workspace).stdout.strip() or "0"
porcelain.commit(
repo,
message=f"state: generation {self.generation} ({phase})".encode("utf-8"),
author=self._author,
committer=self._author,
)
commit_count = sum(1 for _ in repo.get_walker())
if commit_count > self.history_depth:
compact_branch = f"state-compact-{uuid.uuid4().hex[:8]}"
self._git("checkout", "--quiet", "--orphan", compact_branch, cwd=self._workspace)
self._git("rm", "--quiet", "--cached", "-r", ".", cwd=self._workspace, check=False)
self._git("add", "--all", cwd=self._workspace)
self._git(
"commit",
"--quiet",
"-m",
f"state: compacted generation {self.generation}",
cwd=self._workspace,
branch_ref = f"refs/heads/{self.branch}".encode("utf-8")
del repo.refs[branch_ref]
porcelain.add(repo)
porcelain.commit(
repo,
message=f"state: compacted generation {self.generation}".encode("utf-8"),
author=self._author,
committer=self._author,
)
lease = (
f"--force-with-lease=refs/heads/{self.branch}:{self._expected_remote_oid}"
if self._expected_remote_oid
else f"--force-with-lease=refs/heads/{self.branch}:"
current_remote_oid = self._remote_oid()
if current_remote_oid != self._expected_remote_oid:
raise StateSyncError("state branch changed remotely; refusing to overwrite a newer snapshot")
porcelain.push(
repo,
self.remote,
refspecs=f"HEAD:refs/heads/{self.branch}",
force=True,
outstream=io.BytesIO(),
errstream=io.BytesIO(),
**self._auth_kwargs(),
)
self._git("push", "--quiet", lease, "origin", f"HEAD:refs/heads/{self.branch}", cwd=self._workspace)
self._expected_remote_oid = self._git("rev-parse", "HEAD", cwd=self._workspace).stdout.strip()
local_oid = repo.head().decode("ascii")
if self._remote_oid() != local_oid:
raise StateSyncError("state branch verification failed after push")
self._expected_remote_oid = local_oid
self.last_sync_at = manifest["updatedAt"]
self.last_error = None
self.healthy = True

2
modelhub_submmit_api/version.py
View File

@@ -1 +1 @@
AGENT_VERSION = "2026.08.15.2"
AGENT_VERSION = "2026.08.15.3"