183 lines
7.1 KiB
Python
183 lines
7.1 KiB
Python
#!/usr/bin/env python3
|
|
"""Upload FRANKENSTALLM: model, eval reports, source code, and data scripts to Hugging Face.
|
|
|
|
Usage:
|
|
huggingface-cli login
|
|
|
|
# 모델 + README + 평가 결과 + 보고서
|
|
python scripts/upload_to_huggingface.py --repo-id pathcosmos/frankenstallm --create-pr
|
|
|
|
# 위 + 소스 코드 + 데이터 스크립트 (모델/데이터/소스 전부)
|
|
python scripts/upload_to_huggingface.py --repo-id pathcosmos/frankenstallm --with-source --with-data --create-pr
|
|
|
|
# 평가·보고서만
|
|
python scripts/upload_to_huggingface.py --repo-id pathcosmos/frankenstallm --readme-only --create-pr
|
|
"""
|
|
|
|
import argparse
|
|
from pathlib import Path
|
|
|
|
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
|
HF_CHECKPOINT = PROJECT_ROOT / "outputs" / "hf_checkpoint-best-fixed"
|
|
REPORTS_DIR = PROJECT_ROOT / "reports"
|
|
EVAL_RESULTS_DIR = PROJECT_ROOT / "eval" / "results" / "frankenstallm-3b-v2"
|
|
DATA_DIR = PROJECT_ROOT / "data"
|
|
SOURCE_DIRS = ["train", "model", "configs", "scripts", "tokenizer", "eval"]
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description="Upload model, eval reports, source, and data scripts to Hugging Face")
|
|
parser.add_argument("--repo-id", type=str, required=True, help="e.g. pathcosmos/frankenstallm")
|
|
parser.add_argument("--readme-only", action="store_true", help="Only push README + eval results + reports (no model)")
|
|
parser.add_argument("--create-pr", action="store_true", help="Create a Pull Request instead of pushing to main")
|
|
parser.add_argument("--with-source", action="store_true", help="Upload full source code (train, eval, model, configs, scripts, tokenizer)")
|
|
parser.add_argument("--with-data", action="store_true", help="Upload data scripts and DATA_README (no .bin files)")
|
|
args = parser.parse_args()
|
|
create_pr = getattr(args, "create_pr", False)
|
|
|
|
try:
|
|
from huggingface_hub import HfApi, create_repo
|
|
except ImportError:
|
|
print("Install: pip install huggingface_hub")
|
|
raise SystemExit(1)
|
|
|
|
api = HfApi()
|
|
|
|
# 레포 없으면 생성
|
|
# 레포가 없으면 생성 (본인 계정일 때만 성공)
|
|
try:
|
|
create_repo(args.repo_id, repo_type="model", exist_ok=True)
|
|
except Exception as e:
|
|
print(f"Note: create_repo skipped (use Hugging Face website to create repo if needed): {e}")
|
|
|
|
if not args.readme_only:
|
|
if not HF_CHECKPOINT.exists():
|
|
print(f"Checkpoint not found: {HF_CHECKPOINT}")
|
|
raise SystemExit(1)
|
|
print(f"Uploading model from {HF_CHECKPOINT} ...")
|
|
api.upload_folder(
|
|
folder_path=str(HF_CHECKPOINT),
|
|
repo_id=args.repo_id,
|
|
repo_type="model",
|
|
create_pr=create_pr,
|
|
)
|
|
print("Model upload done.")
|
|
|
|
# README는 체크포인트 폴더 것 사용 (이미 평가 요약 포함)
|
|
readme_src = HF_CHECKPOINT / "README.md"
|
|
if readme_src.exists():
|
|
print("Pushing README (model card) ...")
|
|
api.upload_file(
|
|
path_or_fileobj=str(readme_src),
|
|
path_in_repo="README.md",
|
|
repo_id=args.repo_id,
|
|
repo_type="model",
|
|
create_pr=create_pr,
|
|
)
|
|
print("README upload done.")
|
|
else:
|
|
print("No README.md in checkpoint dir; skipping README push.")
|
|
|
|
# 평가 결과 JSON
|
|
results_json = EVAL_RESULTS_DIR / "ollama_benchmark_results.json"
|
|
if results_json.exists():
|
|
print("Pushing ollama_benchmark_results.json ...")
|
|
api.upload_file(
|
|
path_or_fileobj=str(results_json),
|
|
path_in_repo="eval/ollama_benchmark_results.json",
|
|
repo_id=args.repo_id,
|
|
repo_type="model",
|
|
create_pr=create_pr,
|
|
)
|
|
print("Eval results upload done.")
|
|
|
|
# 배포·평가 보고서 (상세 기록)
|
|
for name, src in [
|
|
("2026-03-09_GGUF_DEPLOYMENT_AND_EVAL_REPORT.md", REPORTS_DIR / "2026-03-09_GGUF_DEPLOYMENT_AND_EVAL_REPORT.md"),
|
|
("2026-03-09_ORPO_EVALUATION_REPORT.md", REPORTS_DIR / "2026-03-09_ORPO_EVALUATION_REPORT.md"),
|
|
]:
|
|
if src.exists():
|
|
print(f"Pushing {name} ...")
|
|
api.upload_file(
|
|
path_or_fileobj=str(src),
|
|
path_in_repo=f"eval_reports/{name}",
|
|
repo_id=args.repo_id,
|
|
repo_type="model",
|
|
create_pr=create_pr,
|
|
)
|
|
print("Reports upload done.")
|
|
|
|
# ---------- 소스 코드 (--with-source) ----------
|
|
if getattr(args, "with_source", False):
|
|
print("Uploading source code ...")
|
|
ignore_common = ["**/__pycache__/**", "**/*.pyc", "**/.DS_Store"]
|
|
for dirname in ["train", "model", "configs", "scripts", "tokenizer"]:
|
|
src_dir = PROJECT_ROOT / dirname
|
|
if src_dir.exists():
|
|
api.upload_folder(
|
|
folder_path=str(src_dir),
|
|
path_in_repo=f"source/{dirname}",
|
|
repo_id=args.repo_id,
|
|
repo_type="model",
|
|
ignore_patterns=ignore_common,
|
|
create_pr=create_pr,
|
|
)
|
|
print(f" source/{dirname}/ done.")
|
|
# eval: outputs, results 제외 (대용량)
|
|
eval_dir = PROJECT_ROOT / "eval"
|
|
if eval_dir.exists():
|
|
api.upload_folder(
|
|
folder_path=str(eval_dir),
|
|
path_in_repo="source/eval",
|
|
repo_id=args.repo_id,
|
|
repo_type="model",
|
|
ignore_patterns=ignore_common + ["**/outputs/**", "**/results/**", "**/.compile_cache/**"],
|
|
create_pr=create_pr,
|
|
)
|
|
print(" source/eval/ done.")
|
|
# 루트 문서
|
|
for name in ["README.md", "CLAUDE.md", "requirements.txt", "PROGRESS.md"]:
|
|
src_file = PROJECT_ROOT / name
|
|
if src_file.exists():
|
|
api.upload_file(
|
|
path_or_fileobj=str(src_file),
|
|
path_in_repo=f"source/{name}",
|
|
repo_id=args.repo_id,
|
|
repo_type="model",
|
|
create_pr=create_pr,
|
|
)
|
|
for p in PROJECT_ROOT.glob("PLAN_*.md"):
|
|
api.upload_file(
|
|
path_or_fileobj=str(p),
|
|
path_in_repo=f"source/{p.name}",
|
|
repo_id=args.repo_id,
|
|
repo_type="model",
|
|
create_pr=create_pr,
|
|
)
|
|
print("Source upload done.")
|
|
|
|
# ---------- 데이터 스크립트 (--with-data, .bin 제외) ----------
|
|
if getattr(args, "with_data", False) and DATA_DIR.exists():
|
|
print("Uploading data scripts (no .bin) ...")
|
|
api.upload_folder(
|
|
folder_path=str(DATA_DIR),
|
|
path_in_repo="data",
|
|
repo_id=args.repo_id,
|
|
repo_type="model",
|
|
ignore_patterns=[
|
|
"**/*.bin",
|
|
"**/*.chunk*",
|
|
"**/__pycache__/**",
|
|
"**/code/**",
|
|
"**/*.pyc",
|
|
],
|
|
create_pr=create_pr,
|
|
)
|
|
print("Data scripts upload done.")
|
|
|
|
print(f"Done. https://huggingface.co/{args.repo_id}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|