初始化项目，由ModelHub XC社区提供模型
Model: pathcosmos/frankenstallm Source: Original Platform
This commit is contained in:
ModelHub XC
23
source/scripts/RESTART_GUIDE.md
Normal file
23
source/scripts/RESTART_GUIDE.md
Normal file
@@ -0,0 +1,23 @@
|
||||
# FRANKENSTALLM 3B — Optimization Restart Guide
|
||||
|
||||
## Quick restart (all optimizations applied automatically):
|
||||
```bash
|
||||
bash scripts/apply_optimizations.sh
|
||||
```
|
||||
|
||||
## Validate only (no restart):
|
||||
```bash
|
||||
bash scripts/apply_optimizations.sh --test-only
|
||||
```
|
||||
|
||||
## Manual steps if auto-migration fails:
|
||||
1. Stop: `kill $(cat checkpoints/korean_3b_fp8_run1/train.pid)`
|
||||
2. Migrate: `python3 scripts/migrate_qkv_checkpoint.py checkpoints/korean_3b_fp8_run1/checkpoint-XXXXX`
|
||||
3. Restart: `bash scripts/launch_3b_pretrain.sh`
|
||||
|
||||
## Rollback (undo QKV fusion):
|
||||
```bash
|
||||
CKPT=checkpoints/korean_3b_fp8_run1/checkpoint-XXXXX
|
||||
cp ${CKPT}/model.pt.backup_pre_qkv ${CKPT}/model.pt
|
||||
git checkout model/attention.py # restore original attention code
|
||||
```
|
||||
194
source/scripts/apply_optimizations.sh
Normal file
194
source/scripts/apply_optimizations.sh
Normal file
@@ -0,0 +1,194 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# apply_optimizations.sh — Apply v2 optimizations and restart training
|
||||
#
|
||||
# Optimizations applied:
|
||||
# 1. QKV Projection Fusion (+8-12% throughput)
|
||||
# 2. NUMA CPU Affinity (fix 69% cross-NUMA workers)
|
||||
# 3. Batch size 4→5 (11h saved over full run)
|
||||
# 4. NCCL NVLS algorithm + 256MB buffers
|
||||
# 5. DDP bucket_cap_mb 400→800
|
||||
# 6. DataLoader num_workers 4→6, prefetch_factor 3→4
|
||||
# 7. MADV_RANDOM + WILLNEED for PackedDataset
|
||||
# 8. numactl --interleave=all on torchrun
|
||||
#
|
||||
# Usage:
|
||||
# bash scripts/apply_optimizations.sh # full migration
|
||||
# bash scripts/apply_optimizations.sh --test-only # just validate, don't restart
|
||||
# bash scripts/apply_optimizations.sh --skip-stop # don't stop current training
|
||||
# =============================================================================
|
||||
set -u
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
RUN_NAME="korean_3b_fp8_run1"
|
||||
CKPT_DIR="checkpoints/${RUN_NAME}"
|
||||
PID_FILE="${CKPT_DIR}/train.pid"
|
||||
LOG_FILE="${CKPT_DIR}/train.log"
|
||||
|
||||
TEST_ONLY=false
|
||||
SKIP_STOP=false
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--test-only) TEST_ONLY=true ;;
|
||||
--skip-stop) SKIP_STOP=true ;;
|
||||
esac
|
||||
done
|
||||
|
||||
echo "=================================================================="
|
||||
echo " FRANKENSTALLM 3B — Optimization Migration v2"
|
||||
echo " $(date)"
|
||||
echo "=================================================================="
|
||||
|
||||
# ---- Step 1: Validate all modified files --------------------------------
|
||||
echo ""
|
||||
echo "[1/6] Validating modified files..."
|
||||
ERRORS=0
|
||||
|
||||
for pyfile in model/attention.py train/pretrain.py data/dataset.py scripts/migrate_qkv_checkpoint.py; do
|
||||
if python3 -c "import ast; ast.parse(open('$pyfile').read())" 2>/dev/null; then
|
||||
echo " ✓ $pyfile — syntax OK"
|
||||
else
|
||||
echo " ✗ $pyfile — SYNTAX ERROR"
|
||||
((ERRORS++))
|
||||
fi
|
||||
done
|
||||
|
||||
if bash -n scripts/launch_3b_pretrain.sh 2>/dev/null; then
|
||||
echo " ✓ scripts/launch_3b_pretrain.sh — syntax OK"
|
||||
else
|
||||
echo " ✗ scripts/launch_3b_pretrain.sh — SYNTAX ERROR"
|
||||
((ERRORS++))
|
||||
fi
|
||||
|
||||
# Check YAML
|
||||
python3 -c "
|
||||
import yaml
|
||||
with open('configs/korean_3b_fp8.yaml') as f:
|
||||
cfg = yaml.safe_load(f)
|
||||
assert cfg['train']['batch_size'] == 5, f'batch_size should be 5, got {cfg[\"train\"][\"batch_size\"]}'
|
||||
print(' ✓ configs/korean_3b_fp8.yaml — valid, batch_size=5')
|
||||
" 2>/dev/null || { echo " ✗ configs/korean_3b_fp8.yaml — INVALID"; ((ERRORS++)); }
|
||||
|
||||
if [[ $ERRORS -gt 0 ]]; then
|
||||
echo ""
|
||||
echo "[ERROR] $ERRORS file(s) failed validation. Aborting."
|
||||
exit 1
|
||||
fi
|
||||
echo " All files validated successfully."
|
||||
|
||||
if $TEST_ONLY; then
|
||||
echo ""
|
||||
echo "[INFO] --test-only mode. Exiting without restart."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ---- Step 2: Stop current training (graceful) ---------------------------
|
||||
if ! $SKIP_STOP; then
|
||||
echo ""
|
||||
echo "[2/6] Stopping current training (SIGTERM → emergency checkpoint)..."
|
||||
if [[ -f "$PID_FILE" ]]; then
|
||||
PID=$(cat "$PID_FILE")
|
||||
if kill -0 "$PID" 2>/dev/null; then
|
||||
echo " Sending SIGTERM to PID $PID..."
|
||||
kill "$PID"
|
||||
echo " Waiting for graceful shutdown (up to 120s)..."
|
||||
for i in $(seq 1 120); do
|
||||
if ! kill -0 "$PID" 2>/dev/null; then
|
||||
echo " Process stopped after ${i}s"
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
if kill -0 "$PID" 2>/dev/null; then
|
||||
echo " [WARN] Process still running after 120s. Force killing..."
|
||||
kill -9 "$PID" 2>/dev/null || true
|
||||
sleep 2
|
||||
fi
|
||||
else
|
||||
echo " Process $PID not running."
|
||||
fi
|
||||
else
|
||||
echo " No PID file found."
|
||||
fi
|
||||
|
||||
# Wait for all GPU processes to clear
|
||||
echo " Waiting for GPU processes to terminate..."
|
||||
for i in $(seq 1 30); do
|
||||
if ! pgrep -f "pretrain.py.*korean_3b" >/dev/null 2>&1; then
|
||||
echo " All GPU processes cleared."
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
fi
|
||||
|
||||
# ---- Step 3: Find and migrate latest checkpoint -------------------------
|
||||
echo ""
|
||||
echo "[3/6] Migrating latest checkpoint (QKV fusion)..."
|
||||
LATEST_CKPT=$(ls -d "${CKPT_DIR}"/checkpoint-* 2>/dev/null | sort -V | tail -1)
|
||||
if [[ -z "$LATEST_CKPT" ]]; then
|
||||
echo " [ERROR] No checkpoint found!"
|
||||
exit 1
|
||||
fi
|
||||
echo " Latest checkpoint: $LATEST_CKPT"
|
||||
|
||||
# Backup original model.pt
|
||||
cp "${LATEST_CKPT}/model.pt" "${LATEST_CKPT}/model.pt.backup_pre_qkv"
|
||||
echo " Backup created: model.pt.backup_pre_qkv"
|
||||
|
||||
# Run migration
|
||||
python3 scripts/migrate_qkv_checkpoint.py "$LATEST_CKPT"
|
||||
echo " QKV fusion migration complete."
|
||||
|
||||
# ---- Step 4: Quick validation test (5 steps) ----------------------------
|
||||
echo ""
|
||||
echo "[4/6] Running 5-step validation test..."
|
||||
# Use single GPU for fast test
|
||||
timeout 120 python3 train/pretrain.py \
|
||||
--config configs/korean_3b_fp8.yaml \
|
||||
--train_data data/3b_train.bin \
|
||||
--checkpoint_dir /tmp/frankenstallm_test \
|
||||
--max_steps 5 \
|
||||
--batch_size 5 \
|
||||
--resume "$LATEST_CKPT" \
|
||||
2>&1 | tail -10
|
||||
|
||||
TEST_EXIT=$?
|
||||
if [[ $TEST_EXIT -eq 0 ]]; then
|
||||
echo " ✓ 5-step test passed!"
|
||||
else
|
||||
echo " ✗ 5-step test FAILED (exit code $TEST_EXIT)"
|
||||
echo " [WARN] Restoring original checkpoint..."
|
||||
cp "${LATEST_CKPT}/model.pt.backup_pre_qkv" "${LATEST_CKPT}/model.pt"
|
||||
echo " Original checkpoint restored. Aborting."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ---- Step 5: Clean up test artifacts ------------------------------------
|
||||
echo ""
|
||||
echo "[5/6] Cleaning up test artifacts..."
|
||||
rm -rf /tmp/frankenstallm_test
|
||||
|
||||
# ---- Step 6: Launch full training with optimizations --------------------
|
||||
echo ""
|
||||
echo "[6/6] Launching optimized training..."
|
||||
echo ""
|
||||
echo " Changes applied:"
|
||||
echo " • QKV Projection Fusion (single GEMM)"
|
||||
echo " • NUMA CPU Affinity (cores 0-35→GPU0-3, 36-71→GPU4-7)"
|
||||
echo " • Batch size: 4 → 5"
|
||||
echo " • NCCL: NVLS,Ring algorithm, 256MB buffers"
|
||||
echo " • DDP: bucket_cap_mb 400 → 800"
|
||||
echo " • DataLoader: 4→6 workers, prefetch 3→4"
|
||||
echo " • MADV_RANDOM + WILLNEED for dataset mmap"
|
||||
echo " • numactl --interleave=all on torchrun"
|
||||
echo ""
|
||||
|
||||
bash scripts/launch_3b_pretrain.sh
|
||||
|
||||
echo ""
|
||||
echo "=================================================================="
|
||||
echo " Migration complete! Monitor with:"
|
||||
echo " tail -f ${LOG_FILE}"
|
||||
echo "=================================================================="
|
||||
83
source/scripts/build_3b_dataset.sh
Normal file
83
source/scripts/build_3b_dataset.sh
Normal file
@@ -0,0 +1,83 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
DATA="data"
|
||||
|
||||
echo "=================================================================="
|
||||
echo " 3B 통합 데이터셋 빌드 | 시작: $(date)"
|
||||
echo "=================================================================="
|
||||
|
||||
# 청크 병합 함수
|
||||
merge_chunks() {
|
||||
PREFIX="$1"
|
||||
OUTPUT="$2"
|
||||
CHUNKS=$(ls "${PREFIX}".bin.chunk* 2>/dev/null | sort || true)
|
||||
if [[ -z "$CHUNKS" ]]; then return; fi
|
||||
if [[ -f "$OUTPUT" ]]; then echo " [SKIP] $OUTPUT 이미 존재"; return; fi
|
||||
echo " 청크 병합: $(basename $PREFIX)"
|
||||
cat $CHUNKS > "$OUTPUT"
|
||||
echo " 완료: $(du -sh $OUTPUT | cut -f1)"
|
||||
}
|
||||
|
||||
merge_chunks "$DATA/cosmo_auto_math_text_train" "$DATA/cosmo_auto_math_text_train.bin"
|
||||
merge_chunks "$DATA/cosmo_auto_math_text_val" "$DATA/cosmo_auto_math_text_val.bin"
|
||||
merge_chunks "$DATA/cosmo_web_v2_train" "$DATA/cosmo_web_v2_train.bin"
|
||||
merge_chunks "$DATA/cosmo_web_v2_val" "$DATA/cosmo_web_v2_val.bin"
|
||||
|
||||
TRAIN_FILES=""
|
||||
for f in \
|
||||
"$DATA/korean_train.bin" \
|
||||
"$DATA/hplt_ko_train.bin" \
|
||||
"$DATA/korean_c4_train.bin" \
|
||||
"$DATA/cc100_ko_train.bin" \
|
||||
"$DATA/namuwiki_2023b_train.bin" \
|
||||
"$DATA/korean_namuwiki_train.bin" \
|
||||
"$DATA/wikipedia_ko_train.bin" \
|
||||
"$DATA/korean_wiki_train.bin" \
|
||||
"$DATA/open_web_math_train.bin" \
|
||||
"$DATA/mathpile_train.bin" \
|
||||
"$DATA/cosmo_auto_math_text_train.bin" \
|
||||
"$DATA/cosmo_stories_train.bin" \
|
||||
"$DATA/cosmo_web_v2_train.bin" \
|
||||
"$DATA/cosmo_stanford_train.bin" \
|
||||
"$DATA/cosmo_wikihow_train.bin" \
|
||||
"$DATA/cosmo_openstax_train.bin" \
|
||||
"$DATA/cosmo_khanacademy_train.bin"; do
|
||||
[[ -f "$f" ]] && TRAIN_FILES="$TRAIN_FILES $f"
|
||||
done
|
||||
|
||||
VAL_FILES=""
|
||||
for f in \
|
||||
"$DATA/korean_val.bin" \
|
||||
"$DATA/hplt_ko_val.bin" \
|
||||
"$DATA/korean_c4_val.bin" \
|
||||
"$DATA/cc100_ko_val.bin" \
|
||||
"$DATA/namuwiki_2023b_val.bin" \
|
||||
"$DATA/open_web_math_val.bin" \
|
||||
"$DATA/mathpile_val.bin" \
|
||||
"$DATA/cosmo_auto_math_text_val.bin" \
|
||||
"$DATA/cosmo_stories_val.bin" \
|
||||
"$DATA/cosmo_web_v2_val.bin"; do
|
||||
[[ -f "$f" ]] && VAL_FILES="$VAL_FILES $f"
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "train 파일 병합 → data/3b_train.bin ..."
|
||||
python3 data/merge_bins.py $TRAIN_FILES data/3b_train.bin
|
||||
|
||||
echo ""
|
||||
echo "val 파일 병합 → data/3b_val.bin ..."
|
||||
python3 data/merge_bins.py $VAL_FILES data/3b_val.bin
|
||||
|
||||
echo ""
|
||||
echo "=================================================================="
|
||||
du -sh data/3b_train.bin data/3b_val.bin
|
||||
python3 -c "
|
||||
import os
|
||||
sz = os.path.getsize('data/3b_train.bin')
|
||||
tok = sz // 2
|
||||
print(f'3b_train: {tok/1e9:.2f}B tokens')
|
||||
print(f'60B 달성 에포크: {60/(tok/1e9):.1f}x 반복 필요')
|
||||
"
|
||||
echo "완료: $(date)"
|
||||
echo "=================================================================="
|
||||
178
source/scripts/check_korean_data.sh
Normal file
178
source/scripts/check_korean_data.sh
Normal file
@@ -0,0 +1,178 @@
|
||||
#!/bin/bash
|
||||
|
||||
# 한국어 학습 데이터 현황 확인 스크립트
|
||||
# 용도: 한국어 데이터셋 상태, 토크나이저, 원본 데이터 파일 확인
|
||||
|
||||
set -e
|
||||
|
||||
# 프로젝트 루트 (이 스크립트 실행 위치 기준)
|
||||
PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
cd "${PROJECT_ROOT}"
|
||||
|
||||
echo "=== 한국어 학습 데이터 현황 ==="
|
||||
echo ""
|
||||
|
||||
# ============================================================================
|
||||
# 1. 학습용 바이너리 데이터 확인
|
||||
# ============================================================================
|
||||
echo "[ 학습 바이너리 데이터 ]"
|
||||
|
||||
check_binary_data() {
|
||||
local file=$1
|
||||
local name=$2
|
||||
|
||||
if [ -f "$file" ]; then
|
||||
local size=$(du -h "$file" | cut -f1)
|
||||
|
||||
# Python + numpy memmap으로 토큰 수 계산
|
||||
# 바이너리는 uint32 형태로 저장되어 있음 (4 bytes per token)
|
||||
local token_count=$(python3 -c "
|
||||
import numpy as np
|
||||
try:
|
||||
data = np.memmap('$file', dtype=np.uint32, mode='r')
|
||||
print(len(data))
|
||||
except Exception as e:
|
||||
print('error')
|
||||
" 2>/dev/null || echo "error")
|
||||
|
||||
if [ "$token_count" != "error" ] && [ ! -z "$token_count" ]; then
|
||||
# 토큰 수를 포맷팅 (천 단위 쉼표)
|
||||
local formatted_tokens=$(printf "%'d" "$token_count")
|
||||
|
||||
# 1B 모델 학습 스텝 계산
|
||||
# tokens_per_step = batch_size * grad_accum * seq_len * num_gpus
|
||||
# = 8 * 4 * 4096 * 8 = 1,048,576 tokens/step
|
||||
local tokens_per_step=1048576
|
||||
local estimated_steps=$((token_count / tokens_per_step))
|
||||
|
||||
printf " %-20s : 존재 (%s, %'d 토큰, ~%'d steps)\n" \
|
||||
"$name" "$size" "$token_count" "$estimated_steps"
|
||||
else
|
||||
printf " %-20s : 존재 (%s, 토큰 계산 실패)\n" "$name" "$size"
|
||||
fi
|
||||
else
|
||||
printf " %-20s : 없음\n" "$name"
|
||||
fi
|
||||
}
|
||||
|
||||
check_binary_data "data/korean_train.bin" "korean_train.bin"
|
||||
check_binary_data "data/korean_val.bin" "korean_val.bin"
|
||||
check_binary_data "data/train.bin" "train.bin"
|
||||
check_binary_data "data/val.bin" "val.bin"
|
||||
|
||||
echo ""
|
||||
|
||||
# ============================================================================
|
||||
# 2. 토크나이저 확인
|
||||
# ============================================================================
|
||||
echo "[ 토크나이저 ]"
|
||||
|
||||
check_tokenizer() {
|
||||
local dir=$1
|
||||
local name=$2
|
||||
|
||||
if [ -d "$dir" ]; then
|
||||
local files=$(find "$dir" -type f | wc -l)
|
||||
printf " %-20s : 존재 (%d개 파일)\n" "$name" "$files"
|
||||
else
|
||||
printf " %-20s : 없음\n" "$name"
|
||||
fi
|
||||
}
|
||||
|
||||
check_tokenizer "tokenizer/korean_sp" "korean_sp"
|
||||
check_tokenizer "tokenizer" "default tokenizer"
|
||||
|
||||
echo ""
|
||||
|
||||
# ============================================================================
|
||||
# 3. 원본 데이터 디렉토리 확인
|
||||
# ============================================================================
|
||||
echo "[ 원본 데이터 ]"
|
||||
|
||||
check_raw_data() {
|
||||
local dir=$1
|
||||
local name=$2
|
||||
|
||||
if [ -d "$dir" ]; then
|
||||
local file_count=$(find "$dir" -maxdepth 1 -type f | wc -l)
|
||||
local total_size=$(du -sh "$dir" 2>/dev/null | cut -f1)
|
||||
|
||||
if [ $file_count -eq 0 ]; then
|
||||
printf " %-20s : 없음 (디렉토리만 존재, 0 파일)\n" "$name"
|
||||
else
|
||||
printf " %-20s : %'d 파일 (%s)\n" "$name" "$file_count" "$total_size"
|
||||
fi
|
||||
else
|
||||
printf " %-20s : 없음\n" "$name"
|
||||
fi
|
||||
}
|
||||
|
||||
check_raw_data "data/raw/cc100_ko" "cc100_ko/"
|
||||
check_raw_data "data/raw/c4_ko" "c4_ko/"
|
||||
check_raw_data "data/raw/namuwiki_ko" "namuwiki_ko/"
|
||||
|
||||
# 위키 데이터는 raw/ 직접 하위
|
||||
echo ""
|
||||
echo "[ 위키피디아 데이터 ]"
|
||||
ko_wiki_count=$(find "data/raw" -maxdepth 1 -name "ko_wiki_*.txt" | wc -l)
|
||||
en_wiki_count=$(find "data/raw" -maxdepth 1 -name "en_wiki_*.txt" | wc -l)
|
||||
ko_wiki_size=$(du -sh "data/raw" 2>/dev/null | cut -f1)
|
||||
|
||||
if [ $ko_wiki_count -gt 0 ]; then
|
||||
printf " %-20s : %'d 파일\n" "ko_wiki" "$ko_wiki_count"
|
||||
fi
|
||||
|
||||
if [ $en_wiki_count -gt 0 ]; then
|
||||
printf " %-20s : %'d 파일\n" "en_wiki" "$en_wiki_count"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
|
||||
# ============================================================================
|
||||
# 4. 종합 상태 요약
|
||||
# ============================================================================
|
||||
echo "[ 종합 상태 ]"
|
||||
|
||||
# 학습용 바이너리 데이터 확인
|
||||
binary_ready=false
|
||||
if [ -f "data/korean_train.bin" ] && [ -f "data/korean_val.bin" ]; then
|
||||
binary_ready=true
|
||||
elif [ -f "data/train.bin" ] && [ -f "data/val.bin" ]; then
|
||||
binary_ready=true
|
||||
fi
|
||||
|
||||
# 토크나이저 확인
|
||||
tokenizer_ready=false
|
||||
if [ -d "tokenizer/korean_sp" ] && [ -f "tokenizer/korean_sp/tokenizer.model" ]; then
|
||||
tokenizer_ready=true
|
||||
fi
|
||||
|
||||
# 원본 데이터 확인
|
||||
raw_ready=false
|
||||
if [ -d "data/raw/c4_ko" ] || [ -d "data/raw/namuwiki_ko" ] || [ -d "data/raw/cc100_ko" ]; then
|
||||
count=$(find "data/raw/c4_ko" -maxdepth 1 -type f 2>/dev/null | wc -l)
|
||||
count=$((count + $(find "data/raw/namuwiki_ko" -maxdepth 1 -type f 2>/dev/null | wc -l)))
|
||||
count=$((count + $(find "data/raw/cc100_ko" -maxdepth 1 -type f 2>/dev/null | wc -l)))
|
||||
if [ $count -gt 0 ]; then
|
||||
raw_ready=true
|
||||
fi
|
||||
fi
|
||||
|
||||
printf " 학습용 바이너리 : %s\n" "$([ "$binary_ready" = true ] && echo "✓ 준비됨" || echo "✗ 미준비")"
|
||||
printf " 토크나이저 : %s\n" "$([ "$tokenizer_ready" = true ] && echo "✓ 준비됨" || echo "✗ 미준비")"
|
||||
printf " 원본 데이터 : %s\n" "$([ "$raw_ready" = true ] && echo "✓ 준비됨" || echo "✗ 미준비")"
|
||||
|
||||
echo ""
|
||||
|
||||
# ============================================================================
|
||||
# 5. 학습 설정 파라미터 정보
|
||||
# ============================================================================
|
||||
echo "[ 학습 설정 (1B 모델 기준) ]"
|
||||
echo " 배치 사이즈 : 8"
|
||||
echo " 시퀀스 길이 : 4096"
|
||||
echo " GPU 수 : 8"
|
||||
echo " 그래디언트 누적 : 4"
|
||||
echo " 토큰/스텝 : 8 × 4 × 4096 × 8 = 1,048,576"
|
||||
echo ""
|
||||
|
||||
echo "=== 검사 완료 ==="
|
||||
201
source/scripts/clickhouse-watchdog.sh
Normal file
201
source/scripts/clickhouse-watchdog.sh
Normal file
@@ -0,0 +1,201 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# clickhouse-watchdog.sh — ClickHouse 헬스체크 + 자동 재시작
|
||||
# crontab에 등록하여 1분마다 실행
|
||||
#
|
||||
# Usage:
|
||||
# */1 * * * * /PROJECT/0325120031_A/ghong/taketimes/llm-bang/scripts/clickhouse-watchdog.sh
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# ── 설정 ──────────────────────────────────────────────
|
||||
CH_BIN="/PROJECT/0325120031_A/ghong/taketimes/clickhouse-bin"
|
||||
CH_CONFIG="/PROJECT/0325120031_A/ghong/taketimes/llm-bang/configs/clickhouse-config.xml"
|
||||
TCP_PORT=9000
|
||||
HTTP_PORT=8123
|
||||
HOST="127.0.0.1"
|
||||
|
||||
LOG_DIR="/tmp/clickhouse"
|
||||
LOG_FILE="${LOG_DIR}/watchdog.log"
|
||||
MAX_LOG_SIZE=$((10 * 1024 * 1024)) # 10MB 로테이션
|
||||
|
||||
RESTART_COOLDOWN=180 # 초 — 재시작 후 이 시간 내 재시도 방지
|
||||
LAST_RESTART_FILE="/tmp/clickhouse-last-restart"
|
||||
HEALTH_CHECK_TIMEOUT=5 # 초 — 헬스체크 curl/query 타임아웃
|
||||
|
||||
# ── 함수 ──────────────────────────────────────────────
|
||||
mkdir -p "$LOG_DIR"
|
||||
|
||||
log() {
|
||||
echo "[$(date '+%Y-%m-%d %H:%M:%S')] [clickhouse-watchdog] $*" >> "$LOG_FILE"
|
||||
}
|
||||
|
||||
rotate_log() {
|
||||
local file="$1"
|
||||
if [[ -f "$file" ]] && [[ $(stat -c%s "$file" 2>/dev/null || echo 0) -gt $MAX_LOG_SIZE ]]; then
|
||||
mv "$file" "${file}.old"
|
||||
log "Log rotated: $file"
|
||||
fi
|
||||
}
|
||||
|
||||
is_tcp_port_open() {
|
||||
if command -v ss &>/dev/null; then
|
||||
ss -tlnH "sport = :${TCP_PORT}" 2>/dev/null | grep -q "$TCP_PORT"
|
||||
else
|
||||
(echo > /dev/tcp/"$HOST"/"$TCP_PORT") 2>/dev/null
|
||||
fi
|
||||
}
|
||||
|
||||
is_http_responding() {
|
||||
# HTTP 인터페이스 핑 — ClickHouse는 GET / 에 "Ok.\n" 응답
|
||||
if command -v curl &>/dev/null; then
|
||||
local resp
|
||||
resp=$(curl -s --max-time "$HEALTH_CHECK_TIMEOUT" "http://${HOST}:${HTTP_PORT}/ping" 2>/dev/null || true)
|
||||
[[ "$resp" == "Ok." ]]
|
||||
else
|
||||
# curl 없으면 TCP 포트만 확인
|
||||
(echo > /dev/tcp/"$HOST"/"$HTTP_PORT") 2>/dev/null
|
||||
fi
|
||||
}
|
||||
|
||||
is_process_alive() {
|
||||
# ClickHouse 내부 watchdog 프로세스명: "clickhouse-watchdog" (바이너리 자체)
|
||||
# 이 스크립트(clickhouse-watchdog.sh)와 구분하기 위해 --daemon 플래그 포함 패턴 사용
|
||||
pgrep -f "clickhouse.*server.*--daemon" >/dev/null 2>&1
|
||||
}
|
||||
|
||||
can_execute_query() {
|
||||
# 실제 쿼리 실행으로 서버가 응답하는지 확인
|
||||
local result
|
||||
result=$("$CH_BIN" client --port "$TCP_PORT" --query "SELECT 1" 2>/dev/null || true)
|
||||
[[ "$result" == "1" ]]
|
||||
}
|
||||
|
||||
cooldown_active() {
|
||||
if [[ -f "$LAST_RESTART_FILE" ]]; then
|
||||
local last_restart now diff
|
||||
last_restart=$(cat "$LAST_RESTART_FILE" 2>/dev/null)
|
||||
now=$(date +%s)
|
||||
diff=$(( now - last_restart ))
|
||||
if [[ $diff -lt $RESTART_COOLDOWN ]]; then
|
||||
return 0 # 쿨다운 중
|
||||
fi
|
||||
fi
|
||||
return 1 # 쿨다운 아님
|
||||
}
|
||||
|
||||
stop_existing() {
|
||||
log "Stopping existing ClickHouse processes..."
|
||||
local my_pid=$$
|
||||
local pids
|
||||
|
||||
# 정상 종료 시도 (서버 프로세스)
|
||||
pids=$(pgrep -f "clickhouse.*server.*--daemon" 2>/dev/null | grep -v "^${my_pid}$" || true)
|
||||
if [[ -n "$pids" ]]; then
|
||||
log "Sending TERM to PIDs: $pids"
|
||||
echo "$pids" | xargs kill -TERM 2>/dev/null || true
|
||||
sleep 3
|
||||
# 아직 살아있으면 강제 종료
|
||||
pids=$(pgrep -f "clickhouse.*server.*--daemon" 2>/dev/null | grep -v "^${my_pid}$" || true)
|
||||
if [[ -n "$pids" ]]; then
|
||||
log "Force killing PIDs: $pids"
|
||||
echo "$pids" | xargs kill -9 2>/dev/null || true
|
||||
sleep 2
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
start_server() {
|
||||
log "Starting ClickHouse server (daemon mode)..."
|
||||
|
||||
# 기존 프로세스 정리
|
||||
stop_existing
|
||||
|
||||
# 필요한 디렉토리 생성
|
||||
mkdir -p /tmp/clickhouse/logs
|
||||
mkdir -p /tmp/clickhouse-tmp
|
||||
|
||||
# 데몬 모드로 시작
|
||||
"$CH_BIN" server --config-file="$CH_CONFIG" --daemon
|
||||
|
||||
# 시작 후 대기 + 확인 (최대 15초)
|
||||
local attempts=0
|
||||
local max_attempts=15
|
||||
while [[ $attempts -lt $max_attempts ]]; do
|
||||
sleep 1
|
||||
attempts=$((attempts + 1))
|
||||
if is_tcp_port_open && can_execute_query; then
|
||||
date +%s > "$LAST_RESTART_FILE"
|
||||
log "ClickHouse started successfully (took ${attempts}s)"
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
|
||||
date +%s > "$LAST_RESTART_FILE"
|
||||
log "ERROR: ClickHouse did not respond within ${max_attempts}s after start"
|
||||
return 1
|
||||
}
|
||||
|
||||
# ── 메인 로직 ─────────────────────────────────────────
|
||||
rotate_log "$LOG_FILE"
|
||||
|
||||
# 1) 바이너리 존재 확인
|
||||
if [[ ! -x "$CH_BIN" ]]; then
|
||||
log "FATAL: ClickHouse binary not found or not executable: $CH_BIN"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 2) 프로세스 + 포트 + 쿼리 체크
|
||||
process_ok=false
|
||||
port_ok=false
|
||||
query_ok=false
|
||||
|
||||
if is_process_alive; then
|
||||
process_ok=true
|
||||
fi
|
||||
|
||||
if is_tcp_port_open; then
|
||||
port_ok=true
|
||||
fi
|
||||
|
||||
if $port_ok && can_execute_query; then
|
||||
query_ok=true
|
||||
fi
|
||||
|
||||
# 3) 판단
|
||||
if $process_ok && $port_ok && $query_ok; then
|
||||
# 완전 정상 — 아무것도 안 함
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# HTTP도 확인 (진단 로그용)
|
||||
http_ok=false
|
||||
if is_http_responding; then
|
||||
http_ok=true
|
||||
fi
|
||||
|
||||
# 비정상 상태 로깅
|
||||
if $process_ok && $port_ok && ! $query_ok; then
|
||||
log "WARN: Process alive, port open, but query failed. Possible hung state."
|
||||
elif $process_ok && ! $port_ok; then
|
||||
log "WARN: Process alive but TCP port $TCP_PORT not listening."
|
||||
elif ! $process_ok; then
|
||||
log "WARN: ClickHouse is completely down (no process found)."
|
||||
fi
|
||||
log "Status: process=$process_ok port=$port_ok query=$query_ok http=$http_ok"
|
||||
|
||||
# 4) 쿨다운 체크
|
||||
if cooldown_active; then
|
||||
log "Cooldown active (last restart < ${RESTART_COOLDOWN}s ago). Skipping."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# 5) 재시작
|
||||
log "Attempting ClickHouse restart..."
|
||||
if start_server; then
|
||||
log "ClickHouse restart SUCCESS"
|
||||
else
|
||||
log "ClickHouse restart FAILED"
|
||||
exit 1
|
||||
fi
|
||||
229
source/scripts/convert_3b_gguf.sh
Normal file
229
source/scripts/convert_3b_gguf.sh
Normal file
@@ -0,0 +1,229 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# convert_3b_gguf.sh — 3B 모델 HuggingFace → GGUF 변환 + 다중 양자화
|
||||
#
|
||||
# Usage:
|
||||
# bash scripts/convert_3b_gguf.sh [options]
|
||||
#
|
||||
# Options:
|
||||
# --input_dir DIR HF 포맷 모델 디렉토리 (default: outputs/hf_korean_3b_orpo)
|
||||
# --out_dir DIR GGUF 출력 디렉토리 (default: outputs/gguf)
|
||||
# --checkpoint DIR 커스텀 체크포인트 디렉토리 (지정 시 HF 변환 선행 실행)
|
||||
# --skip_hf_conv HF 변환 단계 건너뜀 (이미 HF 포맷 존재 시)
|
||||
# --skip_quant 양자화 단계 건너뜀 (F16 GGUF만 생성)
|
||||
#
|
||||
# Pipeline:
|
||||
# 1. [선택] 커스텀 체크포인트 → HF transformers 포맷 (convert_to_hf.py)
|
||||
# 2. HF → F16 GGUF (llama.cpp/convert_hf_to_gguf.py)
|
||||
# 3. F16 GGUF → Q4_K_M, Q5_K_M, Q8_0 양자화 (llama-quantize)
|
||||
#
|
||||
# Outputs:
|
||||
# outputs/gguf/frankenstallm-3b-f16.gguf
|
||||
# outputs/gguf/frankenstallm-3b-Q4_K_M.gguf — 권장 (Ollama용)
|
||||
# outputs/gguf/frankenstallm-3b-Q5_K_M.gguf
|
||||
# outputs/gguf/frankenstallm-3b-Q8_0.gguf
|
||||
#
|
||||
# 전제 조건:
|
||||
# - python scripts/convert_to_hf.py 로 HF 변환 완료 (또는 --checkpoint 옵션)
|
||||
# - git, cmake, make 설치
|
||||
# - pip install safetensors
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 인자 파싱
|
||||
# ---------------------------------------------------------------------------
|
||||
INPUT_DIR="outputs/hf_korean_3b_orpo"
|
||||
OUT_DIR="outputs/gguf"
|
||||
CHECKPOINT_DIR=""
|
||||
SKIP_HF_CONV=false
|
||||
SKIP_QUANT=false
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--input_dir) INPUT_DIR="$2"; shift 2 ;;
|
||||
--out_dir) OUT_DIR="$2"; shift 2 ;;
|
||||
--checkpoint) CHECKPOINT_DIR="$2"; shift 2 ;;
|
||||
--skip_hf_conv) SKIP_HF_CONV=true; shift ;;
|
||||
--skip_quant) SKIP_QUANT=true; shift ;;
|
||||
-h|--help)
|
||||
grep '^#' "$0" | head -40 | sed 's/^# \{0,1\}//'
|
||||
exit 0 ;;
|
||||
*)
|
||||
echo "ERROR: 알 수 없는 옵션: $1"
|
||||
echo "Usage: bash scripts/convert_3b_gguf.sh [--input_dir DIR] [--out_dir DIR] [--checkpoint DIR] [--skip_hf_conv] [--skip_quant]"
|
||||
exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
PROJECT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
LLAMA_CPP_DIR="${LLAMA_CPP_DIR:-$PROJECT_DIR/outputs/llama.cpp}"
|
||||
MODEL_NAME="frankenstallm-3b"
|
||||
|
||||
cd "$PROJECT_DIR"
|
||||
|
||||
echo "=================================================================="
|
||||
echo " 3B 모델 GGUF 변환 파이프라인"
|
||||
echo " 입력 HF 디렉토리 : $INPUT_DIR"
|
||||
echo " GGUF 출력 디렉토리: $OUT_DIR"
|
||||
echo " llama.cpp 경로 : $LLAMA_CPP_DIR"
|
||||
echo "=================================================================="
|
||||
echo ""
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Step 0: llama.cpp 존재 여부 확인 / 클론
|
||||
# ---------------------------------------------------------------------------
|
||||
if [[ ! -d "$LLAMA_CPP_DIR" ]]; then
|
||||
echo "[SETUP] llama.cpp 디렉토리가 없습니다."
|
||||
echo " 다음 명령으로 설치하세요:"
|
||||
echo ""
|
||||
echo " git clone --depth 1 https://github.com/ggerganov/llama.cpp $LLAMA_CPP_DIR"
|
||||
echo ""
|
||||
echo " 또는 LLAMA_CPP_DIR 환경변수로 기존 경로를 지정하세요:"
|
||||
echo " LLAMA_CPP_DIR=/path/to/llama.cpp bash scripts/convert_3b_gguf.sh"
|
||||
echo ""
|
||||
read -r -p "지금 자동 클론하시겠습니까? [y/N] " _yn
|
||||
if [[ "${_yn:-N}" =~ ^[Yy]$ ]]; then
|
||||
echo "Cloning llama.cpp ..."
|
||||
git clone --depth 1 https://github.com/ggerganov/llama.cpp "$LLAMA_CPP_DIR"
|
||||
else
|
||||
echo "중단합니다. llama.cpp를 설치한 뒤 다시 실행하세요."
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# llama.cpp Python 의존성
|
||||
echo "[SETUP] llama.cpp Python 의존성 설치 중 ..."
|
||||
pip install -r "$LLAMA_CPP_DIR/requirements.txt" --break-system-packages -q
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Step 1: 커스텀 체크포인트 → HF 포맷 변환 (선택)
|
||||
# ---------------------------------------------------------------------------
|
||||
if [[ -n "$CHECKPOINT_DIR" && "$SKIP_HF_CONV" == "false" ]]; then
|
||||
echo ""
|
||||
echo "[STEP 1] 커스텀 체크포인트 → HF 포맷 변환"
|
||||
echo " 체크포인트: $CHECKPOINT_DIR"
|
||||
echo " 출력 : $INPUT_DIR"
|
||||
echo ""
|
||||
|
||||
if [[ ! -d "$CHECKPOINT_DIR" ]]; then
|
||||
echo "ERROR: 체크포인트 디렉토리를 찾을 수 없습니다: $CHECKPOINT_DIR"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
python "$PROJECT_DIR/scripts/convert_to_hf.py" \
|
||||
--checkpoint "$CHECKPOINT_DIR" \
|
||||
--output "$INPUT_DIR" \
|
||||
--tokenizer "tokenizer/korean_sp/tokenizer.json"
|
||||
|
||||
echo " [OK] HF 변환 완료 → $INPUT_DIR"
|
||||
elif [[ "$SKIP_HF_CONV" == "true" ]]; then
|
||||
echo "[STEP 1] HF 변환 건너뜀 (--skip_hf_conv)"
|
||||
else
|
||||
echo "[STEP 1] 체크포인트 미지정 — HF 디렉토리를 직접 사용합니다."
|
||||
fi
|
||||
|
||||
# HF 디렉토리 최종 검증
|
||||
if [[ ! -d "$INPUT_DIR" ]]; then
|
||||
echo "ERROR: HF 모델 디렉토리를 찾을 수 없습니다: $INPUT_DIR"
|
||||
echo " --checkpoint 옵션으로 체크포인트를 지정하거나,"
|
||||
echo " python scripts/convert_to_hf.py 를 먼저 실행하세요."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! -f "$INPUT_DIR/config.json" ]]; then
|
||||
echo "ERROR: config.json 이 없습니다: $INPUT_DIR/config.json"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "$OUT_DIR"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Step 2: llama.cpp 빌드 (llama-quantize 바이너리)
|
||||
# ---------------------------------------------------------------------------
|
||||
QUANTIZE_BIN="$LLAMA_CPP_DIR/build/bin/llama-quantize"
|
||||
|
||||
if [[ ! -f "$QUANTIZE_BIN" ]]; then
|
||||
echo ""
|
||||
echo "[STEP 2] llama.cpp 빌드 중 (llama-quantize) ..."
|
||||
cmake -S "$LLAMA_CPP_DIR" -B "$LLAMA_CPP_DIR/build" \
|
||||
-DCMAKE_BUILD_TYPE=Release \
|
||||
-DGGML_CUDA=ON \
|
||||
2>&1 | tail -10
|
||||
cmake --build "$LLAMA_CPP_DIR/build" --target llama-quantize -j "$(nproc)" \
|
||||
2>&1 | tail -10
|
||||
echo " [OK] 빌드 완료: $QUANTIZE_BIN"
|
||||
else
|
||||
echo "[STEP 2] llama-quantize 바이너리 이미 존재 — 빌드 건너뜀"
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Step 3: HF → F16 GGUF 변환
|
||||
# ---------------------------------------------------------------------------
|
||||
F16_GGUF="$OUT_DIR/${MODEL_NAME}-f16.gguf"
|
||||
|
||||
echo ""
|
||||
echo "[STEP 3] HF → F16 GGUF 변환"
|
||||
echo " 입력: $INPUT_DIR"
|
||||
echo " 출력: $F16_GGUF"
|
||||
echo ""
|
||||
|
||||
python "$LLAMA_CPP_DIR/convert_hf_to_gguf.py" "$INPUT_DIR" \
|
||||
--outfile "$F16_GGUF" \
|
||||
--outtype f16
|
||||
|
||||
echo " [OK] F16 GGUF 크기: $(du -sh "$F16_GGUF" | cut -f1) ($F16_GGUF)"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Step 4: 다중 양자화 (Q4_K_M, Q5_K_M, Q8_0)
|
||||
# ---------------------------------------------------------------------------
|
||||
if [[ "$SKIP_QUANT" == "true" ]]; then
|
||||
echo ""
|
||||
echo "[STEP 4] 양자화 건너뜀 (--skip_quant)"
|
||||
else
|
||||
echo ""
|
||||
echo "[STEP 4] 다중 양자화 시작 ..."
|
||||
|
||||
if [[ ! -f "$QUANTIZE_BIN" ]]; then
|
||||
echo "[WARN] llama-quantize 바이너리를 찾을 수 없습니다: $QUANTIZE_BIN"
|
||||
echo " 양자화를 건너뜁니다. F16 GGUF만 생성되었습니다."
|
||||
echo " 수동 빌드: cmake --build $LLAMA_CPP_DIR/build --target llama-quantize"
|
||||
else
|
||||
# Q4_K_M — 가장 작은 크기, 품질/속도 균형 (Ollama 기본 권장)
|
||||
Q4KM_GGUF="$OUT_DIR/${MODEL_NAME}-Q4_K_M.gguf"
|
||||
echo " → Q4_K_M 양자화: $Q4KM_GGUF ..."
|
||||
"$QUANTIZE_BIN" "$F16_GGUF" "$Q4KM_GGUF" Q4_K_M
|
||||
echo " 크기: $(du -sh "$Q4KM_GGUF" | cut -f1)"
|
||||
|
||||
# Q5_K_M — 중간 크기, 더 높은 품질
|
||||
Q5KM_GGUF="$OUT_DIR/${MODEL_NAME}-Q5_K_M.gguf"
|
||||
echo " → Q5_K_M 양자화: $Q5KM_GGUF ..."
|
||||
"$QUANTIZE_BIN" "$F16_GGUF" "$Q5KM_GGUF" Q5_K_M
|
||||
echo " 크기: $(du -sh "$Q5KM_GGUF" | cut -f1)"
|
||||
|
||||
# Q8_0 — 가장 높은 품질 (F16 근사)
|
||||
Q8_GGUF="$OUT_DIR/${MODEL_NAME}-Q8_0.gguf"
|
||||
echo " → Q8_0 양자화: $Q8_GGUF ..."
|
||||
"$QUANTIZE_BIN" "$F16_GGUF" "$Q8_GGUF" Q8_0
|
||||
echo " 크기: $(du -sh "$Q8_GGUF" | cut -f1)"
|
||||
|
||||
echo ""
|
||||
echo " [OK] 모든 양자화 완료"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 완료 요약
|
||||
# ---------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "=================================================================="
|
||||
echo " 3B GGUF 변환 완료"
|
||||
echo ""
|
||||
echo " 출력 파일 목록:"
|
||||
ls -lh "$OUT_DIR/${MODEL_NAME}"*.gguf 2>/dev/null | awk '{print " " $5 " " $9}' || \
|
||||
echo " (파일 목록 확인: ls -lh $OUT_DIR/)"
|
||||
echo ""
|
||||
echo " 다음 단계:"
|
||||
echo " bash scripts/deploy_3b_ollama.sh"
|
||||
echo " bash scripts/quality_gate.sh deploy"
|
||||
echo "=================================================================="
|
||||
92
source/scripts/convert_to_gguf.sh
Normal file
92
source/scripts/convert_to_gguf.sh
Normal file
@@ -0,0 +1,92 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# convert_to_gguf.sh — HuggingFace 포맷 모델을 GGUF로 변환 + Q4_K_M 양자화
|
||||
#
|
||||
# Usage:
|
||||
# bash scripts/convert_to_gguf.sh [hf_dir] [out_dir]
|
||||
#
|
||||
# hf_dir : HF 포맷 모델 디렉토리 (default: outputs/hf)
|
||||
# out_dir : GGUF 출력 디렉토리 (default: outputs/gguf)
|
||||
#
|
||||
# Outputs:
|
||||
# outputs/gguf/korean-1b-f16.gguf — F16 GGUF
|
||||
# outputs/gguf/korean-1b-q4km.gguf — Q4_K_M 양자화 (Ollama용)
|
||||
#
|
||||
# 전제 조건:
|
||||
# - python scripts/convert_to_hf.py 로 HF 변환 완료
|
||||
# - git, cmake, make 설치
|
||||
# - pip install safetensors (없으면 pytorch_model.bin으로 fallback)
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
|
||||
HF_DIR="${1:-outputs/hf}"
|
||||
OUT_DIR="${2:-outputs/gguf}"
|
||||
LLAMA_CPP_DIR="outputs/llama.cpp"
|
||||
PROJECT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
|
||||
cd "$PROJECT_DIR"
|
||||
|
||||
# --- Pre-flight check -------------------------------------------------------
|
||||
if [[ ! -d "$HF_DIR" ]]; then
|
||||
echo "ERROR: HF model directory not found: $HF_DIR"
|
||||
echo "Run first: python scripts/convert_to_hf.py --checkpoint <ckpt> --output $HF_DIR"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! -f "$HF_DIR/config.json" ]]; then
|
||||
echo "ERROR: config.json not found in $HF_DIR"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "$OUT_DIR"
|
||||
|
||||
# --- Clone llama.cpp if not present -----------------------------------------
|
||||
if [[ ! -d "$LLAMA_CPP_DIR" ]]; then
|
||||
echo "Cloning llama.cpp ..."
|
||||
git clone --depth 1 https://github.com/ggerganov/llama.cpp "$LLAMA_CPP_DIR"
|
||||
fi
|
||||
|
||||
# Install Python requirements for conversion script
|
||||
echo "Installing llama.cpp Python deps ..."
|
||||
pip install -r "$LLAMA_CPP_DIR/requirements.txt" --break-system-packages -q
|
||||
|
||||
# --- Build llama.cpp (for quantization binary) ------------------------------
|
||||
QUANTIZE_BIN="$LLAMA_CPP_DIR/build/bin/llama-quantize"
|
||||
if [[ ! -f "$QUANTIZE_BIN" ]]; then
|
||||
echo "Building llama.cpp (quantization tool) ..."
|
||||
cmake -S "$LLAMA_CPP_DIR" -B "$LLAMA_CPP_DIR/build" \
|
||||
-DCMAKE_BUILD_TYPE=Release \
|
||||
-DGGML_CUDA=ON \
|
||||
2>&1 | tail -5
|
||||
cmake --build "$LLAMA_CPP_DIR/build" --target llama-quantize -j "$(nproc)" \
|
||||
2>&1 | tail -5
|
||||
fi
|
||||
|
||||
# --- F16 GGUF conversion ---------------------------------------------------
|
||||
F16_GGUF="$OUT_DIR/korean-1b-f16.gguf"
|
||||
echo "Converting to F16 GGUF: $F16_GGUF ..."
|
||||
python "$LLAMA_CPP_DIR/convert_hf_to_gguf.py" "$HF_DIR" \
|
||||
--outfile "$F16_GGUF" \
|
||||
--outtype f16
|
||||
|
||||
echo "F16 GGUF size: $(du -sh "$F16_GGUF" | cut -f1)"
|
||||
|
||||
# --- Q4_K_M quantization ---------------------------------------------------
|
||||
Q4KM_GGUF="$OUT_DIR/korean-1b-q4km.gguf"
|
||||
if [[ -f "$QUANTIZE_BIN" ]]; then
|
||||
echo "Quantizing to Q4_K_M: $Q4KM_GGUF ..."
|
||||
"$QUANTIZE_BIN" "$F16_GGUF" "$Q4KM_GGUF" Q4_K_M
|
||||
echo "Q4_K_M GGUF size: $(du -sh "$Q4KM_GGUF" | cut -f1)"
|
||||
else
|
||||
echo "[WARN] llama-quantize binary not found. Using F16 GGUF for Ollama."
|
||||
echo " Build: cmake --build $LLAMA_CPP_DIR/build --target llama-quantize"
|
||||
cp "$F16_GGUF" "$Q4KM_GGUF"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "=================================================================="
|
||||
echo " GGUF 변환 완료"
|
||||
echo " F16 : $F16_GGUF"
|
||||
echo " Q4KM: $Q4KM_GGUF"
|
||||
echo " 다음 단계: bash scripts/deploy_ollama.sh"
|
||||
echo "=================================================================="
|
||||
262
source/scripts/convert_to_hf.py
Normal file
262
source/scripts/convert_to_hf.py
Normal file
@@ -0,0 +1,262 @@
|
||||
"""
|
||||
Convert custom LLM checkpoint to HuggingFace LlamaForCausalLM format.
|
||||
|
||||
Usage:
|
||||
python scripts/convert_to_hf.py \\
|
||||
--checkpoint checkpoints/korean_1b_fp8_run1/checkpoint-0034000 \\
|
||||
--output outputs/hf \\
|
||||
[--tokenizer tokenizer/korean_sp/tokenizer.json]
|
||||
|
||||
Outputs (in --output directory):
|
||||
config.json — LlamaConfig
|
||||
model.safetensors — converted weights
|
||||
tokenizer.json — tokenizer (copied)
|
||||
tokenizer_config.json
|
||||
generation_config.json
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import shutil
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import torch
|
||||
|
||||
_PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
||||
if str(_PROJECT_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(_PROJECT_ROOT))
|
||||
|
||||
from model.config import LMConfig
|
||||
|
||||
|
||||
def remap_weights(
|
||||
src_state_dict: dict,
|
||||
config: LMConfig,
|
||||
) -> dict:
|
||||
"""
|
||||
Remap custom LLM weight names to HuggingFace LlamaForCausalLM names.
|
||||
|
||||
Handles both FP8 (te.LayerNormMLP / te.Linear) and BF16 (SwiGLU / nn.Linear)
|
||||
checkpoints transparently.
|
||||
"""
|
||||
dst = {}
|
||||
is_fp8 = config.use_fp8
|
||||
|
||||
# --- Token embedding ---
|
||||
dst["model.embed_tokens.weight"] = src_state_dict["embedding.weight"].float()
|
||||
|
||||
for i in range(config.n_layers):
|
||||
pfx = f"layers.{i}"
|
||||
hpfx = f"model.layers.{i}"
|
||||
|
||||
# Attention norm (always RMSNorm)
|
||||
dst[f"{hpfx}.input_layernorm.weight"] = (
|
||||
src_state_dict[f"{pfx}.attn_norm.weight"].float()
|
||||
)
|
||||
|
||||
# Attention projections
|
||||
# Handle fused QKV (te.Linear with qkv_proj) vs separate q/k/v
|
||||
qkv_key = f"{pfx}.attn.qkv_proj.weight"
|
||||
if qkv_key in src_state_dict:
|
||||
# Fused QKV: [Q_dim + K_dim + V_dim, d_model]
|
||||
# GQA: Q = n_heads * head_dim, K = V = n_kv_heads * head_dim
|
||||
qkv = src_state_dict[qkv_key].float()
|
||||
head_dim = config.d_model // config.n_heads
|
||||
q_dim = config.n_heads * head_dim # e.g. 24 * 128 = 3072
|
||||
k_dim = config.n_kv_heads * head_dim # e.g. 8 * 128 = 1024
|
||||
v_dim = config.n_kv_heads * head_dim # e.g. 8 * 128 = 1024
|
||||
assert qkv.shape[0] == q_dim + k_dim + v_dim, (
|
||||
f"QKV shape mismatch: {qkv.shape[0]} != {q_dim}+{k_dim}+{v_dim}"
|
||||
)
|
||||
dst[f"{hpfx}.self_attn.q_proj.weight"] = qkv[:q_dim]
|
||||
dst[f"{hpfx}.self_attn.k_proj.weight"] = qkv[q_dim:q_dim + k_dim]
|
||||
dst[f"{hpfx}.self_attn.v_proj.weight"] = qkv[q_dim + k_dim:]
|
||||
else:
|
||||
# Separate q/k/v projections
|
||||
for src_name, dst_name in [
|
||||
("q_proj", "self_attn.q_proj"),
|
||||
("k_proj", "self_attn.k_proj"),
|
||||
("v_proj", "self_attn.v_proj"),
|
||||
]:
|
||||
w_key = f"{pfx}.attn.{src_name}.weight"
|
||||
if w_key in src_state_dict:
|
||||
dst[f"{hpfx}.{dst_name}.weight"] = src_state_dict[w_key].float()
|
||||
|
||||
# Output projection
|
||||
out_key = f"{pfx}.attn.out_proj.weight"
|
||||
if out_key in src_state_dict:
|
||||
dst[f"{hpfx}.self_attn.o_proj.weight"] = src_state_dict[out_key].float()
|
||||
|
||||
# FFN — FP8 (te.LayerNormMLP) vs BF16 (SwiGLU)
|
||||
if is_fp8 and f"{pfx}.ffn.layer_norm_weight" in src_state_dict:
|
||||
# te.LayerNormMLP: RMSNorm is fused inside
|
||||
dst[f"{hpfx}.post_attention_layernorm.weight"] = (
|
||||
src_state_dict[f"{pfx}.ffn.layer_norm_weight"].float()
|
||||
)
|
||||
# fc1_weight: [2*d_ffn, d_model] — gate and up are concatenated
|
||||
fc1 = src_state_dict[f"{pfx}.ffn.fc1_weight"].float()
|
||||
half = fc1.shape[0] // 2
|
||||
dst[f"{hpfx}.mlp.gate_proj.weight"] = fc1[:half]
|
||||
dst[f"{hpfx}.mlp.up_proj.weight"] = fc1[half:]
|
||||
# fc2_weight: [d_model, d_ffn]
|
||||
dst[f"{hpfx}.mlp.down_proj.weight"] = (
|
||||
src_state_dict[f"{pfx}.ffn.fc2_weight"].float()
|
||||
)
|
||||
else:
|
||||
# Standard SwiGLU (BF16 checkpoint)
|
||||
dst[f"{hpfx}.post_attention_layernorm.weight"] = (
|
||||
src_state_dict[f"{pfx}.ffn_norm.weight"].float()
|
||||
)
|
||||
dst[f"{hpfx}.mlp.gate_proj.weight"] = (
|
||||
src_state_dict[f"{pfx}.ffn.gate_proj.weight"].float()
|
||||
)
|
||||
dst[f"{hpfx}.mlp.up_proj.weight"] = (
|
||||
src_state_dict[f"{pfx}.ffn.up_proj.weight"].float()
|
||||
)
|
||||
dst[f"{hpfx}.mlp.down_proj.weight"] = (
|
||||
src_state_dict[f"{pfx}.ffn.down_proj.weight"].float()
|
||||
)
|
||||
|
||||
# --- Final norm and LM head ---
|
||||
dst["model.norm.weight"] = src_state_dict["norm.weight"].float()
|
||||
# Weight tying: embedding.weight == lm_head.weight in our model.
|
||||
# HF LlamaForCausalLM expects lm_head.weight explicitly.
|
||||
dst["lm_head.weight"] = src_state_dict["embedding.weight"].float().clone()
|
||||
|
||||
return dst
|
||||
|
||||
|
||||
def build_llama_config(config: LMConfig) -> dict:
|
||||
"""Map LMConfig fields to HuggingFace LlamaConfig dict."""
|
||||
return {
|
||||
"architectures": ["LlamaForCausalLM"],
|
||||
"model_type": "llama",
|
||||
"hidden_size": config.d_model,
|
||||
"intermediate_size": config.d_ffn,
|
||||
"num_hidden_layers": config.n_layers,
|
||||
"num_attention_heads": config.n_heads,
|
||||
"num_key_value_heads": config.n_kv_heads,
|
||||
"hidden_act": "silu",
|
||||
"max_position_embeddings": config.max_seq_len,
|
||||
"initializer_range": 0.02,
|
||||
"rms_norm_eps": 1e-5,
|
||||
"vocab_size": config.vocab_size,
|
||||
"rope_theta": config.rope_theta,
|
||||
"rope_scaling": None,
|
||||
"attention_bias": config.bias,
|
||||
"tie_word_embeddings": True,
|
||||
"torch_dtype": "float16",
|
||||
"transformers_version": "4.40.0",
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Convert custom LLM checkpoint to HuggingFace LlamaForCausalLM format."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--checkpoint",
|
||||
required=True,
|
||||
type=Path,
|
||||
help="Path to checkpoint directory (must contain model.pt + config.yaml).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
required=True,
|
||||
type=Path,
|
||||
help="Output directory for HF-format files.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--tokenizer",
|
||||
type=Path,
|
||||
default=Path("tokenizer/korean_sp/tokenizer.json"),
|
||||
help="Path to tokenizer.json (default: tokenizer/korean_sp/tokenizer.json).",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
ckpt_path = args.checkpoint
|
||||
out_path = args.output
|
||||
|
||||
if not ckpt_path.exists():
|
||||
raise FileNotFoundError(f"Checkpoint not found: {ckpt_path}")
|
||||
|
||||
out_path.mkdir(parents=True, exist_ok=True)
|
||||
print(f"Checkpoint : {ckpt_path}")
|
||||
print(f"Output : {out_path}")
|
||||
|
||||
# Load config
|
||||
config = LMConfig.from_yaml(ckpt_path / "config.yaml")
|
||||
print(f"Model : d_model={config.d_model}, n_layers={config.n_layers}, "
|
||||
f"vocab_size={config.vocab_size}, use_fp8={config.use_fp8}")
|
||||
|
||||
# Load weights
|
||||
print("Loading model.pt ...")
|
||||
state_dict = torch.load(
|
||||
ckpt_path / "model.pt",
|
||||
map_location="cpu",
|
||||
weights_only=True,
|
||||
)
|
||||
print(f" Source keys: {len(state_dict)}")
|
||||
|
||||
# Remap
|
||||
print("Remapping weight names ...")
|
||||
hf_state_dict = remap_weights(state_dict, config)
|
||||
print(f" Destination keys: {len(hf_state_dict)}")
|
||||
|
||||
# Save safetensors
|
||||
print("Saving model.safetensors ...")
|
||||
try:
|
||||
from safetensors.torch import save_file
|
||||
save_file(hf_state_dict, out_path / "model.safetensors")
|
||||
except ImportError:
|
||||
print(" [WARN] safetensors not installed; falling back to pytorch_model.bin")
|
||||
torch.save(hf_state_dict, out_path / "pytorch_model.bin")
|
||||
|
||||
# Save config.json
|
||||
llama_cfg = build_llama_config(config)
|
||||
with open(out_path / "config.json", "w", encoding="utf-8") as f:
|
||||
json.dump(llama_cfg, f, indent=2, ensure_ascii=False)
|
||||
print("Saved config.json")
|
||||
|
||||
# Save generation_config.json
|
||||
gen_cfg = {
|
||||
"bos_token_id": 1,
|
||||
"eos_token_id": 2,
|
||||
"pad_token_id": 0,
|
||||
"max_new_tokens": 512,
|
||||
"temperature": 0.8,
|
||||
"top_p": 0.9,
|
||||
"do_sample": True,
|
||||
}
|
||||
with open(out_path / "generation_config.json", "w", encoding="utf-8") as f:
|
||||
json.dump(gen_cfg, f, indent=2, ensure_ascii=False)
|
||||
|
||||
# Copy tokenizer
|
||||
tok_src = args.tokenizer
|
||||
if tok_src.exists():
|
||||
shutil.copy(tok_src, out_path / "tokenizer.json")
|
||||
# Minimal tokenizer_config.json for HF compatibility
|
||||
tok_cfg = {
|
||||
"model_type": "llama",
|
||||
"tokenizer_class": "PreTrainedTokenizerFast",
|
||||
"bos_token": "<s>",
|
||||
"eos_token": "</s>",
|
||||
"unk_token": "<unk>",
|
||||
"pad_token": "<pad>",
|
||||
"clean_up_tokenization_spaces": False,
|
||||
}
|
||||
with open(out_path / "tokenizer_config.json", "w", encoding="utf-8") as f:
|
||||
json.dump(tok_cfg, f, indent=2, ensure_ascii=False)
|
||||
print(f"Copied tokenizer: {tok_src} -> {out_path / 'tokenizer.json'}")
|
||||
else:
|
||||
print(f"[WARN] Tokenizer not found at {tok_src}. Copy manually.")
|
||||
|
||||
print(f"\nDone! HF model saved to: {out_path}")
|
||||
print("Verify: ls -lh", out_path)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
146
source/scripts/deploy_3b_ollama.sh
Normal file
146
source/scripts/deploy_3b_ollama.sh
Normal file
@@ -0,0 +1,146 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# deploy_3b_ollama.sh — 3B GGUF 모델을 Ollama에 등록 & 자동 테스트
|
||||
#
|
||||
# Usage:
|
||||
# bash scripts/deploy_3b_ollama.sh [model_name]
|
||||
#
|
||||
# model_name: Ollama 모델 이름 (default: frankenstallm-3b)
|
||||
#
|
||||
# 전제 조건:
|
||||
# - ollama 설치: https://ollama.com/download
|
||||
# - bash scripts/convert_3b_gguf.sh 실행 완료
|
||||
# - outputs/gguf/frankenstallm-3b-Q4_K_M.gguf 존재
|
||||
# - Modelfile.3b 존재
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
|
||||
MODEL_NAME="${1:-frankenstallm-3b}"
|
||||
PROJECT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
MODELFILE="$PROJECT_DIR/Modelfile.3b"
|
||||
GGUF_PATH="$PROJECT_DIR/outputs/gguf/frankenstallm-3b-Q4_K_M.gguf"
|
||||
|
||||
cd "$PROJECT_DIR"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pre-flight check
|
||||
# ---------------------------------------------------------------------------
|
||||
if ! command -v ollama &> /dev/null; then
|
||||
echo "ERROR: ollama가 설치되어 있지 않습니다."
|
||||
echo "설치: curl -fsSL https://ollama.com/install.sh | sh"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! -f "$GGUF_PATH" ]]; then
|
||||
echo "ERROR: GGUF 파일을 찾을 수 없습니다: $GGUF_PATH"
|
||||
echo "먼저 실행: bash scripts/convert_3b_gguf.sh"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! -f "$MODELFILE" ]]; then
|
||||
echo "ERROR: Modelfile.3b 를 찾을 수 없습니다: $MODELFILE"
|
||||
echo " 프로젝트 루트에 Modelfile.3b 가 있어야 합니다."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "=================================================================="
|
||||
echo " 3B 모델 Ollama 배포"
|
||||
echo " 모델명 : $MODEL_NAME"
|
||||
echo " GGUF : $(du -sh "$GGUF_PATH" | cut -f1) ($GGUF_PATH)"
|
||||
echo " Modelfile: $MODELFILE"
|
||||
echo "=================================================================="
|
||||
echo ""
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Ollama 서버 실행 확인
|
||||
# ---------------------------------------------------------------------------
|
||||
if ! ollama list &>/dev/null; then
|
||||
echo "[WARN] Ollama 서버가 응답하지 않습니다. 백그라운드로 시작합니다 ..."
|
||||
ollama serve &>/tmp/ollama_serve.log &
|
||||
OLLAMA_PID=$!
|
||||
echo " PID: $OLLAMA_PID (로그: /tmp/ollama_serve.log)"
|
||||
# 서버 준비 대기 (최대 15초)
|
||||
for i in $(seq 1 15); do
|
||||
if ollama list &>/dev/null 2>&1; then
|
||||
echo " [OK] Ollama 서버 준비 완료 (${i}초)"
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Ollama 모델 등록
|
||||
# ---------------------------------------------------------------------------
|
||||
echo "[1/2] Ollama 모델 등록 중: $MODEL_NAME ..."
|
||||
ollama create "$MODEL_NAME" -f "$MODELFILE"
|
||||
echo " [OK] 등록 완료"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 자동 테스트 프롬프트 5개 실행
|
||||
# ---------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "[2/2] 자동 테스트 프롬프트 실행 (5개) ..."
|
||||
echo ""
|
||||
|
||||
declare -a TEST_PROMPTS=(
|
||||
"안녕하세요! 간단히 자기소개를 해주세요."
|
||||
"대한민국의 수도는 어디인가요? 그 도시의 특징을 설명해주세요."
|
||||
"파이썬으로 피보나치 수열을 출력하는 함수를 작성해주세요."
|
||||
"인공지능이 사회에 미치는 긍정적인 영향 3가지를 설명해주세요."
|
||||
"오늘 저녁 메뉴로 무엇을 추천해주시겠어요? 이유도 함께 말씀해주세요."
|
||||
)
|
||||
|
||||
PASS_COUNT=0
|
||||
FAIL_COUNT=0
|
||||
TOTAL=${#TEST_PROMPTS[@]}
|
||||
|
||||
for i in "${!TEST_PROMPTS[@]}"; do
|
||||
PROMPT="${TEST_PROMPTS[$i]}"
|
||||
NUM=$((i + 1))
|
||||
echo "--- 테스트 $NUM/$TOTAL ---"
|
||||
echo "프롬프트: $PROMPT"
|
||||
echo ""
|
||||
|
||||
# ollama run: 타임아웃 60초, 응답 첫 300자만 표시
|
||||
if RESPONSE=$(timeout 60 ollama run "$MODEL_NAME" "$PROMPT" 2>&1); then
|
||||
RESP_PREVIEW="${RESPONSE:0:300}"
|
||||
echo "응답: $RESP_PREVIEW"
|
||||
if [[ ${#RESPONSE} -gt 300 ]]; then
|
||||
echo " ... (총 ${#RESPONSE}자)"
|
||||
fi
|
||||
echo "[OK] 테스트 $NUM 성공"
|
||||
PASS_COUNT=$((PASS_COUNT + 1))
|
||||
else
|
||||
EXIT_CODE=$?
|
||||
echo "[FAIL] 테스트 $NUM 실패 (exit code: $EXIT_CODE)"
|
||||
FAIL_COUNT=$((FAIL_COUNT + 1))
|
||||
fi
|
||||
echo ""
|
||||
done
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 결과 요약
|
||||
# ---------------------------------------------------------------------------
|
||||
echo "=================================================================="
|
||||
echo " 배포 & 테스트 완료"
|
||||
echo ""
|
||||
echo " 모델명 : $MODEL_NAME"
|
||||
echo " 테스트 : $PASS_COUNT/$TOTAL 성공 ($FAIL_COUNT 실패)"
|
||||
echo ""
|
||||
if [[ $FAIL_COUNT -eq 0 ]]; then
|
||||
echo " [PASS] 모든 테스트 통과"
|
||||
else
|
||||
echo " [WARN] 일부 테스트 실패 — 로그를 확인하세요"
|
||||
fi
|
||||
echo ""
|
||||
echo " Ollama 사용법:"
|
||||
echo " ollama run $MODEL_NAME"
|
||||
echo " ollama run $MODEL_NAME '질문을 여기에 입력하세요'"
|
||||
echo " ollama rm $MODEL_NAME (삭제)"
|
||||
echo ""
|
||||
echo " Quality Gate:"
|
||||
echo " bash scripts/quality_gate.sh deploy"
|
||||
echo "=================================================================="
|
||||
|
||||
[[ $FAIL_COUNT -gt 0 ]] && exit 1 || exit 0
|
||||
118
source/scripts/deploy_ollama.sh
Normal file
118
source/scripts/deploy_ollama.sh
Normal file
@@ -0,0 +1,118 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# deploy_ollama.sh — FRANKENSTALLM 3B GGUF → Ollama 원클릭 배포
|
||||
#
|
||||
# Usage:
|
||||
# bash scripts/deploy_ollama.sh # 기본 (Q4_K_M)
|
||||
# bash scripts/deploy_ollama.sh --quant Q8_0 # Q8_0 양자화
|
||||
# bash scripts/deploy_ollama.sh --skip_convert # GGUF 이미 존재 시
|
||||
#
|
||||
# Pipeline:
|
||||
# 1. [선택] GGUF 변환 + 양자화 (convert_3b_gguf.sh)
|
||||
# 2. Ollama 설치 확인 / 서버 시작
|
||||
# 3. Modelfile.3b로 모델 등록
|
||||
# 4. 자동 테스트 (5개 프롬프트)
|
||||
# 5. 반복률 검증 (15개 프롬프트)
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
|
||||
QUANT="${QUANT:-Q4_K_M}"
|
||||
MODEL_NAME="frankenstallm-3b"
|
||||
SKIP_CONVERT=false
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--quant) QUANT="$2"; shift 2 ;;
|
||||
--skip_convert) SKIP_CONVERT=true; shift ;;
|
||||
-h|--help)
|
||||
grep '^#' "$0" | head -20 | sed 's/^# \{0,1\}//'
|
||||
exit 0 ;;
|
||||
*) echo "ERROR: 알 수 없는 옵션: $1"; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
PROJECT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
cd "$PROJECT_DIR"
|
||||
|
||||
GGUF_PATH="outputs/gguf/frankenstallm-3b-${QUANT}.gguf"
|
||||
MODELFILE="Modelfile.3b"
|
||||
|
||||
echo "=================================================================="
|
||||
echo " FRANKENSTALLM 3B Ollama 배포"
|
||||
echo " 양자화 : $QUANT"
|
||||
echo " GGUF : $GGUF_PATH"
|
||||
echo " Modelfile: $MODELFILE"
|
||||
echo "=================================================================="
|
||||
|
||||
# ---- Step 1: GGUF 변환 (필요 시) ----
|
||||
if [[ "$SKIP_CONVERT" == "false" ]]; then
|
||||
if [[ ! -f "$GGUF_PATH" ]]; then
|
||||
echo ""
|
||||
echo "[Step 1] GGUF 변환 실행 중 ..."
|
||||
bash scripts/convert_3b_gguf.sh \
|
||||
--input_dir checkpoints/korean_3b_orpo_v1/checkpoint-9840
|
||||
else
|
||||
echo "[Step 1] GGUF 파일 이미 존재 — 변환 건너뜀"
|
||||
fi
|
||||
else
|
||||
echo "[Step 1] 변환 건너뜀 (--skip_convert)"
|
||||
fi
|
||||
|
||||
if [[ ! -f "$GGUF_PATH" ]]; then
|
||||
echo "ERROR: GGUF 파일 없음: $GGUF_PATH"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo " GGUF 크기: $(du -sh "$GGUF_PATH" | cut -f1)"
|
||||
|
||||
# ---- Step 2: Ollama 설치 확인 ----
|
||||
if ! command -v ollama &>/dev/null; then
|
||||
echo ""
|
||||
echo "[Step 2] Ollama 미설치 — 설치 중 ..."
|
||||
curl -fsSL https://ollama.com/install.sh | sh
|
||||
fi
|
||||
|
||||
# Ollama 서버 시작
|
||||
if ! ollama list &>/dev/null 2>&1; then
|
||||
echo "[Step 2] Ollama 서버 시작 중 ..."
|
||||
ollama serve &>/tmp/ollama_serve.log &
|
||||
for i in $(seq 1 15); do
|
||||
if ollama list &>/dev/null 2>&1; then
|
||||
echo " [OK] Ollama 서버 준비 (${i}초)"
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
fi
|
||||
|
||||
# ---- Step 3: 모델 등록 ----
|
||||
echo ""
|
||||
echo "[Step 3] Ollama 모델 등록: $MODEL_NAME"
|
||||
ollama create "$MODEL_NAME" -f "$MODELFILE"
|
||||
echo " [OK] 등록 완료"
|
||||
|
||||
# ---- Step 4: 자동 테스트 ----
|
||||
echo ""
|
||||
echo "[Step 4] 자동 테스트 ..."
|
||||
declare -a QUICK_TESTS=(
|
||||
"대한민국의 수도는?"
|
||||
"인공지능이란 무엇인가요?"
|
||||
"한국의 전통 음식 중에서 김치에 대해 설명해주세요."
|
||||
)
|
||||
|
||||
for prompt in "${QUICK_TESTS[@]}"; do
|
||||
echo " Q: $prompt"
|
||||
RESP=$(timeout 60 ollama run "$MODEL_NAME" "$prompt" 2>&1 || echo "[TIMEOUT/ERROR]")
|
||||
echo " A: ${RESP:0:200}"
|
||||
echo ""
|
||||
done
|
||||
|
||||
# ---- Step 5: 반복률 검증 ----
|
||||
echo "[Step 5] 반복률 검증 (15개 프롬프트) ..."
|
||||
python3 scripts/test_ollama_repetition.py --model "$MODEL_NAME"
|
||||
|
||||
echo ""
|
||||
echo "=================================================================="
|
||||
echo " 배포 완료!"
|
||||
echo " 사용법: ollama run $MODEL_NAME"
|
||||
echo "=================================================================="
|
||||
235
source/scripts/fix_tokenizer_byte_fallback.py
Normal file
235
source/scripts/fix_tokenizer_byte_fallback.py
Normal file
@@ -0,0 +1,235 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fix GGUF newline crash by adding byte-fallback tokens to the tokenizer.
|
||||
|
||||
Problem: The SentencePiece Unigram tokenizer was trained without byte_fallback=True,
|
||||
so characters like \n have no token representation. llama.cpp crashes when it
|
||||
encounters these characters because there's no byte-fallback.
|
||||
|
||||
Fix:
|
||||
1. Add 256 byte-fallback tokens (<0x00> .. <0xFF>) to tokenizer.json
|
||||
2. Resize model embeddings from 64000 -> 64256
|
||||
3. Update config.json vocab_size
|
||||
4. Copy tokenizer.model for proper GGUF conversion
|
||||
|
||||
Usage:
|
||||
python scripts/fix_tokenizer_byte_fallback.py \
|
||||
--input outputs/hf_checkpoint-best \
|
||||
--output outputs/hf_checkpoint-best-fixed \
|
||||
--sp_model tokenizer/korean_sp/tokenizer.model
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
import torch
|
||||
from safetensors.torch import load_file, save_file
|
||||
|
||||
|
||||
BYTE_FALLBACK_COUNT = 256
|
||||
BYTE_TOKEN_TEMPLATE = "<0x{:02X}>"
|
||||
|
||||
|
||||
def fix_tokenizer_json(input_path: Path, output_path: Path):
|
||||
"""Add byte_fallback=True and 256 byte tokens to tokenizer.json."""
|
||||
with open(input_path) as f:
|
||||
tok = json.load(f)
|
||||
|
||||
model = tok["model"]
|
||||
vocab = model["vocab"] # list of [piece, score]
|
||||
original_size = len(vocab)
|
||||
|
||||
# Enable byte_fallback
|
||||
model["byte_fallback"] = True
|
||||
|
||||
# Add 256 byte tokens with very low score (they're fallback only)
|
||||
for i in range(BYTE_FALLBACK_COUNT):
|
||||
byte_token = BYTE_TOKEN_TEMPLATE.format(i)
|
||||
vocab.append([byte_token, 0.0])
|
||||
|
||||
new_size = len(vocab)
|
||||
print(f" Vocab: {original_size} -> {new_size} (+{BYTE_FALLBACK_COUNT} byte tokens)")
|
||||
print(f" byte_fallback: False -> True")
|
||||
|
||||
# Also add byte tokens to added_tokens list
|
||||
added = tok.get("added_tokens", [])
|
||||
for i in range(BYTE_FALLBACK_COUNT):
|
||||
byte_token = BYTE_TOKEN_TEMPLATE.format(i)
|
||||
added.append({
|
||||
"id": original_size + i,
|
||||
"content": byte_token,
|
||||
"single_word": False,
|
||||
"lstrip": False,
|
||||
"rstrip": False,
|
||||
"normalized": False,
|
||||
"special": True,
|
||||
})
|
||||
tok["added_tokens"] = added
|
||||
|
||||
with open(output_path, "w") as f:
|
||||
json.dump(tok, f, ensure_ascii=False, indent=2)
|
||||
|
||||
return original_size, new_size
|
||||
|
||||
|
||||
def fix_config_json(input_path: Path, output_path: Path, new_vocab_size: int):
|
||||
"""Update vocab_size in config.json."""
|
||||
with open(input_path) as f:
|
||||
config = json.load(f)
|
||||
|
||||
old_size = config["vocab_size"]
|
||||
config["vocab_size"] = new_vocab_size
|
||||
print(f" config.json vocab_size: {old_size} -> {new_vocab_size}")
|
||||
|
||||
with open(output_path, "w") as f:
|
||||
json.dump(config, f, indent=2)
|
||||
|
||||
|
||||
def resize_embeddings(input_path: Path, output_path: Path,
|
||||
old_vocab: int, new_vocab: int, tie_embeddings: bool):
|
||||
"""Resize embedding and lm_head weights to accommodate new tokens."""
|
||||
print(f" Loading model weights from {input_path} ...")
|
||||
state_dict = load_file(str(input_path))
|
||||
|
||||
embed_key = "model.embed_tokens.weight"
|
||||
lm_head_key = "lm_head.weight"
|
||||
|
||||
if embed_key not in state_dict:
|
||||
raise KeyError(f"{embed_key} not found in state_dict. Keys: {list(state_dict.keys())[:10]}")
|
||||
|
||||
embed = state_dict[embed_key]
|
||||
print(f" embed_tokens shape: {embed.shape}")
|
||||
|
||||
hidden_size = embed.shape[1]
|
||||
extra = new_vocab - old_vocab
|
||||
|
||||
# Initialize new embeddings as mean of existing (better than random for byte tokens)
|
||||
mean_embed = embed.mean(dim=0, keepdim=True)
|
||||
# Add small noise to avoid identical embeddings
|
||||
noise = torch.randn(extra, hidden_size, dtype=embed.dtype) * 0.01
|
||||
new_rows = mean_embed.expand(extra, -1) + noise
|
||||
|
||||
new_embed = torch.cat([embed, new_rows], dim=0)
|
||||
state_dict[embed_key] = new_embed
|
||||
print(f" embed_tokens resized: {embed.shape} -> {new_embed.shape}")
|
||||
|
||||
if tie_embeddings:
|
||||
# When tie_word_embeddings=True, lm_head shares embed_tokens
|
||||
# Remove lm_head if present (it will be tied automatically)
|
||||
if lm_head_key in state_dict:
|
||||
del state_dict[lm_head_key]
|
||||
print(f" lm_head removed (tie_word_embeddings=True)")
|
||||
else:
|
||||
if lm_head_key in state_dict:
|
||||
lm_head = state_dict[lm_head_key]
|
||||
mean_lm = lm_head.mean(dim=0, keepdim=True)
|
||||
noise_lm = torch.randn(extra, hidden_size, dtype=lm_head.dtype) * 0.01
|
||||
new_lm = torch.cat([lm_head, mean_lm.expand(extra, -1) + noise_lm], dim=0)
|
||||
state_dict[lm_head_key] = new_lm
|
||||
print(f" lm_head resized: {lm_head.shape} -> {new_lm.shape}")
|
||||
|
||||
print(f" Saving to {output_path} ...")
|
||||
save_file(state_dict, str(output_path))
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Fix tokenizer byte-fallback for GGUF")
|
||||
parser.add_argument("--input", type=Path, required=True, help="Input HF checkpoint dir")
|
||||
parser.add_argument("--output", type=Path, required=True, help="Output fixed HF checkpoint dir")
|
||||
parser.add_argument("--sp_model", type=Path, default=None,
|
||||
help="SentencePiece .model file to copy (for GGUF conversion)")
|
||||
args = parser.parse_args()
|
||||
|
||||
input_dir = args.input
|
||||
output_dir = args.output
|
||||
|
||||
if not input_dir.exists():
|
||||
print(f"ERROR: Input directory not found: {input_dir}")
|
||||
return 1
|
||||
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Load config to check tie_word_embeddings
|
||||
with open(input_dir / "config.json") as f:
|
||||
config = json.load(f)
|
||||
old_vocab = config["vocab_size"]
|
||||
new_vocab = old_vocab + BYTE_FALLBACK_COUNT
|
||||
tie_embeddings = config.get("tie_word_embeddings", False)
|
||||
|
||||
print(f"=== Byte-Fallback Fix ===")
|
||||
print(f"Input: {input_dir}")
|
||||
print(f"Output: {output_dir}")
|
||||
print(f"Old vocab: {old_vocab}, New vocab: {new_vocab}")
|
||||
print(f"tie_word_embeddings: {tie_embeddings}")
|
||||
print()
|
||||
|
||||
# 1. Fix tokenizer.json
|
||||
print("[1/4] Fixing tokenizer.json ...")
|
||||
fix_tokenizer_json(
|
||||
input_dir / "tokenizer.json",
|
||||
output_dir / "tokenizer.json",
|
||||
)
|
||||
|
||||
# 2. Fix config.json
|
||||
print("[2/4] Fixing config.json ...")
|
||||
fix_config_json(
|
||||
input_dir / "config.json",
|
||||
output_dir / "config.json",
|
||||
new_vocab,
|
||||
)
|
||||
|
||||
# 3. Resize model weights
|
||||
print("[3/4] Resizing embeddings ...")
|
||||
resize_embeddings(
|
||||
input_dir / "model.safetensors",
|
||||
output_dir / "model.safetensors",
|
||||
old_vocab, new_vocab, tie_embeddings,
|
||||
)
|
||||
|
||||
# 4. Copy other files
|
||||
print("[4/4] Copying remaining files ...")
|
||||
for fname in ["tokenizer_config.json", "generation_config.json"]:
|
||||
src = input_dir / fname
|
||||
if src.exists():
|
||||
shutil.copy2(src, output_dir / fname)
|
||||
print(f" Copied {fname}")
|
||||
|
||||
# Copy SentencePiece model if provided (needed for GGUF conversion)
|
||||
if args.sp_model and args.sp_model.exists():
|
||||
shutil.copy2(args.sp_model, output_dir / "tokenizer.model")
|
||||
print(f" Copied tokenizer.model from {args.sp_model}")
|
||||
elif (input_dir / "tokenizer.model").exists():
|
||||
shutil.copy2(input_dir / "tokenizer.model", output_dir / "tokenizer.model")
|
||||
print(f" Copied tokenizer.model from input dir")
|
||||
|
||||
# Update tokenizer_config.json to add added_tokens_decoder for byte tokens
|
||||
tc_path = output_dir / "tokenizer_config.json"
|
||||
if tc_path.exists():
|
||||
with open(tc_path) as f:
|
||||
tc = json.load(f)
|
||||
added_tokens_decoder = tc.get("added_tokens_decoder", {})
|
||||
for i in range(BYTE_FALLBACK_COUNT):
|
||||
token_id = old_vocab + i
|
||||
byte_token = BYTE_TOKEN_TEMPLATE.format(i)
|
||||
added_tokens_decoder[str(token_id)] = {
|
||||
"content": byte_token,
|
||||
"lstrip": False,
|
||||
"normalized": False,
|
||||
"rstrip": False,
|
||||
"single_word": False,
|
||||
"special": True,
|
||||
}
|
||||
tc["added_tokens_decoder"] = added_tokens_decoder
|
||||
with open(tc_path, "w") as f:
|
||||
json.dump(tc, f, indent=2)
|
||||
print(f" Updated tokenizer_config.json with {BYTE_FALLBACK_COUNT} byte tokens")
|
||||
|
||||
print()
|
||||
print(f"=== Done! Fixed checkpoint at: {output_dir} ===")
|
||||
print(f"Next: python outputs/llama.cpp/convert_hf_to_gguf.py {output_dir} --outfile outputs/gguf/frankenstallm-3b-f16.gguf --outtype f16")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
241
source/scripts/hourly_status.sh
Normal file
241
source/scripts/hourly_status.sh
Normal file
@@ -0,0 +1,241 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# hourly_status.sh — FRANKENSTALLM 3B Hourly Training Status Report (Telegram)
|
||||
# Run: every hour via cron
|
||||
# Sends a rich formatted message with progress, loss, ETA, GPU/disk summary.
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
|
||||
# ─── Paths ───────────────────────────────────────────────────────────────────
|
||||
WORKDIR="${WORKDIR:-$(cd "$(dirname "$0")/.." && pwd)}"
|
||||
CKPT_DIR="$WORKDIR/checkpoints/korean_3b_fp8_run1"
|
||||
LOG_FILE="$CKPT_DIR/train.log"
|
||||
PID_FILE="$CKPT_DIR/train.pid"
|
||||
HOURLY_LOG="$CKPT_DIR/hourly_status.log"
|
||||
NOTIFY="python3 $WORKDIR/scripts/telegram_notify.py"
|
||||
|
||||
TOTAL_STEPS=57000
|
||||
TOTAL_TOKENS_B=114 # billion tokens target (57K steps × batch)
|
||||
|
||||
# ─── Helpers ─────────────────────────────────────────────────────────────────
|
||||
ts() { date '+%Y-%m-%d %H:%M:%S'; }
|
||||
log() { echo "[$(ts)] $*"; }
|
||||
|
||||
# Safely get last matching value from log
|
||||
parse_last() {
|
||||
local pattern="$1"
|
||||
grep -oP "$pattern" "$LOG_FILE" 2>/dev/null | tail -1 || echo ""
|
||||
}
|
||||
|
||||
# ─── Parse training log ───────────────────────────────────────────────────────
|
||||
parse_log() {
|
||||
if [[ ! -f "$LOG_FILE" ]]; then
|
||||
echo "NO_LOG"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Get the last step line
|
||||
LAST_LINE=$(grep -E 'step\s+[0-9]+.*loss' "$LOG_FILE" 2>/dev/null | tail -1 || echo "")
|
||||
if [[ -z "$LAST_LINE" ]]; then
|
||||
echo "NO_STEPS"
|
||||
return 1
|
||||
fi
|
||||
|
||||
CURRENT_STEP=$(echo "$LAST_LINE" | grep -oP 'step\s+\K[0-9]+' || echo "0")
|
||||
CURRENT_LOSS=$(echo "$LAST_LINE" | grep -oP 'loss\s+\K[0-9.]+' || echo "N/A")
|
||||
CURRENT_LR=$(echo "$LAST_LINE" | grep -oP 'lr\s+\K[0-9.e+-]+' || echo "N/A")
|
||||
CURRENT_GNORM=$(echo "$LAST_LINE" | grep -oP 'gnorm\s+\K[0-9.]+' || echo "N/A")
|
||||
CURRENT_TOKPS=$(echo "$LAST_LINE" | grep -oP 'tok/s\s+\K[\d,]+' | tr -d ',' || echo "0")
|
||||
CURRENT_MEM=$(echo "$LAST_LINE" | grep -oP 'mem\s+\K[0-9.]+GB' || echo "N/A")
|
||||
CURRENT_EPOCH=$(echo "$LAST_LINE" | grep -oP 'epoch\s+\K[0-9]+' || echo "0")
|
||||
|
||||
# Log timestamp — parse from the line itself
|
||||
LOG_TS=$(echo "$LAST_LINE" | grep -oP '\[\K[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}' || echo "unknown")
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
# ─── Calculate progress & ETA ─────────────────────────────────────────────────
|
||||
compute_eta() {
|
||||
local step="$1"
|
||||
local tokps="$2"
|
||||
|
||||
# Progress
|
||||
PROGRESS_PCT=$(echo "scale=1; $step * 100 / $TOTAL_STEPS" | bc -l 2>/dev/null || echo "0")
|
||||
|
||||
# Steps remaining
|
||||
STEPS_LEFT=$(( TOTAL_STEPS - step ))
|
||||
|
||||
# Tokens processed so far (approx: step × 2M tokens/step for 3B, bs=4, seqlen=4096, 8gpu)
|
||||
# bs=4, accum=8, 8gpu → effective batch = 4*8*8=256 sequences × 4096 tokens = 1,048,576 ≈ 1M tok/step
|
||||
TOKENS_PROCESSED_B=$(echo "scale=2; $step * 1048576 / 1000000000" | bc -l 2>/dev/null || echo "0")
|
||||
|
||||
# ETA using current tok/s
|
||||
if [[ "$tokps" -gt 0 ]]; then
|
||||
# tokens remaining
|
||||
local tokens_left_b
|
||||
tokens_left_b=$(echo "scale=2; ($TOTAL_STEPS - $step) * 1048576 / 1000000000" | bc -l 2>/dev/null || echo "0")
|
||||
local tokens_left
|
||||
tokens_left=$(echo "scale=0; ($TOTAL_STEPS - $step) * 1048576" | bc -l 2>/dev/null || echo "0")
|
||||
local secs_left
|
||||
secs_left=$(echo "scale=0; $tokens_left / $tokps" | bc -l 2>/dev/null || echo "0")
|
||||
|
||||
ETA_HOURS=$(echo "scale=1; $secs_left / 3600" | bc -l 2>/dev/null || echo "N/A")
|
||||
if [[ "$ETA_HOURS" != "N/A" ]]; then
|
||||
local eta_epoch
|
||||
eta_epoch=$(( $(date +%s) + secs_left ))
|
||||
ETA_DATETIME=$(date -d "@$eta_epoch" '+%m/%d %H:%M' 2>/dev/null || echo "N/A")
|
||||
else
|
||||
ETA_DATETIME="N/A"
|
||||
fi
|
||||
else
|
||||
ETA_HOURS="N/A"
|
||||
ETA_DATETIME="N/A"
|
||||
fi
|
||||
}
|
||||
|
||||
# ─── GPU summary ─────────────────────────────────────────────────────────────
|
||||
get_gpu_summary() {
|
||||
if ! command -v nvidia-smi &>/dev/null; then
|
||||
GPU_SUMMARY="nvidia-smi not available"
|
||||
GPU_AVG_UTIL="N/A"
|
||||
GPU_TOTAL_MEM="N/A"
|
||||
return
|
||||
fi
|
||||
|
||||
local raw
|
||||
raw=$(nvidia-smi --query-gpu=index,utilization.gpu,memory.used,memory.total \
|
||||
--format=csv,noheader,nounits 2>/dev/null || echo "")
|
||||
|
||||
if [[ -z "$raw" ]]; then
|
||||
GPU_SUMMARY="GPU query failed"
|
||||
GPU_AVG_UTIL="N/A"
|
||||
GPU_TOTAL_MEM="N/A"
|
||||
return
|
||||
fi
|
||||
|
||||
# avg util
|
||||
GPU_AVG_UTIL=$(echo "$raw" | awk -F', ' '{sum+=$2; count++} END {printf "%.0f%%", sum/count}')
|
||||
|
||||
# total mem used / total
|
||||
GPU_TOTAL_MEM=$(echo "$raw" | awk -F', ' \
|
||||
'{used+=$3; total+=$4} END {printf "%.1f / %.1f GiB", used/1024, total/1024}')
|
||||
|
||||
# Per-GPU one-liner: "G0:95% 48G | G1:94% 48G | ..."
|
||||
GPU_SUMMARY=$(echo "$raw" | awk -F', ' \
|
||||
'{printf "G%s:%s%% %sMiB | ", $1, $2, $3}' | sed 's/ | $//')
|
||||
}
|
||||
|
||||
# ─── Disk usage ──────────────────────────────────────────────────────────────
|
||||
get_disk_info() {
|
||||
DISK_INFO=$(df -h "$CKPT_DIR" 2>/dev/null | awk 'NR==2 {printf "%s used / %s total (%s)", $3, $2, $5}' || echo "N/A")
|
||||
CKPT_COUNT=$(ls -d "$CKPT_DIR"/checkpoint-* 2>/dev/null | wc -l || echo "0")
|
||||
LAST_CKPT=$(ls -dt "$CKPT_DIR"/checkpoint-* 2>/dev/null | head -1 | xargs basename 2>/dev/null || echo "none")
|
||||
}
|
||||
|
||||
# ─── Process status ───────────────────────────────────────────────────────────
|
||||
get_process_status() {
|
||||
PROC_STATUS="UNKNOWN"
|
||||
if [[ -f "$PID_FILE" ]]; then
|
||||
local pid
|
||||
pid=$(cat "$PID_FILE" 2>/dev/null | tr -d '[:space:]')
|
||||
if [[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null; then
|
||||
PROC_STATUS="RUNNING (PID $pid)"
|
||||
else
|
||||
PROC_STATUS="STOPPED (PID $pid)"
|
||||
fi
|
||||
else
|
||||
PROC_STATUS="NO PID FILE"
|
||||
fi
|
||||
}
|
||||
|
||||
# ─── Build & send message ────────────────────────────────────────────────────
|
||||
build_and_send() {
|
||||
local step="$CURRENT_STEP"
|
||||
local loss="$CURRENT_LOSS"
|
||||
local tokps="$CURRENT_TOKPS"
|
||||
|
||||
# Status icon
|
||||
local status_icon
|
||||
if [[ "$PROC_STATUS" == RUNNING* ]]; then
|
||||
status_icon="✅" # green check
|
||||
else
|
||||
status_icon="❌" # red X
|
||||
fi
|
||||
|
||||
# Progress bar (20 chars)
|
||||
local bar_filled=$(echo "scale=0; $PROGRESS_PCT * 20 / 100" | bc -l 2>/dev/null || echo "0")
|
||||
local bar_empty=$(( 20 - bar_filled ))
|
||||
PROGRESS_BAR=$(printf '%0.s█' $(seq 1 $bar_filled 2>/dev/null) ; printf '%0.s░' $(seq 1 $bar_empty 2>/dev/null)) || PROGRESS_BAR="[$PROGRESS_PCT%]"
|
||||
|
||||
local msg
|
||||
msg="$(cat <<EOF
|
||||
<b>FRANKENSTALLM 3B — Hourly Status</b>
|
||||
<i>$(ts)</i>
|
||||
|
||||
$status_icon <b>Process:</b> $PROC_STATUS
|
||||
|
||||
<b>Progress</b>
|
||||
Step: <code>$step / $TOTAL_STEPS</code> ($PROGRESS_PCT%)
|
||||
Tokens: <code>${TOKENS_PROCESSED_B}B / ${TOTAL_TOKENS_B}B</code>
|
||||
Epoch: <code>$CURRENT_EPOCH</code>
|
||||
Last log: <code>$LOG_TS</code>
|
||||
|
||||
<b>Training Metrics</b>
|
||||
Loss: <code>$loss</code>
|
||||
LR: <code>$CURRENT_LR</code>
|
||||
Gnorm: <code>$CURRENT_GNORM</code>
|
||||
Tok/s: <code>$tokps</code>
|
||||
Mem: <code>$CURRENT_MEM</code>
|
||||
|
||||
<b>ETA</b>
|
||||
Steps left: <code>$STEPS_LEFT</code>
|
||||
Remaining: <code>~$ETA_HOURS h</code>
|
||||
Est. done: <code>$ETA_DATETIME</code>
|
||||
|
||||
<b>GPU</b>
|
||||
Avg util: <code>$GPU_AVG_UTIL</code>
|
||||
Total mem: <code>$GPU_TOTAL_MEM</code>
|
||||
|
||||
<b>Checkpoints</b>
|
||||
Last saved: <code>$LAST_CKPT</code>
|
||||
Total: <code>$CKPT_COUNT</code> checkpoints
|
||||
|
||||
<b>Disk</b>
|
||||
<code>$DISK_INFO</code>
|
||||
EOF
|
||||
)"
|
||||
|
||||
log "Sending hourly status report (step $step)..."
|
||||
$NOTIFY "$msg" || {
|
||||
log "ERROR: Failed to send Telegram message."
|
||||
return 1
|
||||
}
|
||||
log "Status report sent."
|
||||
}
|
||||
|
||||
# ─── Main ────────────────────────────────────────────────────────────────────
|
||||
main() {
|
||||
log "=== Hourly status START ==="
|
||||
|
||||
parse_log || {
|
||||
log "Cannot parse log — sending minimal status."
|
||||
$NOTIFY "<b>FRANKENSTALLM 3B</b> — Status check at $(ts)
|
||||
|
||||
<b>WARNING:</b> Cannot read training log at:
|
||||
<code>$LOG_FILE</code>
|
||||
|
||||
Process status: $(cat "$PID_FILE" 2>/dev/null && echo "(PID found)" || echo "(no PID file)")" || true
|
||||
return 0
|
||||
}
|
||||
|
||||
compute_eta "$CURRENT_STEP" "$CURRENT_TOKPS"
|
||||
get_gpu_summary
|
||||
get_disk_info
|
||||
get_process_status
|
||||
build_and_send
|
||||
|
||||
log "=== Hourly status END ==="
|
||||
}
|
||||
|
||||
main "$@"
|
||||
177
source/scripts/launch_3b_orpo.sh
Normal file
177
source/scripts/launch_3b_orpo.sh
Normal file
@@ -0,0 +1,177 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# launch_3b_orpo.sh — 8-GPU ORPO fine-tuning launcher for Korean 3B LLM
|
||||
#
|
||||
# Usage:
|
||||
# bash scripts/launch_3b_orpo.sh # 기본 실행
|
||||
# bash scripts/launch_3b_orpo.sh --max_steps 200 # 빠른 테스트
|
||||
# RUN_NAME=my_orpo bash scripts/launch_3b_orpo.sh # 이름 지정
|
||||
#
|
||||
# 기반 모델 : eval/outputs/hf_3b_sft_best (SFT v1 best)
|
||||
# 데이터 : data/preference/combined_preference.jsonl
|
||||
# 출력 : checkpoints/korean_3b_orpo_v1/
|
||||
# 로그 : checkpoints/korean_3b_orpo_v1/train.log
|
||||
#
|
||||
# 체크포인트 크기 예상:
|
||||
# model weights: ~6GB (bf16)
|
||||
# optimizer states: ~24GB
|
||||
# 총 ~30GB/개 × max 5개 = 150GB
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
|
||||
# ---- Configurable defaults --------------------------------------------------
|
||||
RUN_NAME="${RUN_NAME:-korean_3b_orpo_v1}"
|
||||
BASE_MODEL="${BASE_MODEL:-eval/outputs/hf_3b_sft_best}"
|
||||
DATA_PATH="${DATA_PATH:-data/preference/combined_preference.jsonl}"
|
||||
OUTPUT_DIR="checkpoints/${RUN_NAME}"
|
||||
CKPT_DIR="checkpoints/${RUN_NAME}"
|
||||
LOG_FILE="${CKPT_DIR}/train.log"
|
||||
NPROC=8
|
||||
MASTER_PORT="${MASTER_PORT:-29502}"
|
||||
|
||||
# ORPO 하이퍼파라미터
|
||||
BATCH_SIZE=4
|
||||
GRAD_ACCUM=4
|
||||
LR=1.2e-5
|
||||
BETA=0.25
|
||||
EPOCHS=2
|
||||
MAX_LENGTH=1536
|
||||
WARMUP_RATIO=0.05
|
||||
WEIGHT_DECAY=0.01
|
||||
EVAL_SPLIT_RATIO=0.05
|
||||
EVAL_STEPS=500
|
||||
EARLY_STOPPING_PATIENCE=3
|
||||
SAVE_TOTAL_LIMIT=5
|
||||
SEED=42
|
||||
|
||||
EXTRA_ARGS="$@"
|
||||
|
||||
# ---- B200 / NVSwitch single-node NCCL tuning --------------------------------
|
||||
# (launch_3b_pretrain.sh와 동일한 NCCL 설정 유지)
|
||||
export NCCL_IB_DISABLE=1
|
||||
export NCCL_PROTO=Simple
|
||||
export NCCL_MIN_NCHANNELS=16
|
||||
export NCCL_MAX_NCHANNELS=16
|
||||
# ORPO forward-backward 패스는 pretrain보다 메모리 변동이 크므로 버퍼 128MB 유지
|
||||
export NCCL_BUFFSIZE=134217728
|
||||
export OMP_NUM_THREADS=9
|
||||
export MKL_NUM_THREADS=9
|
||||
# OOM 방지: 메모리 단편화 완화 (ORPO는 chosen/rejected 동시 forward → 메모리 민감)
|
||||
export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
|
||||
# P2P NVLink 직접 통신 활성화
|
||||
export NCCL_P2P_LEVEL=NVL
|
||||
# Ring + Tree 병행 (3B gradient 크기 기준)
|
||||
export NCCL_ALGO=Ring,Tree
|
||||
|
||||
export PYTHONWARNINGS="ignore::UserWarning:torch.library"
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
# ---- Pre-flight checks ------------------------------------------------------
|
||||
if [[ ! -d "${BASE_MODEL}" ]]; then
|
||||
echo "ERROR: 기반 모델 디렉토리 없음: ${BASE_MODEL}"
|
||||
echo " SFT 완료 후 HF 포맷으로 변환했는지 확인하세요."
|
||||
echo " 예: python scripts/convert_to_hf.py --checkpoint <sft_ckpt> --output ${BASE_MODEL}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! -f "${DATA_PATH}" ]]; then
|
||||
echo "ERROR: 학습 데이터 없음: ${DATA_PATH}"
|
||||
echo " 먼저 데이터 통합 스크립트를 실행하세요:"
|
||||
echo " python data/prepare_preference_combined.py"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! -f "train/orpo.py" ]]; then
|
||||
echo "ERROR: train/orpo.py 없음"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# GPU 메모리 체크
|
||||
GPU_MEM=$(nvidia-smi --query-gpu=memory.total --format=csv,noheader,nounits 2>/dev/null | head -1 || echo "0")
|
||||
if [[ "$GPU_MEM" -gt 0 && "$GPU_MEM" -lt 40000 ]]; then
|
||||
echo "WARNING: GPU 메모리 ${GPU_MEM}MB < 40GB. ORPO 3B 학습에 부족할 수 있음."
|
||||
fi
|
||||
|
||||
# 중복 프로세스 방지
|
||||
EXISTING_PID=$(pgrep -f "orpo.py.*${RUN_NAME}" 2>/dev/null | head -1 || true)
|
||||
if [[ -n "$EXISTING_PID" ]]; then
|
||||
echo "ERROR: 이미 ORPO 프로세스 실행 중 (PID: ${EXISTING_PID})"
|
||||
echo " kill ${EXISTING_PID} 로 먼저 종료하세요."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 디스크 여유 확인 (최소 200GB)
|
||||
AVAIL_KB=$(df /PROJECT 2>/dev/null | awk 'NR==2{print $4}' || echo "0")
|
||||
if [[ -n "$AVAIL_KB" && "$AVAIL_KB" -gt 0 && "$AVAIL_KB" -lt 209715200 ]]; then
|
||||
AVAIL_GB=$(echo "scale=1; $AVAIL_KB / 1048576" | bc 2>/dev/null || echo "?")
|
||||
echo "WARNING: /PROJECT 여유 ${AVAIL_GB}GB < 200GB. 체크포인트 저장 공간 부족 가능."
|
||||
fi
|
||||
|
||||
mkdir -p "${CKPT_DIR}" "${OUTPUT_DIR}"
|
||||
|
||||
# ---- 데이터 레코드 수 확인 --------------------------------------------------
|
||||
DATA_LINES=$(wc -l < "${DATA_PATH}" 2>/dev/null || echo "?")
|
||||
echo " 학습 데이터 레코드 수: ${DATA_LINES}"
|
||||
|
||||
# ---- 유효 배치 크기 계산 ----------------------------------------------------
|
||||
EFF_BATCH=$((BATCH_SIZE * NPROC * GRAD_ACCUM))
|
||||
|
||||
echo "=================================================================="
|
||||
echo " Korean 3B LLM ORPO Fine-Tuning"
|
||||
echo " Run name : ${RUN_NAME}"
|
||||
echo " Base model : ${BASE_MODEL}"
|
||||
echo " Data : ${DATA_PATH} (${DATA_LINES} records)"
|
||||
echo " Output dir : ${OUTPUT_DIR}"
|
||||
echo " CKPT dir : ${CKPT_DIR}"
|
||||
echo " Log file : ${LOG_FILE}"
|
||||
echo " Epochs : ${EPOCHS}"
|
||||
echo " LR : ${LR}"
|
||||
echo " Beta (ORPO) : ${BETA}"
|
||||
echo " Batch : ${BATCH_SIZE} (local) × ${NPROC} GPU × ${GRAD_ACCUM} accum = ${EFF_BATCH}"
|
||||
echo " Max length : ${MAX_LENGTH}"
|
||||
echo " Weight decay : ${WEIGHT_DECAY}"
|
||||
echo " Eval steps : ${EVAL_STEPS}"
|
||||
echo " Early stop : patience=${EARLY_STOPPING_PATIENCE}"
|
||||
echo " Started : $(date)"
|
||||
echo "=================================================================="
|
||||
|
||||
torchrun \
|
||||
--nproc_per_node=${NPROC} \
|
||||
--master_port=${MASTER_PORT} \
|
||||
train/orpo.py \
|
||||
--model_path "${BASE_MODEL}" \
|
||||
--custom_data_path "${DATA_PATH}" \
|
||||
--output_dir "${OUTPUT_DIR}" \
|
||||
--epochs ${EPOCHS} \
|
||||
--lr ${LR} \
|
||||
--beta ${BETA} \
|
||||
--batch_size ${BATCH_SIZE} \
|
||||
--gradient_accumulation_steps ${GRAD_ACCUM} \
|
||||
--max_length ${MAX_LENGTH} \
|
||||
--weight_decay ${WEIGHT_DECAY} \
|
||||
--eval_split_ratio ${EVAL_SPLIT_RATIO} \
|
||||
--eval_steps ${EVAL_STEPS} \
|
||||
--early_stopping_patience ${EARLY_STOPPING_PATIENCE} \
|
||||
--save_total_limit ${SAVE_TOTAL_LIMIT} \
|
||||
${EXTRA_ARGS} \
|
||||
2>&1 | tee "${LOG_FILE}" \
|
||||
| grep -v "UserWarning" \
|
||||
| grep -v "Warning only once" \
|
||||
| grep -v "Overriding a previously" \
|
||||
| grep -v "dispatch key:" \
|
||||
| grep -v "previous kernel:" \
|
||||
| grep -v "new kernel:" \
|
||||
| grep -v "operator: flash_attn" \
|
||||
| grep -v "registered at /usr/local" \
|
||||
| grep -v "self.m.impl"
|
||||
|
||||
EXIT_CODE=$?
|
||||
echo "=================================================================="
|
||||
echo " Done : $(date)"
|
||||
echo " Exit code: ${EXIT_CODE}"
|
||||
if [[ "${EXIT_CODE}" -eq 0 ]]; then
|
||||
echo " 모델 저장 위치: ${OUTPUT_DIR}"
|
||||
fi
|
||||
echo "=================================================================="
|
||||
exit $EXIT_CODE
|
||||
258
source/scripts/launch_3b_pretrain.sh
Normal file
258
source/scripts/launch_3b_pretrain.sh
Normal file
@@ -0,0 +1,258 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# launch_3b_pretrain.sh — 8-GPU FP8 pretraining launcher for Korean 3B LLM
|
||||
#
|
||||
# Features:
|
||||
# - SIGHUP 방어: SSH 끊김 시 자동으로 nohup+setsid로 세션 보호
|
||||
# - Graceful shutdown: SIGTERM 시 Python 시그널 핸들러가 비상 체크포인트 저장
|
||||
# - 자동 resume: 최신 체크포인트에서 자동 재개
|
||||
# - PID 파일: 프로세스 모니터링 및 제어용
|
||||
# - grep 파이프라인 exit code 보호 (|| true)
|
||||
#
|
||||
# Usage:
|
||||
# bash scripts/launch_3b_pretrain.sh # full run (60B tokens)
|
||||
# bash scripts/launch_3b_pretrain.sh --max_steps 500 # quick test
|
||||
# bash scripts/launch_3b_pretrain.sh --resume checkpoints/korean_3b_fp8_run1/checkpoint-0010000
|
||||
# MAX_STEPS=95000 bash scripts/launch_3b_pretrain.sh # 100B tokens
|
||||
#
|
||||
# 모니터링:
|
||||
# tail -f checkpoints/korean_3b_fp8_run1/train.log
|
||||
# cat checkpoints/korean_3b_fp8_run1/train.pid
|
||||
#
|
||||
# 중지 (비상 체크포인트 자동 저장):
|
||||
# kill $(cat checkpoints/korean_3b_fp8_run1/train.pid)
|
||||
#
|
||||
# 강제 종료 (체크포인트 저장 없음):
|
||||
# kill -9 $(cat checkpoints/korean_3b_fp8_run1/train.pid)
|
||||
# =============================================================================
|
||||
|
||||
# -u: 미정의 변수 에러
|
||||
# NOTE: -e, -o pipefail 의도적 제거
|
||||
# 이전 문제: grep 파이프라인에서 모든 라인이 필터링되면 exit code 1 반환
|
||||
# → pipefail이 이를 스크립트 실패로 전파 → 학습 중단
|
||||
# 해결: set -e/pipefail 제거 + grep 체인에 || true 추가
|
||||
set -u
|
||||
|
||||
# ---- Configurable defaults --------------------------------------------------
|
||||
RUN_NAME="${RUN_NAME:-korean_3b_fp8_run1}"
|
||||
CONFIG="${CONFIG:-configs/korean_3b_fp8.yaml}"
|
||||
TRAIN_DATA="${TRAIN_DATA:-data/3b_train.bin}"
|
||||
VAL_DATA="${VAL_DATA:-data/3b_val.bin}"
|
||||
CKPT_DIR="checkpoints/${RUN_NAME}"
|
||||
LOG_FILE="${CKPT_DIR}/train.log"
|
||||
NPROC=8
|
||||
MASTER_PORT="${MASTER_PORT:-29501}"
|
||||
|
||||
MAX_STEPS="${MAX_STEPS:-57000}"
|
||||
BATCH_SIZE=5
|
||||
GRAD_ACCUM=8
|
||||
WARMUP_STEPS=2000
|
||||
SEED=42
|
||||
|
||||
# ---- B200 / NVSwitch single-node NCCL tuning (3B optimized, v2) ----------
|
||||
export NCCL_IB_DISABLE=1
|
||||
export NCCL_ALGO=NVLS,Ring # NVSwitch hardware reduction first (was Ring,Tree)
|
||||
export NCCL_PROTO=Simple
|
||||
export NCCL_NVLS_ENABLE=1 # NVLink SHARP — hardware-accelerated all-reduce
|
||||
export NCCL_MIN_NCHANNELS=32 # raise minimum for NVSwitch headroom (was 16)
|
||||
export NCCL_MAX_NCHANNELS=32
|
||||
export NCCL_BUFFSIZE=268435456 # 256MB (was 128MB) — reduces bucket pipeline stalls
|
||||
export NCCL_P2P_LEVEL=NVL
|
||||
export NCCL_NET_GDR_LEVEL=0
|
||||
export OMP_NUM_THREADS=4
|
||||
export MKL_NUM_THREADS=4
|
||||
export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
|
||||
# Triton/Inductor cache on executable filesystem (not /tmp which is noexec)
|
||||
export TRITON_CUDACRT_PATH=/usr/local/cuda/include
|
||||
export TRITON_PTXAS_PATH=/usr/local/cuda/bin/ptxas
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
mkdir -p "${CKPT_DIR}"
|
||||
|
||||
# ---- Session protection (SIGHUP 방어) ---------------------------------------
|
||||
# tmux/screen 없이 실행 시, 자동으로 nohup + setsid로 래핑하여
|
||||
# SSH 끊김(SIGHUP)으로부터 학습 프로세스를 보호합니다.
|
||||
#
|
||||
# 작동 원리:
|
||||
# 1. tmux/screen/이미 보호됨 여부 확인
|
||||
# 2. 미보호 상태이면 _LAUNCH_PROTECTED=1 설정 후 nohup setsid로 자기 자신을 재실행
|
||||
# 3. 재실행된 프로세스는 새로운 세션 리더가 되어 터미널과 분리됨
|
||||
# 4. 원래 셸은 PID와 모니터링 명령을 출력하고 즉시 종료
|
||||
PID_FILE="${CKPT_DIR}/train.pid"
|
||||
|
||||
if [[ -z "${_LAUNCH_PROTECTED:-}" ]] && [[ -z "${TMUX:-}" ]] && [[ -z "${STY:-}" ]]; then
|
||||
export _LAUNCH_PROTECTED=1
|
||||
NOHUP_LOG="${CKPT_DIR}/launch_$(date +%Y%m%d_%H%M%S).log"
|
||||
|
||||
echo "=================================================================="
|
||||
echo " SIGHUP PROTECTION ACTIVATED"
|
||||
echo " tmux/screen 미감지 → 세션 보호 모드 자동 활성화 (nohup + setsid)"
|
||||
echo " SSH 끊어져도 학습이 계속됩니다."
|
||||
echo "=================================================================="
|
||||
echo ""
|
||||
|
||||
# 자기 자신을 세션 보호 모드로 재실행
|
||||
nohup setsid bash "$0" "$@" > "${NOHUP_LOG}" 2>&1 &
|
||||
BG_PID=$!
|
||||
echo "${BG_PID}" > "${PID_FILE}"
|
||||
|
||||
echo " PID : ${BG_PID}"
|
||||
echo " PID 파일 : ${PID_FILE}"
|
||||
echo " Launch 로그 : ${NOHUP_LOG}"
|
||||
echo " 학습 로그 : ${LOG_FILE}"
|
||||
echo ""
|
||||
echo " 모니터링:"
|
||||
echo " tail -f ${LOG_FILE}"
|
||||
echo ""
|
||||
echo " 중지 (비상 체크포인트 자동 저장):"
|
||||
echo " kill \$(cat ${PID_FILE})"
|
||||
echo ""
|
||||
echo " 강제 종료:"
|
||||
echo " kill -9 \$(cat ${PID_FILE})"
|
||||
echo "=================================================================="
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ---- Cleanup on exit --------------------------------------------------------
|
||||
PREWARM_PID=""
|
||||
|
||||
cleanup() {
|
||||
rm -f "${PID_FILE}" 2>/dev/null || true
|
||||
if [[ -n "${PREWARM_PID:-}" ]]; then
|
||||
kill "${PREWARM_PID}" 2>/dev/null || true
|
||||
fi
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
# PID 파일 기록 (tmux/screen 내에서 실행 시에도 PID 추적 가능)
|
||||
echo "$$" > "${PID_FILE}"
|
||||
|
||||
# ---- Pre-flight checks ------------------------------------------------------
|
||||
if [[ ! -f "${CONFIG}" ]]; then
|
||||
echo "[ERROR] Config not found: ${CONFIG}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! -f "${TRAIN_DATA}" ]]; then
|
||||
echo "[ERROR] Training data not found: ${TRAIN_DATA}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# GPU 메모리 체크 (3B는 최소 80GB/GPU 권장, B200=192GB → OK)
|
||||
GPU_MEM=$(nvidia-smi --query-gpu=memory.total --format=csv,noheader,nounits 2>/dev/null | head -1 || echo "0")
|
||||
if [[ "$GPU_MEM" -gt 0 && "$GPU_MEM" -lt 80000 ]]; then
|
||||
echo "[WARN] GPU memory ${GPU_MEM}MB < 80GB. 3B 학습에 부족할 수 있음."
|
||||
fi
|
||||
|
||||
# 중복 프로세스 방지
|
||||
EXISTING_PID=$(pgrep -f "pretrain.py.*korean_3b" 2>/dev/null | head -1 || true)
|
||||
if [[ -n "$EXISTING_PID" ]]; then
|
||||
echo "[ERROR] 이미 3B pretrain 프로세스 실행 중 (PID: ${EXISTING_PID})"
|
||||
echo " kill ${EXISTING_PID} 로 먼저 종료하세요."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 디스크 여유 확인 (최소 1TB 필요)
|
||||
AVAIL_KB=$(df /PROJECT 2>/dev/null | awk 'NR==2{print $4}')
|
||||
if [[ -n "${AVAIL_KB:-}" ]] && [[ "$AVAIL_KB" -lt 1073741824 ]]; then
|
||||
AVAIL_TB=$(echo "scale=1; $AVAIL_KB / 1073741824" | bc 2>/dev/null || echo "?")
|
||||
echo "[WARN] /PROJECT 여유 ${AVAIL_TB}TB < 1TB. 체크포인트 저장 공간 부족 가능."
|
||||
fi
|
||||
|
||||
# ---- Resume detection -------------------------------------------------------
|
||||
RESUME_ARG=""
|
||||
EXTRA_ARGS="${*:-}"
|
||||
if [[ ! "${EXTRA_ARGS}" =~ "--resume" ]]; then
|
||||
# 가장 최근 체크포인트 자동 감지
|
||||
LATEST_CKPT=$(ls -d "${CKPT_DIR}"/checkpoint-* 2>/dev/null | sort -V | tail -1 || true)
|
||||
if [[ -n "$LATEST_CKPT" ]]; then
|
||||
echo "[INFO] 자동 resume 감지: ${LATEST_CKPT}"
|
||||
RESUME_ARG="--resume ${LATEST_CKPT}"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ---- Banner ------------------------------------------------------------------
|
||||
SESSION_TYPE="direct"
|
||||
[[ -n "${TMUX:-}" ]] && SESSION_TYPE="tmux"
|
||||
[[ -n "${STY:-}" ]] && SESSION_TYPE="screen"
|
||||
[[ -n "${_LAUNCH_PROTECTED:-}" ]] && SESSION_TYPE="protected (nohup+setsid)"
|
||||
|
||||
echo "=================================================================="
|
||||
echo " Korean 3B LLM Pre-Training (FP8)"
|
||||
echo " Run name : ${RUN_NAME}"
|
||||
echo " Config : ${CONFIG}"
|
||||
echo " CKPT dir : ${CKPT_DIR}"
|
||||
echo " Log file : ${LOG_FILE}"
|
||||
echo " Max steps : ${MAX_STEPS}"
|
||||
echo " Batch : ${BATCH_SIZE} (local) x ${NPROC} GPU x ${GRAD_ACCUM} accum"
|
||||
echo " Eff tokens : $((BATCH_SIZE * NPROC * GRAD_ACCUM * 4096)) tokens/step (~1M)"
|
||||
echo " Total tokens: ~$((MAX_STEPS * BATCH_SIZE * NPROC * GRAD_ACCUM * 4096 / 1000000000))B"
|
||||
echo " Resume : ${RESUME_ARG:-none (fresh start)}"
|
||||
echo " Session : ${SESSION_TYPE}"
|
||||
echo " PID : $$ (file: ${PID_FILE})"
|
||||
echo " Started : $(date)"
|
||||
echo "=================================================================="
|
||||
|
||||
export PYTHONWARNINGS="ignore::UserWarning:torch.library"
|
||||
|
||||
# ---- Pre-warm OS page cache (NUMA-interleaved, non-blocking) ---------------
|
||||
if [[ -f "${TRAIN_DATA}" ]]; then
|
||||
echo "[INFO] Pre-warming page cache for ${TRAIN_DATA} (NUMA interleaved)..."
|
||||
numactl --interleave=all dd if="${TRAIN_DATA}" of=/dev/null bs=16M 2>/dev/null &
|
||||
PREWARM_PID=$!
|
||||
fi
|
||||
|
||||
# ---- Launch training ---------------------------------------------------------
|
||||
# grep 파이프라인 보호:
|
||||
# 문제: grep -v 가 매칭 라인이 없으면 exit code 1 반환
|
||||
# 해결: { ... || true; } 래핑으로 파이프라인 exit code를 항상 0으로 보장
|
||||
# torchrun의 실제 exit code는 PIPESTATUS[0]으로 별도 캡처
|
||||
numactl --interleave=all \
|
||||
torchrun \
|
||||
--nproc_per_node=${NPROC} \
|
||||
--master_port=${MASTER_PORT} \
|
||||
train/pretrain.py \
|
||||
--config "${CONFIG}" \
|
||||
--train_data "${TRAIN_DATA}" \
|
||||
--val_data "${VAL_DATA}" \
|
||||
--checkpoint_dir "${CKPT_DIR}" \
|
||||
--log_file "${LOG_FILE}" \
|
||||
--max_steps ${MAX_STEPS} \
|
||||
--batch_size ${BATCH_SIZE} \
|
||||
--grad_accum ${GRAD_ACCUM} \
|
||||
--warmup_steps ${WARMUP_STEPS} \
|
||||
--seed ${SEED} \
|
||||
${RESUME_ARG} \
|
||||
${EXTRA_ARGS} \
|
||||
2>&1 | { grep -v "UserWarning" \
|
||||
| grep -v "Warning only once" \
|
||||
| grep -v "Overriding a previously" \
|
||||
| grep -v "dispatch key:" \
|
||||
| grep -v "previous kernel:" \
|
||||
| grep -v "new kernel:" \
|
||||
| grep -v "operator: flash_attn" \
|
||||
| grep -v "registered at /usr/local" \
|
||||
| grep -v "self.m.impl" \
|
||||
|| true; }
|
||||
|
||||
EXIT_CODE=${PIPESTATUS[0]}
|
||||
|
||||
# ---- Exit summary ------------------------------------------------------------
|
||||
echo ""
|
||||
echo "=================================================================="
|
||||
echo " Finished : $(date)"
|
||||
echo " Exit code : ${EXIT_CODE}"
|
||||
if [[ ${EXIT_CODE} -eq 0 ]]; then
|
||||
echo " Status : SUCCESS (학습 완료 또는 graceful shutdown)"
|
||||
elif [[ ${EXIT_CODE} -eq 143 ]]; then
|
||||
echo " Status : TERMINATED (SIGTERM — 비상 체크포인트 저장됨)"
|
||||
elif [[ ${EXIT_CODE} -eq 137 ]]; then
|
||||
echo " Status : KILLED (SIGKILL — 강제 종료, 체크포인트 미저장)"
|
||||
elif [[ ${EXIT_CODE} -eq 1 ]]; then
|
||||
echo " Status : ERROR (${LOG_FILE} 확인 필요)"
|
||||
else
|
||||
echo " Status : FAILED (exit code ${EXIT_CODE}, ${LOG_FILE} 확인)"
|
||||
fi
|
||||
echo "=================================================================="
|
||||
exit ${EXIT_CODE}
|
||||
145
source/scripts/launch_3b_sft.sh
Normal file
145
source/scripts/launch_3b_sft.sh
Normal file
@@ -0,0 +1,145 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# launch_3b_sft.sh — 8-GPU FP8 SFT launcher for 3B Korean LLM
|
||||
#
|
||||
# Usage:
|
||||
# bash scripts/launch_3b_sft.sh
|
||||
# bash scripts/launch_3b_sft.sh --max_steps 200 # quick test
|
||||
# bash scripts/launch_3b_sft.sh --resume checkpoints/korean_3b_sft_v1/checkpoint-0002000
|
||||
#
|
||||
# Base model : checkpoints/korean_3b_fp8_run1/checkpoint-XXXXXX (기본값)
|
||||
# --base_checkpoint 인자로 덮어쓸 수 있음
|
||||
# SFT data : data/sft_combined/train_filtered.jsonl
|
||||
# (먼저 scripts/prepare_sft_combined.sh → data/filter_sft_v2.py 실행)
|
||||
#
|
||||
# Effective batch: 2 (local) × 8 GPU × 4 (grad_accum) = 64 samples/step
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
|
||||
# ---- Configurable defaults --------------------------------------------------
|
||||
RUN_NAME="${RUN_NAME:-korean_3b_sft_v1}"
|
||||
CONFIG="${CONFIG:-configs/korean_3b_sft.yaml}"
|
||||
BASE_CHECKPOINT="${BASE_CHECKPOINT:-checkpoints/korean_3b_fp8_run1/checkpoint-0057000}"
|
||||
SFT_DATA="${SFT_DATA:-data/sft_combined/train_filtered.jsonl}"
|
||||
VAL_DATA="${VAL_DATA:-data/sft_combined/val_filtered.jsonl}"
|
||||
CKPT_DIR="checkpoints/${RUN_NAME}"
|
||||
LOG_FILE="${CKPT_DIR}/train.log"
|
||||
NPROC=8
|
||||
MASTER_PORT="${MASTER_PORT:-29503}"
|
||||
|
||||
MAX_STEPS=33000
|
||||
BATCH_SIZE=2
|
||||
GRAD_ACCUM=4
|
||||
LR="1.0e-5"
|
||||
WARMUP_STEPS=500
|
||||
SEED=42
|
||||
|
||||
EXTRA_ARGS="$@"
|
||||
|
||||
# ---- B200 / NVSwitch NCCL tuning (same as pretrain) -------------------------
|
||||
export NCCL_IB_DISABLE=1
|
||||
export NCCL_ALGO=Ring
|
||||
export NCCL_PROTO=Simple
|
||||
export NCCL_MIN_NCHANNELS=16
|
||||
export NCCL_MAX_NCHANNELS=16
|
||||
export NCCL_BUFFSIZE=67108864
|
||||
export OMP_NUM_THREADS=4
|
||||
export MKL_NUM_THREADS=4
|
||||
|
||||
# 3B 모델 VRAM 절약 — 동적 메모리 세그먼트 확장 허용
|
||||
export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
# ---- Pre-flight checks ------------------------------------------------------
|
||||
if [[ ! -d "${BASE_CHECKPOINT}" ]]; then
|
||||
echo "=================================================================="
|
||||
echo " ERROR: Base checkpoint 디렉토리를 찾을 수 없습니다."
|
||||
echo " 경로: ${BASE_CHECKPOINT}"
|
||||
echo ""
|
||||
echo " --base_checkpoint 인자로 실제 경로를 지정하거나"
|
||||
echo " BASE_CHECKPOINT 환경변수를 설정하세요."
|
||||
echo " 예: bash scripts/launch_3b_sft.sh --base_checkpoint checkpoints/korean_3b_fp8_run1/checkpoint-0057000"
|
||||
echo "=================================================================="
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! -f "${SFT_DATA}" ]]; then
|
||||
echo "=================================================================="
|
||||
echo " ERROR: SFT 학습 데이터를 찾을 수 없습니다: ${SFT_DATA}"
|
||||
echo ""
|
||||
echo " 데이터 준비 순서:"
|
||||
echo " 1. bash scripts/prepare_sft_combined.sh"
|
||||
echo " 2. python data/filter_sft_v2.py \\"
|
||||
echo " --input data/sft_combined/train.jsonl \\"
|
||||
echo " --output data/sft_combined/train_filtered.jsonl"
|
||||
echo "=================================================================="
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# val 파일 없으면 원본 val.jsonl 로 폴백
|
||||
if [[ ! -f "${VAL_DATA}" ]]; then
|
||||
VAL_FALLBACK="data/sft_combined/val.jsonl"
|
||||
if [[ -f "${VAL_FALLBACK}" ]]; then
|
||||
VAL_DATA="${VAL_FALLBACK}"
|
||||
echo "[INFO] val_filtered 없음, 폴백: ${VAL_DATA}"
|
||||
else
|
||||
echo "ERROR: VAL_DATA 파일을 찾을 수 없습니다: ${VAL_DATA}"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
mkdir -p "${CKPT_DIR}"
|
||||
|
||||
echo "=================================================================="
|
||||
echo " 3B SFT Fine-Tuning"
|
||||
echo " Run name : ${RUN_NAME}"
|
||||
echo " Config : ${CONFIG}"
|
||||
echo " Base checkpoint : ${BASE_CHECKPOINT}"
|
||||
echo " SFT data : ${SFT_DATA}"
|
||||
echo " Val data : ${VAL_DATA}"
|
||||
echo " CKPT dir : ${CKPT_DIR}"
|
||||
echo " Log file : ${LOG_FILE}"
|
||||
echo " Max steps : ${MAX_STEPS}"
|
||||
echo " Batch size : ${BATCH_SIZE} (local) × ${NPROC} GPU × ${GRAD_ACCUM} grad_accum = $((BATCH_SIZE * NPROC * GRAD_ACCUM)) eff_batch"
|
||||
echo " Learning rate : ${LR}"
|
||||
echo " Warmup : ${WARMUP_STEPS} steps"
|
||||
echo " Master port : ${MASTER_PORT}"
|
||||
echo " ALLOC_CONF : ${PYTORCH_CUDA_ALLOC_CONF}"
|
||||
echo " Started : $(date)"
|
||||
echo "=================================================================="
|
||||
|
||||
export PYTHONWARNINGS="ignore::UserWarning:torch.library"
|
||||
|
||||
torchrun \
|
||||
--nproc_per_node=${NPROC} \
|
||||
--master_port=${MASTER_PORT} \
|
||||
train/sft.py \
|
||||
--config "${CONFIG}" \
|
||||
--base_checkpoint "${BASE_CHECKPOINT}" \
|
||||
--sft_data "${SFT_DATA}" \
|
||||
--val_data "${VAL_DATA}" \
|
||||
--checkpoint_dir "${CKPT_DIR}" \
|
||||
--log_file "${LOG_FILE}" \
|
||||
--max_steps ${MAX_STEPS} \
|
||||
--batch_size ${BATCH_SIZE} \
|
||||
--grad_accum ${GRAD_ACCUM} \
|
||||
--lr ${LR} \
|
||||
--warmup_steps ${WARMUP_STEPS} \
|
||||
--seed ${SEED} \
|
||||
--use_fp8 \
|
||||
${EXTRA_ARGS} \
|
||||
2>&1 | grep -v "UserWarning" \
|
||||
| grep -v "Warning only once" \
|
||||
| grep -v "Overriding a previously" \
|
||||
| grep -v "dispatch key:" \
|
||||
| grep -v "previous kernel:" \
|
||||
| grep -v "new kernel:" \
|
||||
| grep -v "operator: flash_attn" \
|
||||
| grep -v "registered at /usr/local" \
|
||||
| grep -v "self.m.impl" \
|
||||
| tee -a "${LOG_FILE}"
|
||||
|
||||
echo "=================================================================="
|
||||
echo " 3B SFT Done : $(date)"
|
||||
echo "=================================================================="
|
||||
156
source/scripts/launch_3b_sft_v2.sh
Normal file
156
source/scripts/launch_3b_sft_v2.sh
Normal file
@@ -0,0 +1,156 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# launch_3b_sft_v2.sh — 8-GPU FP8 SFT v2 launcher for 3B Korean LLM
|
||||
#
|
||||
# SFT v2 improvements over v1:
|
||||
# - LR: 1e-5 → 5e-5 (5x, resolve underfitting)
|
||||
# - Effective batch: 64 → 256 (4x)
|
||||
# - Data mixing: 70% SFT + 30% pretrain (forgetting prevention)
|
||||
# - Weight decay: 0.01 → 0.05
|
||||
# - Warmup: 500 → 2000 steps
|
||||
# - Max steps: 33000 → 15000
|
||||
#
|
||||
# Usage:
|
||||
# bash scripts/launch_3b_sft_v2.sh
|
||||
# bash scripts/launch_3b_sft_v2.sh --max_steps 200 # quick test
|
||||
# bash scripts/launch_3b_sft_v2.sh --resume checkpoints/korean_3b_sft_v2/checkpoint-0002000
|
||||
#
|
||||
# Effective batch: 4 (local) x 8 GPU x 8 (grad_accum) = 256 samples/step
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
|
||||
# ---- Configurable defaults --------------------------------------------------
|
||||
RUN_NAME="${RUN_NAME:-korean_3b_sft_v2}"
|
||||
CONFIG="${CONFIG:-configs/korean_3b_sft_v2.yaml}"
|
||||
BASE_CHECKPOINT="${BASE_CHECKPOINT:-checkpoints/korean_3b_fp8_run1/checkpoint-0057000}"
|
||||
SFT_DATA="${SFT_DATA:-data/sft_combined/train_filtered.jsonl}"
|
||||
VAL_DATA="${VAL_DATA:-data/sft_combined/val_filtered.jsonl}"
|
||||
PRETRAIN_DATA="${PRETRAIN_DATA:-data/3b_train.bin}"
|
||||
CKPT_DIR="checkpoints/${RUN_NAME}"
|
||||
LOG_FILE="${CKPT_DIR}/train.log"
|
||||
NPROC=8
|
||||
MASTER_PORT="${MASTER_PORT:-29504}"
|
||||
|
||||
MAX_STEPS=15000
|
||||
BATCH_SIZE=4
|
||||
GRAD_ACCUM=8
|
||||
LR="5.0e-5"
|
||||
WARMUP_STEPS=2000
|
||||
WEIGHT_DECAY=0.05
|
||||
PRETRAIN_MIX_RATIO=0.3
|
||||
SEED=42
|
||||
|
||||
EXTRA_ARGS="$@"
|
||||
|
||||
# ---- B200 / NVSwitch NCCL tuning (same as pretrain) -------------------------
|
||||
export NCCL_IB_DISABLE=1
|
||||
export NCCL_ALGO=Ring
|
||||
export NCCL_PROTO=Simple
|
||||
export NCCL_MIN_NCHANNELS=16
|
||||
export NCCL_MAX_NCHANNELS=16
|
||||
export NCCL_BUFFSIZE=67108864
|
||||
export OMP_NUM_THREADS=4
|
||||
export MKL_NUM_THREADS=4
|
||||
|
||||
# 3B + bs=4 VRAM allocation
|
||||
export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
# ---- Pre-flight checks ------------------------------------------------------
|
||||
if [[ ! -d "${BASE_CHECKPOINT}" ]]; then
|
||||
echo "=================================================================="
|
||||
echo " ERROR: Base checkpoint not found: ${BASE_CHECKPOINT}"
|
||||
echo " Set BASE_CHECKPOINT env var or use --base_checkpoint CLI arg."
|
||||
echo "=================================================================="
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! -f "${SFT_DATA}" ]]; then
|
||||
echo "=================================================================="
|
||||
echo " ERROR: SFT data not found: ${SFT_DATA}"
|
||||
echo " Run: bash scripts/prepare_sft_combined.sh"
|
||||
echo "=================================================================="
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! -f "${PRETRAIN_DATA}" ]]; then
|
||||
echo "=================================================================="
|
||||
echo " ERROR: Pretrain data not found: ${PRETRAIN_DATA}"
|
||||
echo " Set PRETRAIN_DATA env var to the correct path."
|
||||
echo "=================================================================="
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# val fallback
|
||||
if [[ ! -f "${VAL_DATA}" ]]; then
|
||||
VAL_FALLBACK="data/sft_combined/val.jsonl"
|
||||
if [[ -f "${VAL_FALLBACK}" ]]; then
|
||||
VAL_DATA="${VAL_FALLBACK}"
|
||||
echo "[INFO] val_filtered not found, fallback: ${VAL_DATA}"
|
||||
else
|
||||
echo "ERROR: VAL_DATA not found: ${VAL_DATA}"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
mkdir -p "${CKPT_DIR}"
|
||||
|
||||
echo "=================================================================="
|
||||
echo " 3B SFT v2 Fine-Tuning"
|
||||
echo " Run name : ${RUN_NAME}"
|
||||
echo " Config : ${CONFIG}"
|
||||
echo " Base checkpoint : ${BASE_CHECKPOINT}"
|
||||
echo " SFT data : ${SFT_DATA}"
|
||||
echo " Pretrain data : ${PRETRAIN_DATA}"
|
||||
echo " Val data : ${VAL_DATA}"
|
||||
echo " CKPT dir : ${CKPT_DIR}"
|
||||
echo " Log file : ${LOG_FILE}"
|
||||
echo " Max steps : ${MAX_STEPS}"
|
||||
echo " Batch size : ${BATCH_SIZE} (local) x ${NPROC} GPU x ${GRAD_ACCUM} grad_accum = $((BATCH_SIZE * NPROC * GRAD_ACCUM)) eff_batch"
|
||||
echo " Learning rate : ${LR}"
|
||||
echo " Weight decay : ${WEIGHT_DECAY}"
|
||||
echo " Warmup : ${WARMUP_STEPS} steps"
|
||||
echo " Data mixing : $((100 - ${PRETRAIN_MIX_RATIO%.*}0))% SFT + ${PRETRAIN_MIX_RATIO}00% pretrain"
|
||||
echo " Master port : ${MASTER_PORT}"
|
||||
echo " ALLOC_CONF : ${PYTORCH_CUDA_ALLOC_CONF}"
|
||||
echo " Started : $(date)"
|
||||
echo "=================================================================="
|
||||
|
||||
export PYTHONWARNINGS="ignore::UserWarning:torch.library"
|
||||
|
||||
torchrun \
|
||||
--nproc_per_node=${NPROC} \
|
||||
--master_port=${MASTER_PORT} \
|
||||
train/sft.py \
|
||||
--config "${CONFIG}" \
|
||||
--base_checkpoint "${BASE_CHECKPOINT}" \
|
||||
--sft_data "${SFT_DATA}" \
|
||||
--val_data "${VAL_DATA}" \
|
||||
--pretrain_data "${PRETRAIN_DATA}" \
|
||||
--pretrain_mix_ratio ${PRETRAIN_MIX_RATIO} \
|
||||
--checkpoint_dir "${CKPT_DIR}" \
|
||||
--log_file "${LOG_FILE}" \
|
||||
--max_steps ${MAX_STEPS} \
|
||||
--batch_size ${BATCH_SIZE} \
|
||||
--grad_accum ${GRAD_ACCUM} \
|
||||
--lr ${LR} \
|
||||
--weight_decay ${WEIGHT_DECAY} \
|
||||
--warmup_steps ${WARMUP_STEPS} \
|
||||
--seed ${SEED} \
|
||||
--use_fp8 \
|
||||
${EXTRA_ARGS} \
|
||||
2>&1 | grep -v "UserWarning" \
|
||||
| grep -v "Warning only once" \
|
||||
| grep -v "Overriding a previously" \
|
||||
| grep -v "dispatch key:" \
|
||||
| grep -v "previous kernel:" \
|
||||
| grep -v "new kernel:" \
|
||||
| grep -v "operator: flash_attn" \
|
||||
| grep -v "registered at /usr/local" \
|
||||
| grep -v "self.m.impl" \
|
||||
| tee -a "${LOG_FILE}"
|
||||
|
||||
echo "=================================================================="
|
||||
echo " 3B SFT v2 Done : $(date)"
|
||||
echo "=================================================================="
|
||||
94
source/scripts/launch_fp8.sh
Normal file
94
source/scripts/launch_fp8.sh
Normal file
@@ -0,0 +1,94 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# launch_fp8.sh — 8-GPU FP8 pretraining launcher for B200
|
||||
#
|
||||
# Usage:
|
||||
# bash scripts/launch_fp8.sh # full run
|
||||
# bash scripts/launch_fp8.sh --max_steps 500 # quick test
|
||||
# bash scripts/launch_fp8.sh --resume checkpoints/small_fp8_run1/checkpoint-0001000
|
||||
#
|
||||
# Config is read from configs/small_fp8.yaml (model) + CLI args (train).
|
||||
# Logs: checkpoints/<RUN_NAME>/train.log
|
||||
# checkpoints/<RUN_NAME>/tensorboard/
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
|
||||
# ---- Configurable defaults --------------------------------------------------
|
||||
RUN_NAME="${RUN_NAME:-small_fp8_run1}"
|
||||
CONFIG="${CONFIG:-configs/small_fp8.yaml}"
|
||||
TRAIN_DATA="${TRAIN_DATA:-data/train.bin}"
|
||||
VAL_DATA="${VAL_DATA:-data/val.bin}"
|
||||
CKPT_DIR="checkpoints/${RUN_NAME}"
|
||||
LOG_FILE="${CKPT_DIR}/train.log"
|
||||
NPROC=8
|
||||
MASTER_PORT="${MASTER_PORT:-29500}"
|
||||
|
||||
# ---- Defaults that can be overridden via extra CLI args --------------------
|
||||
MAX_STEPS=100000
|
||||
BATCH_SIZE=8
|
||||
GRAD_ACCUM=4
|
||||
WARMUP_STEPS=2000
|
||||
SEED=42
|
||||
|
||||
# ---- Pass remaining CLI args directly to pretrain.py ----------------------
|
||||
EXTRA_ARGS="$@"
|
||||
|
||||
# ---- B200 / NVSwitch single-node NCCL tuning --------------------------------
|
||||
# Single-node NVSwitch (NV18 full-mesh): disable IB to prevent NCCL probing.
|
||||
export NCCL_IB_DISABLE=1
|
||||
# Use Ring algorithm for large gradient tensors (128M-70B model range).
|
||||
export NCCL_ALGO=Ring
|
||||
# Simple protocol is optimal for NVLink bulk transfers (vs LL/LL128 for IB).
|
||||
export NCCL_PROTO=Simple
|
||||
# More channels → better NVSwitch saturation for large all-reduce payloads.
|
||||
export NCCL_MIN_NCHANNELS=16
|
||||
export NCCL_MAX_NCHANNELS=16
|
||||
# Larger NCCL buffer (64 MB) reduces ring synchronisation overhead.
|
||||
export NCCL_BUFFSIZE=67108864
|
||||
# CPU thread limits (72 cores ÷ 8 ranks = 9; use 4 for DataLoader headroom).
|
||||
export OMP_NUM_THREADS=4
|
||||
export MKL_NUM_THREADS=4
|
||||
|
||||
# ---- Setup ------------------------------------------------------------------
|
||||
mkdir -p "${CKPT_DIR}"
|
||||
cd "$(dirname "$0")/.." # always run from project root
|
||||
|
||||
echo "=================================================================="
|
||||
echo " Run name : ${RUN_NAME}"
|
||||
echo " Config : ${CONFIG}"
|
||||
echo " CKPT dir : ${CKPT_DIR}"
|
||||
echo " Log file : ${LOG_FILE}"
|
||||
echo " Started : $(date)"
|
||||
echo "=================================================================="
|
||||
|
||||
# Suppress the harmless flash_attn kernel override warning from all ranks.
|
||||
export PYTHONWARNINGS="ignore::UserWarning:torch.library"
|
||||
|
||||
torchrun \
|
||||
--nproc_per_node=${NPROC} \
|
||||
--master_port=${MASTER_PORT} \
|
||||
train/pretrain.py \
|
||||
--config "${CONFIG}" \
|
||||
--train_data "${TRAIN_DATA}" \
|
||||
--val_data "${VAL_DATA}" \
|
||||
--checkpoint_dir "${CKPT_DIR}" \
|
||||
--log_file "${LOG_FILE}" \
|
||||
--max_steps ${MAX_STEPS} \
|
||||
--batch_size ${BATCH_SIZE} \
|
||||
--grad_accum ${GRAD_ACCUM} \
|
||||
--warmup_steps ${WARMUP_STEPS} \
|
||||
--seed ${SEED} \
|
||||
${EXTRA_ARGS} \
|
||||
2>&1 | grep -v "UserWarning" \
|
||||
| grep -v "Warning only once" \
|
||||
| grep -v "Overriding a previously" \
|
||||
| grep -v "dispatch key:" \
|
||||
| grep -v "previous kernel:" \
|
||||
| grep -v "new kernel:" \
|
||||
| grep -v "operator: flash_attn" \
|
||||
| grep -v "registered at /usr/local" \
|
||||
| grep -v "self.m.impl"
|
||||
|
||||
echo "=================================================================="
|
||||
echo " Done : $(date)"
|
||||
echo "=================================================================="
|
||||
62
source/scripts/launch_hybrid_3b.sh
Normal file
62
source/scripts/launch_hybrid_3b.sh
Normal file
@@ -0,0 +1,62 @@
|
||||
#!/bin/bash
|
||||
# ============================================================================
|
||||
# FRANKENSTALLM-H 3B: Hybrid Mamba-2 + Transformer 학습 런치 스크립트
|
||||
# ============================================================================
|
||||
#
|
||||
# 사용법:
|
||||
# nohup setsid bash scripts/launch_hybrid_3b.sh > logs/hybrid_3b.log 2>&1 &
|
||||
#
|
||||
# SIGHUP 방어: nohup + setsid 조합으로 SSH 끊김에도 학습 유지
|
||||
# ============================================================================
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# ---- 환경 변수 ----
|
||||
export OMP_NUM_THREADS=4
|
||||
export NCCL_ALGO=NVLS # NVSwitch 최적 알고리즘
|
||||
export NCCL_IB_DISABLE=1 # InfiniBand 비활성 (단일 노드)
|
||||
export NCCL_P2P_LEVEL=NVL # NVLink P2P
|
||||
export NCCL_NET_GDR_LEVEL=0 # GPU Direct RDMA 비활성 (단일 노드)
|
||||
|
||||
# ---- 경로 ----
|
||||
PROJECT_ROOT="/PROJECT/0325120031_A/ghong/taketimes/llm-bang"
|
||||
CONFIG="${PROJECT_ROOT}/configs/hybrid_3b.yaml"
|
||||
TRAIN_DATA="${PROJECT_ROOT}/data/3b_train.bin"
|
||||
VAL_DATA="${PROJECT_ROOT}/data/3b_val.bin"
|
||||
CKPT_DIR="${PROJECT_ROOT}/checkpoints/hybrid_3b_run1"
|
||||
LOG_FILE="${PROJECT_ROOT}/logs/hybrid_3b_train.log"
|
||||
|
||||
# ---- 디렉토리 생성 ----
|
||||
mkdir -p "${CKPT_DIR}"
|
||||
mkdir -p "$(dirname ${LOG_FILE})"
|
||||
|
||||
cd "${PROJECT_ROOT}"
|
||||
|
||||
echo "============================================"
|
||||
echo " FRANKENSTALLM-H 3B Hybrid Training"
|
||||
echo " Config: ${CONFIG}"
|
||||
echo " Data: ${TRAIN_DATA}"
|
||||
echo " Checkpoint: ${CKPT_DIR}"
|
||||
echo " Started: $(date '+%Y-%m-%d %H:%M:%S')"
|
||||
echo "============================================"
|
||||
|
||||
# ---- 학습 실행 (8 GPU DDP) ----
|
||||
torchrun \
|
||||
--nproc_per_node=8 \
|
||||
--master_port=29500 \
|
||||
train/pretrain.py \
|
||||
--config "${CONFIG}" \
|
||||
--train_data "${TRAIN_DATA}" \
|
||||
--val_data "${VAL_DATA}" \
|
||||
--checkpoint_dir "${CKPT_DIR}" \
|
||||
--batch_size 4 \
|
||||
--lr 2e-4 \
|
||||
--weight_decay 0.1 \
|
||||
--warmup_steps 2000 \
|
||||
--grad_accum 8 \
|
||||
--max_steps 57000 \
|
||||
--log_file "${LOG_FILE}" \
|
||||
--use_fp8 \
|
||||
"$@"
|
||||
|
||||
echo "Training finished at $(date '+%Y-%m-%d %H:%M:%S')"
|
||||
133
source/scripts/launch_korean_1b.sh
Normal file
133
source/scripts/launch_korean_1b.sh
Normal file
@@ -0,0 +1,133 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# launch_korean_1b.sh — 8-GPU FP8 pretraining launcher for 1B Korean LLM
|
||||
#
|
||||
# Usage:
|
||||
# bash scripts/launch_korean_1b.sh # full run
|
||||
# bash scripts/launch_korean_1b.sh --max_steps 500 # quick test
|
||||
# bash scripts/launch_korean_1b.sh --resume checkpoints/korean_1b_fp8_run1/checkpoint-0010000
|
||||
#
|
||||
# Config is read from configs/korean_1b_fp8.yaml (model) + CLI args (train).
|
||||
# Effective batch size: 8 (local) × 8 GPU × 4 (grad_accum) × 4096 (seq_len)
|
||||
# = 1,048,576 tokens / step
|
||||
# Logs: checkpoints/<RUN_NAME>/train.log
|
||||
# checkpoints/<RUN_NAME>/tensorboard/
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
|
||||
# ---- Configurable defaults --------------------------------------------------
|
||||
RUN_NAME="${RUN_NAME:-korean_1b_fp8_run1}"
|
||||
CONFIG="${CONFIG:-configs/korean_1b_fp8.yaml}"
|
||||
TRAIN_DATA="${TRAIN_DATA:-data/korean_train.bin}"
|
||||
VAL_DATA="${VAL_DATA:-data/korean_val.bin}"
|
||||
CKPT_DIR="checkpoints/${RUN_NAME}"
|
||||
LOG_FILE="${CKPT_DIR}/train.log"
|
||||
NPROC=8
|
||||
MASTER_PORT="${MASTER_PORT:-29501}"
|
||||
|
||||
# ---- Defaults that can be overridden via extra CLI args --------------------
|
||||
MAX_STEPS=34000 # 4 에포크 × 8.91B tokens = 35.6B (Muennighoff 2023: 4에포크 초과 시 val loss 상승)
|
||||
BATCH_SIZE=8
|
||||
GRAD_ACCUM=4
|
||||
WARMUP_STEPS=2000 # 34k steps의 5.9% (기존 4000 = 11.8%로 과도)
|
||||
SEED=42
|
||||
|
||||
# ---- Pass remaining CLI args directly to pretrain.py ----------------------
|
||||
EXTRA_ARGS="$@"
|
||||
|
||||
# ---- B200 / NVSwitch single-node NCCL tuning --------------------------------
|
||||
# Single-node NVSwitch (NV18 full-mesh): disable IB to prevent NCCL probing.
|
||||
export NCCL_IB_DISABLE=1
|
||||
# Use Ring algorithm for large gradient tensors (128M-70B model range).
|
||||
export NCCL_ALGO=Ring
|
||||
# Simple protocol is optimal for NVLink bulk transfers (vs LL/LL128 for IB).
|
||||
export NCCL_PROTO=Simple
|
||||
# More channels → better NVSwitch saturation for large all-reduce payloads.
|
||||
export NCCL_MIN_NCHANNELS=16
|
||||
export NCCL_MAX_NCHANNELS=16
|
||||
# Larger NCCL buffer (64 MB) reduces ring synchronisation overhead.
|
||||
export NCCL_BUFFSIZE=67108864
|
||||
# CPU thread limits (72 cores ÷ 8 ranks = 9; use 4 for DataLoader headroom).
|
||||
export OMP_NUM_THREADS=4
|
||||
export MKL_NUM_THREADS=4
|
||||
|
||||
# ---- Setup ------------------------------------------------------------------
|
||||
cd "$(dirname "$0")/.." # always run from project root
|
||||
|
||||
# ---- Pre-flight check: Korean data must exist before launching --------------
|
||||
if [[ ! -f "${TRAIN_DATA}" ]]; then
|
||||
echo "=================================================================="
|
||||
echo " ERROR: Training data not found: ${TRAIN_DATA}"
|
||||
echo ""
|
||||
echo " You need to run the Korean data pipeline first."
|
||||
echo " Example steps:"
|
||||
echo " 1. Download / prepare raw Korean corpus"
|
||||
echo " 2. Tokenise and pack into binary format:"
|
||||
echo " python data/prepare_korean.py --output data/korean_train.bin"
|
||||
echo " 3. Re-run this script once the file exists."
|
||||
echo "=================================================================="
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! -f "${VAL_DATA}" ]]; then
|
||||
echo "=================================================================="
|
||||
echo " ERROR: Validation data not found: ${VAL_DATA}"
|
||||
echo ""
|
||||
echo " You need to run the Korean data pipeline first."
|
||||
echo " Example steps:"
|
||||
echo " 1. Download / prepare raw Korean corpus"
|
||||
echo " 2. Tokenise and pack into binary format (val split):"
|
||||
echo " python data/prepare_korean.py --output_val data/korean_val.bin"
|
||||
echo " 3. Re-run this script once the file exists."
|
||||
echo "=================================================================="
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "${CKPT_DIR}"
|
||||
|
||||
echo "=================================================================="
|
||||
echo " Run name : ${RUN_NAME}"
|
||||
echo " Config : ${CONFIG}"
|
||||
echo " Train data : ${TRAIN_DATA}"
|
||||
echo " Val data : ${VAL_DATA}"
|
||||
echo " CKPT dir : ${CKPT_DIR}"
|
||||
echo " Log file : ${LOG_FILE}"
|
||||
echo " Max steps : ${MAX_STEPS}"
|
||||
echo " Batch size : ${BATCH_SIZE} (local) × ${NPROC} GPU × ${GRAD_ACCUM} grad_accum"
|
||||
echo " Warmup : ${WARMUP_STEPS} steps"
|
||||
echo " Master port : ${MASTER_PORT}"
|
||||
echo " Started : $(date)"
|
||||
echo "=================================================================="
|
||||
|
||||
# Suppress the harmless flash_attn kernel override warning from all ranks.
|
||||
export PYTHONWARNINGS="ignore::UserWarning:torch.library"
|
||||
|
||||
torchrun \
|
||||
--nproc_per_node=${NPROC} \
|
||||
--master_port=${MASTER_PORT} \
|
||||
train/pretrain.py \
|
||||
--config "${CONFIG}" \
|
||||
--train_data "${TRAIN_DATA}" \
|
||||
--val_data "${VAL_DATA}" \
|
||||
--checkpoint_dir "${CKPT_DIR}" \
|
||||
--log_file "${LOG_FILE}" \
|
||||
--max_steps ${MAX_STEPS} \
|
||||
--batch_size ${BATCH_SIZE} \
|
||||
--grad_accum ${GRAD_ACCUM} \
|
||||
--warmup_steps ${WARMUP_STEPS} \
|
||||
--seed ${SEED} \
|
||||
${EXTRA_ARGS} \
|
||||
2>&1 | grep -v "UserWarning" \
|
||||
| grep -v "Warning only once" \
|
||||
| grep -v "Overriding a previously" \
|
||||
| grep -v "dispatch key:" \
|
||||
| grep -v "previous kernel:" \
|
||||
| grep -v "new kernel:" \
|
||||
| grep -v "operator: flash_attn" \
|
||||
| grep -v "registered at /usr/local" \
|
||||
| grep -v "self.m.impl" \
|
||||
| tee -a "${LOG_FILE}"
|
||||
|
||||
echo "=================================================================="
|
||||
echo " Done : $(date)"
|
||||
echo "=================================================================="
|
||||
115
source/scripts/launch_korean_3b.sh
Normal file
115
source/scripts/launch_korean_3b.sh
Normal file
@@ -0,0 +1,115 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# launch_korean_3b.sh — 8-GPU FP8 pretraining launcher for 3B Korean LLM
|
||||
#
|
||||
# Usage:
|
||||
# bash scripts/launch_korean_3b.sh # full run (~60B tokens)
|
||||
# bash scripts/launch_korean_3b.sh --max_steps 50 # quick benchmark
|
||||
# bash scripts/launch_korean_3b.sh --resume checkpoints/korean_3b_fp8_run1/checkpoint-XXXXX
|
||||
#
|
||||
# Effective batch size: 8 (local) × 8 GPU × 4 (grad_accum) × 4096 (seq_len)
|
||||
# = 1,048,576 tokens / step
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
|
||||
RUN_NAME="${RUN_NAME:-korean_3b_fp8_run1}"
|
||||
CONFIG="${CONFIG:-configs/3b_pretrain.yaml}"
|
||||
CKPT_DIR="checkpoints/${RUN_NAME}"
|
||||
LOG_FILE="${CKPT_DIR}/train.log"
|
||||
NPROC=8
|
||||
MASTER_PORT="${MASTER_PORT:-29502}"
|
||||
|
||||
MAX_STEPS=57000
|
||||
BATCH_SIZE=4
|
||||
GRAD_ACCUM=8
|
||||
LR=1.5e-4
|
||||
WARMUP_STEPS=2000
|
||||
SEED=42
|
||||
|
||||
EXTRA_ARGS="$@"
|
||||
|
||||
# ---- B200 / NVSwitch NCCL tuning -------------------------------------------
|
||||
export NCCL_IB_DISABLE=1
|
||||
export NCCL_ALGO=Ring
|
||||
export NCCL_PROTO=Simple
|
||||
export NCCL_MIN_NCHANNELS=16
|
||||
export NCCL_MAX_NCHANNELS=16
|
||||
export NCCL_BUFFSIZE=67108864
|
||||
export OMP_NUM_THREADS=4
|
||||
export MKL_NUM_THREADS=4
|
||||
|
||||
# cd FIRST — 이후 상대경로 체크가 프로젝트 루트 기준으로 동작
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
# TRAIN_DATA fallback: cd 이후에 상대경로 체크
|
||||
if [[ -f "data/merged_3b_train.bin" ]]; then
|
||||
TRAIN_DATA="${TRAIN_DATA:-data/merged_3b_train.bin}"
|
||||
echo "Using merged training data: data/merged_3b_train.bin"
|
||||
elif [[ -f "data/korean_train.bin" ]]; then
|
||||
TRAIN_DATA="${TRAIN_DATA:-data/korean_train.bin}"
|
||||
echo "Using fallback training data: data/korean_train.bin"
|
||||
else
|
||||
echo "ERROR: No training data found (data/merged_3b_train.bin or data/korean_train.bin)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# VAL_DATA fallback: cd 이후에 상대경로 체크
|
||||
VAL_DATA="${VAL_DATA:-data/merged_3b_val.bin}"
|
||||
if [[ ! -f "${VAL_DATA}" ]]; then
|
||||
VAL_DATA="data/korean_val.bin"
|
||||
fi
|
||||
|
||||
if [[ ! -f "${TRAIN_DATA}" ]]; then
|
||||
echo "ERROR: Training data not found: ${TRAIN_DATA}"
|
||||
exit 1
|
||||
fi
|
||||
if [[ ! -f "${VAL_DATA}" ]]; then
|
||||
echo "ERROR: Validation data not found: ${VAL_DATA}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "${CKPT_DIR}"
|
||||
|
||||
echo "=================================================================="
|
||||
echo " Run name : ${RUN_NAME}"
|
||||
echo " Config : ${CONFIG}"
|
||||
echo " Train data : ${TRAIN_DATA}"
|
||||
echo " CKPT dir : ${CKPT_DIR}"
|
||||
echo " Max steps : ${MAX_STEPS}"
|
||||
echo " LR : ${LR}"
|
||||
echo " Batch size : ${BATCH_SIZE} (local) × ${NPROC} GPU × ${GRAD_ACCUM} grad_accum"
|
||||
echo " Started : $(date)"
|
||||
echo "=================================================================="
|
||||
|
||||
export PYTHONWARNINGS="ignore::UserWarning:torch.library"
|
||||
|
||||
torchrun \
|
||||
--nproc_per_node=${NPROC} \
|
||||
--master_port=${MASTER_PORT} \
|
||||
train/pretrain.py \
|
||||
--config "${CONFIG}" \
|
||||
--train_data "${TRAIN_DATA}" \
|
||||
--val_data "${VAL_DATA}" \
|
||||
--checkpoint_dir "${CKPT_DIR}" \
|
||||
--log_file "${LOG_FILE}" \
|
||||
--max_steps ${MAX_STEPS} \
|
||||
--batch_size ${BATCH_SIZE} \
|
||||
--lr ${LR} \
|
||||
--grad_accum ${GRAD_ACCUM} \
|
||||
--warmup_steps ${WARMUP_STEPS} \
|
||||
--seed ${SEED} \
|
||||
${EXTRA_ARGS} \
|
||||
2>&1 | grep -v "UserWarning" \
|
||||
| grep -v "Warning only once" \
|
||||
| grep -v "Overriding a previously" \
|
||||
| grep -v "dispatch key:" \
|
||||
| grep -v "previous kernel:" \
|
||||
| grep -v "new kernel:" \
|
||||
| grep -v "operator: flash_attn" \
|
||||
| grep -v "registered at /usr/local" \
|
||||
| grep -v "self.m.impl" \
|
||||
| tee -a "${LOG_FILE}"
|
||||
|
||||
echo "=================================================================="
|
||||
echo " Done : $(date)"
|
||||
echo "=================================================================="
|
||||
111
source/scripts/launch_sft.sh
Normal file
111
source/scripts/launch_sft.sh
Normal file
@@ -0,0 +1,111 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# launch_sft.sh — 8-GPU FP8 SFT launcher for 1B Korean LLM
|
||||
#
|
||||
# Usage:
|
||||
# bash scripts/launch_sft.sh
|
||||
# bash scripts/launch_sft.sh --max_steps 500 # quick test
|
||||
# bash scripts/launch_sft.sh --resume checkpoints/korean_1b_sft/checkpoint-0001000
|
||||
#
|
||||
# Base model: checkpoints/korean_1b_fp8_run1/checkpoint-0034000
|
||||
# SFT data: data/sft/train.jsonl
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
|
||||
# ---- Configurable defaults --------------------------------------------------
|
||||
RUN_NAME="${RUN_NAME:-korean_1b_sft}"
|
||||
BASE_CHECKPOINT="${BASE_CHECKPOINT:-checkpoints/korean_1b_fp8_run1/checkpoint-0034000}"
|
||||
SFT_DATA="${SFT_DATA:-data/sft/train.jsonl}"
|
||||
VAL_DATA="${VAL_DATA:-data/sft/val.jsonl}"
|
||||
CKPT_DIR="checkpoints/${RUN_NAME}"
|
||||
LOG_FILE="${CKPT_DIR}/train.log"
|
||||
NPROC=8
|
||||
MASTER_PORT="${MASTER_PORT:-29502}"
|
||||
|
||||
MAX_STEPS=9000
|
||||
BATCH_SIZE=4
|
||||
GRAD_ACCUM=2
|
||||
LR="2.0e-5"
|
||||
WARMUP_STEPS=300
|
||||
SEED=42
|
||||
|
||||
EXTRA_ARGS="$@"
|
||||
|
||||
# ---- B200 / NVSwitch NCCL tuning (same as pretrain) -------------------------
|
||||
export NCCL_IB_DISABLE=1
|
||||
export NCCL_ALGO=Ring
|
||||
export NCCL_PROTO=Simple
|
||||
export NCCL_MIN_NCHANNELS=16
|
||||
export NCCL_MAX_NCHANNELS=16
|
||||
export NCCL_BUFFSIZE=67108864
|
||||
export OMP_NUM_THREADS=4
|
||||
export MKL_NUM_THREADS=4
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
# ---- Pre-flight checks ------------------------------------------------------
|
||||
if [[ ! -d "${BASE_CHECKPOINT}" ]]; then
|
||||
echo "ERROR: Base checkpoint not found: ${BASE_CHECKPOINT}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! -f "${SFT_DATA}" ]]; then
|
||||
echo "=================================================================="
|
||||
echo " ERROR: SFT training data not found: ${SFT_DATA}"
|
||||
echo ""
|
||||
echo " Run the data preparation script first:"
|
||||
echo " python data/prepare_sft_data.py"
|
||||
echo "=================================================================="
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "${CKPT_DIR}"
|
||||
|
||||
echo "=================================================================="
|
||||
echo " SFT Fine-Tuning"
|
||||
echo " Run name : ${RUN_NAME}"
|
||||
echo " Base checkpoint : ${BASE_CHECKPOINT}"
|
||||
echo " SFT data : ${SFT_DATA}"
|
||||
echo " CKPT dir : ${CKPT_DIR}"
|
||||
echo " Log file : ${LOG_FILE}"
|
||||
echo " Max steps : ${MAX_STEPS}"
|
||||
echo " Batch size : ${BATCH_SIZE} (local) × ${NPROC} GPU × ${GRAD_ACCUM} grad_accum"
|
||||
echo " Learning rate : ${LR}"
|
||||
echo " Warmup : ${WARMUP_STEPS} steps"
|
||||
echo " Master port : ${MASTER_PORT}"
|
||||
echo " Started : $(date)"
|
||||
echo "=================================================================="
|
||||
|
||||
export PYTHONWARNINGS="ignore::UserWarning:torch.library"
|
||||
|
||||
torchrun \
|
||||
--nproc_per_node=${NPROC} \
|
||||
--master_port=${MASTER_PORT} \
|
||||
train/sft.py \
|
||||
--base_checkpoint "${BASE_CHECKPOINT}" \
|
||||
--sft_data "${SFT_DATA}" \
|
||||
--checkpoint_dir "${CKPT_DIR}" \
|
||||
--log_file "${LOG_FILE}" \
|
||||
--max_steps ${MAX_STEPS} \
|
||||
--batch_size ${BATCH_SIZE} \
|
||||
--grad_accum ${GRAD_ACCUM} \
|
||||
--lr ${LR} \
|
||||
--warmup_steps ${WARMUP_STEPS} \
|
||||
--seed ${SEED} \
|
||||
--use_fp8 \
|
||||
--val_data "${VAL_DATA}" \
|
||||
${EXTRA_ARGS} \
|
||||
2>&1 | grep -v "UserWarning" \
|
||||
| grep -v "Warning only once" \
|
||||
| grep -v "Overriding a previously" \
|
||||
| grep -v "dispatch key:" \
|
||||
| grep -v "previous kernel:" \
|
||||
| grep -v "new kernel:" \
|
||||
| grep -v "operator: flash_attn" \
|
||||
| grep -v "registered at /usr/local" \
|
||||
| grep -v "self.m.impl" \
|
||||
| tee -a "${LOG_FILE}"
|
||||
|
||||
echo "=================================================================="
|
||||
echo " SFT Done : $(date)"
|
||||
echo "=================================================================="
|
||||
230
source/scripts/migrate_qkv_checkpoint.py
Normal file
230
source/scripts/migrate_qkv_checkpoint.py
Normal file
@@ -0,0 +1,230 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Migrate checkpoint from separate Q/K/V projections to fused QKV.
|
||||
|
||||
Usage:
|
||||
python3 scripts/migrate_qkv_checkpoint.py <checkpoint_dir>
|
||||
|
||||
Migrates both model.pt AND optimizer.pt:
|
||||
- model.pt: q_proj/k_proj/v_proj weights → qkv_proj weight
|
||||
- optimizer.pt: exp_avg/exp_avg_sq states fused, param indices re-mapped
|
||||
|
||||
The concatenation order is [Q ; K ; V] along the output (dim-0) axis,
|
||||
which matches the split in MultiHeadAttention.forward:
|
||||
q, k, v = qkv.split([_q_dim, _kv_dim, _kv_dim], dim=-1)
|
||||
|
||||
Optimizer layout (group 0 = weight_decay, per layer × 28):
|
||||
[i*6+0] q_proj.weight [3072, 3072]
|
||||
[i*6+1] k_proj.weight [1024, 3072]
|
||||
[i*6+2] v_proj.weight [1024, 3072]
|
||||
[i*6+3] out_proj.weight [3072, 3072]
|
||||
[i*6+4] fc1_weight [16384, 3072]
|
||||
[i*6+5] fc2_weight [3072, 8192]
|
||||
After fusion: indices 0,1,2 → single qkv_proj → 4 params per layer.
|
||||
"""
|
||||
import sys
|
||||
import torch
|
||||
from pathlib import Path
|
||||
|
||||
N_LAYERS = 28
|
||||
OLD_PARAMS_PER_LAYER = 6 # q, k, v, out, fc1, fc2
|
||||
NEW_PARAMS_PER_LAYER = 4 # qkv, out, fc1, fc2
|
||||
|
||||
|
||||
def migrate_model(state: dict) -> dict:
|
||||
"""Fuse Q/K/V projection weights into QKV in model state dict."""
|
||||
new_state: dict = {}
|
||||
layers_done: set = set()
|
||||
|
||||
for key, val in state.items():
|
||||
if ".q_proj." not in key and ".k_proj." not in key and ".v_proj." not in key:
|
||||
new_state[key] = val
|
||||
continue
|
||||
|
||||
if ".q_proj." not in key:
|
||||
continue
|
||||
|
||||
prefix = key.rsplit(".", 2)[0]
|
||||
suffix = key.rsplit(".", 1)[-1]
|
||||
|
||||
tag = (prefix, suffix)
|
||||
if tag in layers_done:
|
||||
continue
|
||||
layers_done.add(tag)
|
||||
|
||||
q_key = f"{prefix}.q_proj.{suffix}"
|
||||
k_key = f"{prefix}.k_proj.{suffix}"
|
||||
v_key = f"{prefix}.v_proj.{suffix}"
|
||||
|
||||
missing = [k for k in (q_key, k_key, v_key) if k not in state]
|
||||
if missing:
|
||||
raise KeyError(f"Expected keys not found in checkpoint: {missing}")
|
||||
|
||||
q_w, k_w, v_w = state[q_key], state[k_key], state[v_key]
|
||||
fused = torch.cat([q_w, k_w, v_w], dim=0)
|
||||
fused_key = f"{prefix}.qkv_proj.{suffix}"
|
||||
new_state[fused_key] = fused
|
||||
print(f" Fused {fused_key}: {list(fused.shape)}"
|
||||
f" (q={list(q_w.shape)}, k={list(k_w.shape)}, v={list(v_w.shape)})")
|
||||
|
||||
leaked = [k for k in new_state if ".q_proj." in k or ".k_proj." in k or ".v_proj." in k]
|
||||
if leaked:
|
||||
raise RuntimeError(f"BUG: old projection keys still present: {leaked}")
|
||||
|
||||
return new_state
|
||||
|
||||
|
||||
def migrate_optimizer(opt_state: dict) -> dict:
|
||||
"""Fuse optimizer states for Q/K/V → QKV and re-index parameters.
|
||||
|
||||
The optimizer has 2 param groups:
|
||||
Group 0 (weight_decay): 168 = 28 layers × 6 (q,k,v,out,fc1,fc2)
|
||||
Group 1 (no weight_decay): 58 = norms + embedding
|
||||
|
||||
We fuse q,k,v entries in group 0 (indices i*6+0,1,2 → one entry per layer).
|
||||
Group 0 shrinks from 168 to 112 (28 layers × 4 params).
|
||||
Group 1 stays at 58. Total: 170.
|
||||
"""
|
||||
old_state = opt_state["state"]
|
||||
old_groups = opt_state["param_groups"]
|
||||
|
||||
group0_count = len(old_groups[0]["params"])
|
||||
expected_g0 = N_LAYERS * OLD_PARAMS_PER_LAYER
|
||||
if group0_count != expected_g0:
|
||||
raise ValueError(
|
||||
f"Group 0 has {group0_count} params, expected {expected_g0}. "
|
||||
f"Cannot auto-detect QKV layout."
|
||||
)
|
||||
|
||||
# Validate shapes for first layer
|
||||
shapes = []
|
||||
for j in range(OLD_PARAMS_PER_LAYER):
|
||||
idx = old_groups[0]["params"][j]
|
||||
shapes.append(list(old_state[idx]["exp_avg"].shape))
|
||||
expected_shapes = [[3072, 3072], [1024, 3072], [1024, 3072],
|
||||
[3072, 3072], [16384, 3072], [3072, 8192]]
|
||||
if shapes != expected_shapes:
|
||||
raise ValueError(
|
||||
f"Layer 0 shapes {shapes} don't match expected {expected_shapes}. "
|
||||
f"Cannot auto-detect QKV layout."
|
||||
)
|
||||
print(f" Shape validation passed for layer 0.")
|
||||
|
||||
new_state_entries = {}
|
||||
new_idx = 0
|
||||
|
||||
# --- Group 0: fuse q/k/v per layer ---
|
||||
for layer_i in range(N_LAYERS):
|
||||
base = layer_i * OLD_PARAMS_PER_LAYER
|
||||
q_opt_idx = old_groups[0]["params"][base + 0]
|
||||
k_opt_idx = old_groups[0]["params"][base + 1]
|
||||
v_opt_idx = old_groups[0]["params"][base + 2]
|
||||
|
||||
q_entry = old_state[q_opt_idx]
|
||||
k_entry = old_state[k_opt_idx]
|
||||
v_entry = old_state[v_opt_idx]
|
||||
|
||||
# Fuse QKV
|
||||
fused_entry = {"step": q_entry["step"]}
|
||||
for field in ["exp_avg", "exp_avg_sq"]:
|
||||
if field in q_entry:
|
||||
fused_entry[field] = torch.cat(
|
||||
[q_entry[field], k_entry[field], v_entry[field]], dim=0
|
||||
)
|
||||
new_state_entries[new_idx] = fused_entry
|
||||
if layer_i == 0:
|
||||
print(f" Layer 0 QKV fused: exp_avg {list(fused_entry['exp_avg'].shape)}")
|
||||
new_idx += 1
|
||||
|
||||
# Copy remaining params (out, fc1, fc2)
|
||||
for offset in [3, 4, 5]:
|
||||
opt_idx = old_groups[0]["params"][base + offset]
|
||||
new_state_entries[new_idx] = old_state[opt_idx]
|
||||
new_idx += 1
|
||||
|
||||
new_group0_count = new_idx # should be N_LAYERS * NEW_PARAMS_PER_LAYER = 112
|
||||
print(f" Group 0: {group0_count} → {new_group0_count} params")
|
||||
|
||||
# --- Group 1: copy as-is (norms, embedding — no QKV) ---
|
||||
group1_count = len(old_groups[1]["params"])
|
||||
for j in range(group1_count):
|
||||
opt_idx = old_groups[1]["params"][j]
|
||||
if opt_idx in old_state:
|
||||
new_state_entries[new_idx] = old_state[opt_idx]
|
||||
new_idx += 1
|
||||
print(f" Group 1: {group1_count} → {group1_count} params (unchanged)")
|
||||
|
||||
# Build new param_groups
|
||||
new_groups = []
|
||||
g0 = {k: v for k, v in old_groups[0].items() if k != "params"}
|
||||
g0["params"] = list(range(0, new_group0_count))
|
||||
new_groups.append(g0)
|
||||
|
||||
g1 = {k: v for k, v in old_groups[1].items() if k != "params"}
|
||||
g1["params"] = list(range(new_group0_count, new_group0_count + group1_count))
|
||||
new_groups.append(g1)
|
||||
|
||||
total = new_group0_count + group1_count
|
||||
print(f" Total: {len(old_state)} → {total} optimizer params")
|
||||
|
||||
return {"state": new_state_entries, "param_groups": new_groups}
|
||||
|
||||
|
||||
def migrate(ckpt_dir: Path) -> None:
|
||||
model_path = ckpt_dir / "model.pt"
|
||||
opt_path = ckpt_dir / "optimizer.pt"
|
||||
|
||||
if not model_path.exists():
|
||||
raise FileNotFoundError(f"model.pt not found in {ckpt_dir}")
|
||||
|
||||
# --- Model migration ---
|
||||
print(f"[1/2] Migrating model weights from {model_path} ...")
|
||||
state = torch.load(model_path, map_location="cpu", weights_only=True)
|
||||
|
||||
has_old = any(".q_proj." in k for k in state)
|
||||
has_new = any(".qkv_proj." in k for k in state)
|
||||
|
||||
if has_new and not has_old:
|
||||
print(" Model already migrated. Skipping.")
|
||||
elif has_old:
|
||||
new_model_state = migrate_model(state)
|
||||
torch.save(new_model_state, model_path)
|
||||
print(f" Model saved.")
|
||||
else:
|
||||
raise RuntimeError("Model state has neither q_proj nor qkv_proj keys!")
|
||||
|
||||
# --- Optimizer migration ---
|
||||
if opt_path.exists():
|
||||
print(f"\n[2/2] Migrating optimizer states from {opt_path} ...")
|
||||
opt = torch.load(opt_path, map_location="cpu", weights_only=True)
|
||||
|
||||
# Check if already migrated
|
||||
total_params = sum(len(pg["params"]) for pg in opt["param_groups"])
|
||||
expected_old = N_LAYERS * OLD_PARAMS_PER_LAYER + 58 # 168 + 58 = 226
|
||||
expected_new = N_LAYERS * NEW_PARAMS_PER_LAYER + 58 # 112 + 58 = 170
|
||||
|
||||
if total_params == expected_old:
|
||||
opt_backup = ckpt_dir / "optimizer.pt.backup_pre_qkv"
|
||||
if not opt_backup.exists():
|
||||
torch.save(opt, opt_backup)
|
||||
print(f" Backup: {opt_backup}")
|
||||
new_opt = migrate_optimizer(opt)
|
||||
torch.save(new_opt, opt_path)
|
||||
print(f" Optimizer saved.")
|
||||
elif total_params == expected_new:
|
||||
print(f" Optimizer already migrated ({total_params} params). Skipping.")
|
||||
else:
|
||||
print(f" [WARN] Unexpected param count {total_params} "
|
||||
f"(expected old={expected_old} or new={expected_new}). "
|
||||
f"Deleting optimizer.pt — optimizer will restart fresh.")
|
||||
opt_path.unlink()
|
||||
else:
|
||||
print("\n[2/2] No optimizer.pt found. Optimizer will restart fresh.")
|
||||
|
||||
print("\nMigration complete!")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) != 2:
|
||||
print(__doc__)
|
||||
sys.exit(1)
|
||||
migrate(Path(sys.argv[1]))
|
||||
316
source/scripts/monitor_3b.sh
Normal file
316
source/scripts/monitor_3b.sh
Normal file
@@ -0,0 +1,316 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# monitor_3b.sh — 3B 학습 실시간 모니터링 + 이상 감지 + 자동 체크포인트 정리
|
||||
#
|
||||
# Usage:
|
||||
# bash scripts/monitor_3b.sh # 기본 감시
|
||||
# bash scripts/monitor_3b.sh --check-once # 1회 검사
|
||||
# bash scripts/monitor_3b.sh --auto-cleanup # 자동 오래된 체크포인트 삭제
|
||||
#
|
||||
# 3B 특화 사항:
|
||||
# - 체크포인트 27GB/개 → 디스크 감시 강화
|
||||
# - NCCL hang 감지 + 자동 재시작 옵션
|
||||
# - 예상 완료 시간 실시간 계산
|
||||
# - 프로세스 중복 실행 방지
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
|
||||
# ---- Configuration ----------------------------------------------------------
|
||||
RUN_NAME="${RUN_NAME:-korean_3b_fp8_run1}"
|
||||
LOG_FILE="${1:-checkpoints/${RUN_NAME}/train.log}"
|
||||
CKPT_DIR="checkpoints/${RUN_NAME}"
|
||||
CHECK_INTERVAL=60 # 3B는 step 간격 더 김 → 60초
|
||||
ZERO_LOSS_THRESHOLD=3
|
||||
GNORM_WARN=10.0
|
||||
GNORM_CRITICAL=50.0
|
||||
LOSS_SPIKE_FACTOR=3.0
|
||||
STALL_TIMEOUT=600 # 10분 (3B는 step 더 오래 걸림)
|
||||
DISK_WARN_PCT=85
|
||||
DISK_CRITICAL_PCT=92
|
||||
GPU_UTIL_WARN=50
|
||||
MAX_CHECKPOINTS=15 # 최대 보관 체크포인트 수 (15 × 27GB = 405GB)
|
||||
CHECK_ONCE=false
|
||||
AUTO_CLEANUP=false
|
||||
AUTO_RESTART=false
|
||||
|
||||
# Parse args
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--check-once) CHECK_ONCE=true ;;
|
||||
--auto-cleanup) AUTO_CLEANUP=true ;;
|
||||
--auto-restart) AUTO_RESTART=true ;;
|
||||
esac
|
||||
done
|
||||
# Fix LOG_FILE if first arg was a flag
|
||||
if [[ "$LOG_FILE" == --* ]]; then
|
||||
LOG_FILE="checkpoints/${RUN_NAME}/train.log"
|
||||
fi
|
||||
|
||||
# ---- Colors -----------------------------------------------------------------
|
||||
RED='\033[0;31m'; YELLOW='\033[1;33m'; GREEN='\033[0;32m'
|
||||
CYAN='\033[0;36m'; MAGENTA='\033[0;35m'; NC='\033[0m'
|
||||
|
||||
timestamp() { date '+%Y-%m-%d %H:%M:%S'; }
|
||||
|
||||
alert() {
|
||||
local level="$1" msg="$2"
|
||||
case "$level" in
|
||||
CRITICAL) echo -e "${RED}🔴 [$(timestamp)] [CRITICAL] ${msg}${NC}" ;;
|
||||
WARNING) echo -e "${YELLOW}🟠 [$(timestamp)] [WARNING] ${msg}${NC}" ;;
|
||||
INFO) echo -e "${CYAN}🟡 [$(timestamp)] [INFO] ${msg}${NC}" ;;
|
||||
OK) echo -e "${GREEN}✅ [$(timestamp)] [OK] ${msg}${NC}" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# ---- Parse metrics ----------------------------------------------------------
|
||||
parse_metrics() {
|
||||
local n="${1:-20}"
|
||||
[[ -f "$LOG_FILE" ]] || return
|
||||
tail -n "$n" "$LOG_FILE" | grep "step.*loss.*gnorm" || true
|
||||
}
|
||||
|
||||
extract_field() {
|
||||
echo "$1" | grep -oP "${2}\s+\K[0-9]+\.[0-9e+\-]+" | head -1
|
||||
}
|
||||
|
||||
extract_step() {
|
||||
echo "$1" | grep -oP "step\s+\K[0-9]+" | head -1
|
||||
}
|
||||
|
||||
# ---- Check: Loss = 0 -------------------------------------------------------
|
||||
check_loss_zero() {
|
||||
local lines
|
||||
lines=$(parse_metrics "$ZERO_LOSS_THRESHOLD")
|
||||
[[ -z "$lines" ]] && return 0
|
||||
local zero_count=0
|
||||
while IFS= read -r line; do
|
||||
local loss=$(extract_field "$line" "loss")
|
||||
if [[ -n "$loss" ]] && (( $(echo "$loss < 0.001" | bc -l 2>/dev/null || echo 0) )); then
|
||||
((zero_count++))
|
||||
fi
|
||||
done <<< "$lines"
|
||||
if [[ $zero_count -ge $ZERO_LOSS_THRESHOLD ]]; then
|
||||
alert CRITICAL "Loss가 ${zero_count}회 연속 ~0! Labels 버그. 즉시 중단!"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# ---- Check: Loss spike -----------------------------------------------------
|
||||
check_loss_spike() {
|
||||
local lines=$(parse_metrics 20)
|
||||
[[ -z "$lines" ]] && return 0
|
||||
local losses=()
|
||||
while IFS= read -r line; do
|
||||
local loss=$(extract_field "$line" "loss")
|
||||
[[ -n "$loss" ]] && losses+=("$loss")
|
||||
done <<< "$lines"
|
||||
local count=${#losses[@]}
|
||||
[[ $count -lt 5 ]] && return 0
|
||||
local last="${losses[$((count-1))]}"
|
||||
local sum=0
|
||||
for ((i=0; i<count-1; i++)); do
|
||||
sum=$(echo "$sum + ${losses[$i]}" | bc -l 2>/dev/null || echo "$sum")
|
||||
done
|
||||
local avg=$(echo "$sum / ($count - 1)" | bc -l 2>/dev/null || echo "0")
|
||||
if [[ "$avg" != "0" ]]; then
|
||||
local ratio=$(echo "$last / $avg" | bc -l 2>/dev/null || echo "1")
|
||||
if (( $(echo "$ratio > $LOSS_SPIKE_FACTOR" | bc -l 2>/dev/null || echo 0) )); then
|
||||
alert WARNING "Loss spike! 현재=${last}, 평균=${avg}, 비율=${ratio}x"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
# ---- Check: Gradient norm ---------------------------------------------------
|
||||
check_gnorm() {
|
||||
local lines=$(parse_metrics 5)
|
||||
[[ -z "$lines" ]] && return 0
|
||||
local gnorm=$(extract_field "$(echo "$lines" | tail -1)" "gnorm")
|
||||
[[ -z "$gnorm" ]] && return 0
|
||||
if (( $(echo "$gnorm > $GNORM_CRITICAL" | bc -l 2>/dev/null || echo 0) )); then
|
||||
alert CRITICAL "GNorm=${gnorm} > ${GNORM_CRITICAL}! 발산 직전."
|
||||
elif (( $(echo "$gnorm > $GNORM_WARN" | bc -l 2>/dev/null || echo 0) )); then
|
||||
alert WARNING "GNorm=${gnorm} 불안정."
|
||||
fi
|
||||
}
|
||||
|
||||
# ---- Check: Stall / NCCL hang ----------------------------------------------
|
||||
check_stall() {
|
||||
[[ ! -f "$LOG_FILE" ]] && return 0
|
||||
local last_mod=$(stat -c %Y "$LOG_FILE" 2>/dev/null || echo 0)
|
||||
local now=$(date +%s)
|
||||
local diff=$((now - last_mod))
|
||||
if [[ $diff -gt $STALL_TIMEOUT ]]; then
|
||||
alert CRITICAL "로그 ${diff}초 ($(( diff/60 ))분) 멈춤! NCCL hang 가능성."
|
||||
# NCCL hang 자동 재시작
|
||||
if $AUTO_RESTART; then
|
||||
alert WARNING "자동 재시작 시도..."
|
||||
local pid=$(pgrep -f "pretrain.py.*korean_3b" | head -1 || true)
|
||||
if [[ -n "$pid" ]]; then
|
||||
kill -9 "$pid" 2>/dev/null || true
|
||||
sleep 5
|
||||
alert INFO "이전 프로세스 종료. launch_3b_pretrain.sh 재실행 필요."
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
# ---- Check: Disk (3B 강화) --------------------------------------------------
|
||||
check_disk() {
|
||||
local usage=$(df /PROJECT 2>/dev/null | awk 'NR==2 {print $5}' | tr -d '%')
|
||||
if [[ -n "$usage" && "$usage" -gt "$DISK_CRITICAL_PCT" ]]; then
|
||||
alert CRITICAL "디스크 ${usage}% > ${DISK_CRITICAL_PCT}%! 즉시 정리 필요!"
|
||||
$AUTO_CLEANUP && cleanup_old_checkpoints
|
||||
elif [[ -n "$usage" && "$usage" -gt "$DISK_WARN_PCT" ]]; then
|
||||
alert WARNING "디스크 ${usage}% > ${DISK_WARN_PCT}%. 체크포인트 정리 권장."
|
||||
fi
|
||||
}
|
||||
|
||||
# ---- Check: GPU utilization -------------------------------------------------
|
||||
check_gpu() {
|
||||
command -v nvidia-smi &>/dev/null || return 0
|
||||
local low=0 total=0
|
||||
while IFS= read -r util; do
|
||||
((total++))
|
||||
[[ "$util" -lt "$GPU_UTIL_WARN" ]] && ((low++))
|
||||
done < <(nvidia-smi --query-gpu=utilization.gpu --format=csv,noheader,nounits 2>/dev/null)
|
||||
[[ $total -gt 0 && $low -gt 0 ]] && alert INFO "${low}/${total} GPU util < ${GPU_UTIL_WARN}%"
|
||||
}
|
||||
|
||||
# ---- Check: 체크포인트 무결성 -----------------------------------------------
|
||||
check_checkpoint_integrity() {
|
||||
local latest=$(ls -d "${CKPT_DIR}"/checkpoint-* 2>/dev/null | sort -V | tail -1 || true)
|
||||
[[ -z "$latest" ]] && return 0
|
||||
# 최소 파일 존재 확인
|
||||
if [[ ! -f "${latest}/model.pt" ]] && [[ ! -f "${latest}/model.safetensors" ]]; then
|
||||
alert WARNING "최근 체크포인트에 모델 파일 없음: ${latest}"
|
||||
fi
|
||||
# 크기 확인 (3B model.pt는 최소 2GB)
|
||||
local size=$(du -sb "${latest}" 2>/dev/null | awk '{print $1}')
|
||||
if [[ -n "$size" && "$size" -lt 2000000000 ]]; then
|
||||
alert WARNING "체크포인트 크기 비정상 (${size} bytes < 2GB): ${latest}"
|
||||
fi
|
||||
}
|
||||
|
||||
# ---- Cleanup: 오래된 체크포인트 자동 삭제 ------------------------------------
|
||||
cleanup_old_checkpoints() {
|
||||
local ckpts=($(ls -d "${CKPT_DIR}"/checkpoint-* 2>/dev/null | sort -V))
|
||||
local count=${#ckpts[@]}
|
||||
if [[ $count -le $MAX_CHECKPOINTS ]]; then
|
||||
alert OK "체크포인트 ${count}개 ≤ ${MAX_CHECKPOINTS}. 정리 불필요."
|
||||
return
|
||||
fi
|
||||
# 이정표 체크포인트 보존 (매 10K step)
|
||||
local deletable=()
|
||||
local preserved=()
|
||||
for ckpt in "${ckpts[@]}"; do
|
||||
local step_num=$(basename "$ckpt" | grep -oP '\d+' || echo "0")
|
||||
if (( step_num % 10000 == 0 && step_num > 0 )); then
|
||||
preserved+=("$ckpt")
|
||||
else
|
||||
deletable+=("$ckpt")
|
||||
fi
|
||||
done
|
||||
# 최근 MAX_CHECKPOINTS개는 무조건 보존
|
||||
local n_deletable=${#deletable[@]}
|
||||
local total_keep=$(( ${#preserved[@]} + MAX_CHECKPOINTS ))
|
||||
local to_delete=$(( count - total_keep ))
|
||||
[[ $to_delete -le 0 ]] && { alert OK "정리 불필요 (이정표 ${#preserved[@]}개 + 최근 ${MAX_CHECKPOINTS}개 보존)."; return; }
|
||||
alert INFO "${count}개 체크포인트 → ${to_delete}개 삭제 (이정표 ${#preserved[@]}개 영구 보존)"
|
||||
local deleted=0
|
||||
for ckpt in "${deletable[@]}"; do
|
||||
[[ $deleted -ge $to_delete ]] && break
|
||||
local ckpt_size=$(du -sh "$ckpt" 2>/dev/null | awk '{print $1}')
|
||||
echo " 삭제: $ckpt (${ckpt_size})"
|
||||
rm -rf "$ckpt"
|
||||
((deleted++))
|
||||
done
|
||||
alert OK "체크포인트 정리 완료. (${deleted}개 삭제)"
|
||||
}
|
||||
|
||||
# ---- ETA 계산 ---------------------------------------------------------------
|
||||
estimate_eta() {
|
||||
[[ ! -f "$LOG_FILE" ]] && return
|
||||
# 최근 step 번호 + 시간
|
||||
local lines=$(parse_metrics 50)
|
||||
[[ -z "$lines" ]] && return
|
||||
local last_line=$(echo "$lines" | tail -1)
|
||||
local first_line=$(echo "$lines" | head -1)
|
||||
local cur_step=$(extract_step "$last_line")
|
||||
local max_steps=$(grep -oP "max_steps.*?(\d+)" "${CKPT_DIR}/train.log" 2>/dev/null | head -1 | grep -oP '\d+$' || echo "57000")
|
||||
|
||||
[[ -z "$cur_step" || "$cur_step" == "0" ]] && return
|
||||
|
||||
# step/sec from log timestamps (approximate)
|
||||
local remaining=$((max_steps - cur_step))
|
||||
if [[ $remaining -le 0 ]]; then
|
||||
echo -e "${MAGENTA}📊 진행: ${cur_step}/${max_steps} (완료!)${NC}"
|
||||
return
|
||||
fi
|
||||
|
||||
# 파일 수정 시간 기반 rough ETA
|
||||
local first_time=$(head -20 "$LOG_FILE" 2>/dev/null | grep -oP '^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}' | head -1 || true)
|
||||
if [[ -n "$first_time" ]]; then
|
||||
local start_epoch=$(date -d "$first_time" +%s 2>/dev/null || echo 0)
|
||||
local now=$(date +%s)
|
||||
if [[ $start_epoch -gt 0 && $cur_step -gt 0 ]]; then
|
||||
local elapsed=$((now - start_epoch))
|
||||
local sec_per_step=$(echo "$elapsed / $cur_step" | bc -l 2>/dev/null || echo "0")
|
||||
local eta_sec=$(echo "$remaining * $sec_per_step" | bc 2>/dev/null | cut -d. -f1 || echo "0")
|
||||
local eta_hours=$(echo "$eta_sec / 3600" | bc 2>/dev/null || echo "?")
|
||||
local pct=$(echo "scale=1; $cur_step * 100 / $max_steps" | bc 2>/dev/null || echo "?")
|
||||
echo -e "${MAGENTA}📊 진행: ${cur_step}/${max_steps} (${pct}%) | 남은 시간: ~${eta_hours}h | ${sec_per_step}s/step${NC}"
|
||||
fi
|
||||
else
|
||||
echo -e "${MAGENTA}📊 진행: ${cur_step}/${max_steps}${NC}"
|
||||
fi
|
||||
}
|
||||
|
||||
# ---- Status summary ---------------------------------------------------------
|
||||
print_status() {
|
||||
local lines=$(parse_metrics 1)
|
||||
[[ -n "$lines" ]] && echo -e "${GREEN}최근:${NC} $lines"
|
||||
estimate_eta
|
||||
if command -v nvidia-smi &>/dev/null; then
|
||||
echo -e "${CYAN}GPU:${NC}"
|
||||
nvidia-smi --query-gpu=index,memory.used,memory.total,utilization.gpu,temperature.gpu \
|
||||
--format=csv,noheader 2>/dev/null | head -8
|
||||
fi
|
||||
local ckpt_count=$(ls -d "${CKPT_DIR}"/checkpoint-* 2>/dev/null | wc -l)
|
||||
local ckpt_size=$(du -sh "${CKPT_DIR}" 2>/dev/null | awk '{print $1}')
|
||||
echo -e "${CYAN}체크포인트:${NC} ${ckpt_count}개 (${ckpt_size})"
|
||||
local disk=$(df -h /PROJECT 2>/dev/null | awk 'NR==2 {print $3"/"$2" ("$5")"}')
|
||||
echo -e "${CYAN}디스크:${NC} ${disk}"
|
||||
}
|
||||
|
||||
# ---- Main -------------------------------------------------------------------
|
||||
echo "=================================================================="
|
||||
echo " 3B Training Monitor"
|
||||
echo " Run: ${RUN_NAME}"
|
||||
echo " Log: ${LOG_FILE}"
|
||||
echo " Interval: ${CHECK_INTERVAL}s"
|
||||
echo " Auto-cleanup: ${AUTO_CLEANUP} | Auto-restart: ${AUTO_RESTART}"
|
||||
echo " Ctrl+C to stop"
|
||||
echo "=================================================================="
|
||||
|
||||
run_all_checks() {
|
||||
check_loss_zero || true
|
||||
check_loss_spike || true
|
||||
check_gnorm || true
|
||||
check_stall || true
|
||||
check_disk || true
|
||||
check_gpu || true
|
||||
check_checkpoint_integrity || true
|
||||
echo "---"
|
||||
print_status
|
||||
echo ""
|
||||
}
|
||||
|
||||
if $CHECK_ONCE; then
|
||||
run_all_checks
|
||||
exit 0
|
||||
fi
|
||||
|
||||
while true; do
|
||||
run_all_checks
|
||||
sleep "$CHECK_INTERVAL"
|
||||
done
|
||||
244
source/scripts/monitor_training.sh
Normal file
244
source/scripts/monitor_training.sh
Normal file
@@ -0,0 +1,244 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# monitor_training.sh — SFT 학습 실시간 모니터링 + 이상 감지
|
||||
#
|
||||
# Usage:
|
||||
# bash scripts/monitor_training.sh # 기본 로그 경로
|
||||
# bash scripts/monitor_training.sh /path/to/train.log # 커스텀 경로
|
||||
# bash scripts/monitor_training.sh --check-once # 1회 검사 후 종료
|
||||
#
|
||||
# 감시 항목:
|
||||
# 🔴 loss = 0.0000 (3 step 연속) → Labels 버그
|
||||
# 🔴 gnorm > 50.0 → 발산 직전
|
||||
# 🔴 로그 5분 이상 멈춤 → Hang
|
||||
# 🟠 loss spike (3× 이동평균) → Bad batch / LR
|
||||
# 🟠 gnorm > 10.0 → 불안정
|
||||
# 🟠 디스크 > 80% → 정리 필요
|
||||
# 🟡 GPU util < 50% → 병목
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
|
||||
# ---- Configuration ----------------------------------------------------------
|
||||
LOG_FILE="${1:-checkpoints/korean_1b_sft/train.log}"
|
||||
CHECK_INTERVAL=30 # 초 단위 폴링 간격
|
||||
ZERO_LOSS_THRESHOLD=3 # N회 연속 loss=0이면 경고
|
||||
GNORM_WARN=10.0
|
||||
GNORM_CRITICAL=50.0
|
||||
LOSS_SPIKE_FACTOR=3.0 # 이동평균 대비 N배 이상이면 spike
|
||||
STALL_TIMEOUT=300 # 초 (5분) 로그 멈춤 감지
|
||||
DISK_WARN_PCT=80
|
||||
GPU_UTIL_WARN=50
|
||||
CHECK_ONCE=false
|
||||
|
||||
if [[ "${1:-}" == "--check-once" ]]; then
|
||||
CHECK_ONCE=true
|
||||
LOG_FILE="${2:-checkpoints/korean_1b_sft/train.log}"
|
||||
fi
|
||||
|
||||
# ---- Colors -----------------------------------------------------------------
|
||||
RED='\033[0;31m'
|
||||
YELLOW='\033[1;33m'
|
||||
GREEN='\033[0;32m'
|
||||
CYAN='\033[0;36m'
|
||||
NC='\033[0m'
|
||||
|
||||
# ---- Helper -----------------------------------------------------------------
|
||||
timestamp() { date '+%Y-%m-%d %H:%M:%S'; }
|
||||
|
||||
alert() {
|
||||
local level="$1" msg="$2"
|
||||
case "$level" in
|
||||
CRITICAL) echo -e "${RED}🔴 [$(timestamp)] [CRITICAL] ${msg}${NC}" ;;
|
||||
WARNING) echo -e "${YELLOW}🟠 [$(timestamp)] [WARNING] ${msg}${NC}" ;;
|
||||
INFO) echo -e "${CYAN}🟡 [$(timestamp)] [INFO] ${msg}${NC}" ;;
|
||||
OK) echo -e "${GREEN}✅ [$(timestamp)] [OK] ${msg}${NC}" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# ---- Parse last N log lines -------------------------------------------------
|
||||
parse_metrics() {
|
||||
# 로그 형식: [timestamp] [INFO] step XXXX | loss X.XXXX | lr X.XXe-XX | gnorm X.XXX | ...
|
||||
local n="${1:-20}"
|
||||
if [[ ! -f "$LOG_FILE" ]]; then
|
||||
echo ""
|
||||
return
|
||||
fi
|
||||
tail -n "$n" "$LOG_FILE" | grep "step.*loss.*gnorm" || true
|
||||
}
|
||||
|
||||
extract_field() {
|
||||
# $1=line, $2=field name (loss, gnorm, lr)
|
||||
echo "$1" | grep -oP "${2}\s+\K[0-9]+\.[0-9e+\-]+" | head -1
|
||||
}
|
||||
|
||||
# ---- Check functions --------------------------------------------------------
|
||||
|
||||
check_loss_zero() {
|
||||
local lines
|
||||
lines=$(parse_metrics "$ZERO_LOSS_THRESHOLD")
|
||||
if [[ -z "$lines" ]]; then return; fi
|
||||
|
||||
local zero_count=0
|
||||
while IFS= read -r line; do
|
||||
local loss
|
||||
loss=$(extract_field "$line" "loss")
|
||||
if [[ -n "$loss" ]]; then
|
||||
# loss < 0.001
|
||||
if (( $(echo "$loss < 0.001" | bc -l 2>/dev/null || echo 0) )); then
|
||||
((zero_count++))
|
||||
fi
|
||||
fi
|
||||
done <<< "$lines"
|
||||
|
||||
if [[ $zero_count -ge $ZERO_LOSS_THRESHOLD ]]; then
|
||||
alert CRITICAL "Loss가 ${zero_count}회 연속 ~0! Labels 버그 가능성. 즉시 학습 중단!"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
check_loss_spike() {
|
||||
local lines
|
||||
lines=$(parse_metrics 20)
|
||||
if [[ -z "$lines" ]]; then return 0; fi
|
||||
|
||||
local losses=()
|
||||
while IFS= read -r line; do
|
||||
local loss
|
||||
loss=$(extract_field "$line" "loss")
|
||||
[[ -n "$loss" ]] && losses+=("$loss")
|
||||
done <<< "$lines"
|
||||
|
||||
local count=${#losses[@]}
|
||||
if [[ $count -lt 5 ]]; then return 0; fi
|
||||
|
||||
# 마지막 값과 이전 평균 비교
|
||||
local last_loss="${losses[$((count-1))]}"
|
||||
local sum=0
|
||||
for ((i=0; i<count-1; i++)); do
|
||||
sum=$(echo "$sum + ${losses[$i]}" | bc -l 2>/dev/null || echo "$sum")
|
||||
done
|
||||
local avg=$(echo "$sum / ($count - 1)" | bc -l 2>/dev/null || echo "0")
|
||||
|
||||
if [[ "$avg" != "0" ]]; then
|
||||
local ratio=$(echo "$last_loss / $avg" | bc -l 2>/dev/null || echo "1")
|
||||
if (( $(echo "$ratio > $LOSS_SPIKE_FACTOR" | bc -l 2>/dev/null || echo 0) )); then
|
||||
alert WARNING "Loss spike 감지! 현재=${last_loss}, 평균=${avg}, 비율=${ratio}x"
|
||||
fi
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
check_gnorm() {
|
||||
local lines
|
||||
lines=$(parse_metrics 5)
|
||||
if [[ -z "$lines" ]]; then return 0; fi
|
||||
|
||||
local last_line
|
||||
last_line=$(echo "$lines" | tail -1)
|
||||
local gnorm
|
||||
gnorm=$(extract_field "$last_line" "gnorm")
|
||||
|
||||
if [[ -z "$gnorm" ]]; then return 0; fi
|
||||
|
||||
if (( $(echo "$gnorm > $GNORM_CRITICAL" | bc -l 2>/dev/null || echo 0) )); then
|
||||
alert CRITICAL "GNorm=${gnorm} > ${GNORM_CRITICAL}! 발산 직전. 학습 중단 고려."
|
||||
elif (( $(echo "$gnorm > $GNORM_WARN" | bc -l 2>/dev/null || echo 0) )); then
|
||||
alert WARNING "GNorm=${gnorm} > ${GNORM_WARN}. 불안정 징후."
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
check_stall() {
|
||||
if [[ ! -f "$LOG_FILE" ]]; then
|
||||
alert INFO "로그 파일 없음: ${LOG_FILE}"
|
||||
return 0
|
||||
fi
|
||||
|
||||
local last_modified
|
||||
last_modified=$(stat -c %Y "$LOG_FILE" 2>/dev/null || echo 0)
|
||||
local now
|
||||
now=$(date +%s)
|
||||
local diff=$((now - last_modified))
|
||||
|
||||
if [[ $diff -gt $STALL_TIMEOUT ]]; then
|
||||
alert CRITICAL "로그가 ${diff}초 ($(( diff/60 ))분) 동안 업데이트 없음! Hang 가능성."
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
check_disk() {
|
||||
local usage
|
||||
usage=$(df /PROJECT 2>/dev/null | awk 'NR==2 {print $5}' | tr -d '%')
|
||||
if [[ -n "$usage" && "$usage" -gt "$DISK_WARN_PCT" ]]; then
|
||||
alert WARNING "디스크 사용률 ${usage}% > ${DISK_WARN_PCT}%. 체크포인트 정리 필요."
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
check_gpu() {
|
||||
if ! command -v nvidia-smi &>/dev/null; then return 0; fi
|
||||
|
||||
local low_util=0
|
||||
local total_gpus=0
|
||||
while IFS= read -r util; do
|
||||
((total_gpus++))
|
||||
if [[ "$util" -lt "$GPU_UTIL_WARN" ]]; then
|
||||
((low_util++))
|
||||
fi
|
||||
done < <(nvidia-smi --query-gpu=utilization.gpu --format=csv,noheader,nounits 2>/dev/null)
|
||||
|
||||
if [[ $total_gpus -gt 0 && $low_util -gt 0 ]]; then
|
||||
alert INFO "${low_util}/${total_gpus} GPU utilization < ${GPU_UTIL_WARN}%. 데이터 로딩 병목?"
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# ---- Status summary ---------------------------------------------------------
|
||||
print_status() {
|
||||
local lines
|
||||
lines=$(parse_metrics 1)
|
||||
if [[ -n "$lines" ]]; then
|
||||
echo -e "${GREEN}최근 로그:${NC} $lines"
|
||||
fi
|
||||
|
||||
if command -v nvidia-smi &>/dev/null; then
|
||||
echo -e "${CYAN}GPU 메모리:${NC}"
|
||||
nvidia-smi --query-gpu=index,memory.used,memory.total,utilization.gpu \
|
||||
--format=csv,noheader 2>/dev/null | head -8
|
||||
fi
|
||||
|
||||
local disk
|
||||
disk=$(df -h /PROJECT 2>/dev/null | awk 'NR==2 {print "사용: "$3"/"$2" ("$5")"}')
|
||||
echo -e "${CYAN}디스크:${NC} ${disk}"
|
||||
}
|
||||
|
||||
# ---- Main loop --------------------------------------------------------------
|
||||
echo "=================================================================="
|
||||
echo " SFT Training Monitor"
|
||||
echo " Log file: ${LOG_FILE}"
|
||||
echo " Check interval: ${CHECK_INTERVAL}s"
|
||||
echo " Press Ctrl+C to stop"
|
||||
echo "=================================================================="
|
||||
|
||||
run_all_checks() {
|
||||
check_loss_zero || true
|
||||
check_loss_spike || true
|
||||
check_gnorm || true
|
||||
check_stall || true
|
||||
check_disk || true
|
||||
check_gpu || true
|
||||
echo "---"
|
||||
print_status
|
||||
echo ""
|
||||
}
|
||||
|
||||
if $CHECK_ONCE; then
|
||||
run_all_checks
|
||||
exit 0
|
||||
fi
|
||||
|
||||
while true; do
|
||||
run_all_checks
|
||||
sleep "$CHECK_INTERVAL"
|
||||
done
|
||||
243
source/scripts/openclaw-watchdog.sh
Normal file
243
source/scripts/openclaw-watchdog.sh
Normal file
@@ -0,0 +1,243 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# openclaw-watchdog.sh — OpenClaw Gateway 헬스체크 + 자동 재시작
|
||||
# crontab에 등록하여 1분마다 실행
|
||||
#
|
||||
# Usage:
|
||||
# */1 * * * * /PROJECT/0325120031_A/ghong/taketimes/llm-bang/scripts/openclaw-watchdog.sh
|
||||
#
|
||||
# 변경이력:
|
||||
# 2026-03-01 네트워크 체크를 ICMP→HTTP로 변경 (ICMP 차단 환경 대응)
|
||||
# 다중 엔드포인트 fallback, 게이트웨이 HTTP 응답 체크 추가
|
||||
# setsid 분리 실행, 상세 로깅 강화
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# ── 설정 ──────────────────────────────────────────────
|
||||
RNTIER_HOME="REDACTED_RNTIER_PATH"
|
||||
OPENCLAW_BIN="${RNTIER_HOME}/.npm-global/bin/openclaw"
|
||||
GATEWAY_PORT=18789
|
||||
GATEWAY_HOST="127.0.0.1"
|
||||
PID_FILE="/tmp/openclaw-gateway.pid"
|
||||
LOG_DIR="/tmp/openclaw"
|
||||
LOG_FILE="${LOG_DIR}/watchdog.log"
|
||||
GATEWAY_LOG="${LOG_DIR}/gateway.log"
|
||||
MAX_LOG_SIZE=$((10 * 1024 * 1024)) # 10MB 로테이션
|
||||
RESTART_COOLDOWN=120 # 초 — 재시작 후 이 시간 내 재시도 방지
|
||||
LAST_RESTART_FILE="/tmp/openclaw-last-restart"
|
||||
CONSECUTIVE_FAIL_FILE="/tmp/openclaw-consecutive-fails"
|
||||
|
||||
# 환경변수 — openclaw가 config를 찾을 수 있도록
|
||||
export PATH="${RNTIER_HOME}/.npm-global/bin:/usr/bin:/usr/local/bin:/bin:$PATH"
|
||||
export HOME="/home/ghong"
|
||||
export OPENCLAW_STATE_DIR="${RNTIER_HOME}/.openclaw"
|
||||
export OPENCLAW_CONFIG_PATH="${RNTIER_HOME}/.openclaw/openclaw.json"
|
||||
|
||||
# ── 함수 ──────────────────────────────────────────────
|
||||
mkdir -p "$LOG_DIR"
|
||||
|
||||
log() {
|
||||
echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" >> "$LOG_FILE"
|
||||
}
|
||||
|
||||
rotate_log() {
|
||||
local file="$1"
|
||||
if [[ -f "$file" ]] && [[ $(stat -c%s "$file" 2>/dev/null || echo 0) -gt $MAX_LOG_SIZE ]]; then
|
||||
mv "$file" "${file}.old"
|
||||
log "Log rotated: $file"
|
||||
fi
|
||||
}
|
||||
|
||||
# 게이트웨이의 실제 엔드포인트로 로컬 HTTP 응답 체크
|
||||
check_gateway_http() {
|
||||
if command -v curl &>/dev/null; then
|
||||
curl -sf --max-time 5 -o /dev/null "http://${GATEWAY_HOST}:${GATEWAY_PORT}/__openclaw__/canvas/" 2>/dev/null
|
||||
return $?
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
|
||||
is_port_open() {
|
||||
if command -v ss &>/dev/null; then
|
||||
ss -tlnH "sport = :${GATEWAY_PORT}" 2>/dev/null | grep -q "$GATEWAY_PORT"
|
||||
else
|
||||
(echo > /dev/tcp/"$GATEWAY_HOST"/"$GATEWAY_PORT") 2>/dev/null
|
||||
fi
|
||||
}
|
||||
|
||||
is_process_alive() {
|
||||
if [[ -f "$PID_FILE" ]]; then
|
||||
local pid
|
||||
pid=$(cat "$PID_FILE" 2>/dev/null)
|
||||
if [[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null; then
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
pgrep -f "openclaw.*gateway" >/dev/null 2>&1
|
||||
}
|
||||
|
||||
# 네트워크 체크 — DNS 해석 기반
|
||||
# 이 서버는 ICMP(ping)과 아웃바운드 HTTPS(curl)가 모두 차단됨.
|
||||
# 단, DNS 해석은 가능하고 게이트웨이(Node.js)는 long-polling으로 통신 가능.
|
||||
# 따라서 DNS 해석 성공 여부로 "네트워크 자체가 살아있는지" 판단한다.
|
||||
check_network() {
|
||||
# 방법1: getent (가장 빠르고 가벼움)
|
||||
if command -v getent &>/dev/null; then
|
||||
getent hosts api.telegram.org >/dev/null 2>&1 && return 0
|
||||
getent hosts api.anthropic.com >/dev/null 2>&1 && return 0
|
||||
fi
|
||||
# 방법2: nslookup
|
||||
if command -v nslookup &>/dev/null; then
|
||||
nslookup -timeout=5 api.telegram.org >/dev/null 2>&1 && return 0
|
||||
fi
|
||||
# 방법3: /dev/tcp 로 DNS 서버(168.126.63.1) 포트 53 확인
|
||||
(echo > /dev/tcp/168.126.63.1/53) 2>/dev/null && return 0
|
||||
return 1
|
||||
}
|
||||
|
||||
cooldown_active() {
|
||||
if [[ -f "$LAST_RESTART_FILE" ]]; then
|
||||
local last_restart now diff
|
||||
last_restart=$(cat "$LAST_RESTART_FILE" 2>/dev/null)
|
||||
now=$(date +%s)
|
||||
diff=$(( now - last_restart ))
|
||||
if [[ $diff -lt $RESTART_COOLDOWN ]]; then
|
||||
return 0 # 쿨다운 중
|
||||
fi
|
||||
fi
|
||||
return 1 # 쿨다운 아님
|
||||
}
|
||||
|
||||
get_consecutive_fails() {
|
||||
if [[ -f "$CONSECUTIVE_FAIL_FILE" ]]; then
|
||||
cat "$CONSECUTIVE_FAIL_FILE" 2>/dev/null || echo 0
|
||||
else
|
||||
echo 0
|
||||
fi
|
||||
}
|
||||
|
||||
set_consecutive_fails() {
|
||||
echo "$1" > "$CONSECUTIVE_FAIL_FILE"
|
||||
}
|
||||
|
||||
start_gateway() {
|
||||
log "ACTION: Starting OpenClaw gateway on port $GATEWAY_PORT..."
|
||||
|
||||
# 기존 좀비 프로세스 정리
|
||||
local old_pids
|
||||
old_pids=$(pgrep -f "openclaw.*gateway" 2>/dev/null || true)
|
||||
if [[ -n "$old_pids" ]]; then
|
||||
log "ACTION: Killing stale gateway processes: $old_pids"
|
||||
echo "$old_pids" | xargs kill -9 2>/dev/null || true
|
||||
sleep 2
|
||||
fi
|
||||
|
||||
# 게이트웨이 시작 — setsid로 완전 분리 (부모 프로세스 시그널 전파 방지)
|
||||
setsid nohup "$OPENCLAW_BIN" gateway run \
|
||||
--port "$GATEWAY_PORT" \
|
||||
--bind loopback \
|
||||
>> "$GATEWAY_LOG" 2>&1 < /dev/null &
|
||||
|
||||
local new_pid=$!
|
||||
echo "$new_pid" > "$PID_FILE"
|
||||
date +%s > "$LAST_RESTART_FILE"
|
||||
|
||||
log "ACTION: Gateway launched with PID $new_pid (setsid)"
|
||||
|
||||
# 8초 대기 후 확인 (Telegram provider 초기화에 시간 필요)
|
||||
sleep 8
|
||||
if kill -0 "$new_pid" 2>/dev/null; then
|
||||
log "OK: Gateway PID $new_pid is alive after startup"
|
||||
if is_port_open; then
|
||||
log "OK: Port $GATEWAY_PORT is listening"
|
||||
else
|
||||
log "WARN: Gateway alive but port $GATEWAY_PORT not yet listening (may need more time)"
|
||||
fi
|
||||
return 0
|
||||
else
|
||||
log "ERROR: Gateway PID $new_pid died immediately after start"
|
||||
log "ERROR: Last 10 lines of gateway.log:"
|
||||
tail -10 "$GATEWAY_LOG" 2>/dev/null | while read -r line; do
|
||||
log " | $line"
|
||||
done
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# ── 메인 로직 ─────────────────────────────────────────
|
||||
rotate_log "$LOG_FILE"
|
||||
rotate_log "$GATEWAY_LOG"
|
||||
|
||||
# 오래된 openclaw 로그 파일 정리 (7일 이상)
|
||||
find "$LOG_DIR" -name "openclaw-*.log" -mtime +7 -delete 2>/dev/null || true
|
||||
|
||||
# 1) 프로세스 + 포트 체크를 먼저 수행 (게이트웨이가 살아있으면 네트워크 체크 불필요)
|
||||
process_ok=false
|
||||
port_ok=false
|
||||
http_ok=false
|
||||
|
||||
if is_process_alive; then
|
||||
process_ok=true
|
||||
fi
|
||||
|
||||
if is_port_open; then
|
||||
port_ok=true
|
||||
fi
|
||||
|
||||
if $port_ok && check_gateway_http; then
|
||||
http_ok=true
|
||||
fi
|
||||
|
||||
# 2) 게이트웨이 정상이면 바로 종료
|
||||
if $process_ok && $port_ok; then
|
||||
if $http_ok; then
|
||||
# 완전 정상
|
||||
set_consecutive_fails 0
|
||||
exit 0
|
||||
fi
|
||||
# 프로세스+포트 OK인데 HTTP 응답 없음 → hung 가능성
|
||||
fails=$(get_consecutive_fails)
|
||||
fails=$((fails + 1))
|
||||
set_consecutive_fails "$fails"
|
||||
log "WARN: Process alive, port open, but HTTP not responding (consecutive: $fails)"
|
||||
if [[ $fails -lt 3 ]]; then
|
||||
log "INFO: Waiting more cycles before restart (transient check, $fails/3)"
|
||||
exit 0
|
||||
fi
|
||||
log "WARN: HTTP unresponsive for $fails consecutive checks — proceeding to restart"
|
||||
fi
|
||||
|
||||
# 3) 게이트웨이가 비정상 — 네트워크 체크 후 재시작 여부 판단
|
||||
if $process_ok && ! $port_ok; then
|
||||
log "WARN: Process alive but port $GATEWAY_PORT not listening. Possible hung state."
|
||||
fi
|
||||
|
||||
if ! $process_ok && ! $port_ok; then
|
||||
log "WARN: Gateway is completely down (no process, no port)."
|
||||
fi
|
||||
|
||||
if ! $process_ok && $port_ok; then
|
||||
log "WARN: No known gateway process but port $GATEWAY_PORT is in use. Stale process?"
|
||||
fi
|
||||
|
||||
# 4) 네트워크 체크 — DNS 기반 (게이트웨이가 죽었을 때만 실행)
|
||||
if ! check_network; then
|
||||
log "WARN: Network unreachable (DNS resolution failed). Skipping gateway restart."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# 5) 쿨다운 체크
|
||||
if cooldown_active; then
|
||||
log "INFO: Cooldown active (last restart < ${RESTART_COOLDOWN}s ago). Skipping."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# 6) 재시작
|
||||
log "ACTION: Attempting gateway restart..."
|
||||
if start_gateway; then
|
||||
log "OK: Gateway restart SUCCESS"
|
||||
set_consecutive_fails 0
|
||||
else
|
||||
log "ERROR: Gateway restart FAILED"
|
||||
exit 1
|
||||
fi
|
||||
127
source/scripts/orpo_eval_watchdog.sh
Normal file
127
source/scripts/orpo_eval_watchdog.sh
Normal file
@@ -0,0 +1,127 @@
|
||||
#!/bin/bash
|
||||
# =============================================================================
|
||||
# ORPO Training Completion Watchdog
|
||||
# =============================================================================
|
||||
# Monitors the ORPO training process. When it finishes, automatically launches
|
||||
# the comprehensive evaluation pipeline.
|
||||
#
|
||||
# Usage:
|
||||
# nohup bash scripts/orpo_eval_watchdog.sh > checkpoints/korean_3b_orpo_v1/watchdog.log 2>&1 &
|
||||
# =============================================================================
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
PROJECT_ROOT="/PROJECT/0325120031_A/ghong/taketimes/llm-bang"
|
||||
TRAIN_LOG="${PROJECT_ROOT}/checkpoints/korean_3b_orpo_v1/train.log"
|
||||
TRAIN_PID=$(pgrep -f "train/orpo.py.*korean_3b_orpo_v1" | head -1)
|
||||
|
||||
echo "=============================================="
|
||||
echo " ORPO Eval Watchdog Started"
|
||||
echo "=============================================="
|
||||
echo " Time : $(date '+%Y-%m-%d %H:%M:%S')"
|
||||
echo " Train PID : ${TRAIN_PID:-NOT FOUND}"
|
||||
echo " Train Log : ${TRAIN_LOG}"
|
||||
echo "=============================================="
|
||||
|
||||
if [ -z "${TRAIN_PID}" ]; then
|
||||
echo "[WARN] Training process not found. Checking if already completed..."
|
||||
# Check if training already finished by looking for final output
|
||||
if grep -q "Training completed" "${TRAIN_LOG}" 2>/dev/null || \
|
||||
grep -q "Saving model checkpoint" "${TRAIN_LOG}" 2>/dev/null; then
|
||||
echo "[INFO] Training appears to have already completed."
|
||||
else
|
||||
echo "[ERROR] No training process and no completion marker found. Exiting."
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
echo "[INFO] Watching training PID ${TRAIN_PID}..."
|
||||
echo ""
|
||||
|
||||
# Poll every 60 seconds
|
||||
while kill -0 "${TRAIN_PID}" 2>/dev/null; do
|
||||
# Get current step
|
||||
CURRENT_STEP=$(grep -oP '\d+/9840' "${TRAIN_LOG}" 2>/dev/null | tail -1 || echo "?/?")
|
||||
LATEST_LOSS=$(grep "'loss':" "${TRAIN_LOG}" 2>/dev/null | tail -1 | grep -oP "'loss': '([^']+)'" | sed "s/'loss': '//;s/'//" || echo "?")
|
||||
echo "[$(date '+%H:%M:%S')] Step ${CURRENT_STEP} | Loss: ${LATEST_LOSS} | PID ${TRAIN_PID} running"
|
||||
sleep 60
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "=============================================="
|
||||
echo "[INFO] Training process ${TRAIN_PID} has ended."
|
||||
echo "[INFO] Time: $(date '+%Y-%m-%d %H:%M:%S')"
|
||||
echo "=============================================="
|
||||
fi
|
||||
|
||||
# Wait a moment for any final I/O
|
||||
sleep 10
|
||||
|
||||
# Get final training stats
|
||||
echo ""
|
||||
echo "[INFO] Final training stats:"
|
||||
grep "eval_loss" "${TRAIN_LOG}" | tail -1 | tr ',' '\n' | head -10
|
||||
echo ""
|
||||
|
||||
# Detect the latest checkpoint
|
||||
LATEST_CKPT=$(ls -d ${PROJECT_ROOT}/checkpoints/korean_3b_orpo_v1/checkpoint-* 2>/dev/null | sort -t- -k2 -n | tail -1)
|
||||
echo "[INFO] Latest checkpoint: ${LATEST_CKPT}"
|
||||
|
||||
if [ -z "${LATEST_CKPT}" ]; then
|
||||
echo "[ERROR] No checkpoint found. Cannot proceed with evaluation."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Send telegram notification (if available)
|
||||
python3 -c "
|
||||
import os, urllib.request, urllib.parse, json
|
||||
token = os.environ.get('TELEGRAM_BOT_TOKEN', '')
|
||||
chat_id = os.environ.get('TELEGRAM_CHAT_ID', '')
|
||||
if token and chat_id:
|
||||
msg = '🏁 ORPO 학습 완료! 자동 평가 시작합니다.\nCheckpoint: ${LATEST_CKPT##*/}'
|
||||
url = f'https://api.telegram.org/bot{token}/sendMessage'
|
||||
data = urllib.parse.urlencode({'chat_id': chat_id, 'text': msg}).encode()
|
||||
urllib.request.urlopen(url, data, timeout=10)
|
||||
print('[INFO] Telegram notification sent.')
|
||||
else:
|
||||
print('[INFO] Telegram not configured, skipping notification.')
|
||||
" 2>/dev/null || true
|
||||
|
||||
# ============================================================================
|
||||
# Launch evaluation pipeline
|
||||
# ============================================================================
|
||||
echo ""
|
||||
echo "=============================================="
|
||||
echo " Starting ORPO Evaluation Pipeline"
|
||||
echo " Time: $(date '+%Y-%m-%d %H:%M:%S')"
|
||||
echo "=============================================="
|
||||
|
||||
cd "${PROJECT_ROOT}"
|
||||
|
||||
python3 eval/orpo_eval_pipeline.py \
|
||||
--checkpoint "${LATEST_CKPT}" \
|
||||
2>&1 | tee -a checkpoints/korean_3b_orpo_v1/eval.log
|
||||
|
||||
EVAL_EXIT=$?
|
||||
|
||||
echo ""
|
||||
echo "=============================================="
|
||||
echo " Evaluation Complete"
|
||||
echo " Exit code: ${EVAL_EXIT}"
|
||||
echo " Time: $(date '+%Y-%m-%d %H:%M:%S')"
|
||||
echo "=============================================="
|
||||
|
||||
# Send completion notification
|
||||
python3 -c "
|
||||
import os, urllib.request, urllib.parse
|
||||
token = os.environ.get('TELEGRAM_BOT_TOKEN', '')
|
||||
chat_id = os.environ.get('TELEGRAM_CHAT_ID', '')
|
||||
if token and chat_id:
|
||||
exit_code = ${EVAL_EXIT}
|
||||
status = '✅ 성공' if exit_code == 0 else '❌ 실패'
|
||||
msg = f'ORPO 평가 완료: {status}\nExit code: {exit_code}\n보고서: reports/ 확인'
|
||||
url = f'https://api.telegram.org/bot{token}/sendMessage'
|
||||
data = urllib.parse.urlencode({'chat_id': chat_id, 'text': msg}).encode()
|
||||
urllib.request.urlopen(url, data, timeout=10)
|
||||
" 2>/dev/null || true
|
||||
|
||||
exit ${EVAL_EXIT}
|
||||
166
source/scripts/orpo_hp_sweep.sh
Normal file
166
source/scripts/orpo_hp_sweep.sh
Normal file
@@ -0,0 +1,166 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# orpo_hp_sweep.sh — ORPO Hyperparameter Sweep (200 steps each)
|
||||
#
|
||||
# 각 설정을 200 steps씩 돌려서 최적 조합을 찾는 스크립트.
|
||||
# 결과는 sweep_results/ 디렉토리에 저장됨.
|
||||
#
|
||||
# Usage:
|
||||
# bash scripts/orpo_hp_sweep.sh # 전체 sweep (6 runs)
|
||||
# bash scripts/orpo_hp_sweep.sh --dry-run # 설정만 출력
|
||||
# =============================================================================
|
||||
set -uo pipefail
|
||||
# NOTE: set +e — individual runs may fail; we log failures and continue the sweep
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
SWEEP_STEPS=200
|
||||
SWEEP_DIR="checkpoints/orpo_sweep"
|
||||
RESULTS_FILE="${SWEEP_DIR}/sweep_results.jsonl"
|
||||
BASE_MODEL="eval/outputs/hf_3b_sft_best"
|
||||
DATA_PATH="data/preference/combined_preference.jsonl"
|
||||
NPROC=8
|
||||
MASTER_PORT_BASE=29510
|
||||
|
||||
# B200 NCCL tuning (NVSwitch mesh — let NCCL auto-detect proto/channels/algo)
|
||||
export NCCL_IB_DISABLE=1
|
||||
export NCCL_BUFFSIZE=134217728
|
||||
export OMP_NUM_THREADS=9
|
||||
export MKL_NUM_THREADS=9
|
||||
export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
|
||||
export NCCL_P2P_LEVEL=NVL
|
||||
export PYTHONWARNINGS="ignore::UserWarning:torch.library"
|
||||
|
||||
mkdir -p "${SWEEP_DIR}"
|
||||
declare -a FAILED_RUNS=()
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Sweep configurations: (name, beta, lr, max_length, batch_size, grad_accum)
|
||||
# ---------------------------------------------------------------------------
|
||||
# 핵심 탐색 축:
|
||||
# 1. beta: 반복 억제 강도 (0.15 vs 0.25 vs 0.35)
|
||||
# 2. lr: 수렴 속도 (5e-6 vs 8e-6 vs 1.2e-5)
|
||||
# 3. max_length: VRAM vs 커버리지 (1024 vs 1536)
|
||||
|
||||
declare -a CONFIGS=(
|
||||
# name beta lr max_len bs accum
|
||||
"baseline_b015_lr8e6 0.15 8e-6 1536 4 4"
|
||||
"baseline_b025_lr8e6 0.25 8e-6 1536 4 4"
|
||||
"strong_b035_lr8e6 0.35 8e-6 1536 4 4"
|
||||
"fast_b025_lr12e6 0.25 1.2e-5 1536 4 4"
|
||||
"conserv_b025_lr5e6 0.25 5e-6 1536 4 4"
|
||||
"short_b025_lr8e6 0.25 8e-6 1024 4 4"
|
||||
)
|
||||
|
||||
DRY_RUN=false
|
||||
if [[ "${1:-}" == "--dry-run" ]]; then
|
||||
DRY_RUN=true
|
||||
fi
|
||||
|
||||
echo "=================================================================="
|
||||
echo " ORPO Hyperparameter Sweep"
|
||||
echo " Configs: ${#CONFIGS[@]}"
|
||||
echo " Steps each: ${SWEEP_STEPS}"
|
||||
echo " Results: ${RESULTS_FILE}"
|
||||
echo "=================================================================="
|
||||
|
||||
for i in "${!CONFIGS[@]}"; do
|
||||
read -r NAME BETA LR MAX_LEN BS ACCUM <<< "${CONFIGS[$i]}"
|
||||
PORT=$((MASTER_PORT_BASE + i))
|
||||
OUTPUT="${SWEEP_DIR}/${NAME}"
|
||||
|
||||
echo ""
|
||||
echo "--- Run $((i+1))/${#CONFIGS[@]}: ${NAME} ---"
|
||||
echo " beta=${BETA} lr=${LR} max_length=${MAX_LEN} bs=${BS} accum=${ACCUM}"
|
||||
|
||||
if [[ "${DRY_RUN}" == "true" ]]; then
|
||||
echo " [DRY RUN] skipping"
|
||||
continue
|
||||
fi
|
||||
|
||||
mkdir -p "${OUTPUT}"
|
||||
START_TIME=$(date +%s)
|
||||
|
||||
torchrun \
|
||||
--nproc_per_node=${NPROC} \
|
||||
--master_port=${PORT} \
|
||||
train/orpo.py \
|
||||
--model_path "${BASE_MODEL}" \
|
||||
--custom_data_path "${DATA_PATH}" \
|
||||
--output_dir "${OUTPUT}" \
|
||||
--max_steps ${SWEEP_STEPS} \
|
||||
--lr ${LR} \
|
||||
--beta ${BETA} \
|
||||
--batch_size ${BS} \
|
||||
--gradient_accumulation_steps ${ACCUM} \
|
||||
--max_length ${MAX_LEN} \
|
||||
\
|
||||
--weight_decay 0.01 \
|
||||
--warmup_ratio 0.05 \
|
||||
--eval_split_ratio 0.05 \
|
||||
--eval_steps 100 \
|
||||
--early_stopping_patience 100 \
|
||||
--save_steps 200 \
|
||||
--save_total_limit 1 \
|
||||
--logging_steps 10 \
|
||||
--report_to none \
|
||||
--dataset_num_proc 64 \
|
||||
--dataloader_num_workers 4 \
|
||||
--no_load_best \
|
||||
2>&1 | tee "${OUTPUT}/train.log"
|
||||
RUN_EXIT=$?
|
||||
|
||||
END_TIME=$(date +%s)
|
||||
ELAPSED=$((END_TIME - START_TIME))
|
||||
|
||||
if [[ ${RUN_EXIT} -ne 0 ]]; then
|
||||
echo " [ERROR] Run ${NAME} failed with exit code ${RUN_EXIT} after ${ELAPSED}s"
|
||||
echo "{\"name\":\"${NAME}\",\"beta\":${BETA},\"lr\":\"${LR}\",\"max_length\":${MAX_LEN},\"status\":\"FAILED\",\"exit_code\":${RUN_EXIT},\"elapsed_s\":${ELAPSED}}" >> "${RESULTS_FILE}"
|
||||
FAILED_RUNS+=("${NAME}")
|
||||
continue
|
||||
fi
|
||||
|
||||
# Extract final metrics from log
|
||||
FINAL_LOSS=$(grep -oP "'loss': '[\d.]+'" "${OUTPUT}/train.log" | tail -1 | grep -oP "[\d.]+" || echo "N/A")
|
||||
EVAL_LOSS=$(grep -oP "'eval_loss': '[\d.]+'" "${OUTPUT}/train.log" | tail -1 | grep -oP "[\d.]+" || echo "N/A")
|
||||
MARGIN=$(grep -oP "'rewards/margins': '[-\d.]+'" "${OUTPUT}/train.log" | tail -1 | grep -oP "[-\d.]+" || echo "N/A")
|
||||
|
||||
# Save result
|
||||
echo "{\"name\":\"${NAME}\",\"beta\":${BETA},\"lr\":\"${LR}\",\"max_length\":${MAX_LEN},\"status\":\"OK\",\"loss\":\"${FINAL_LOSS}\",\"eval_loss\":\"${EVAL_LOSS}\",\"margin\":\"${MARGIN}\",\"elapsed_s\":${ELAPSED}}" >> "${RESULTS_FILE}"
|
||||
|
||||
echo " -> loss=${FINAL_LOSS} eval_loss=${EVAL_LOSS} margin=${MARGIN} time=${ELAPSED}s"
|
||||
|
||||
# Cleanup weights to save disk (keep logs)
|
||||
rm -rf "${OUTPUT}/checkpoint-"* "${OUTPUT}/emergency_checkpoint" 2>/dev/null || true
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "=================================================================="
|
||||
echo " Sweep Complete!"
|
||||
echo " Results: ${RESULTS_FILE}"
|
||||
if [[ -f "${RESULTS_FILE}" ]]; then
|
||||
echo ""
|
||||
echo " Summary:"
|
||||
cat "${RESULTS_FILE}" | python3 -c "
|
||||
import sys, json
|
||||
results = [json.loads(l) for l in sys.stdin]
|
||||
results.sort(key=lambda r: float(r.get('eval_loss', '999')))
|
||||
print(f' {\"Name\":<25} {\"Beta\":>6} {\"LR\":>10} {\"Loss\":>8} {\"EvalLoss\":>10} {\"Margin\":>8} {\"Time\":>6}')
|
||||
print(f' {\"-\"*25} {\"-\"*6} {\"-\"*10} {\"-\"*8} {\"-\"*10} {\"-\"*8} {\"-\"*6}')
|
||||
for r in results:
|
||||
print(f' {r[\"name\"]:<25} {r[\"beta\"]:>6} {r[\"lr\"]:>10} {r[\"loss\"]:>8} {r[\"eval_loss\"]:>10} {r[\"margin\"]:>8} {r[\"elapsed_s\"]:>5}s')
|
||||
print()
|
||||
best = results[0]
|
||||
print(f' BEST: {best[\"name\"]} (eval_loss={best[\"eval_loss\"]})')
|
||||
" 2>/dev/null || cat "${RESULTS_FILE}"
|
||||
fi
|
||||
|
||||
# Report failed runs
|
||||
if [[ ${#FAILED_RUNS[@]} -gt 0 ]]; then
|
||||
echo ""
|
||||
echo " FAILED RUNS (${#FAILED_RUNS[@]}):"
|
||||
for fname in "${FAILED_RUNS[@]}"; do
|
||||
echo " - ${fname}"
|
||||
done
|
||||
fi
|
||||
echo "=================================================================="
|
||||
414
source/scripts/prepare_3b_data.sh
Normal file
414
source/scripts/prepare_3b_data.sh
Normal file
@@ -0,0 +1,414 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# prepare_3b_data.sh — 3B 모델 학습 데이터 전체 파이프라인
|
||||
#
|
||||
# 사용법:
|
||||
# bash scripts/prepare_3b_data.sh [--step N] [--jobs 72]
|
||||
#
|
||||
# 스텝:
|
||||
# 1 = CulturaX 토큰화
|
||||
# 2 = cc100 해제 + 토큰화
|
||||
# 3 = OSCAR 토큰화
|
||||
# 4 = korean_webtext 토큰화
|
||||
# 5 = HPLT 한국어 추출 + 토큰화
|
||||
# 6 = textbooks + finepdfs + kovast 토큰화
|
||||
# 7 = 전체 병합
|
||||
# 8 = train/val split 검증
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
cd "${PROJECT_ROOT}"
|
||||
|
||||
# ─── 설정 ────────────────────────────────────────────────────────────────
|
||||
DATA_DIR="data"
|
||||
EXTRA_DIR="data/korean_extra"
|
||||
TOKENIZER="tokenizer/tokenizer.json"
|
||||
VAL_SPLIT=0.002
|
||||
SEED=42
|
||||
JOBS=72
|
||||
FROM_STEP=0
|
||||
LOG_FILE="data/prepare_3b.log"
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--step) FROM_STEP="$2"; shift 2 ;;
|
||||
--jobs) JOBS="$2"; shift 2 ;;
|
||||
*) echo "Unknown arg: $1"; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
mkdir -p "$(dirname "$LOG_FILE")"
|
||||
exec > >(tee -a "$LOG_FILE") 2>&1
|
||||
|
||||
log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"; }
|
||||
|
||||
# ─── 토큰화 헬퍼 (parquet → bin) ─────────────────────────────────────────
|
||||
tokenize_parquet() {
|
||||
local name="$1"
|
||||
local input_pattern="$2"
|
||||
local text_col="$3"
|
||||
local output="${DATA_DIR}/${name}_train.bin"
|
||||
|
||||
if [[ -f "$output" && $FROM_STEP -le 0 ]]; then
|
||||
log "[SKIP] $output already exists ($(du -h "$output" | cut -f1))"
|
||||
return
|
||||
fi
|
||||
|
||||
log "[START] Tokenizing $name from parquet..."
|
||||
python3 - <<PYEOF
|
||||
import glob, os, sys
|
||||
import numpy as np
|
||||
from tokenizers import Tokenizer
|
||||
import pyarrow.parquet as pq
|
||||
from tqdm import tqdm
|
||||
from concurrent.futures import ProcessPoolExecutor
|
||||
import multiprocessing as mp
|
||||
|
||||
tokenizer_path = "${TOKENIZER}"
|
||||
input_pattern = "${input_pattern}"
|
||||
text_col = "${text_col}"
|
||||
output_train = "${output}"
|
||||
output_val = output_train.replace("_train.bin", "_val.bin")
|
||||
val_split = ${VAL_SPLIT}
|
||||
seed = ${SEED}
|
||||
|
||||
files = sorted(glob.glob(input_pattern))
|
||||
print(f"Found {len(files)} parquet files")
|
||||
|
||||
tokenizer = Tokenizer.from_file(tokenizer_path)
|
||||
|
||||
all_tokens = []
|
||||
total_docs = 0
|
||||
|
||||
for f in tqdm(files, desc="${name}"):
|
||||
try:
|
||||
table = pq.read_table(f, columns=[text_col])
|
||||
for text in table.column(text_col):
|
||||
t = text.as_py()
|
||||
if t and len(t) > 50:
|
||||
ids = tokenizer.encode(t).ids
|
||||
all_tokens.extend(ids)
|
||||
total_docs += 1
|
||||
except Exception as e:
|
||||
print(f"Error processing {f}: {e}", file=sys.stderr)
|
||||
continue
|
||||
|
||||
print(f"Total: {total_docs:,} docs, {len(all_tokens):,} tokens")
|
||||
|
||||
# Split
|
||||
import random
|
||||
random.seed(seed)
|
||||
random.shuffle(all_tokens) # Not ideal but matches existing code
|
||||
n_val = int(len(all_tokens) * val_split)
|
||||
val_tokens = all_tokens[:n_val]
|
||||
train_tokens = all_tokens[n_val:]
|
||||
|
||||
np.array(train_tokens, dtype=np.uint16).tofile(output_train)
|
||||
np.array(val_tokens, dtype=np.uint16).tofile(output_val)
|
||||
print(f"Saved: {output_train} ({len(train_tokens):,} tokens)")
|
||||
print(f"Saved: {output_val} ({len(val_tokens):,} tokens)")
|
||||
PYEOF
|
||||
log "[DONE] $name → $output"
|
||||
}
|
||||
|
||||
# ─── Step 1: CulturaX ────────────────────────────────────────────────────
|
||||
if [[ $FROM_STEP -le 1 ]]; then
|
||||
log "=== Step 1: CulturaX 토큰화 ==="
|
||||
tokenize_parquet "culturax" \
|
||||
"${EXTRA_DIR}/culturax_ko/ko/*.parquet" \
|
||||
"text"
|
||||
fi
|
||||
|
||||
# ─── Step 2: cc100 해제 + 토큰화 ─────────────────────────────────────────
|
||||
if [[ $FROM_STEP -le 2 ]]; then
|
||||
log "=== Step 2: cc100 해제 + 토큰화 ==="
|
||||
CC100_XZ="${EXTRA_DIR}/cc100_ko/ko.txt.xz"
|
||||
CC100_TXT="${EXTRA_DIR}/cc100_ko/ko.txt"
|
||||
CC100_OUT="${DATA_DIR}/cc100_train.bin"
|
||||
|
||||
if [[ -f "$CC100_OUT" && $FROM_STEP -le 0 ]]; then
|
||||
log "[SKIP] cc100 already tokenized"
|
||||
else
|
||||
# 해제
|
||||
if [[ ! -f "$CC100_TXT" ]]; then
|
||||
log "Decompressing cc100 xz (14GB → 54GB)..."
|
||||
xz -dk "$CC100_XZ"
|
||||
log "Decompression done"
|
||||
fi
|
||||
|
||||
# 토큰화 (대용량 → 스트리밍)
|
||||
log "Tokenizing cc100 (54GB text)..."
|
||||
python3 - <<'PYEOF'
|
||||
import numpy as np
|
||||
from tokenizers import Tokenizer
|
||||
from tqdm import tqdm
|
||||
import random
|
||||
|
||||
tokenizer = Tokenizer.from_file("tokenizer/tokenizer.json")
|
||||
input_file = "data/korean_extra/cc100_ko/ko.txt"
|
||||
output_train = "data/cc100_train.bin"
|
||||
output_val = "data/cc100_val.bin"
|
||||
|
||||
# Stream tokenize in chunks
|
||||
all_tokens = []
|
||||
doc_buffer = []
|
||||
doc_count = 0
|
||||
|
||||
with open(input_file, 'r', encoding='utf-8', errors='replace') as f:
|
||||
for line in tqdm(f, desc="cc100", unit=" lines"):
|
||||
line = line.strip()
|
||||
if not line:
|
||||
# Document boundary
|
||||
if doc_buffer:
|
||||
text = '\n'.join(doc_buffer)
|
||||
if len(text) > 50:
|
||||
ids = tokenizer.encode(text).ids
|
||||
all_tokens.extend(ids)
|
||||
doc_count += 1
|
||||
doc_buffer = []
|
||||
else:
|
||||
doc_buffer.append(line)
|
||||
|
||||
# Last doc
|
||||
if doc_buffer:
|
||||
text = '\n'.join(doc_buffer)
|
||||
if len(text) > 50:
|
||||
all_tokens.extend(tokenizer.encode(text).ids)
|
||||
doc_count += 1
|
||||
|
||||
print(f"Total: {doc_count:,} docs, {len(all_tokens):,} tokens")
|
||||
|
||||
# Split
|
||||
n_val = int(len(all_tokens) * 0.002)
|
||||
np.array(all_tokens[n_val:], dtype=np.uint16).tofile(output_train)
|
||||
np.array(all_tokens[:n_val], dtype=np.uint16).tofile(output_val)
|
||||
print(f"Saved train: {len(all_tokens)-n_val:,} tokens")
|
||||
print(f"Saved val: {n_val:,} tokens")
|
||||
PYEOF
|
||||
log "[DONE] cc100"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ─── Step 3: OSCAR ───────────────────────────────────────────────────────
|
||||
if [[ $FROM_STEP -le 3 ]]; then
|
||||
log "=== Step 3: OSCAR 토큰화 ==="
|
||||
OSCAR_OUT="${DATA_DIR}/oscar_train.bin"
|
||||
|
||||
if [[ -f "$OSCAR_OUT" && $FROM_STEP -le 0 ]]; then
|
||||
log "[SKIP] OSCAR already tokenized"
|
||||
else
|
||||
python3 - <<'PYEOF'
|
||||
import glob, numpy as np
|
||||
from tokenizers import Tokenizer
|
||||
import pyarrow.parquet as pq
|
||||
from tqdm import tqdm
|
||||
|
||||
tokenizer = Tokenizer.from_file("tokenizer/tokenizer.json")
|
||||
files = sorted(glob.glob("data/korean_extra/oscar_ko/data/kor_Hang/*.parquet"))
|
||||
all_tokens = []
|
||||
doc_count = 0
|
||||
|
||||
for f in tqdm(files, desc="OSCAR"):
|
||||
table = pq.read_table(f, columns=['text'])
|
||||
for row in table.column('text'):
|
||||
if row is None:
|
||||
continue
|
||||
parts = row.as_py()
|
||||
if parts:
|
||||
text = '\n'.join(item['text'] for item in parts if item and item.get('text'))
|
||||
if len(text) > 50:
|
||||
all_tokens.extend(tokenizer.encode(text).ids)
|
||||
doc_count += 1
|
||||
|
||||
print(f"OSCAR: {doc_count:,} docs, {len(all_tokens):,} tokens")
|
||||
n_val = int(len(all_tokens) * 0.002)
|
||||
np.array(all_tokens[n_val:], dtype=np.uint16).tofile("data/oscar_train.bin")
|
||||
np.array(all_tokens[:n_val], dtype=np.uint16).tofile("data/oscar_val.bin")
|
||||
PYEOF
|
||||
log "[DONE] OSCAR"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ─── Step 4: korean_webtext ──────────────────────────────────────────────
|
||||
if [[ $FROM_STEP -le 4 ]]; then
|
||||
log "=== Step 4: korean_webtext 토큰화 ==="
|
||||
tokenize_parquet "webtext" \
|
||||
"${EXTRA_DIR}/korean_webtext/data/*.parquet" \
|
||||
"text"
|
||||
fi
|
||||
|
||||
# ─── Step 5: HPLT 한국어 추출 + 토큰화 ──────────────────────────────────
|
||||
if [[ $FROM_STEP -le 5 ]]; then
|
||||
log "=== Step 5: HPLT 한국어 추출 + 토큰화 ==="
|
||||
HPLT_OUT="${DATA_DIR}/hplt_ko_train.bin"
|
||||
|
||||
if [[ -f "$HPLT_OUT" && $FROM_STEP -le 0 ]]; then
|
||||
log "[SKIP] HPLT already tokenized"
|
||||
else
|
||||
python3 - <<'PYEOF'
|
||||
import glob, numpy as np
|
||||
from tokenizers import Tokenizer
|
||||
import pyarrow.parquet as pq
|
||||
from tqdm import tqdm
|
||||
|
||||
tokenizer = Tokenizer.from_file("tokenizer/tokenizer.json")
|
||||
files = sorted(glob.glob("data/korean_extra/hplt_ko/en-ko/*.parquet"))
|
||||
all_tokens = []
|
||||
doc_count = 0
|
||||
|
||||
for f in tqdm(files, desc="HPLT"):
|
||||
table = pq.read_table(f, columns=['tgt_doc'])
|
||||
for row in table.column('tgt_doc'):
|
||||
d = row.as_py()
|
||||
if d and d.get('sentences'):
|
||||
text = '\n'.join(s for s in d['sentences'] if s)
|
||||
if len(text) > 50:
|
||||
all_tokens.extend(tokenizer.encode(text).ids)
|
||||
doc_count += 1
|
||||
|
||||
print(f"HPLT Korean: {doc_count:,} docs, {len(all_tokens):,} tokens")
|
||||
n_val = int(len(all_tokens) * 0.002)
|
||||
np.array(all_tokens[n_val:], dtype=np.uint16).tofile("data/hplt_ko_train.bin")
|
||||
np.array(all_tokens[:n_val], dtype=np.uint16).tofile("data/hplt_ko_val.bin")
|
||||
PYEOF
|
||||
log "[DONE] HPLT"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ─── Step 6: textbooks + finepdfs + kovast ───────────────────────────────
|
||||
if [[ $FROM_STEP -le 6 ]]; then
|
||||
log "=== Step 6: 기타 소스 토큰화 ==="
|
||||
EXTRA_OUT="${DATA_DIR}/extra_misc_train.bin"
|
||||
|
||||
if [[ -f "$EXTRA_OUT" && $FROM_STEP -le 0 ]]; then
|
||||
log "[SKIP] extra_misc already tokenized"
|
||||
else
|
||||
python3 - <<'PYEOF'
|
||||
import glob, numpy as np, os
|
||||
from tokenizers import Tokenizer
|
||||
import pyarrow.parquet as pq
|
||||
from tqdm import tqdm
|
||||
|
||||
tokenizer = Tokenizer.from_file("tokenizer/tokenizer.json")
|
||||
all_tokens = []
|
||||
doc_count = 0
|
||||
|
||||
# korean_textbooks (MMLU-style: look for text columns)
|
||||
tb_files = glob.glob("data/korean_extra/korean_textbooks/**/*.parquet", recursive=True)
|
||||
for f in tqdm(tb_files, desc="textbooks"):
|
||||
try:
|
||||
table = pq.read_table(f)
|
||||
# Try common text columns
|
||||
for col in ['question', 'text', 'input', 'instruction']:
|
||||
if col in table.column_names:
|
||||
for val in table.column(col):
|
||||
t = val.as_py()
|
||||
if t and len(t) > 20:
|
||||
all_tokens.extend(tokenizer.encode(t).ids)
|
||||
doc_count += 1
|
||||
break
|
||||
except:
|
||||
continue
|
||||
|
||||
# finepdfs
|
||||
pdf_files = glob.glob("data/korean_extra/finepdfs_edu_ko/*.parquet")
|
||||
for f in tqdm(pdf_files, desc="finepdfs"):
|
||||
try:
|
||||
table = pq.read_table(f)
|
||||
for col in ['text', 'content']:
|
||||
if col in table.column_names:
|
||||
for val in table.column(col):
|
||||
t = val.as_py()
|
||||
if t and len(t) > 50:
|
||||
all_tokens.extend(tokenizer.encode(t).ids)
|
||||
doc_count += 1
|
||||
break
|
||||
except:
|
||||
continue
|
||||
|
||||
print(f"Extra: {doc_count:,} docs, {len(all_tokens):,} tokens")
|
||||
n_val = int(len(all_tokens) * 0.002)
|
||||
np.array(all_tokens[n_val:], dtype=np.uint16).tofile("data/extra_misc_train.bin")
|
||||
np.array(all_tokens[:n_val], dtype=np.uint16).tofile("data/extra_misc_val.bin")
|
||||
PYEOF
|
||||
log "[DONE] extra_misc"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ─── Step 7: 전체 병합 ──────────────────────────────────────────────────
|
||||
if [[ $FROM_STEP -le 7 ]]; then
|
||||
log "=== Step 7: 전체 병합 ==="
|
||||
|
||||
TRAIN_BINS=""
|
||||
for f in \
|
||||
"${DATA_DIR}/korean_train.bin" \
|
||||
"${DATA_DIR}/culturax_train.bin" \
|
||||
"${DATA_DIR}/cc100_train.bin" \
|
||||
"${DATA_DIR}/oscar_train.bin" \
|
||||
"${DATA_DIR}/webtext_train.bin" \
|
||||
"${DATA_DIR}/hplt_ko_train.bin" \
|
||||
"${DATA_DIR}/extra_misc_train.bin"; do
|
||||
if [[ -f "$f" ]]; then
|
||||
TRAIN_BINS="$TRAIN_BINS $f"
|
||||
log " Including: $f ($(du -h "$f" | cut -f1))"
|
||||
else
|
||||
log " [WARN] Missing: $f"
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ -n "$TRAIN_BINS" ]]; then
|
||||
python3 data/merge_bins.py $TRAIN_BINS "${DATA_DIR}/merged_3b_train.bin"
|
||||
log "[DONE] merged_3b_train.bin created"
|
||||
fi
|
||||
|
||||
# Val 병합
|
||||
VAL_BINS=""
|
||||
for f in \
|
||||
"${DATA_DIR}/korean_val.bin" \
|
||||
"${DATA_DIR}/culturax_val.bin" \
|
||||
"${DATA_DIR}/cc100_val.bin" \
|
||||
"${DATA_DIR}/oscar_val.bin" \
|
||||
"${DATA_DIR}/webtext_val.bin" \
|
||||
"${DATA_DIR}/hplt_ko_val.bin" \
|
||||
"${DATA_DIR}/extra_misc_val.bin"; do
|
||||
if [[ -f "$f" ]]; then
|
||||
VAL_BINS="$VAL_BINS $f"
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ -n "$VAL_BINS" ]]; then
|
||||
python3 data/merge_bins.py $VAL_BINS "${DATA_DIR}/merged_3b_val.bin"
|
||||
log "[DONE] merged_3b_val.bin created"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ─── Step 8: 검증 ────────────────────────────────────────────────────────
|
||||
if [[ $FROM_STEP -le 8 ]]; then
|
||||
log "=== Step 8: 최종 검증 ==="
|
||||
python3 - <<'PYEOF'
|
||||
import os, glob
|
||||
import numpy as np
|
||||
|
||||
print("=== 토큰화 결과 ===")
|
||||
total_train = 0
|
||||
total_val = 0
|
||||
for f in sorted(glob.glob("data/*_train.bin") + glob.glob("data/train.bin")):
|
||||
n = os.path.getsize(f) // 2
|
||||
total_train += n
|
||||
print(f" {os.path.basename(f):30s}: {n:>15,} tokens ({os.path.getsize(f)/1e9:.2f} GB)")
|
||||
|
||||
for f in sorted(glob.glob("data/*_val.bin") + glob.glob("data/val.bin")):
|
||||
n = os.path.getsize(f) // 2
|
||||
total_val += n
|
||||
|
||||
print(f"\n Total train: {total_train:,} tokens ({total_train/1e9:.1f}B)")
|
||||
print(f" Total val: {total_val:,} tokens ({total_val/1e6:.1f}M)")
|
||||
print(f"\n 3B Chinchilla minimum: 60B tokens")
|
||||
print(f" Epochs needed for 60B: {60e9/total_train:.1f}")
|
||||
print(f" Epochs needed for 100B: {100e9/total_train:.1f}")
|
||||
PYEOF
|
||||
fi
|
||||
|
||||
log "=== 파이프라인 완료 ==="
|
||||
264
source/scripts/prepare_sft_combined.sh
Normal file
264
source/scripts/prepare_sft_combined.sh
Normal file
@@ -0,0 +1,264 @@
|
||||
#!/usr/bin/env bash
|
||||
# prepare_sft_combined.sh — 3B SFT용 전체 데이터 통합
|
||||
# 모든 SFT 데이터를 하나의 train/val 파일로 합침
|
||||
#
|
||||
# 업데이트 (2026-03-02): sft_extra 신규 소스 추가
|
||||
# - nayohan_Evol-Instruct-Code-80k-v1-ko (코드 instruction)
|
||||
# - FreedomIntelligence_alpaca-gpt4-korean (GPT-4 alpaca 한국어)
|
||||
# - FreedomIntelligence_evol-instruct-korean (evol-instruct 한국어)
|
||||
# - coastral_korean-writing-style-instruct (한국어 글쓰기 스타일)
|
||||
# - maywell_ko_wikidata_QA (위키데이터 QA)
|
||||
# - OpenAssistant_oasst1_ko (OASST1 한국어, 트리 재구성)
|
||||
# - Bllossom_evol-instruct-ko (존재 확인 후 로드)
|
||||
set -euo pipefail
|
||||
BASE="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
OUT_DIR="$BASE/data/sft_combined"
|
||||
mkdir -p "$OUT_DIR"
|
||||
|
||||
python3 << 'PYEOF'
|
||||
import json, random, os, glob, hashlib
|
||||
from collections import defaultdict
|
||||
|
||||
BASE = "/PROJECT/0325120031_A/ghong/taketimes/llm-bang/data"
|
||||
OUT_TRAIN = f"{BASE}/sft_combined/train.jsonl"
|
||||
OUT_VAL = f"{BASE}/sft_combined/val.jsonl"
|
||||
VAL_RATIO = 0.02
|
||||
SEED = 42
|
||||
|
||||
# SFT 소스 파일 목록 (chat 포맷으로 변환 가능한 것들)
|
||||
SOURCES = [
|
||||
# (path, fmt) fmt: "messages" | "auto" | "oasst"
|
||||
(f"{BASE}/sft/train.jsonl", "messages"),
|
||||
(f"{BASE}/sft_extra/ultrachat_200k/train_sft.jsonl", "messages"),
|
||||
(f"{BASE}/sft_extra/open_korean_instructions/train.jsonl", "messages"),
|
||||
(f"{BASE}/sft_extra/korean_instruction_mix/train.jsonl", "messages"),
|
||||
(f"{BASE}/sft_extra/openhermes_2.5/train.jsonl", "messages"),
|
||||
(f"{BASE}/sft_extra/magpie_reasoning_v2/train.jsonl", "messages"),
|
||||
(f"{BASE}/sft_extra/magpie_reasoning_ko/train.jsonl", "messages"),
|
||||
(f"{BASE}/sft_extra/reasoning_r1_1.4m/train.jsonl", "messages"),
|
||||
(f"{BASE}/sft_extra/lemon-mint_smol-koreantalk.jsonl", "auto"),
|
||||
(f"{BASE}/sft_extra/dbdu_ShareGPT-74k-ko.jsonl", "auto"),
|
||||
(f"{BASE}/sft_extra/ko_lima/data.jsonl", "auto"),
|
||||
(f"{BASE}/sft_extra/koalpaca_v1_1a/data.jsonl", "auto"),
|
||||
(f"{BASE}/sft_extra/kullm_v2/data.jsonl", "auto"),
|
||||
(f"{BASE}/sft_extra/kuotient_orca-math-word-problems-193k-korean.jsonl", "auto"),
|
||||
(f"{BASE}/sft_extra/kyujinpy_KOR-OpenOrca-Platypus-v3/data.jsonl", "auto"),
|
||||
(f"{BASE}/sft_extra/nlp-with-deeplearning_Ko.WizardLM_evol_instruct_V2_196k.jsonl", "auto"),
|
||||
(f"{BASE}/sft_extra/AI-MO_NuminaMath-CoT/data.jsonl", "auto"),
|
||||
(f"{BASE}/sft_extra/zwhe99_DeepMath-103K/data.jsonl", "auto"),
|
||||
# ---- 신규 소스 (2026-03-02) ----
|
||||
(f"{BASE}/sft_extra/nayohan_Evol-Instruct-Code-80k-v1-ko/data.jsonl", "auto"),
|
||||
(f"{BASE}/sft_extra/FreedomIntelligence_alpaca-gpt4-korean.jsonl", "auto"),
|
||||
(f"{BASE}/sft_extra/FreedomIntelligence_evol-instruct-korean.jsonl", "auto"),
|
||||
(f"{BASE}/sft_extra/coastral_korean-writing-style-instruct.jsonl", "auto"),
|
||||
(f"{BASE}/sft_extra/maywell_ko_wikidata_QA.jsonl", "auto"),
|
||||
(f"{BASE}/sft_extra/OpenAssistant_oasst1_ko.jsonl", "oasst"),
|
||||
(f"{BASE}/sft_extra/Bllossom_evol-instruct-ko/data.jsonl", "auto"),
|
||||
]
|
||||
|
||||
def to_messages(obj):
|
||||
"""다양한 포맷을 통일된 messages 포맷으로 변환"""
|
||||
# 이미 messages 포맷
|
||||
if 'messages' in obj and isinstance(obj['messages'], list):
|
||||
return obj['messages']
|
||||
# conversations 포맷
|
||||
if 'conversations' in obj:
|
||||
msgs = []
|
||||
for turn in obj['conversations']:
|
||||
role = turn.get('from', turn.get('role', ''))
|
||||
content = turn.get('value', turn.get('content', ''))
|
||||
if role in ('human', 'user', 'prompter'):
|
||||
msgs.append({'role': 'user', 'content': content})
|
||||
elif role in ('gpt', 'assistant', 'bot'):
|
||||
msgs.append({'role': 'assistant', 'content': content})
|
||||
return msgs if len(msgs) >= 2 else None
|
||||
# instruction/output 포맷
|
||||
if 'instruction' in obj:
|
||||
instruction = obj['instruction']
|
||||
inp = obj.get('input', '')
|
||||
output = obj.get('output', obj.get('response', ''))
|
||||
if not output: return None
|
||||
user_content = instruction + ('\n\n' + inp if inp else '')
|
||||
return [{'role': 'user', 'content': user_content}, {'role': 'assistant', 'content': output}]
|
||||
# question/answer 포맷
|
||||
if 'question' in obj and 'answer' in obj:
|
||||
return [{'role': 'user', 'content': obj['question']}, {'role': 'assistant', 'content': obj['answer']}]
|
||||
# prompt/response
|
||||
if 'prompt' in obj and ('response' in obj or 'completion' in obj):
|
||||
resp = obj.get('response', obj.get('completion', ''))
|
||||
return [{'role': 'user', 'content': obj['prompt']}, {'role': 'assistant', 'content': resp}]
|
||||
# problem/solution
|
||||
if 'problem' in obj and 'solution' in obj:
|
||||
return [{'role': 'user', 'content': obj['problem']}, {'role': 'assistant', 'content': obj['solution']}]
|
||||
return None
|
||||
|
||||
|
||||
def load_oasst(path):
|
||||
"""
|
||||
OpenAssistant OASST1 flat message 포맷을 대화 트리로 재구성.
|
||||
각 루트(prompter) 메시지에서 best-ranked assistant 응답(rank=0.0)을
|
||||
따라 단일 대화 스레드를 추출한다.
|
||||
deleted=True 메시지와 review_result=False 메시지는 제외.
|
||||
"""
|
||||
nodes = {} # message_id → obj
|
||||
children = defaultdict(list) # parent_id → [child_obj, ...]
|
||||
|
||||
with open(path, 'r', errors='replace') as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
obj = json.loads(line)
|
||||
except Exception:
|
||||
continue
|
||||
if obj.get('deleted', False):
|
||||
continue
|
||||
if obj.get('review_result') is False:
|
||||
continue
|
||||
mid = obj.get('message_id')
|
||||
if mid:
|
||||
nodes[mid] = obj
|
||||
pid = obj.get('parent_id')
|
||||
if pid:
|
||||
children[pid].append(obj)
|
||||
|
||||
# 자식 목록을 rank 오름차순 정렬 (rank=null은 뒤로)
|
||||
def sort_key(c):
|
||||
r = c.get('rank')
|
||||
mid = c.get('message_id', '')
|
||||
return (1, 0, mid) if r is None else (0, r, mid)
|
||||
for pid in children:
|
||||
children[pid].sort(key=sort_key)
|
||||
|
||||
samples = []
|
||||
|
||||
def build_thread(node, current_msgs):
|
||||
"""재귀적으로 대화 스레드를 따라 samples에 추가."""
|
||||
role = node.get('role', '')
|
||||
text = node.get('text', '')
|
||||
if role == 'prompter':
|
||||
mapped_role = 'user'
|
||||
elif role == 'assistant':
|
||||
mapped_role = 'assistant'
|
||||
else:
|
||||
return
|
||||
|
||||
msgs = current_msgs + [{'role': mapped_role, 'content': text}]
|
||||
|
||||
# 유효한 user→assistant 쌍이 있을 때만 샘플 추가
|
||||
if mapped_role == 'assistant' and len(msgs) >= 2:
|
||||
samples.append({'messages': msgs})
|
||||
|
||||
# 자식 중 best (rank=0.0) 하나만 따라간다 (가장 품질 높은 경로)
|
||||
kids = children.get(node.get('message_id'), [])
|
||||
if kids:
|
||||
build_thread(kids[0], msgs)
|
||||
|
||||
# 루트 노드: parent_id가 없는 prompter 메시지
|
||||
roots = [n for n in nodes.values() if n.get('parent_id') is None and n.get('role') == 'prompter']
|
||||
for root in roots:
|
||||
build_thread(root, [])
|
||||
|
||||
return samples
|
||||
|
||||
|
||||
random.seed(SEED)
|
||||
all_samples = []
|
||||
|
||||
for path, fmt in SOURCES:
|
||||
if not os.path.exists(path):
|
||||
print(f"[SKIP] {path}")
|
||||
continue
|
||||
|
||||
if fmt == "oasst":
|
||||
samples = load_oasst(path)
|
||||
all_samples.extend(samples)
|
||||
print(f"[LOADED] {os.path.basename(path)}: {len(samples):,} samples (oasst tree)")
|
||||
continue
|
||||
|
||||
count = 0
|
||||
with open(path, 'r', errors='replace') as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line: continue
|
||||
try:
|
||||
obj = json.loads(line)
|
||||
except Exception:
|
||||
continue
|
||||
if fmt == "messages":
|
||||
msgs = obj.get('messages') or obj.get('conversations')
|
||||
if msgs:
|
||||
all_samples.append({'messages': msgs})
|
||||
count += 1
|
||||
else: # auto detect
|
||||
msgs = to_messages(obj)
|
||||
if msgs and len(msgs) >= 2:
|
||||
all_samples.append({'messages': msgs})
|
||||
count += 1
|
||||
print(f"[LOADED] {os.path.basename(path)}: {count:,} samples")
|
||||
if count == 0:
|
||||
print(f"[WARN] {os.path.basename(path)}: 0 samples extracted (format detection may have failed)")
|
||||
|
||||
print(f"\n총 샘플: {len(all_samples):,}")
|
||||
|
||||
# ---- Deduplication (MD5 of first user message) ----
|
||||
seen_hashes = set()
|
||||
unique_samples = []
|
||||
dup_count = 0
|
||||
for s in all_samples:
|
||||
msgs = s.get('messages', [])
|
||||
first_user = next((m['content'] for m in msgs if m.get('role') == 'user'), '')
|
||||
h = hashlib.md5(first_user.encode('utf-8', errors='replace')).hexdigest()
|
||||
if h in seen_hashes:
|
||||
dup_count += 1
|
||||
continue
|
||||
seen_hashes.add(h)
|
||||
unique_samples.append(s)
|
||||
|
||||
print(f"[DEDUP] 제거: {dup_count:,}, 남은 샘플: {len(unique_samples):,}")
|
||||
all_samples = unique_samples
|
||||
|
||||
# ---- Format validation ----
|
||||
def validate_messages(msgs):
|
||||
"""Check messages have valid role/content structure."""
|
||||
if not isinstance(msgs, list) or len(msgs) < 2:
|
||||
return False
|
||||
for m in msgs:
|
||||
if not isinstance(m, dict):
|
||||
return False
|
||||
if m.get('role') not in ('user', 'assistant', 'system'):
|
||||
return False
|
||||
if not isinstance(m.get('content'), str):
|
||||
return False
|
||||
return True
|
||||
|
||||
valid_samples = []
|
||||
invalid_count = 0
|
||||
for s in all_samples:
|
||||
if validate_messages(s.get('messages', [])):
|
||||
valid_samples.append(s)
|
||||
else:
|
||||
invalid_count += 1
|
||||
|
||||
print(f"[VALIDATE] 유효하지 않은 포맷 제거: {invalid_count:,}, 남은 샘플: {len(valid_samples):,}")
|
||||
all_samples = valid_samples
|
||||
|
||||
random.shuffle(all_samples)
|
||||
|
||||
n_val = int(len(all_samples) * VAL_RATIO)
|
||||
val_samples = all_samples[:n_val]
|
||||
train_samples = all_samples[n_val:]
|
||||
|
||||
os.makedirs(os.path.dirname(OUT_TRAIN), exist_ok=True)
|
||||
with open(OUT_TRAIN, 'w') as f:
|
||||
for s in train_samples:
|
||||
f.write(json.dumps(s, ensure_ascii=False) + '\n')
|
||||
with open(OUT_VAL, 'w') as f:
|
||||
for s in val_samples:
|
||||
f.write(json.dumps(s, ensure_ascii=False) + '\n')
|
||||
|
||||
print(f"[DONE] train: {len(train_samples):,} → {OUT_TRAIN}")
|
||||
print(f"[DONE] val: {len(val_samples):,} → {OUT_VAL}")
|
||||
PYEOF
|
||||
echo "SFT 데이터 병합 완료"
|
||||
518
source/scripts/quality_gate.sh
Normal file
518
source/scripts/quality_gate.sh
Normal file
@@ -0,0 +1,518 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# quality_gate.sh — Phase 완료 자동 품질 게이트 검증
|
||||
#
|
||||
# Usage:
|
||||
# bash scripts/quality_gate.sh <phase>
|
||||
#
|
||||
# Phases:
|
||||
# pretrain — 사전학습 게이트 (val_loss, loss 단조 감소)
|
||||
# sft — SFT 게이트 (val_loss 수렴, 반복률, KoBEST)
|
||||
# orpo — ORPO 게이트 (반복률, KoBEST, chosen > rejected)
|
||||
# deploy — 배포 게이트 (GGUF perplexity, Ollama 응답)
|
||||
# all — 모든 게이트 순차 실행
|
||||
#
|
||||
# Exit codes:
|
||||
# 0 — 게이트 통과
|
||||
# 1 — 게이트 실패 (기준 미달)
|
||||
# 2 — 필수 파일 / 의존성 없음 (실행 불가)
|
||||
# =============================================================================
|
||||
set -uo pipefail
|
||||
|
||||
PROJECT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 색상 출력 헬퍼
|
||||
# ---------------------------------------------------------------------------
|
||||
_RED='\033[0;31m'
|
||||
_GREEN='\033[0;32m'
|
||||
_YELLOW='\033[1;33m'
|
||||
_BLUE='\033[0;34m'
|
||||
_NC='\033[0m'
|
||||
|
||||
log_info() { echo -e "${_BLUE}[INFO]${_NC} $*"; }
|
||||
log_ok() { echo -e "${_GREEN}[PASS]${_NC} $*"; }
|
||||
log_warn() { echo -e "${_YELLOW}[WARN]${_NC} $*"; }
|
||||
log_fail() { echo -e "${_RED}[FAIL]${_NC} $*"; }
|
||||
log_skip() { echo -e " [SKIP] $*"; }
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 유틸리티: Python 한 줄 표현식 평가 (부동소수점 비교)
|
||||
# ---------------------------------------------------------------------------
|
||||
py_eval() {
|
||||
python3 -c "import sys; sys.exit(0 if ($1) else 1)"
|
||||
}
|
||||
|
||||
py_value() {
|
||||
python3 -c "print($1)"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 유틸리티: JSON에서 값 추출
|
||||
# ---------------------------------------------------------------------------
|
||||
json_get() {
|
||||
local file="$1" key="$2"
|
||||
python3 -c "
|
||||
import json, sys
|
||||
try:
|
||||
d = json.load(open('$file'))
|
||||
keys = '$key'.split('.')
|
||||
for k in keys:
|
||||
d = d[k]
|
||||
print(d)
|
||||
except Exception as e:
|
||||
print('NOT_FOUND')
|
||||
sys.exit(1)
|
||||
"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 게이트 결과 집계
|
||||
# ---------------------------------------------------------------------------
|
||||
GATE_PASS=0
|
||||
GATE_FAIL=0
|
||||
GATE_SKIP=0
|
||||
|
||||
record_pass() { GATE_PASS=$((GATE_PASS + 1)); log_ok "$*"; }
|
||||
record_fail() { GATE_FAIL=$((GATE_FAIL + 1)); log_fail "$*"; }
|
||||
record_skip() { GATE_SKIP=$((GATE_SKIP + 1)); log_skip "$*"; }
|
||||
|
||||
# =============================================================================
|
||||
# Gate 1: Pretrain
|
||||
# =============================================================================
|
||||
gate_pretrain() {
|
||||
echo ""
|
||||
echo "=================================================================="
|
||||
echo " Gate: PRETRAIN"
|
||||
echo " 기준: val_loss < 2.5 | loss 단조 감소 확인"
|
||||
echo "=================================================================="
|
||||
|
||||
# 최신 체크포인트 디렉토리 탐색
|
||||
CKPT_BASE="$PROJECT_DIR/checkpoints"
|
||||
METRICS_FILE=""
|
||||
|
||||
# metrics.json 또는 train_log.jsonl 탐색
|
||||
for candidate in \
|
||||
"$CKPT_BASE/korean_3b_fp8_pretrain/metrics.json" \
|
||||
"$CKPT_BASE/korean_3b_pretrain/metrics.json" \
|
||||
"$PROJECT_DIR/outputs/pretrain_metrics.json" \
|
||||
"$PROJECT_DIR/logs/pretrain_metrics.json"
|
||||
do
|
||||
if [[ -f "$candidate" ]]; then
|
||||
METRICS_FILE="$candidate"
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ -z "$METRICS_FILE" ]]; then
|
||||
log_warn "사전학습 메트릭 파일을 찾을 수 없습니다."
|
||||
log_warn "찾는 경로: $CKPT_BASE/korean_3b_*/metrics.json"
|
||||
log_warn "메트릭 파일이 없으면 학습 스크립트에서 아래 형식으로 저장하세요:"
|
||||
log_warn ' {"val_loss": 2.3, "loss_history": [3.1, 2.8, 2.5, 2.3]}'
|
||||
record_skip "메트릭 파일 없음 — 게이트 건너뜀"
|
||||
return 0
|
||||
fi
|
||||
|
||||
log_info "메트릭 파일: $METRICS_FILE"
|
||||
|
||||
# val_loss 확인
|
||||
VAL_LOSS=$(json_get "$METRICS_FILE" "val_loss" 2>/dev/null || echo "NOT_FOUND")
|
||||
if [[ "$VAL_LOSS" == "NOT_FOUND" ]]; then
|
||||
record_skip "val_loss 키 없음 — 건너뜀"
|
||||
else
|
||||
log_info "val_loss = $VAL_LOSS (기준: < 2.5)"
|
||||
if py_eval "$VAL_LOSS < 2.5" 2>/dev/null; then
|
||||
record_pass "val_loss $VAL_LOSS < 2.5"
|
||||
else
|
||||
record_fail "val_loss $VAL_LOSS >= 2.5 (기준 미달)"
|
||||
fi
|
||||
fi
|
||||
|
||||
# loss 단조 감소 확인 (loss_history)
|
||||
python3 - "$METRICS_FILE" <<'PYEOF'
|
||||
import json, sys
|
||||
|
||||
metrics_file = sys.argv[1]
|
||||
try:
|
||||
d = json.load(open(metrics_file))
|
||||
history = d.get("loss_history", [])
|
||||
except Exception as e:
|
||||
print(f"[SKIP] loss_history 읽기 실패: {e}")
|
||||
sys.exit(0)
|
||||
|
||||
if len(history) < 2:
|
||||
print(f"[SKIP] loss_history 데이터 부족 ({len(history)}개)")
|
||||
sys.exit(0)
|
||||
|
||||
# 전체 추세가 감소하는지 확인 (처음 1/4 vs 마지막 1/4 평균 비교)
|
||||
n = len(history)
|
||||
q = max(1, n // 4)
|
||||
early_avg = sum(history[:q]) / q
|
||||
late_avg = sum(history[-q:]) / q
|
||||
|
||||
if late_avg < early_avg:
|
||||
print(f"[PASS] loss 단조 감소 확인: 초기 avg={early_avg:.4f} → 최근 avg={late_avg:.4f}")
|
||||
sys.exit(0)
|
||||
else:
|
||||
print(f"[FAIL] loss 감소 미확인: 초기 avg={early_avg:.4f}, 최근 avg={late_avg:.4f}")
|
||||
sys.exit(1)
|
||||
PYEOF
|
||||
local mono_exit=$?
|
||||
if [[ $mono_exit -eq 0 ]]; then
|
||||
GATE_PASS=$((GATE_PASS + 1))
|
||||
elif [[ $mono_exit -eq 1 ]]; then
|
||||
GATE_FAIL=$((GATE_FAIL + 1))
|
||||
fi
|
||||
# exit 0 (SKIP) 는 이미 처리됨
|
||||
}
|
||||
|
||||
# =============================================================================
|
||||
# Gate 2: SFT
|
||||
# =============================================================================
|
||||
gate_sft() {
|
||||
echo ""
|
||||
echo "=================================================================="
|
||||
echo " Gate: SFT"
|
||||
echo " 기준: val_loss 수렴 | 반복률 < 15% | KoBEST > 55%"
|
||||
echo "=================================================================="
|
||||
|
||||
METRICS_FILE=""
|
||||
for candidate in \
|
||||
"$PROJECT_DIR/outputs/sft_metrics.json" \
|
||||
"$PROJECT_DIR/logs/sft_metrics.json" \
|
||||
"$PROJECT_DIR/checkpoints/sft/metrics.json"
|
||||
do
|
||||
if [[ -f "$candidate" ]]; then
|
||||
METRICS_FILE="$candidate"
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ -z "$METRICS_FILE" ]]; then
|
||||
log_warn "SFT 메트릭 파일을 찾을 수 없습니다."
|
||||
log_warn ' {"val_loss": 1.8, "rep_rate": 0.08, "kobest_score": 0.62}'
|
||||
record_skip "SFT 메트릭 파일 없음 — 게이트 건너뜀"
|
||||
return 0
|
||||
fi
|
||||
|
||||
log_info "메트릭 파일: $METRICS_FILE"
|
||||
|
||||
# val_loss 수렴 (상대 변화율 < 1% — 마지막 두 체크포인트)
|
||||
python3 - "$METRICS_FILE" <<'PYEOF'
|
||||
import json, sys
|
||||
|
||||
metrics_file = sys.argv[1]
|
||||
try:
|
||||
d = json.load(open(metrics_file))
|
||||
history = d.get("val_loss_history", [])
|
||||
except Exception as e:
|
||||
print(f"[SKIP] val_loss_history 읽기 실패: {e}")
|
||||
sys.exit(0)
|
||||
|
||||
if len(history) < 2:
|
||||
# 단일 val_loss만 있으면 단순 확인
|
||||
val_loss = d.get("val_loss")
|
||||
if val_loss is not None:
|
||||
print(f"[INFO] val_loss = {val_loss} (수렴 히스토리 없음 — 단일 값 확인 건너뜀)")
|
||||
sys.exit(0)
|
||||
|
||||
last = history[-1]
|
||||
second = history[-2]
|
||||
rel_change = abs(last - second) / max(abs(second), 1e-9)
|
||||
|
||||
if rel_change < 0.01:
|
||||
print(f"[PASS] val_loss 수렴 (상대변화율 {rel_change*100:.3f}% < 1%): {second:.4f} → {last:.4f}")
|
||||
sys.exit(0)
|
||||
else:
|
||||
print(f"[FAIL] val_loss 미수렴 (상대변화율 {rel_change*100:.3f}% >= 1%): {second:.4f} → {last:.4f}")
|
||||
sys.exit(1)
|
||||
PYEOF
|
||||
local conv_exit=$?
|
||||
[[ $conv_exit -eq 0 ]] && GATE_PASS=$((GATE_PASS + 1)) || GATE_FAIL=$((GATE_FAIL + 1))
|
||||
|
||||
# 반복률 확인
|
||||
REP_RATE=$(json_get "$METRICS_FILE" "rep_rate" 2>/dev/null || echo "NOT_FOUND")
|
||||
if [[ "$REP_RATE" == "NOT_FOUND" ]]; then
|
||||
record_skip "rep_rate 키 없음 — 건너뜀"
|
||||
else
|
||||
REP_PCT=$(py_value "$REP_RATE * 100")
|
||||
log_info "반복률 = ${REP_PCT}% (기준: < 15%)"
|
||||
if py_eval "$REP_RATE < 0.15" 2>/dev/null; then
|
||||
record_pass "반복률 ${REP_PCT}% < 15%"
|
||||
else
|
||||
record_fail "반복률 ${REP_PCT}% >= 15% (기준 미달)"
|
||||
fi
|
||||
fi
|
||||
|
||||
# KoBEST 확인
|
||||
KOBEST=$(json_get "$METRICS_FILE" "kobest_score" 2>/dev/null || echo "NOT_FOUND")
|
||||
if [[ "$KOBEST" == "NOT_FOUND" ]]; then
|
||||
record_skip "kobest_score 키 없음 — 건너뜀"
|
||||
else
|
||||
KOBEST_PCT=$(py_value "$KOBEST * 100")
|
||||
log_info "KoBEST = ${KOBEST_PCT}% (기준: > 55%)"
|
||||
if py_eval "$KOBEST > 0.55" 2>/dev/null; then
|
||||
record_pass "KoBEST ${KOBEST_PCT}% > 55%"
|
||||
else
|
||||
record_fail "KoBEST ${KOBEST_PCT}% <= 55% (기준 미달)"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
# =============================================================================
|
||||
# Gate 3: ORPO
|
||||
# =============================================================================
|
||||
gate_orpo() {
|
||||
echo ""
|
||||
echo "=================================================================="
|
||||
echo " Gate: ORPO"
|
||||
echo " 기준: 반복률 < 5% | KoBEST > 60% | chosen > rejected 90%+"
|
||||
echo "=================================================================="
|
||||
|
||||
METRICS_FILE=""
|
||||
for candidate in \
|
||||
"$PROJECT_DIR/outputs/orpo_metrics.json" \
|
||||
"$PROJECT_DIR/logs/orpo_metrics.json" \
|
||||
"$PROJECT_DIR/checkpoints/orpo/metrics.json"
|
||||
do
|
||||
if [[ -f "$candidate" ]]; then
|
||||
METRICS_FILE="$candidate"
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ -z "$METRICS_FILE" ]]; then
|
||||
log_warn "ORPO 메트릭 파일을 찾을 수 없습니다."
|
||||
log_warn ' {"rep_rate": 0.03, "kobest_score": 0.63, "chosen_win_rate": 0.92}'
|
||||
record_skip "ORPO 메트릭 파일 없음 — 게이트 건너뜀"
|
||||
return 0
|
||||
fi
|
||||
|
||||
log_info "메트릭 파일: $METRICS_FILE"
|
||||
|
||||
# 반복률 (더 엄격: < 5%)
|
||||
REP_RATE=$(json_get "$METRICS_FILE" "rep_rate" 2>/dev/null || echo "NOT_FOUND")
|
||||
if [[ "$REP_RATE" == "NOT_FOUND" ]]; then
|
||||
record_skip "rep_rate 키 없음 — 건너뜀"
|
||||
else
|
||||
REP_PCT=$(py_value "$REP_RATE * 100")
|
||||
log_info "반복률 = ${REP_PCT}% (기준: < 5%)"
|
||||
if py_eval "$REP_RATE < 0.05" 2>/dev/null; then
|
||||
record_pass "반복률 ${REP_PCT}% < 5%"
|
||||
else
|
||||
record_fail "반복률 ${REP_PCT}% >= 5% (기준 미달)"
|
||||
fi
|
||||
fi
|
||||
|
||||
# KoBEST (더 엄격: > 60%)
|
||||
KOBEST=$(json_get "$METRICS_FILE" "kobest_score" 2>/dev/null || echo "NOT_FOUND")
|
||||
if [[ "$KOBEST" == "NOT_FOUND" ]]; then
|
||||
record_skip "kobest_score 키 없음 — 건너뜀"
|
||||
else
|
||||
KOBEST_PCT=$(py_value "$KOBEST * 100")
|
||||
log_info "KoBEST = ${KOBEST_PCT}% (기준: > 60%)"
|
||||
if py_eval "$KOBEST > 0.60" 2>/dev/null; then
|
||||
record_pass "KoBEST ${KOBEST_PCT}% > 60%"
|
||||
else
|
||||
record_fail "KoBEST ${KOBEST_PCT}% <= 60% (기준 미달)"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Chosen win rate (chosen log-prob > rejected log-prob 비율)
|
||||
CHOSEN_WIN=$(json_get "$METRICS_FILE" "chosen_win_rate" 2>/dev/null || echo "NOT_FOUND")
|
||||
if [[ "$CHOSEN_WIN" == "NOT_FOUND" ]]; then
|
||||
record_skip "chosen_win_rate 키 없음 — 건너뜀"
|
||||
else
|
||||
WIN_PCT=$(py_value "$CHOSEN_WIN * 100")
|
||||
log_info "Chosen win rate = ${WIN_PCT}% (기준: >= 90%)"
|
||||
if py_eval "$CHOSEN_WIN >= 0.90" 2>/dev/null; then
|
||||
record_pass "Chosen win rate ${WIN_PCT}% >= 90%"
|
||||
else
|
||||
record_fail "Chosen win rate ${WIN_PCT}% < 90% (기준 미달)"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
# =============================================================================
|
||||
# Gate 4: Deploy
|
||||
# =============================================================================
|
||||
gate_deploy() {
|
||||
echo ""
|
||||
echo "=================================================================="
|
||||
echo " Gate: DEPLOY"
|
||||
echo " 기준: Q4_K_M perplexity < F16 × 1.05 | Ollama 5개 프롬프트 응답"
|
||||
echo "=================================================================="
|
||||
|
||||
local MODEL_NAME="frankenstallm-3b"
|
||||
local GGUF_DIR="$PROJECT_DIR/outputs/gguf"
|
||||
local F16_GGUF="$GGUF_DIR/${MODEL_NAME}-f16.gguf"
|
||||
local Q4KM_GGUF="$GGUF_DIR/${MODEL_NAME}-Q4_K_M.gguf"
|
||||
|
||||
# --- GGUF 파일 존재 확인 ---
|
||||
if [[ ! -f "$Q4KM_GGUF" ]]; then
|
||||
log_warn "Q4_K_M GGUF 파일 없음: $Q4KM_GGUF"
|
||||
log_warn "먼저 실행: bash scripts/convert_3b_gguf.sh"
|
||||
record_skip "GGUF 파일 없음 — perplexity 게이트 건너뜀"
|
||||
else
|
||||
# perplexity 측정 (llama-perplexity 또는 Python fallback)
|
||||
LLAMA_PPL_BIN="$PROJECT_DIR/outputs/llama.cpp/build/bin/llama-perplexity"
|
||||
|
||||
if [[ ! -f "$LLAMA_PPL_BIN" ]]; then
|
||||
log_warn "llama-perplexity 바이너리 없음 — 빌드 시도 중 ..."
|
||||
cmake --build "$PROJECT_DIR/outputs/llama.cpp/build" \
|
||||
--target llama-perplexity -j "$(nproc)" &>/dev/null || true
|
||||
fi
|
||||
|
||||
# 샘플 텍스트로 perplexity 비교
|
||||
SAMPLE_TEXT="$PROJECT_DIR/outputs/gguf/ppl_sample.txt"
|
||||
if [[ ! -f "$SAMPLE_TEXT" ]]; then
|
||||
# 짧은 한국어 샘플 생성
|
||||
cat > "$SAMPLE_TEXT" <<'SAMPLE'
|
||||
인공지능은 현대 사회에서 매우 중요한 기술로 자리잡고 있습니다.
|
||||
기계 학습과 딥러닝의 발전으로 인해 다양한 분야에서 혁신이 이루어지고 있습니다.
|
||||
자연어 처리 기술의 발전은 인간과 컴퓨터의 상호작용 방식을 근본적으로 변화시키고 있습니다.
|
||||
한국어는 교착어로서 특유의 형태론적 특성을 가지고 있어 자연어 처리에 독특한 도전을 제시합니다.
|
||||
대규모 언어 모델의 등장으로 기계 번역, 텍스트 요약, 질의응답 등의 성능이 크게 향상되었습니다.
|
||||
SAMPLE
|
||||
fi
|
||||
|
||||
if [[ -f "$LLAMA_PPL_BIN" && -f "$F16_GGUF" ]]; then
|
||||
log_info "Perplexity 측정 중 (F16 vs Q4_K_M) ..."
|
||||
|
||||
PPL_F16=$(timeout 120 "$LLAMA_PPL_BIN" -m "$F16_GGUF" -f "$SAMPLE_TEXT" 2>&1 \
|
||||
| grep -oP "Perplexity: \K[0-9.]+" | head -1 || echo "0")
|
||||
PPL_Q4=$(timeout 120 "$LLAMA_PPL_BIN" -m "$Q4KM_GGUF" -f "$SAMPLE_TEXT" 2>&1 \
|
||||
| grep -oP "Perplexity: \K[0-9.]+" | head -1 || echo "0")
|
||||
|
||||
if [[ "$PPL_F16" == "0" || "$PPL_Q4" == "0" ]]; then
|
||||
record_skip "Perplexity 측정 실패 — 건너뜀"
|
||||
else
|
||||
THRESHOLD=$(py_value "$PPL_F16 * 1.05")
|
||||
log_info "F16 PPL = $PPL_F16 | Q4_K_M PPL = $PPL_Q4 | 기준: < $THRESHOLD"
|
||||
if py_eval "$PPL_Q4 < $PPL_F16 * 1.05" 2>/dev/null; then
|
||||
record_pass "Q4_K_M PPL $PPL_Q4 < F16 PPL × 1.05 ($THRESHOLD)"
|
||||
else
|
||||
record_fail "Q4_K_M PPL $PPL_Q4 >= F16 PPL × 1.05 ($THRESHOLD)"
|
||||
fi
|
||||
fi
|
||||
else
|
||||
record_skip "llama-perplexity 또는 F16 GGUF 없음 — perplexity 게이트 건너뜀"
|
||||
fi
|
||||
fi
|
||||
|
||||
# --- Ollama 응답 테스트 ---
|
||||
if ! command -v ollama &>/dev/null; then
|
||||
record_skip "ollama 없음 — 응답 테스트 건너뜀"
|
||||
return 0
|
||||
fi
|
||||
|
||||
if ! ollama list 2>/dev/null | grep -q "$MODEL_NAME"; then
|
||||
log_warn "Ollama에 $MODEL_NAME 모델이 등록되지 않았습니다."
|
||||
log_warn "먼저 실행: bash scripts/deploy_3b_ollama.sh"
|
||||
record_skip "Ollama 모델 미등록 — 응답 테스트 건너뜀"
|
||||
return 0
|
||||
fi
|
||||
|
||||
log_info "Ollama 응답 테스트 (5개 프롬프트) ..."
|
||||
|
||||
declare -a PROMPTS=(
|
||||
"안녕하세요."
|
||||
"1 더하기 1은 무엇인가요?"
|
||||
"파이썬이란 무엇인가요?"
|
||||
"한국의 수도는 어디인가요?"
|
||||
"오늘 날씨가 좋네요."
|
||||
)
|
||||
|
||||
local PASS=0 FAIL=0
|
||||
for i in "${!PROMPTS[@]}"; do
|
||||
local PROMPT="${PROMPTS[$i]}"
|
||||
local NUM=$((i + 1))
|
||||
if RESP=$(timeout 45 ollama run "$MODEL_NAME" "$PROMPT" 2>&1) && [[ -n "$RESP" ]]; then
|
||||
log_ok " 프롬프트 $NUM 응답 OK (${#RESP}자)"
|
||||
PASS=$((PASS + 1))
|
||||
else
|
||||
log_fail " 프롬프트 $NUM 응답 실패"
|
||||
FAIL=$((FAIL + 1))
|
||||
fi
|
||||
done
|
||||
|
||||
log_info "Ollama 응답: $PASS/5 성공"
|
||||
if [[ $FAIL -eq 0 ]]; then
|
||||
record_pass "Ollama 5개 프롬프트 모두 응답 성공"
|
||||
else
|
||||
record_fail "Ollama 응답 실패 $FAIL/5"
|
||||
fi
|
||||
}
|
||||
|
||||
# =============================================================================
|
||||
# 최종 요약 출력
|
||||
# =============================================================================
|
||||
print_summary() {
|
||||
local phase="$1"
|
||||
local TOTAL=$((GATE_PASS + GATE_FAIL + GATE_SKIP))
|
||||
echo ""
|
||||
echo "=================================================================="
|
||||
echo " Quality Gate 결과: $phase"
|
||||
echo " PASS: $GATE_PASS | FAIL: $GATE_FAIL | SKIP: $GATE_SKIP | TOTAL: $TOTAL"
|
||||
echo "=================================================================="
|
||||
|
||||
if [[ $GATE_FAIL -eq 0 ]]; then
|
||||
echo -e "${_GREEN} [GATE PASSED]${_NC} 모든 검증 기준 통과"
|
||||
echo ""
|
||||
return 0
|
||||
else
|
||||
echo -e "${_RED} [GATE FAILED]${_NC} ${GATE_FAIL}개 검증 기준 미달"
|
||||
echo " 실패 항목을 수정한 후 다시 실행하세요."
|
||||
echo ""
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# =============================================================================
|
||||
# 진입점
|
||||
# =============================================================================
|
||||
PHASE="${1:-}"
|
||||
|
||||
if [[ -z "$PHASE" ]]; then
|
||||
echo "Usage: bash scripts/quality_gate.sh <phase>"
|
||||
echo " phase: pretrain | sft | orpo | deploy | all"
|
||||
exit 2
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "=================================================================="
|
||||
echo " Quality Gate 검증 시작: $PHASE"
|
||||
echo " 프로젝트: $PROJECT_DIR"
|
||||
echo " 시각 : $(date '+%Y-%m-%d %H:%M:%S')"
|
||||
echo "=================================================================="
|
||||
|
||||
case "$PHASE" in
|
||||
pretrain)
|
||||
gate_pretrain
|
||||
print_summary "pretrain"
|
||||
;;
|
||||
sft)
|
||||
gate_sft
|
||||
print_summary "sft"
|
||||
;;
|
||||
orpo)
|
||||
gate_orpo
|
||||
print_summary "orpo"
|
||||
;;
|
||||
deploy)
|
||||
gate_deploy
|
||||
print_summary "deploy"
|
||||
;;
|
||||
all)
|
||||
gate_pretrain
|
||||
gate_sft
|
||||
gate_orpo
|
||||
gate_deploy
|
||||
print_summary "all"
|
||||
;;
|
||||
*)
|
||||
echo "ERROR: 알 수 없는 phase: $PHASE"
|
||||
echo "Usage: bash scripts/quality_gate.sh <pretrain|sft|orpo|deploy|all>"
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
23
source/scripts/run_eval.sh
Normal file
23
source/scripts/run_eval.sh
Normal file
@@ -0,0 +1,23 @@
|
||||
#!/usr/bin/env bash
|
||||
# Usage: bash scripts/run_eval.sh <checkpoint_dir>
|
||||
# Example: bash scripts/run_eval.sh checkpoints/korean_1b_fp8_run1/checkpoint-0200000
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
|
||||
|
||||
CHECKPOINT="${1:?Usage: bash scripts/run_eval.sh <checkpoint_dir>}"
|
||||
|
||||
echo "=== Perplexity Evaluation ==="
|
||||
python "$PROJECT_DIR/eval/perplexity.py" \
|
||||
--checkpoint "$CHECKPOINT" \
|
||||
--data "$PROJECT_DIR/data/korean_val.bin" \
|
||||
--device cuda:0
|
||||
|
||||
echo ""
|
||||
echo "=== Text Generation ==="
|
||||
python "$PROJECT_DIR/eval/generate.py" \
|
||||
--checkpoint "$CHECKPOINT" \
|
||||
--prompt "안녕하세요, 저는" \
|
||||
--max_new_tokens 200 \
|
||||
--device cuda:0
|
||||
236
source/scripts/run_eval_full.sh
Normal file
236
source/scripts/run_eval_full.sh
Normal file
@@ -0,0 +1,236 @@
|
||||
#!/usr/bin/env bash
|
||||
# ============================================================
|
||||
# run_eval_full.sh — 전체 한국어 벤치마크 평가 (목표: 1.5-3시간)
|
||||
#
|
||||
# 사용법:
|
||||
# bash scripts/run_eval_full.sh [CHECKPOINT_DIR] [OUTPUT_DIR]
|
||||
#
|
||||
# 예시:
|
||||
# bash scripts/run_eval_full.sh \
|
||||
# checkpoints/korean_1b_sft/checkpoint-0005000 \
|
||||
# eval/outputs/full_5000
|
||||
#
|
||||
# 태스크:
|
||||
# - KoBEST (5): boolq, copa, hellaswag, sentineg, wic
|
||||
# - HAE-RAE Bench (5): general_knowledge, history, loan_word, rare_word, standard_nomenclature
|
||||
# - Global MMLU Korean: 57개 도메인
|
||||
# - PAWS-Ko: 패러프레이즈 탐지
|
||||
# - KorMedMCQA: 한국어 의학 MCQ (선택)
|
||||
#
|
||||
# 총 예상 샘플: ~15,000개
|
||||
# 1B 모델 @ 8×B200 기준: 약 1.5-3시간
|
||||
# ============================================================
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
|
||||
|
||||
# ─── 인자 처리 ────────────────────────────────────────────
|
||||
CHECKPOINT="${1:-checkpoints/korean_1b_sft/checkpoint-0005000}"
|
||||
TIMESTAMP="$(date +%Y%m%d_%H%M%S)"
|
||||
OUTPUT_DIR="${2:-eval/outputs/full_${TIMESTAMP}}"
|
||||
|
||||
[[ "$CHECKPOINT" != /* ]] && CHECKPOINT="$PROJECT_DIR/$CHECKPOINT"
|
||||
[[ "$OUTPUT_DIR" != /* ]] && OUTPUT_DIR="$PROJECT_DIR/$OUTPUT_DIR"
|
||||
|
||||
# ─── 설정 ────────────────────────────────────────────────
|
||||
HF_MODEL_DIR="$PROJECT_DIR/outputs/hf_$(basename "$CHECKPOINT")"
|
||||
TOKENIZER="$PROJECT_DIR/tokenizer/korean_sp/tokenizer.json"
|
||||
|
||||
# GPU 설정: 단일 GPU 또는 tensor parallel
|
||||
# lm-eval의 hf backend는 기본 단일 GPU 사용
|
||||
# 멀티 GPU: --model_args "pretrained=...,parallelize=True" (자동 device_map)
|
||||
USE_MULTI_GPU="${USE_MULTI_GPU:-0}"
|
||||
if [ "$USE_MULTI_GPU" = "1" ]; then
|
||||
MODEL_EXTRA_ARGS=",parallelize=True"
|
||||
echo "▶ 멀티 GPU 모드 활성화 (device_map=auto)"
|
||||
else
|
||||
MODEL_EXTRA_ARGS=""
|
||||
CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0}"
|
||||
fi
|
||||
|
||||
BATCH_SIZE="${BATCH_SIZE:-auto}"
|
||||
NUM_FEWSHOT="${NUM_FEWSHOT:-0}"
|
||||
|
||||
# ─── 태스크 정의 ─────────────────────────────────────────
|
||||
# Core Korean tasks (항상 실행)
|
||||
TASKS_CORE="kobest,haerae,paws_ko"
|
||||
|
||||
# Extended tasks (시간 있을 때)
|
||||
TASKS_EXTENDED="global_mmlu_ko"
|
||||
|
||||
# 선택적 태스크
|
||||
TASKS_OPTIONAL="kormedmcqa" # 한국어 의학 MCQ
|
||||
|
||||
# 전체 실행 태스크
|
||||
TASKS="${TASKS_CORE},${TASKS_EXTENDED}"
|
||||
|
||||
# ─── 의존성 확인 ─────────────────────────────────────────
|
||||
check_dep() {
|
||||
python3 -c "import $1" 2>/dev/null || { echo "❌ $1 not found. pip install $2"; exit 1; }
|
||||
}
|
||||
check_dep lm_eval lm-eval
|
||||
check_dep transformers transformers
|
||||
check_dep safetensors safetensors
|
||||
|
||||
echo "=================================================="
|
||||
echo " Ko-LLM Full Benchmark Evaluation"
|
||||
echo "=================================================="
|
||||
echo " Checkpoint : $CHECKPOINT"
|
||||
echo " HF output : $HF_MODEL_DIR"
|
||||
echo " Tasks : $TASKS"
|
||||
echo " Few-shot : $NUM_FEWSHOT"
|
||||
echo " Batch size : $BATCH_SIZE"
|
||||
echo " Output : $OUTPUT_DIR"
|
||||
echo " Multi-GPU : $USE_MULTI_GPU"
|
||||
echo " Start time : $(date)"
|
||||
echo "=================================================="
|
||||
|
||||
mkdir -p "$OUTPUT_DIR"
|
||||
LOG_FILE="$OUTPUT_DIR/eval_full.log"
|
||||
|
||||
# ─── Step 1: HF 포맷 변환 ───────────────────────────────
|
||||
echo ""
|
||||
echo "▶ [1/3] 커스텀 체크포인트 → HF 포맷 변환..."
|
||||
|
||||
if [ ! -f "$HF_MODEL_DIR/config.json" ]; then
|
||||
python3 "$PROJECT_DIR/scripts/convert_to_hf.py" \
|
||||
--checkpoint "$CHECKPOINT" \
|
||||
--output "$HF_MODEL_DIR" \
|
||||
--tokenizer "$TOKENIZER" \
|
||||
2>&1 | tee -a "$LOG_FILE"
|
||||
echo "✅ HF 변환 완료: $HF_MODEL_DIR"
|
||||
else
|
||||
echo " ↳ HF 모델 이미 존재, 변환 스킵: $HF_MODEL_DIR"
|
||||
fi
|
||||
|
||||
# ─── Step 2: 전체 평가 ──────────────────────────────────
|
||||
echo ""
|
||||
echo "▶ [2/3] lm-eval 전체 평가 시작..."
|
||||
echo " ↳ 로그: $LOG_FILE"
|
||||
START_TIME=$(date +%s)
|
||||
|
||||
if [ "$USE_MULTI_GPU" = "1" ]; then
|
||||
python3 -m lm_eval \
|
||||
--model hf \
|
||||
--model_args "pretrained=$HF_MODEL_DIR,dtype=float16,parallelize=True" \
|
||||
--tasks "$TASKS" \
|
||||
--num_fewshot "$NUM_FEWSHOT" \
|
||||
--batch_size "$BATCH_SIZE" \
|
||||
--output_path "$OUTPUT_DIR" \
|
||||
--log_samples \
|
||||
--verbosity INFO \
|
||||
2>&1 | tee -a "$LOG_FILE"
|
||||
else
|
||||
CUDA_VISIBLE_DEVICES="$CUDA_VISIBLE_DEVICES" python3 -m lm_eval \
|
||||
--model hf \
|
||||
--model_args "pretrained=$HF_MODEL_DIR,dtype=float16" \
|
||||
--tasks "$TASKS" \
|
||||
--num_fewshot "$NUM_FEWSHOT" \
|
||||
--batch_size "$BATCH_SIZE" \
|
||||
--output_path "$OUTPUT_DIR" \
|
||||
--log_samples \
|
||||
--verbosity INFO \
|
||||
2>&1 | tee -a "$LOG_FILE"
|
||||
fi
|
||||
|
||||
END_TIME=$(date +%s)
|
||||
ELAPSED=$(( END_TIME - START_TIME ))
|
||||
echo ""
|
||||
echo "✅ 평가 완료! 소요: $((ELAPSED/60))분 $((ELAPSED%60))초"
|
||||
|
||||
# ─── Step 3: 결과 요약 리포트 생성 ─────────────────────
|
||||
echo ""
|
||||
echo "▶ [3/3] 결과 리포트 생성..."
|
||||
|
||||
python3 - "$OUTPUT_DIR" "$CHECKPOINT" <<'PYEOF'
|
||||
import json, glob, sys, os
|
||||
from datetime import datetime
|
||||
|
||||
output_dir = sys.argv[1]
|
||||
checkpoint = sys.argv[2] if len(sys.argv) > 2 else "unknown"
|
||||
|
||||
results_files = sorted(glob.glob(f"{output_dir}/**/*.json", recursive=True))
|
||||
results_files = [f for f in results_files if "samples_" not in os.path.basename(f)]
|
||||
|
||||
report_lines = [
|
||||
f"# Ko-LLM Full Eval Report",
|
||||
f"Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}",
|
||||
f"Checkpoint: {checkpoint}",
|
||||
"",
|
||||
]
|
||||
|
||||
all_results = {}
|
||||
for rf in results_files:
|
||||
try:
|
||||
with open(rf) as f:
|
||||
data = json.load(f)
|
||||
results = data.get("results", {})
|
||||
if results:
|
||||
all_results.update(results)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# KoBEST 요약
|
||||
kobest_tasks = [k for k in all_results if k.startswith("kobest_")]
|
||||
if kobest_tasks:
|
||||
report_lines.append("## KoBEST")
|
||||
report_lines.append("| Task | Metric | Score |")
|
||||
report_lines.append("|------|--------|-------|")
|
||||
for task in sorted(kobest_tasks):
|
||||
metrics = all_results[task]
|
||||
for key, val in metrics.items():
|
||||
if "stderr" not in key and isinstance(val, (int, float)):
|
||||
report_lines.append(f"| {task} | {key} | {val:.4f} |")
|
||||
|
||||
# HAE-RAE 요약
|
||||
haerae_tasks = [k for k in all_results if k.startswith("haerae")]
|
||||
if haerae_tasks:
|
||||
report_lines.append("\n## HAE-RAE Bench")
|
||||
report_lines.append("| Task | Metric | Score |")
|
||||
report_lines.append("|------|--------|-------|")
|
||||
for task in sorted(haerae_tasks):
|
||||
metrics = all_results[task]
|
||||
for key, val in metrics.items():
|
||||
if "stderr" not in key and isinstance(val, (int, float)):
|
||||
report_lines.append(f"| {task} | {key} | {val:.4f} |")
|
||||
|
||||
# MMLU Ko 요약 (상위 레벨만)
|
||||
mmlu_top = {k: v for k, v in all_results.items()
|
||||
if k.startswith("global_mmlu_ko") and "_" not in k.replace("global_mmlu_ko", "")}
|
||||
if mmlu_top:
|
||||
report_lines.append("\n## Global MMLU (Korean)")
|
||||
for task, metrics in mmlu_top.items():
|
||||
for key, val in metrics.items():
|
||||
if "stderr" not in key and isinstance(val, (int, float)):
|
||||
report_lines.append(f"- {task} {key}: {val:.4f}")
|
||||
|
||||
# 기타
|
||||
other_tasks = [k for k in all_results
|
||||
if not k.startswith("kobest_")
|
||||
and not k.startswith("haerae")
|
||||
and not k.startswith("global_mmlu_ko")]
|
||||
if other_tasks:
|
||||
report_lines.append("\n## 기타 태스크")
|
||||
for task in sorted(other_tasks):
|
||||
metrics = all_results[task]
|
||||
for key, val in metrics.items():
|
||||
if "stderr" not in key and isinstance(val, (int, float)):
|
||||
report_lines.append(f"- {task} | {key}: {val:.4f}")
|
||||
|
||||
report_path = os.path.join(output_dir, "SUMMARY.md")
|
||||
with open(report_path, "w") as f:
|
||||
f.write("\n".join(report_lines))
|
||||
|
||||
print("\n".join(report_lines))
|
||||
print(f"\n📄 리포트 저장: {report_path}")
|
||||
PYEOF
|
||||
|
||||
echo ""
|
||||
echo "=================================================="
|
||||
echo "✅ 전체 평가 완료!"
|
||||
echo " 결과 디렉토리: $OUTPUT_DIR"
|
||||
echo " 요약 리포트 : $OUTPUT_DIR/SUMMARY.md"
|
||||
echo " 전체 로그 : $LOG_FILE"
|
||||
echo " 완료 시각 : $(date)"
|
||||
echo "=================================================="
|
||||
150
source/scripts/run_eval_quick.sh
Normal file
150
source/scripts/run_eval_quick.sh
Normal file
@@ -0,0 +1,150 @@
|
||||
#!/usr/bin/env bash
|
||||
# ============================================================
|
||||
# run_eval_quick.sh — 빠른 평가 체크 (목표: 20-30분)
|
||||
#
|
||||
# 사용법:
|
||||
# bash scripts/run_eval_quick.sh [CHECKPOINT_DIR] [OUTPUT_DIR]
|
||||
#
|
||||
# 예시:
|
||||
# bash scripts/run_eval_quick.sh \
|
||||
# checkpoints/korean_1b_sft/checkpoint-0005000 \
|
||||
# eval/outputs/quick_5000
|
||||
#
|
||||
# 태스크: kobest_boolq, kobest_copa, haerae_general_knowledge,
|
||||
# haerae_history, paws_ko
|
||||
# ============================================================
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
|
||||
|
||||
# ─── 인자 처리 ────────────────────────────────────────────
|
||||
CHECKPOINT="${1:-checkpoints/korean_1b_sft/checkpoint-0005000}"
|
||||
TIMESTAMP="$(date +%Y%m%d_%H%M%S)"
|
||||
OUTPUT_DIR="${2:-eval/outputs/quick_${TIMESTAMP}}"
|
||||
|
||||
# 상대 경로 → 절대 경로
|
||||
[[ "$CHECKPOINT" != /* ]] && CHECKPOINT="$PROJECT_DIR/$CHECKPOINT"
|
||||
[[ "$OUTPUT_DIR" != /* ]] && OUTPUT_DIR="$PROJECT_DIR/$OUTPUT_DIR"
|
||||
|
||||
# ─── 설정 ────────────────────────────────────────────────
|
||||
HF_MODEL_DIR="$PROJECT_DIR/outputs/hf_$(basename "$CHECKPOINT")"
|
||||
TOKENIZER="$PROJECT_DIR/tokenizer/korean_sp/tokenizer.json"
|
||||
DEVICE="${CUDA_VISIBLE_DEVICES:-0}" # 기본: GPU 0번만 사용
|
||||
BATCH_SIZE="auto"
|
||||
|
||||
# 빠른 체크 태스크 (약 2,000 샘플, ~20분)
|
||||
TASKS="kobest_boolq,kobest_copa,haerae_general_knowledge,haerae_history,paws_ko"
|
||||
|
||||
# ─── 의존성 확인 ─────────────────────────────────────────
|
||||
check_dep() {
|
||||
python3 -c "import $1" 2>/dev/null || { echo "❌ $1 not found. pip install $2"; exit 1; }
|
||||
}
|
||||
check_dep lm_eval lm-eval
|
||||
check_dep transformers transformers
|
||||
check_dep safetensors safetensors
|
||||
|
||||
echo "=================================================="
|
||||
echo " Ko-LLM Quick Eval"
|
||||
echo "=================================================="
|
||||
echo " Checkpoint : $CHECKPOINT"
|
||||
echo " HF output : $HF_MODEL_DIR"
|
||||
echo " Tasks : $TASKS"
|
||||
echo " Output : $OUTPUT_DIR"
|
||||
echo " Device : cuda:$DEVICE"
|
||||
echo "=================================================="
|
||||
|
||||
mkdir -p "$OUTPUT_DIR"
|
||||
|
||||
# ─── Step 1: HF 포맷 변환 ───────────────────────────────
|
||||
if [ ! -f "$HF_MODEL_DIR/config.json" ]; then
|
||||
echo ""
|
||||
echo "▶ Step 1: 커스텀 체크포인트 → HF 포맷 변환..."
|
||||
python3 "$PROJECT_DIR/scripts/convert_to_hf.py" \
|
||||
--checkpoint "$CHECKPOINT" \
|
||||
--output "$HF_MODEL_DIR" \
|
||||
--tokenizer "$TOKENIZER"
|
||||
echo "✅ HF 변환 완료: $HF_MODEL_DIR"
|
||||
else
|
||||
echo "▶ Step 1: HF 모델 이미 존재, 변환 스킵"
|
||||
echo " $HF_MODEL_DIR"
|
||||
fi
|
||||
|
||||
# ─── Step 2: lm-eval 실행 ───────────────────────────────
|
||||
echo ""
|
||||
echo "▶ Step 2: lm-eval 평가 시작..."
|
||||
START_TIME=$(date +%s)
|
||||
|
||||
CUDA_VISIBLE_DEVICES="$DEVICE" python3 -m lm_eval \
|
||||
--model hf \
|
||||
--model_args "pretrained=$HF_MODEL_DIR,dtype=float16" \
|
||||
--tasks "$TASKS" \
|
||||
--num_fewshot 0 \
|
||||
--batch_size "$BATCH_SIZE" \
|
||||
--output_path "$OUTPUT_DIR" \
|
||||
--log_samples \
|
||||
--verbosity INFO \
|
||||
2>&1 | tee "$OUTPUT_DIR/eval.log"
|
||||
|
||||
END_TIME=$(date +%s)
|
||||
ELAPSED=$(( END_TIME - START_TIME ))
|
||||
|
||||
echo ""
|
||||
echo "=================================================="
|
||||
echo "✅ 평가 완료!"
|
||||
echo " 소요시간: $((ELAPSED / 60))분 $((ELAPSED % 60))초"
|
||||
echo " 결과 저장: $OUTPUT_DIR"
|
||||
echo "=================================================="
|
||||
|
||||
# ─── Step 3: 결과 요약 출력 ─────────────────────────────
|
||||
echo ""
|
||||
echo "▶ Step 3: 결과 요약"
|
||||
python3 - <<'PYEOF'
|
||||
import json, glob, sys, os
|
||||
|
||||
output_dir = sys.argv[1] if len(sys.argv) > 1 else "."
|
||||
results_files = glob.glob(f"{output_dir}/**/*.json", recursive=True)
|
||||
results_files = [f for f in results_files if "results" in f.lower()]
|
||||
|
||||
if not results_files:
|
||||
print("결과 JSON 파일 없음. eval.log 확인하세요.")
|
||||
sys.exit(0)
|
||||
|
||||
for rf in results_files:
|
||||
try:
|
||||
with open(rf) as f:
|
||||
data = json.load(f)
|
||||
results = data.get("results", {})
|
||||
print(f"\n{'='*50}")
|
||||
print(f"Task Results (from {os.path.basename(rf)})")
|
||||
print(f"{'='*50}")
|
||||
for task, metrics in results.items():
|
||||
print(f"\n{task}:")
|
||||
for key, val in metrics.items():
|
||||
if "stderr" not in key and isinstance(val, (int, float)):
|
||||
print(f" {key}: {val:.4f}")
|
||||
except Exception as e:
|
||||
print(f"파싱 실패: {rf}: {e}")
|
||||
PYEOF
|
||||
python3 - "$OUTPUT_DIR" <<'PYEOF'
|
||||
import json, glob, sys, os
|
||||
output_dir = sys.argv[1] if len(sys.argv) > 1 else "."
|
||||
results_files = glob.glob(f"{output_dir}/**/*.json", recursive=True)
|
||||
results_files = [f for f in results_files if "results" in os.path.basename(f)]
|
||||
if not results_files:
|
||||
# try finding any json
|
||||
results_files = glob.glob(f"{output_dir}/*.json")
|
||||
for rf in results_files[:3]:
|
||||
try:
|
||||
with open(rf) as f:
|
||||
data = json.load(f)
|
||||
results = data.get("results", {})
|
||||
print(f"\n{'='*50}\nTask Results: {os.path.basename(rf)}\n{'='*50}")
|
||||
for task, metrics in results.items():
|
||||
print(f"\n{task}:")
|
||||
for key, val in metrics.items():
|
||||
if "stderr" not in key and isinstance(val, (int, float)):
|
||||
print(f" {key}: {val:.4f}")
|
||||
except Exception as e:
|
||||
print(f"파싱 실패: {rf}: {e}")
|
||||
PYEOF
|
||||
26
source/scripts/run_pretrain.sh
Normal file
26
source/scripts/run_pretrain.sh
Normal file
@@ -0,0 +1,26 @@
|
||||
#!/bin/bash
|
||||
# Usage: bash scripts/run_pretrain.sh [additional torchrun args]
|
||||
# Runs 8-GPU DDP pretraining via torchrun.
|
||||
#
|
||||
# Any extra arguments are forwarded verbatim to pretrain.py.
|
||||
# Examples:
|
||||
# bash scripts/run_pretrain.sh --max_steps 200000
|
||||
# bash scripts/run_pretrain.sh --resume checkpoints/checkpoint-0010000
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
|
||||
|
||||
torchrun \
|
||||
--nproc_per_node=8 \
|
||||
--master_port=29500 \
|
||||
"$PROJECT_DIR/train/pretrain.py" \
|
||||
--config "$PROJECT_DIR/configs/small.yaml" \
|
||||
--train_data "$PROJECT_DIR/data/train.bin" \
|
||||
--val_data "$PROJECT_DIR/data/val.bin" \
|
||||
--checkpoint_dir "$PROJECT_DIR/checkpoints" \
|
||||
--batch_size 8 \
|
||||
--grad_accum 4 \
|
||||
--warmup_steps 2000 \
|
||||
"$@"
|
||||
44
source/scripts/start-gateway.sh
Normal file
44
source/scripts/start-gateway.sh
Normal file
@@ -0,0 +1,44 @@
|
||||
#!/usr/bin/env bash
|
||||
# start-gateway.sh — OpenClaw 게이트웨이 직접 시작 (독립 프로세스)
|
||||
set -euo pipefail
|
||||
|
||||
RNTIER_HOME="${RNTIER_HOME:-$HOME}"
|
||||
export PATH="${RNTIER_HOME}/.npm-global/bin:/usr/bin:/usr/local/bin:/bin:$PATH"
|
||||
export HOME="${HOME:-/home/ghong}"
|
||||
export OPENCLAW_STATE_DIR="${RNTIER_HOME}/.openclaw"
|
||||
export OPENCLAW_CONFIG_PATH="${RNTIER_HOME}/.openclaw/openclaw.json"
|
||||
|
||||
LOG_DIR="/tmp/openclaw"
|
||||
GATEWAY_LOG="${LOG_DIR}/gateway.log"
|
||||
PID_FILE="/tmp/openclaw-gateway.pid"
|
||||
|
||||
mkdir -p "$LOG_DIR"
|
||||
|
||||
# 기존 프로세스 정리
|
||||
pkill -f "openclaw.*gateway" 2>/dev/null || true
|
||||
sleep 2
|
||||
|
||||
# 게이트웨이 시작 — setsid로 완전 분리
|
||||
setsid nohup "${RNTIER_HOME}/.npm-global/bin/openclaw" gateway run \
|
||||
--port 18789 \
|
||||
--bind loopback \
|
||||
>> "$GATEWAY_LOG" 2>&1 < /dev/null &
|
||||
|
||||
PID=$!
|
||||
echo "$PID" > "$PID_FILE"
|
||||
date +%s > /tmp/openclaw-last-restart
|
||||
|
||||
echo "[$(date)] Gateway launched with PID $PID"
|
||||
|
||||
# 10초 대기 후 상태 확인
|
||||
sleep 10
|
||||
|
||||
if kill -0 "$PID" 2>/dev/null; then
|
||||
echo "[$(date)] OK: Gateway PID $PID is alive"
|
||||
ss -tlnH "sport = :18789" 2>/dev/null && echo "[$(date)] OK: Port 18789 is listening" || echo "[$(date)] WARN: Port 18789 not yet listening"
|
||||
else
|
||||
echo "[$(date)] FAIL: Gateway PID $PID died"
|
||||
echo "--- Last 20 lines of gateway.log ---"
|
||||
tail -20 "$GATEWAY_LOG" 2>/dev/null
|
||||
exit 1
|
||||
fi
|
||||
168
source/scripts/telegram_notify.py
Normal file
168
source/scripts/telegram_notify.py
Normal file
@@ -0,0 +1,168 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Standalone Telegram notification helper for FRANKENSTALLM 3B training.
|
||||
|
||||
Usage:
|
||||
python3 scripts/telegram_notify.py "Your message here"
|
||||
python3 scripts/telegram_notify.py "<b>Bold</b> message" --parse-mode HTML
|
||||
|
||||
Function API:
|
||||
from scripts.telegram_notify import send_telegram
|
||||
send_telegram("message text")
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import urllib.request
|
||||
import urllib.parse
|
||||
import urllib.error
|
||||
import logging
|
||||
from typing import Optional
|
||||
|
||||
# ─── Configuration ────────────────────────────────────────────────────────────
|
||||
BOT_TOKEN = os.environ.get("TELEGRAM_BOT_TOKEN", "")
|
||||
CHAT_ID = os.environ.get("TELEGRAM_CHAT_ID", "")
|
||||
TIMEOUT = 15 # seconds
|
||||
MAX_MSG_LEN = 4096 # Telegram limit
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.WARNING,
|
||||
format="%(asctime)s [telegram_notify] %(levelname)s: %(message)s",
|
||||
)
|
||||
log = logging.getLogger("telegram_notify")
|
||||
|
||||
|
||||
def send_telegram(
|
||||
message: str,
|
||||
parse_mode: str = "HTML",
|
||||
token: str = BOT_TOKEN,
|
||||
chat_id: str = CHAT_ID,
|
||||
disable_web_page_preview: bool = True,
|
||||
) -> bool:
|
||||
"""
|
||||
Send a Telegram message via Bot API using urllib (curl-free).
|
||||
|
||||
Args:
|
||||
message: Text to send (HTML or Markdown depending on parse_mode).
|
||||
parse_mode: "HTML" or "Markdown" or "" (plain).
|
||||
token: Bot token (defaults to module-level BOT_TOKEN).
|
||||
chat_id: Recipient chat/channel ID.
|
||||
disable_web_page_preview: Suppress link previews.
|
||||
|
||||
Returns:
|
||||
True on success, False on any error.
|
||||
"""
|
||||
if not message:
|
||||
log.warning("Empty message — skipping send.")
|
||||
return False
|
||||
|
||||
# Truncate if over Telegram limit, with notice
|
||||
if len(message) > MAX_MSG_LEN:
|
||||
truncated_notice = "\n\n<i>[message truncated]</i>" if parse_mode == "HTML" else "\n\n[message truncated]"
|
||||
message = message[: MAX_MSG_LEN - len(truncated_notice)] + truncated_notice
|
||||
|
||||
url = f"https://api.telegram.org/bot{token}/sendMessage"
|
||||
|
||||
payload: dict = {
|
||||
"chat_id": chat_id,
|
||||
"text": message,
|
||||
"disable_web_page_preview": disable_web_page_preview,
|
||||
}
|
||||
if parse_mode:
|
||||
payload["parse_mode"] = parse_mode
|
||||
|
||||
data = urllib.parse.urlencode(payload).encode("utf-8")
|
||||
|
||||
try:
|
||||
req = urllib.request.Request(
|
||||
url,
|
||||
data=data,
|
||||
method="POST",
|
||||
headers={"Content-Type": "application/x-www-form-urlencoded"},
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=TIMEOUT) as resp:
|
||||
body = resp.read().decode("utf-8")
|
||||
result = json.loads(body)
|
||||
if result.get("ok"):
|
||||
return True
|
||||
else:
|
||||
log.error("Telegram API error: %s", result.get("description", result))
|
||||
return False
|
||||
|
||||
except urllib.error.HTTPError as e:
|
||||
try:
|
||||
err_body = e.read().decode("utf-8")
|
||||
except Exception:
|
||||
err_body = str(e)
|
||||
log.error("HTTP %d from Telegram: %s", e.code, err_body)
|
||||
return False
|
||||
|
||||
except urllib.error.URLError as e:
|
||||
log.error("Network error sending Telegram message: %s", e.reason)
|
||||
return False
|
||||
|
||||
except json.JSONDecodeError as e:
|
||||
log.error("Failed to parse Telegram response: %s", e)
|
||||
return False
|
||||
|
||||
except Exception as e: # noqa: BLE001
|
||||
log.error("Unexpected error in send_telegram: %s", e)
|
||||
return False
|
||||
|
||||
|
||||
def send_telegram_safe(message: str, **kwargs) -> bool:
|
||||
"""
|
||||
Wrapper that catches ALL exceptions — guaranteed never to crash the caller.
|
||||
Suitable for embedding in training loops where stability is critical.
|
||||
"""
|
||||
try:
|
||||
return send_telegram(message, **kwargs)
|
||||
except Exception as e: # noqa: BLE001
|
||||
log.error("send_telegram_safe caught unhandled exception: %s", e)
|
||||
return False
|
||||
|
||||
|
||||
# ─── CLI entry point ──────────────────────────────────────────────────────────
|
||||
if __name__ == "__main__":
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Send a Telegram message from the command line."
|
||||
)
|
||||
parser.add_argument("message", nargs="?", help="Message text to send")
|
||||
parser.add_argument(
|
||||
"--parse-mode",
|
||||
default="HTML",
|
||||
choices=["HTML", "Markdown", "MarkdownV2", ""],
|
||||
help="Telegram parse_mode (default: HTML)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--token", default=BOT_TOKEN, help="Override bot token"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--chat-id", default=CHAT_ID, help="Override chat ID"
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
# Allow piped stdin if no positional arg given
|
||||
if args.message is None:
|
||||
if not sys.stdin.isatty():
|
||||
args.message = sys.stdin.read().strip()
|
||||
else:
|
||||
parser.print_help()
|
||||
sys.exit(1)
|
||||
|
||||
ok = send_telegram(
|
||||
args.message,
|
||||
parse_mode=args.parse_mode,
|
||||
token=args.token,
|
||||
chat_id=args.chat_id,
|
||||
)
|
||||
|
||||
if ok:
|
||||
print("Telegram message sent successfully.")
|
||||
sys.exit(0)
|
||||
else:
|
||||
print("ERROR: Failed to send Telegram message.", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
148
source/scripts/test_ollama_repetition.py
Normal file
148
source/scripts/test_ollama_repetition.py
Normal file
@@ -0,0 +1,148 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
test_ollama_repetition.py — Ollama 배포 모델 반복률 검증
|
||||
|
||||
ORPO eval과 동일한 프롬프트로 Ollama API 호출 후 n-gram 반복률 + EOS 종료율 측정.
|
||||
목표: 3-gram rep < 3% (한국어 자연 반복 고려), EOS 종료율 > 95%
|
||||
|
||||
Usage:
|
||||
python scripts/test_ollama_repetition.py [--model frankenstallm-3b] [--host localhost:11434]
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
import sys
|
||||
from collections import Counter
|
||||
|
||||
# ORPO eval에서 사용한 15개 한국어 프롬프트
|
||||
TEST_PROMPTS = [
|
||||
"대한민국의 수도는 어디인가요?",
|
||||
"인공지능이란 무엇인가요?",
|
||||
"한국의 전통 음식 중에서 김치에 대해 설명해주세요.",
|
||||
"프로그래밍을 배우려면 어떻게 해야 하나요?",
|
||||
"지구 온난화의 원인과 대책에 대해 설명해주세요.",
|
||||
"한국어의 특징을 3가지 설명해주세요.",
|
||||
"좋은 리더의 자질에 대해 논해주세요.",
|
||||
"우주 탐사의 의미와 중요성을 설명해주세요.",
|
||||
"건강한 생활 습관 5가지를 추천해주세요.",
|
||||
"인터넷이 현대 사회에 미친 영향을 분석해주세요.",
|
||||
"한국의 교육 제도의 장단점을 설명해주세요.",
|
||||
"환경 보호를 위해 개인이 할 수 있는 일을 알려주세요.",
|
||||
"4차 산업혁명이 일자리에 미치는 영향을 분석해주세요.",
|
||||
"독서의 중요성과 효과적인 독서 방법을 알려주세요.",
|
||||
"한국 문화의 세계화에 대해 어떻게 생각하시나요?",
|
||||
]
|
||||
|
||||
|
||||
def compute_ngram_repetition(text: str, n: int) -> float:
|
||||
"""n-gram 반복률 계산 (0.0 ~ 1.0)"""
|
||||
tokens = text.split()
|
||||
if len(tokens) < n:
|
||||
return 0.0
|
||||
ngrams = [tuple(tokens[i:i+n]) for i in range(len(tokens) - n + 1)]
|
||||
if not ngrams:
|
||||
return 0.0
|
||||
counts = Counter(ngrams)
|
||||
repeated = sum(c - 1 for c in counts.values() if c > 1)
|
||||
return repeated / len(ngrams)
|
||||
|
||||
|
||||
def call_ollama(prompt: str, model: str, host: str, timeout: int = 120) -> dict:
|
||||
"""Ollama API 호출"""
|
||||
url = f"http://{host}/api/generate"
|
||||
payload = json.dumps({
|
||||
"model": model,
|
||||
"prompt": prompt,
|
||||
"stream": False,
|
||||
}).encode("utf-8")
|
||||
|
||||
req = urllib.request.Request(url, data=payload, headers={"Content-Type": "application/json"})
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
return json.loads(resp.read().decode("utf-8"))
|
||||
except urllib.error.URLError as e:
|
||||
return {"error": str(e), "response": ""}
|
||||
except Exception as e:
|
||||
return {"error": str(e), "response": ""}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Ollama 반복률 검증")
|
||||
parser.add_argument("--model", default="frankenstallm-3b", help="Ollama 모델 이름")
|
||||
parser.add_argument("--host", default="localhost:11434", help="Ollama 서버 주소")
|
||||
args = parser.parse_args()
|
||||
|
||||
print(f"{'='*70}")
|
||||
print(f" Ollama 반복률 검증: {args.model}")
|
||||
print(f" 서버: {args.host}")
|
||||
print(f" 프롬프트: {len(TEST_PROMPTS)}개")
|
||||
print(f"{'='*70}\n")
|
||||
|
||||
results = []
|
||||
for i, prompt in enumerate(TEST_PROMPTS, 1):
|
||||
print(f"[{i:2d}/{len(TEST_PROMPTS)}] {prompt[:40]}...")
|
||||
resp = call_ollama(prompt, args.model, args.host)
|
||||
|
||||
if "error" in resp and resp["error"]:
|
||||
print(f" ERROR: {resp['error']}")
|
||||
results.append({"prompt": prompt, "error": resp["error"]})
|
||||
continue
|
||||
|
||||
text = resp.get("response", "")
|
||||
eos_done = resp.get("done", False)
|
||||
|
||||
rep1 = compute_ngram_repetition(text, 1)
|
||||
rep2 = compute_ngram_repetition(text, 2)
|
||||
rep3 = compute_ngram_repetition(text, 3)
|
||||
rep4 = compute_ngram_repetition(text, 4)
|
||||
|
||||
results.append({
|
||||
"prompt": prompt,
|
||||
"response_len": len(text),
|
||||
"word_count": len(text.split()),
|
||||
"eos_done": eos_done,
|
||||
"rep1": rep1, "rep2": rep2, "rep3": rep3, "rep4": rep4,
|
||||
})
|
||||
|
||||
preview = text[:100].replace("\n", " ")
|
||||
print(f" 응답: {preview}...")
|
||||
print(f" 길이: {len(text)}자, EOS: {eos_done}, "
|
||||
f"rep(1/2/3/4): {rep1:.2%}/{rep2:.2%}/{rep3:.2%}/{rep4:.2%}")
|
||||
print()
|
||||
|
||||
# --- Summary ---
|
||||
valid = [r for r in results if "error" not in r or not r.get("error")]
|
||||
if not valid:
|
||||
print("ERROR: 유효한 응답 없음")
|
||||
sys.exit(1)
|
||||
|
||||
avg_rep3 = sum(r["rep3"] for r in valid) / len(valid)
|
||||
eos_rate = sum(1 for r in valid if r["eos_done"]) / len(valid)
|
||||
errors = len(results) - len(valid)
|
||||
|
||||
print(f"{'='*70}")
|
||||
print(f" 결과 요약")
|
||||
print(f"{'='*70}")
|
||||
print(f" 유효 응답: {len(valid)}/{len(results)} (에러: {errors})")
|
||||
print(f" 평균 3-gram 반복률: {avg_rep3:.2%} (목표: < 3%)")
|
||||
print(f" EOS 종료율: {eos_rate:.0%} (목표: > 95%)")
|
||||
print()
|
||||
|
||||
# Pass/Fail
|
||||
# 한국어는 조사/접속사 자연 반복으로 어절 기준 3-gram rep 1.5~2%가 자연 floor
|
||||
# 퇴행적 반복(30%+)과 구별하여 3% 기준 적용
|
||||
rep_pass = avg_rep3 < 0.03
|
||||
eos_pass = eos_rate > 0.95
|
||||
overall = rep_pass and eos_pass
|
||||
|
||||
print(f" 3-gram 반복률: {'PASS ✓' if rep_pass else 'FAIL ✗'} ({avg_rep3:.2%})")
|
||||
print(f" EOS 종료율: {'PASS ✓' if eos_pass else 'FAIL ✗'} ({eos_rate:.0%})")
|
||||
print(f" 종합: {'PASS ✓' if overall else 'FAIL ✗'}")
|
||||
print(f"{'='*70}")
|
||||
|
||||
sys.exit(0 if overall else 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
292
source/scripts/training_watchdog.sh
Normal file
292
source/scripts/training_watchdog.sh
Normal file
@@ -0,0 +1,292 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# training_watchdog.sh — FRANKENSTALLM 3B Cron-based Training Watchdog
|
||||
# Run: every 10 minutes via cron
|
||||
# Alerts via Telegram only when problems are detected.
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
|
||||
# ─── Paths ───────────────────────────────────────────────────────────────────
|
||||
WORKDIR="${WORKDIR:-$(cd "$(dirname "$0")/.." && pwd)}"
|
||||
CKPT_DIR="$WORKDIR/checkpoints/korean_3b_fp8_run1"
|
||||
LOG_FILE="$CKPT_DIR/train.log"
|
||||
PID_FILE="$CKPT_DIR/train.pid"
|
||||
WATCHDOG_LOG="$CKPT_DIR/watchdog.log"
|
||||
STATE_FILE="$CKPT_DIR/watchdog.state" # persists last-good step/time
|
||||
NOTIFY="python3 $WORKDIR/scripts/telegram_notify.py"
|
||||
|
||||
# ─── Thresholds ──────────────────────────────────────────────────────────────
|
||||
LOSS_SPIKE_THRESHOLD="5.0" # alert if loss > this value
|
||||
LOSS_NAN_PATTERN="nan|inf|NaN|Inf"
|
||||
STALL_SECONDS=900 # 15 min without new log line → stalled
|
||||
DISK_WARN_PCT=85 # alert if disk usage >= this %
|
||||
GPU_UTIL_WARN_PCT=20 # alert if avg GPU util drops below this %
|
||||
MIN_TOKPS=5000 # alert if tok/s drops below this
|
||||
TOTAL_STEPS=57000
|
||||
WAIT_COUNT_FILE="/tmp/frankenstallm-wait-count" # 대기 횟수 파일
|
||||
MAX_WAIT_COUNT=10 # 이 횟수 초과 시 알림 후 cron 해제
|
||||
|
||||
# ─── Helpers ─────────────────────────────────────────────────────────────────
|
||||
ts() { date '+%Y-%m-%d %H:%M:%S'; }
|
||||
|
||||
log_msg() {
|
||||
echo "[$(ts)] $*"
|
||||
}
|
||||
|
||||
send_alert() {
|
||||
local level="$1"
|
||||
local msg="$2"
|
||||
log_msg "ALERT[$level]: $msg"
|
||||
$NOTIFY "<b>[FRANKENSTALLM ALERT] $level</b>
|
||||
|
||||
$msg
|
||||
|
||||
<i>$(ts) | watchdog check</i>" || true
|
||||
}
|
||||
|
||||
# ─── 1. Process alive check ──────────────────────────────────────────────────
|
||||
check_process() {
|
||||
if [[ ! -f "$PID_FILE" ]]; then
|
||||
# 대기 모드: PID 파일 없으면 학습 미시작 상태로 카운트
|
||||
local wait_count=0
|
||||
[[ -f "$WAIT_COUNT_FILE" ]] && wait_count=$(cat "$WAIT_COUNT_FILE" 2>/dev/null || echo 0)
|
||||
wait_count=$(( wait_count + 1 ))
|
||||
echo "$wait_count" > "$WAIT_COUNT_FILE"
|
||||
log_msg "Training not started yet (waiting ${wait_count}/${MAX_WAIT_COUNT})."
|
||||
|
||||
if (( wait_count > MAX_WAIT_COUNT )); then
|
||||
send_alert "WAIT_TIMEOUT" "학습이 <b>${wait_count}회</b> 체크 동안 시작되지 않았습니다 (~$((wait_count * 10))분).
|
||||
|
||||
PID 파일 없음: <code>$PID_FILE</code>
|
||||
|
||||
Watchdog cron을 자동 해제합니다. 학습 시작 후 직접 재등록하세요:
|
||||
<code>crontab -e</code>"
|
||||
# cron에서 training_watchdog 제거
|
||||
crontab -l 2>/dev/null | grep -v "training_watchdog" | crontab -
|
||||
rm -f "$WAIT_COUNT_FILE"
|
||||
log_msg "Watchdog cron entry removed after ${wait_count} waits."
|
||||
fi
|
||||
return 1
|
||||
fi
|
||||
# 학습 시작됨 → 대기 카운터 초기화
|
||||
rm -f "$WAIT_COUNT_FILE"
|
||||
|
||||
local pid
|
||||
pid=$(cat "$PID_FILE" 2>/dev/null | tr -d '[:space:]')
|
||||
|
||||
if [[ -z "$pid" ]]; then
|
||||
send_alert "PROCESS" "PID file is empty: $PID_FILE"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! kill -0 "$pid" 2>/dev/null; then
|
||||
# Check if it completed normally (step == TOTAL_STEPS)
|
||||
local last_step
|
||||
last_step=$(grep -oP 'step\s+\K[0-9]+' "$LOG_FILE" 2>/dev/null | tail -1)
|
||||
if [[ "$last_step" == "$TOTAL_STEPS" ]]; then
|
||||
log_msg "Training COMPLETED at step $TOTAL_STEPS — process exit is expected."
|
||||
send_alert "COMPLETE" "Training completed normally at step <code>$TOTAL_STEPS/$TOTAL_STEPS</code>."
|
||||
else
|
||||
send_alert "CRASH" "Training process (PID $pid) is NOT running.
|
||||
Last logged step: <code>${last_step:-unknown}</code>/$TOTAL_STEPS
|
||||
|
||||
Check log: <code>tail -50 $LOG_FILE</code>"
|
||||
fi
|
||||
return 1
|
||||
fi
|
||||
|
||||
log_msg "Process PID $pid is alive."
|
||||
return 0
|
||||
}
|
||||
|
||||
# ─── 2. Stall detection ──────────────────────────────────────────────────────
|
||||
check_stall() {
|
||||
if [[ ! -f "$LOG_FILE" ]]; then
|
||||
send_alert "STALL" "Log file not found: $LOG_FILE"
|
||||
return 1
|
||||
fi
|
||||
|
||||
local log_mtime now elapsed
|
||||
log_mtime=$(stat -c '%Y' "$LOG_FILE" 2>/dev/null || echo 0)
|
||||
now=$(date +%s)
|
||||
elapsed=$(( now - log_mtime ))
|
||||
|
||||
if (( elapsed >= STALL_SECONDS )); then
|
||||
local mins=$(( elapsed / 60 ))
|
||||
send_alert "STALL" "No log activity for <b>${mins} minutes</b> (threshold: $(( STALL_SECONDS/60 ))min).
|
||||
Log last modified: <code>$(date -d "@$log_mtime" '+%Y-%m-%d %H:%M:%S')</code>
|
||||
Training may be hung or extremely slow."
|
||||
return 1
|
||||
fi
|
||||
|
||||
log_msg "Log freshness OK: last update ${elapsed}s ago."
|
||||
return 0
|
||||
}
|
||||
|
||||
# ─── 3. Loss anomaly check ───────────────────────────────────────────────────
|
||||
check_loss() {
|
||||
if [[ ! -f "$LOG_FILE" ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Get last step line
|
||||
local last_line
|
||||
last_line=$(grep -E 'step\s+[0-9]+.*loss' "$LOG_FILE" 2>/dev/null | tail -1)
|
||||
|
||||
if [[ -z "$last_line" ]]; then
|
||||
log_msg "No step lines found in log yet."
|
||||
return 0
|
||||
fi
|
||||
|
||||
local loss step
|
||||
loss=$(echo "$last_line" | grep -oP 'loss\s+\K[0-9.eE+\-naifNIF]+' || echo "")
|
||||
step=$(echo "$last_line" | grep -oP 'step\s+\K[0-9]+' || echo "0")
|
||||
|
||||
if [[ -z "$loss" ]]; then
|
||||
log_msg "Could not parse loss from: $last_line"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# NaN/Inf check
|
||||
if echo "$loss" | grep -qiE "$LOSS_NAN_PATTERN"; then
|
||||
send_alert "LOSS_NAN" "Loss is <b>$loss</b> at step <code>$step</code>.
|
||||
Training has diverged — NaN/Inf detected.
|
||||
|
||||
Last log line:
|
||||
<code>${last_line}</code>"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Spike check (only after warmup, step > 500)
|
||||
if (( step > 500 )); then
|
||||
local loss_int
|
||||
loss_int=$(echo "$loss >= $LOSS_SPIKE_THRESHOLD" | bc -l 2>/dev/null || echo 0)
|
||||
if [[ "$loss_int" == "1" ]]; then
|
||||
send_alert "LOSS_SPIKE" "Loss spike detected: <b>$loss</b> at step <code>$step</code> (threshold: $LOSS_SPIKE_THRESHOLD).
|
||||
|
||||
Last log line:
|
||||
<code>${last_line}</code>"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
|
||||
log_msg "Loss OK: $loss at step $step."
|
||||
return 0
|
||||
}
|
||||
|
||||
# ─── 4. Throughput check ─────────────────────────────────────────────────────
|
||||
check_throughput() {
|
||||
if [[ ! -f "$LOG_FILE" ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
local last_line
|
||||
last_line=$(grep -E 'step\s+[0-9]+.*tok/s' "$LOG_FILE" 2>/dev/null | tail -1)
|
||||
[[ -z "$last_line" ]] && return 0
|
||||
|
||||
# tok/s may be formatted with commas: 36,321
|
||||
local tokps step
|
||||
tokps=$(echo "$last_line" | grep -oP 'tok/s\s+\K[\d,]+' | tr -d ',' || echo "")
|
||||
step=$(echo "$last_line" | grep -oP 'step\s+\K[0-9]+' || echo "0")
|
||||
|
||||
if [[ -z "$tokps" ]]; then
|
||||
log_msg "Could not parse tok/s from last log line."
|
||||
return 0
|
||||
fi
|
||||
|
||||
if (( step > 100 && tokps < MIN_TOKPS )); then
|
||||
send_alert "THROUGHPUT" "Throughput dropped to <b>${tokps} tok/s</b> at step <code>$step</code> (min: ${MIN_TOKPS}).
|
||||
GPU may be throttling, NCCL stalled, or a data worker is slow."
|
||||
return 1
|
||||
fi
|
||||
|
||||
log_msg "Throughput OK: ${tokps} tok/s at step $step."
|
||||
return 0
|
||||
}
|
||||
|
||||
# ─── 5. GPU utilization check ────────────────────────────────────────────────
|
||||
check_gpu() {
|
||||
if ! command -v nvidia-smi &>/dev/null; then
|
||||
log_msg "nvidia-smi not available — skipping GPU check."
|
||||
return 0
|
||||
fi
|
||||
|
||||
local avg_util
|
||||
avg_util=$(nvidia-smi --query-gpu=utilization.gpu --format=csv,noheader,nounits 2>/dev/null \
|
||||
| awk '{sum+=$1; count++} END {if(count>0) printf "%.0f", sum/count; else print 0}')
|
||||
|
||||
if [[ -z "$avg_util" || "$avg_util" == "0" ]]; then
|
||||
log_msg "GPU util query returned 0 or empty — possibly all idle."
|
||||
# Only alert if process is also running
|
||||
local pid
|
||||
pid=$(cat "$PID_FILE" 2>/dev/null | tr -d '[:space:]')
|
||||
if [[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null; then
|
||||
send_alert "GPU_IDLE" "All 8× B200 GPUs show <b>0% utilization</b> while training process is alive.
|
||||
Possible NCCL hang or data pipeline stall."
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
fi
|
||||
|
||||
if (( avg_util < GPU_UTIL_WARN_PCT )); then
|
||||
local gpu_details
|
||||
gpu_details=$(nvidia-smi --query-gpu=index,utilization.gpu,memory.used,memory.total \
|
||||
--format=csv,noheader 2>/dev/null | head -8 || echo "unavailable")
|
||||
send_alert "GPU_LOW" "Average GPU utilization: <b>${avg_util}%</b> (threshold: ${GPU_UTIL_WARN_PCT}%).
|
||||
|
||||
GPU details:
|
||||
<code>${gpu_details}</code>"
|
||||
return 1
|
||||
fi
|
||||
|
||||
log_msg "GPU utilization OK: ${avg_util}% average."
|
||||
return 0
|
||||
}
|
||||
|
||||
# ─── 6. Disk space check ─────────────────────────────────────────────────────
|
||||
check_disk() {
|
||||
local usage_pct
|
||||
usage_pct=$(df "$CKPT_DIR" 2>/dev/null | awk 'NR==2 {gsub(/%/,"",$5); print $5}')
|
||||
|
||||
if [[ -z "$usage_pct" ]]; then
|
||||
log_msg "Could not determine disk usage for $CKPT_DIR."
|
||||
return 0
|
||||
fi
|
||||
|
||||
if (( usage_pct >= DISK_WARN_PCT )); then
|
||||
local avail
|
||||
avail=$(df -h "$CKPT_DIR" 2>/dev/null | awk 'NR==2 {print $4}')
|
||||
send_alert "DISK" "Disk usage at <b>${usage_pct}%</b> (threshold: ${DISK_WARN_PCT}%).
|
||||
Available: <b>${avail}</b> on partition containing checkpoints.
|
||||
|
||||
Risk: checkpoint saves may fail. Consider deleting old checkpoints."
|
||||
return 1
|
||||
fi
|
||||
|
||||
log_msg "Disk usage OK: ${usage_pct}% used."
|
||||
return 0
|
||||
}
|
||||
|
||||
# ─── Main ────────────────────────────────────────────────────────────────────
|
||||
main() {
|
||||
log_msg "=== Watchdog check START ==="
|
||||
|
||||
local issues=0
|
||||
|
||||
check_process || (( issues++ )) || true
|
||||
check_stall || (( issues++ )) || true
|
||||
check_loss || (( issues++ )) || true
|
||||
check_throughput || (( issues++ )) || true
|
||||
check_gpu || (( issues++ )) || true
|
||||
check_disk || (( issues++ )) || true
|
||||
|
||||
if (( issues == 0 )); then
|
||||
log_msg "All checks passed — no alerts sent."
|
||||
else
|
||||
log_msg "Watchdog found $issues issue(s) — alerts sent."
|
||||
fi
|
||||
|
||||
log_msg "=== Watchdog check END ==="
|
||||
}
|
||||
|
||||
main "$@"
|
||||
182
source/scripts/upload_to_huggingface.py
Normal file
182
source/scripts/upload_to_huggingface.py
Normal file
@@ -0,0 +1,182 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Upload FRANKENSTALLM: model, eval reports, source code, and data scripts to Hugging Face.
|
||||
|
||||
Usage:
|
||||
huggingface-cli login
|
||||
|
||||
# 모델 + README + 평가 결과 + 보고서
|
||||
python scripts/upload_to_huggingface.py --repo-id pathcosmos/frankenstallm --create-pr
|
||||
|
||||
# 위 + 소스 코드 + 데이터 스크립트 (모델/데이터/소스 전부)
|
||||
python scripts/upload_to_huggingface.py --repo-id pathcosmos/frankenstallm --with-source --with-data --create-pr
|
||||
|
||||
# 평가·보고서만
|
||||
python scripts/upload_to_huggingface.py --repo-id pathcosmos/frankenstallm --readme-only --create-pr
|
||||
"""
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
||||
HF_CHECKPOINT = PROJECT_ROOT / "outputs" / "hf_checkpoint-best-fixed"
|
||||
REPORTS_DIR = PROJECT_ROOT / "reports"
|
||||
EVAL_RESULTS_DIR = PROJECT_ROOT / "eval" / "results" / "frankenstallm-3b-v2"
|
||||
DATA_DIR = PROJECT_ROOT / "data"
|
||||
SOURCE_DIRS = ["train", "model", "configs", "scripts", "tokenizer", "eval"]
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Upload model, eval reports, source, and data scripts to Hugging Face")
|
||||
parser.add_argument("--repo-id", type=str, required=True, help="e.g. pathcosmos/frankenstallm")
|
||||
parser.add_argument("--readme-only", action="store_true", help="Only push README + eval results + reports (no model)")
|
||||
parser.add_argument("--create-pr", action="store_true", help="Create a Pull Request instead of pushing to main")
|
||||
parser.add_argument("--with-source", action="store_true", help="Upload full source code (train, eval, model, configs, scripts, tokenizer)")
|
||||
parser.add_argument("--with-data", action="store_true", help="Upload data scripts and DATA_README (no .bin files)")
|
||||
args = parser.parse_args()
|
||||
create_pr = getattr(args, "create_pr", False)
|
||||
|
||||
try:
|
||||
from huggingface_hub import HfApi, create_repo
|
||||
except ImportError:
|
||||
print("Install: pip install huggingface_hub")
|
||||
raise SystemExit(1)
|
||||
|
||||
api = HfApi()
|
||||
|
||||
# 레포 없으면 생성
|
||||
# 레포가 없으면 생성 (본인 계정일 때만 성공)
|
||||
try:
|
||||
create_repo(args.repo_id, repo_type="model", exist_ok=True)
|
||||
except Exception as e:
|
||||
print(f"Note: create_repo skipped (use Hugging Face website to create repo if needed): {e}")
|
||||
|
||||
if not args.readme_only:
|
||||
if not HF_CHECKPOINT.exists():
|
||||
print(f"Checkpoint not found: {HF_CHECKPOINT}")
|
||||
raise SystemExit(1)
|
||||
print(f"Uploading model from {HF_CHECKPOINT} ...")
|
||||
api.upload_folder(
|
||||
folder_path=str(HF_CHECKPOINT),
|
||||
repo_id=args.repo_id,
|
||||
repo_type="model",
|
||||
create_pr=create_pr,
|
||||
)
|
||||
print("Model upload done.")
|
||||
|
||||
# README는 체크포인트 폴더 것 사용 (이미 평가 요약 포함)
|
||||
readme_src = HF_CHECKPOINT / "README.md"
|
||||
if readme_src.exists():
|
||||
print("Pushing README (model card) ...")
|
||||
api.upload_file(
|
||||
path_or_fileobj=str(readme_src),
|
||||
path_in_repo="README.md",
|
||||
repo_id=args.repo_id,
|
||||
repo_type="model",
|
||||
create_pr=create_pr,
|
||||
)
|
||||
print("README upload done.")
|
||||
else:
|
||||
print("No README.md in checkpoint dir; skipping README push.")
|
||||
|
||||
# 평가 결과 JSON
|
||||
results_json = EVAL_RESULTS_DIR / "ollama_benchmark_results.json"
|
||||
if results_json.exists():
|
||||
print("Pushing ollama_benchmark_results.json ...")
|
||||
api.upload_file(
|
||||
path_or_fileobj=str(results_json),
|
||||
path_in_repo="eval/ollama_benchmark_results.json",
|
||||
repo_id=args.repo_id,
|
||||
repo_type="model",
|
||||
create_pr=create_pr,
|
||||
)
|
||||
print("Eval results upload done.")
|
||||
|
||||
# 배포·평가 보고서 (상세 기록)
|
||||
for name, src in [
|
||||
("2026-03-09_GGUF_DEPLOYMENT_AND_EVAL_REPORT.md", REPORTS_DIR / "2026-03-09_GGUF_DEPLOYMENT_AND_EVAL_REPORT.md"),
|
||||
("2026-03-09_ORPO_EVALUATION_REPORT.md", REPORTS_DIR / "2026-03-09_ORPO_EVALUATION_REPORT.md"),
|
||||
]:
|
||||
if src.exists():
|
||||
print(f"Pushing {name} ...")
|
||||
api.upload_file(
|
||||
path_or_fileobj=str(src),
|
||||
path_in_repo=f"eval_reports/{name}",
|
||||
repo_id=args.repo_id,
|
||||
repo_type="model",
|
||||
create_pr=create_pr,
|
||||
)
|
||||
print("Reports upload done.")
|
||||
|
||||
# ---------- 소스 코드 (--with-source) ----------
|
||||
if getattr(args, "with_source", False):
|
||||
print("Uploading source code ...")
|
||||
ignore_common = ["**/__pycache__/**", "**/*.pyc", "**/.DS_Store"]
|
||||
for dirname in ["train", "model", "configs", "scripts", "tokenizer"]:
|
||||
src_dir = PROJECT_ROOT / dirname
|
||||
if src_dir.exists():
|
||||
api.upload_folder(
|
||||
folder_path=str(src_dir),
|
||||
path_in_repo=f"source/{dirname}",
|
||||
repo_id=args.repo_id,
|
||||
repo_type="model",
|
||||
ignore_patterns=ignore_common,
|
||||
create_pr=create_pr,
|
||||
)
|
||||
print(f" source/{dirname}/ done.")
|
||||
# eval: outputs, results 제외 (대용량)
|
||||
eval_dir = PROJECT_ROOT / "eval"
|
||||
if eval_dir.exists():
|
||||
api.upload_folder(
|
||||
folder_path=str(eval_dir),
|
||||
path_in_repo="source/eval",
|
||||
repo_id=args.repo_id,
|
||||
repo_type="model",
|
||||
ignore_patterns=ignore_common + ["**/outputs/**", "**/results/**", "**/.compile_cache/**"],
|
||||
create_pr=create_pr,
|
||||
)
|
||||
print(" source/eval/ done.")
|
||||
# 루트 문서
|
||||
for name in ["README.md", "CLAUDE.md", "requirements.txt", "PROGRESS.md"]:
|
||||
src_file = PROJECT_ROOT / name
|
||||
if src_file.exists():
|
||||
api.upload_file(
|
||||
path_or_fileobj=str(src_file),
|
||||
path_in_repo=f"source/{name}",
|
||||
repo_id=args.repo_id,
|
||||
repo_type="model",
|
||||
create_pr=create_pr,
|
||||
)
|
||||
for p in PROJECT_ROOT.glob("PLAN_*.md"):
|
||||
api.upload_file(
|
||||
path_or_fileobj=str(p),
|
||||
path_in_repo=f"source/{p.name}",
|
||||
repo_id=args.repo_id,
|
||||
repo_type="model",
|
||||
create_pr=create_pr,
|
||||
)
|
||||
print("Source upload done.")
|
||||
|
||||
# ---------- 데이터 스크립트 (--with-data, .bin 제외) ----------
|
||||
if getattr(args, "with_data", False) and DATA_DIR.exists():
|
||||
print("Uploading data scripts (no .bin) ...")
|
||||
api.upload_folder(
|
||||
folder_path=str(DATA_DIR),
|
||||
path_in_repo="data",
|
||||
repo_id=args.repo_id,
|
||||
repo_type="model",
|
||||
ignore_patterns=[
|
||||
"**/*.bin",
|
||||
"**/*.chunk*",
|
||||
"**/__pycache__/**",
|
||||
"**/code/**",
|
||||
"**/*.pyc",
|
||||
],
|
||||
create_pr=create_pr,
|
||||
)
|
||||
print("Data scripts upload done.")
|
||||
|
||||
print(f"Done. https://huggingface.co/{args.repo_id}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user