317 lines
12 KiB
Bash
317 lines
12 KiB
Bash
#!/usr/bin/env bash
|
||
# =============================================================================
|
||
# monitor_3b.sh — 3B 학습 실시간 모니터링 + 이상 감지 + 자동 체크포인트 정리
|
||
#
|
||
# Usage:
|
||
# bash scripts/monitor_3b.sh # 기본 감시
|
||
# bash scripts/monitor_3b.sh --check-once # 1회 검사
|
||
# bash scripts/monitor_3b.sh --auto-cleanup # 자동 오래된 체크포인트 삭제
|
||
#
|
||
# 3B 특화 사항:
|
||
# - 체크포인트 27GB/개 → 디스크 감시 강화
|
||
# - NCCL hang 감지 + 자동 재시작 옵션
|
||
# - 예상 완료 시간 실시간 계산
|
||
# - 프로세스 중복 실행 방지
|
||
# =============================================================================
|
||
set -euo pipefail
|
||
|
||
# ---- Configuration ----------------------------------------------------------
|
||
RUN_NAME="${RUN_NAME:-korean_3b_fp8_run1}"
|
||
LOG_FILE="${1:-checkpoints/${RUN_NAME}/train.log}"
|
||
CKPT_DIR="checkpoints/${RUN_NAME}"
|
||
CHECK_INTERVAL=60 # 3B는 step 간격 더 김 → 60초
|
||
ZERO_LOSS_THRESHOLD=3
|
||
GNORM_WARN=10.0
|
||
GNORM_CRITICAL=50.0
|
||
LOSS_SPIKE_FACTOR=3.0
|
||
STALL_TIMEOUT=600 # 10분 (3B는 step 더 오래 걸림)
|
||
DISK_WARN_PCT=85
|
||
DISK_CRITICAL_PCT=92
|
||
GPU_UTIL_WARN=50
|
||
MAX_CHECKPOINTS=15 # 최대 보관 체크포인트 수 (15 × 27GB = 405GB)
|
||
CHECK_ONCE=false
|
||
AUTO_CLEANUP=false
|
||
AUTO_RESTART=false
|
||
|
||
# Parse args
|
||
for arg in "$@"; do
|
||
case "$arg" in
|
||
--check-once) CHECK_ONCE=true ;;
|
||
--auto-cleanup) AUTO_CLEANUP=true ;;
|
||
--auto-restart) AUTO_RESTART=true ;;
|
||
esac
|
||
done
|
||
# Fix LOG_FILE if first arg was a flag
|
||
if [[ "$LOG_FILE" == --* ]]; then
|
||
LOG_FILE="checkpoints/${RUN_NAME}/train.log"
|
||
fi
|
||
|
||
# ---- Colors -----------------------------------------------------------------
|
||
RED='\033[0;31m'; YELLOW='\033[1;33m'; GREEN='\033[0;32m'
|
||
CYAN='\033[0;36m'; MAGENTA='\033[0;35m'; NC='\033[0m'
|
||
|
||
timestamp() { date '+%Y-%m-%d %H:%M:%S'; }
|
||
|
||
alert() {
|
||
local level="$1" msg="$2"
|
||
case "$level" in
|
||
CRITICAL) echo -e "${RED}🔴 [$(timestamp)] [CRITICAL] ${msg}${NC}" ;;
|
||
WARNING) echo -e "${YELLOW}🟠 [$(timestamp)] [WARNING] ${msg}${NC}" ;;
|
||
INFO) echo -e "${CYAN}🟡 [$(timestamp)] [INFO] ${msg}${NC}" ;;
|
||
OK) echo -e "${GREEN}✅ [$(timestamp)] [OK] ${msg}${NC}" ;;
|
||
esac
|
||
}
|
||
|
||
# ---- Parse metrics ----------------------------------------------------------
|
||
parse_metrics() {
|
||
local n="${1:-20}"
|
||
[[ -f "$LOG_FILE" ]] || return
|
||
tail -n "$n" "$LOG_FILE" | grep "step.*loss.*gnorm" || true
|
||
}
|
||
|
||
extract_field() {
|
||
echo "$1" | grep -oP "${2}\s+\K[0-9]+\.[0-9e+\-]+" | head -1
|
||
}
|
||
|
||
extract_step() {
|
||
echo "$1" | grep -oP "step\s+\K[0-9]+" | head -1
|
||
}
|
||
|
||
# ---- Check: Loss = 0 -------------------------------------------------------
|
||
check_loss_zero() {
|
||
local lines
|
||
lines=$(parse_metrics "$ZERO_LOSS_THRESHOLD")
|
||
[[ -z "$lines" ]] && return 0
|
||
local zero_count=0
|
||
while IFS= read -r line; do
|
||
local loss=$(extract_field "$line" "loss")
|
||
if [[ -n "$loss" ]] && (( $(echo "$loss < 0.001" | bc -l 2>/dev/null || echo 0) )); then
|
||
((zero_count++))
|
||
fi
|
||
done <<< "$lines"
|
||
if [[ $zero_count -ge $ZERO_LOSS_THRESHOLD ]]; then
|
||
alert CRITICAL "Loss가 ${zero_count}회 연속 ~0! Labels 버그. 즉시 중단!"
|
||
return 1
|
||
fi
|
||
}
|
||
|
||
# ---- Check: Loss spike -----------------------------------------------------
|
||
check_loss_spike() {
|
||
local lines=$(parse_metrics 20)
|
||
[[ -z "$lines" ]] && return 0
|
||
local losses=()
|
||
while IFS= read -r line; do
|
||
local loss=$(extract_field "$line" "loss")
|
||
[[ -n "$loss" ]] && losses+=("$loss")
|
||
done <<< "$lines"
|
||
local count=${#losses[@]}
|
||
[[ $count -lt 5 ]] && return 0
|
||
local last="${losses[$((count-1))]}"
|
||
local sum=0
|
||
for ((i=0; i<count-1; i++)); do
|
||
sum=$(echo "$sum + ${losses[$i]}" | bc -l 2>/dev/null || echo "$sum")
|
||
done
|
||
local avg=$(echo "$sum / ($count - 1)" | bc -l 2>/dev/null || echo "0")
|
||
if [[ "$avg" != "0" ]]; then
|
||
local ratio=$(echo "$last / $avg" | bc -l 2>/dev/null || echo "1")
|
||
if (( $(echo "$ratio > $LOSS_SPIKE_FACTOR" | bc -l 2>/dev/null || echo 0) )); then
|
||
alert WARNING "Loss spike! 현재=${last}, 평균=${avg}, 비율=${ratio}x"
|
||
fi
|
||
fi
|
||
}
|
||
|
||
# ---- Check: Gradient norm ---------------------------------------------------
|
||
check_gnorm() {
|
||
local lines=$(parse_metrics 5)
|
||
[[ -z "$lines" ]] && return 0
|
||
local gnorm=$(extract_field "$(echo "$lines" | tail -1)" "gnorm")
|
||
[[ -z "$gnorm" ]] && return 0
|
||
if (( $(echo "$gnorm > $GNORM_CRITICAL" | bc -l 2>/dev/null || echo 0) )); then
|
||
alert CRITICAL "GNorm=${gnorm} > ${GNORM_CRITICAL}! 발산 직전."
|
||
elif (( $(echo "$gnorm > $GNORM_WARN" | bc -l 2>/dev/null || echo 0) )); then
|
||
alert WARNING "GNorm=${gnorm} 불안정."
|
||
fi
|
||
}
|
||
|
||
# ---- Check: Stall / NCCL hang ----------------------------------------------
|
||
check_stall() {
|
||
[[ ! -f "$LOG_FILE" ]] && return 0
|
||
local last_mod=$(stat -c %Y "$LOG_FILE" 2>/dev/null || echo 0)
|
||
local now=$(date +%s)
|
||
local diff=$((now - last_mod))
|
||
if [[ $diff -gt $STALL_TIMEOUT ]]; then
|
||
alert CRITICAL "로그 ${diff}초 ($(( diff/60 ))분) 멈춤! NCCL hang 가능성."
|
||
# NCCL hang 자동 재시작
|
||
if $AUTO_RESTART; then
|
||
alert WARNING "자동 재시작 시도..."
|
||
local pid=$(pgrep -f "pretrain.py.*korean_3b" | head -1 || true)
|
||
if [[ -n "$pid" ]]; then
|
||
kill -9 "$pid" 2>/dev/null || true
|
||
sleep 5
|
||
alert INFO "이전 프로세스 종료. launch_3b_pretrain.sh 재실행 필요."
|
||
fi
|
||
fi
|
||
fi
|
||
}
|
||
|
||
# ---- Check: Disk (3B 강화) --------------------------------------------------
|
||
check_disk() {
|
||
local usage=$(df /PROJECT 2>/dev/null | awk 'NR==2 {print $5}' | tr -d '%')
|
||
if [[ -n "$usage" && "$usage" -gt "$DISK_CRITICAL_PCT" ]]; then
|
||
alert CRITICAL "디스크 ${usage}% > ${DISK_CRITICAL_PCT}%! 즉시 정리 필요!"
|
||
$AUTO_CLEANUP && cleanup_old_checkpoints
|
||
elif [[ -n "$usage" && "$usage" -gt "$DISK_WARN_PCT" ]]; then
|
||
alert WARNING "디스크 ${usage}% > ${DISK_WARN_PCT}%. 체크포인트 정리 권장."
|
||
fi
|
||
}
|
||
|
||
# ---- Check: GPU utilization -------------------------------------------------
|
||
check_gpu() {
|
||
command -v nvidia-smi &>/dev/null || return 0
|
||
local low=0 total=0
|
||
while IFS= read -r util; do
|
||
((total++))
|
||
[[ "$util" -lt "$GPU_UTIL_WARN" ]] && ((low++))
|
||
done < <(nvidia-smi --query-gpu=utilization.gpu --format=csv,noheader,nounits 2>/dev/null)
|
||
[[ $total -gt 0 && $low -gt 0 ]] && alert INFO "${low}/${total} GPU util < ${GPU_UTIL_WARN}%"
|
||
}
|
||
|
||
# ---- Check: 체크포인트 무결성 -----------------------------------------------
|
||
check_checkpoint_integrity() {
|
||
local latest=$(ls -d "${CKPT_DIR}"/checkpoint-* 2>/dev/null | sort -V | tail -1 || true)
|
||
[[ -z "$latest" ]] && return 0
|
||
# 최소 파일 존재 확인
|
||
if [[ ! -f "${latest}/model.pt" ]] && [[ ! -f "${latest}/model.safetensors" ]]; then
|
||
alert WARNING "최근 체크포인트에 모델 파일 없음: ${latest}"
|
||
fi
|
||
# 크기 확인 (3B model.pt는 최소 2GB)
|
||
local size=$(du -sb "${latest}" 2>/dev/null | awk '{print $1}')
|
||
if [[ -n "$size" && "$size" -lt 2000000000 ]]; then
|
||
alert WARNING "체크포인트 크기 비정상 (${size} bytes < 2GB): ${latest}"
|
||
fi
|
||
}
|
||
|
||
# ---- Cleanup: 오래된 체크포인트 자동 삭제 ------------------------------------
|
||
cleanup_old_checkpoints() {
|
||
local ckpts=($(ls -d "${CKPT_DIR}"/checkpoint-* 2>/dev/null | sort -V))
|
||
local count=${#ckpts[@]}
|
||
if [[ $count -le $MAX_CHECKPOINTS ]]; then
|
||
alert OK "체크포인트 ${count}개 ≤ ${MAX_CHECKPOINTS}. 정리 불필요."
|
||
return
|
||
fi
|
||
# 이정표 체크포인트 보존 (매 10K step)
|
||
local deletable=()
|
||
local preserved=()
|
||
for ckpt in "${ckpts[@]}"; do
|
||
local step_num=$(basename "$ckpt" | grep -oP '\d+' || echo "0")
|
||
if (( step_num % 10000 == 0 && step_num > 0 )); then
|
||
preserved+=("$ckpt")
|
||
else
|
||
deletable+=("$ckpt")
|
||
fi
|
||
done
|
||
# 최근 MAX_CHECKPOINTS개는 무조건 보존
|
||
local n_deletable=${#deletable[@]}
|
||
local total_keep=$(( ${#preserved[@]} + MAX_CHECKPOINTS ))
|
||
local to_delete=$(( count - total_keep ))
|
||
[[ $to_delete -le 0 ]] && { alert OK "정리 불필요 (이정표 ${#preserved[@]}개 + 최근 ${MAX_CHECKPOINTS}개 보존)."; return; }
|
||
alert INFO "${count}개 체크포인트 → ${to_delete}개 삭제 (이정표 ${#preserved[@]}개 영구 보존)"
|
||
local deleted=0
|
||
for ckpt in "${deletable[@]}"; do
|
||
[[ $deleted -ge $to_delete ]] && break
|
||
local ckpt_size=$(du -sh "$ckpt" 2>/dev/null | awk '{print $1}')
|
||
echo " 삭제: $ckpt (${ckpt_size})"
|
||
rm -rf "$ckpt"
|
||
((deleted++))
|
||
done
|
||
alert OK "체크포인트 정리 완료. (${deleted}개 삭제)"
|
||
}
|
||
|
||
# ---- ETA 계산 ---------------------------------------------------------------
|
||
estimate_eta() {
|
||
[[ ! -f "$LOG_FILE" ]] && return
|
||
# 최근 step 번호 + 시간
|
||
local lines=$(parse_metrics 50)
|
||
[[ -z "$lines" ]] && return
|
||
local last_line=$(echo "$lines" | tail -1)
|
||
local first_line=$(echo "$lines" | head -1)
|
||
local cur_step=$(extract_step "$last_line")
|
||
local max_steps=$(grep -oP "max_steps.*?(\d+)" "${CKPT_DIR}/train.log" 2>/dev/null | head -1 | grep -oP '\d+$' || echo "57000")
|
||
|
||
[[ -z "$cur_step" || "$cur_step" == "0" ]] && return
|
||
|
||
# step/sec from log timestamps (approximate)
|
||
local remaining=$((max_steps - cur_step))
|
||
if [[ $remaining -le 0 ]]; then
|
||
echo -e "${MAGENTA}📊 진행: ${cur_step}/${max_steps} (완료!)${NC}"
|
||
return
|
||
fi
|
||
|
||
# 파일 수정 시간 기반 rough ETA
|
||
local first_time=$(head -20 "$LOG_FILE" 2>/dev/null | grep -oP '^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}' | head -1 || true)
|
||
if [[ -n "$first_time" ]]; then
|
||
local start_epoch=$(date -d "$first_time" +%s 2>/dev/null || echo 0)
|
||
local now=$(date +%s)
|
||
if [[ $start_epoch -gt 0 && $cur_step -gt 0 ]]; then
|
||
local elapsed=$((now - start_epoch))
|
||
local sec_per_step=$(echo "$elapsed / $cur_step" | bc -l 2>/dev/null || echo "0")
|
||
local eta_sec=$(echo "$remaining * $sec_per_step" | bc 2>/dev/null | cut -d. -f1 || echo "0")
|
||
local eta_hours=$(echo "$eta_sec / 3600" | bc 2>/dev/null || echo "?")
|
||
local pct=$(echo "scale=1; $cur_step * 100 / $max_steps" | bc 2>/dev/null || echo "?")
|
||
echo -e "${MAGENTA}📊 진행: ${cur_step}/${max_steps} (${pct}%) | 남은 시간: ~${eta_hours}h | ${sec_per_step}s/step${NC}"
|
||
fi
|
||
else
|
||
echo -e "${MAGENTA}📊 진행: ${cur_step}/${max_steps}${NC}"
|
||
fi
|
||
}
|
||
|
||
# ---- Status summary ---------------------------------------------------------
|
||
print_status() {
|
||
local lines=$(parse_metrics 1)
|
||
[[ -n "$lines" ]] && echo -e "${GREEN}최근:${NC} $lines"
|
||
estimate_eta
|
||
if command -v nvidia-smi &>/dev/null; then
|
||
echo -e "${CYAN}GPU:${NC}"
|
||
nvidia-smi --query-gpu=index,memory.used,memory.total,utilization.gpu,temperature.gpu \
|
||
--format=csv,noheader 2>/dev/null | head -8
|
||
fi
|
||
local ckpt_count=$(ls -d "${CKPT_DIR}"/checkpoint-* 2>/dev/null | wc -l)
|
||
local ckpt_size=$(du -sh "${CKPT_DIR}" 2>/dev/null | awk '{print $1}')
|
||
echo -e "${CYAN}체크포인트:${NC} ${ckpt_count}개 (${ckpt_size})"
|
||
local disk=$(df -h /PROJECT 2>/dev/null | awk 'NR==2 {print $3"/"$2" ("$5")"}')
|
||
echo -e "${CYAN}디스크:${NC} ${disk}"
|
||
}
|
||
|
||
# ---- Main -------------------------------------------------------------------
|
||
echo "=================================================================="
|
||
echo " 3B Training Monitor"
|
||
echo " Run: ${RUN_NAME}"
|
||
echo " Log: ${LOG_FILE}"
|
||
echo " Interval: ${CHECK_INTERVAL}s"
|
||
echo " Auto-cleanup: ${AUTO_CLEANUP} | Auto-restart: ${AUTO_RESTART}"
|
||
echo " Ctrl+C to stop"
|
||
echo "=================================================================="
|
||
|
||
run_all_checks() {
|
||
check_loss_zero || true
|
||
check_loss_spike || true
|
||
check_gnorm || true
|
||
check_stall || true
|
||
check_disk || true
|
||
check_gpu || true
|
||
check_checkpoint_integrity || true
|
||
echo "---"
|
||
print_status
|
||
echo ""
|
||
}
|
||
|
||
if $CHECK_ONCE; then
|
||
run_all_checks
|
||
exit 0
|
||
fi
|
||
|
||
while true; do
|
||
run_all_checks
|
||
sleep "$CHECK_INTERVAL"
|
||
done
|