初始化项目，由ModelHub XC社区提供模型
Model: pathcosmos/frankenstallm Source: Original Platform
This commit is contained in:
ModelHub XC
241
source/scripts/hourly_status.sh
Normal file
241
source/scripts/hourly_status.sh
Normal file
@@ -0,0 +1,241 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# hourly_status.sh — FRANKENSTALLM 3B Hourly Training Status Report (Telegram)
|
||||
# Run: every hour via cron
|
||||
# Sends a rich formatted message with progress, loss, ETA, GPU/disk summary.
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
|
||||
# ─── Paths ───────────────────────────────────────────────────────────────────
|
||||
WORKDIR="${WORKDIR:-$(cd "$(dirname "$0")/.." && pwd)}"
|
||||
CKPT_DIR="$WORKDIR/checkpoints/korean_3b_fp8_run1"
|
||||
LOG_FILE="$CKPT_DIR/train.log"
|
||||
PID_FILE="$CKPT_DIR/train.pid"
|
||||
HOURLY_LOG="$CKPT_DIR/hourly_status.log"
|
||||
NOTIFY="python3 $WORKDIR/scripts/telegram_notify.py"
|
||||
|
||||
TOTAL_STEPS=57000
|
||||
TOTAL_TOKENS_B=114 # billion tokens target (57K steps × batch)
|
||||
|
||||
# ─── Helpers ─────────────────────────────────────────────────────────────────
|
||||
ts() { date '+%Y-%m-%d %H:%M:%S'; }
|
||||
log() { echo "[$(ts)] $*"; }
|
||||
|
||||
# Safely get last matching value from log
|
||||
parse_last() {
|
||||
local pattern="$1"
|
||||
grep -oP "$pattern" "$LOG_FILE" 2>/dev/null | tail -1 || echo ""
|
||||
}
|
||||
|
||||
# ─── Parse training log ───────────────────────────────────────────────────────
|
||||
parse_log() {
|
||||
if [[ ! -f "$LOG_FILE" ]]; then
|
||||
echo "NO_LOG"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Get the last step line
|
||||
LAST_LINE=$(grep -E 'step\s+[0-9]+.*loss' "$LOG_FILE" 2>/dev/null | tail -1 || echo "")
|
||||
if [[ -z "$LAST_LINE" ]]; then
|
||||
echo "NO_STEPS"
|
||||
return 1
|
||||
fi
|
||||
|
||||
CURRENT_STEP=$(echo "$LAST_LINE" | grep -oP 'step\s+\K[0-9]+' || echo "0")
|
||||
CURRENT_LOSS=$(echo "$LAST_LINE" | grep -oP 'loss\s+\K[0-9.]+' || echo "N/A")
|
||||
CURRENT_LR=$(echo "$LAST_LINE" | grep -oP 'lr\s+\K[0-9.e+-]+' || echo "N/A")
|
||||
CURRENT_GNORM=$(echo "$LAST_LINE" | grep -oP 'gnorm\s+\K[0-9.]+' || echo "N/A")
|
||||
CURRENT_TOKPS=$(echo "$LAST_LINE" | grep -oP 'tok/s\s+\K[\d,]+' | tr -d ',' || echo "0")
|
||||
CURRENT_MEM=$(echo "$LAST_LINE" | grep -oP 'mem\s+\K[0-9.]+GB' || echo "N/A")
|
||||
CURRENT_EPOCH=$(echo "$LAST_LINE" | grep -oP 'epoch\s+\K[0-9]+' || echo "0")
|
||||
|
||||
# Log timestamp — parse from the line itself
|
||||
LOG_TS=$(echo "$LAST_LINE" | grep -oP '\[\K[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}' || echo "unknown")
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
# ─── Calculate progress & ETA ─────────────────────────────────────────────────
|
||||
compute_eta() {
|
||||
local step="$1"
|
||||
local tokps="$2"
|
||||
|
||||
# Progress
|
||||
PROGRESS_PCT=$(echo "scale=1; $step * 100 / $TOTAL_STEPS" | bc -l 2>/dev/null || echo "0")
|
||||
|
||||
# Steps remaining
|
||||
STEPS_LEFT=$(( TOTAL_STEPS - step ))
|
||||
|
||||
# Tokens processed so far (approx: step × 2M tokens/step for 3B, bs=4, seqlen=4096, 8gpu)
|
||||
# bs=4, accum=8, 8gpu → effective batch = 4*8*8=256 sequences × 4096 tokens = 1,048,576 ≈ 1M tok/step
|
||||
TOKENS_PROCESSED_B=$(echo "scale=2; $step * 1048576 / 1000000000" | bc -l 2>/dev/null || echo "0")
|
||||
|
||||
# ETA using current tok/s
|
||||
if [[ "$tokps" -gt 0 ]]; then
|
||||
# tokens remaining
|
||||
local tokens_left_b
|
||||
tokens_left_b=$(echo "scale=2; ($TOTAL_STEPS - $step) * 1048576 / 1000000000" | bc -l 2>/dev/null || echo "0")
|
||||
local tokens_left
|
||||
tokens_left=$(echo "scale=0; ($TOTAL_STEPS - $step) * 1048576" | bc -l 2>/dev/null || echo "0")
|
||||
local secs_left
|
||||
secs_left=$(echo "scale=0; $tokens_left / $tokps" | bc -l 2>/dev/null || echo "0")
|
||||
|
||||
ETA_HOURS=$(echo "scale=1; $secs_left / 3600" | bc -l 2>/dev/null || echo "N/A")
|
||||
if [[ "$ETA_HOURS" != "N/A" ]]; then
|
||||
local eta_epoch
|
||||
eta_epoch=$(( $(date +%s) + secs_left ))
|
||||
ETA_DATETIME=$(date -d "@$eta_epoch" '+%m/%d %H:%M' 2>/dev/null || echo "N/A")
|
||||
else
|
||||
ETA_DATETIME="N/A"
|
||||
fi
|
||||
else
|
||||
ETA_HOURS="N/A"
|
||||
ETA_DATETIME="N/A"
|
||||
fi
|
||||
}
|
||||
|
||||
# ─── GPU summary ─────────────────────────────────────────────────────────────
|
||||
get_gpu_summary() {
|
||||
if ! command -v nvidia-smi &>/dev/null; then
|
||||
GPU_SUMMARY="nvidia-smi not available"
|
||||
GPU_AVG_UTIL="N/A"
|
||||
GPU_TOTAL_MEM="N/A"
|
||||
return
|
||||
fi
|
||||
|
||||
local raw
|
||||
raw=$(nvidia-smi --query-gpu=index,utilization.gpu,memory.used,memory.total \
|
||||
--format=csv,noheader,nounits 2>/dev/null || echo "")
|
||||
|
||||
if [[ -z "$raw" ]]; then
|
||||
GPU_SUMMARY="GPU query failed"
|
||||
GPU_AVG_UTIL="N/A"
|
||||
GPU_TOTAL_MEM="N/A"
|
||||
return
|
||||
fi
|
||||
|
||||
# avg util
|
||||
GPU_AVG_UTIL=$(echo "$raw" | awk -F', ' '{sum+=$2; count++} END {printf "%.0f%%", sum/count}')
|
||||
|
||||
# total mem used / total
|
||||
GPU_TOTAL_MEM=$(echo "$raw" | awk -F', ' \
|
||||
'{used+=$3; total+=$4} END {printf "%.1f / %.1f GiB", used/1024, total/1024}')
|
||||
|
||||
# Per-GPU one-liner: "G0:95% 48G | G1:94% 48G | ..."
|
||||
GPU_SUMMARY=$(echo "$raw" | awk -F', ' \
|
||||
'{printf "G%s:%s%% %sMiB | ", $1, $2, $3}' | sed 's/ | $//')
|
||||
}
|
||||
|
||||
# ─── Disk usage ──────────────────────────────────────────────────────────────
|
||||
get_disk_info() {
|
||||
DISK_INFO=$(df -h "$CKPT_DIR" 2>/dev/null | awk 'NR==2 {printf "%s used / %s total (%s)", $3, $2, $5}' || echo "N/A")
|
||||
CKPT_COUNT=$(ls -d "$CKPT_DIR"/checkpoint-* 2>/dev/null | wc -l || echo "0")
|
||||
LAST_CKPT=$(ls -dt "$CKPT_DIR"/checkpoint-* 2>/dev/null | head -1 | xargs basename 2>/dev/null || echo "none")
|
||||
}
|
||||
|
||||
# ─── Process status ───────────────────────────────────────────────────────────
|
||||
get_process_status() {
|
||||
PROC_STATUS="UNKNOWN"
|
||||
if [[ -f "$PID_FILE" ]]; then
|
||||
local pid
|
||||
pid=$(cat "$PID_FILE" 2>/dev/null | tr -d '[:space:]')
|
||||
if [[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null; then
|
||||
PROC_STATUS="RUNNING (PID $pid)"
|
||||
else
|
||||
PROC_STATUS="STOPPED (PID $pid)"
|
||||
fi
|
||||
else
|
||||
PROC_STATUS="NO PID FILE"
|
||||
fi
|
||||
}
|
||||
|
||||
# ─── Build & send message ────────────────────────────────────────────────────
|
||||
build_and_send() {
|
||||
local step="$CURRENT_STEP"
|
||||
local loss="$CURRENT_LOSS"
|
||||
local tokps="$CURRENT_TOKPS"
|
||||
|
||||
# Status icon
|
||||
local status_icon
|
||||
if [[ "$PROC_STATUS" == RUNNING* ]]; then
|
||||
status_icon="✅" # green check
|
||||
else
|
||||
status_icon="❌" # red X
|
||||
fi
|
||||
|
||||
# Progress bar (20 chars)
|
||||
local bar_filled=$(echo "scale=0; $PROGRESS_PCT * 20 / 100" | bc -l 2>/dev/null || echo "0")
|
||||
local bar_empty=$(( 20 - bar_filled ))
|
||||
PROGRESS_BAR=$(printf '%0.s█' $(seq 1 $bar_filled 2>/dev/null) ; printf '%0.s░' $(seq 1 $bar_empty 2>/dev/null)) || PROGRESS_BAR="[$PROGRESS_PCT%]"
|
||||
|
||||
local msg
|
||||
msg="$(cat <<EOF
|
||||
<b>FRANKENSTALLM 3B — Hourly Status</b>
|
||||
<i>$(ts)</i>
|
||||
|
||||
$status_icon <b>Process:</b> $PROC_STATUS
|
||||
|
||||
<b>Progress</b>
|
||||
Step: <code>$step / $TOTAL_STEPS</code> ($PROGRESS_PCT%)
|
||||
Tokens: <code>${TOKENS_PROCESSED_B}B / ${TOTAL_TOKENS_B}B</code>
|
||||
Epoch: <code>$CURRENT_EPOCH</code>
|
||||
Last log: <code>$LOG_TS</code>
|
||||
|
||||
<b>Training Metrics</b>
|
||||
Loss: <code>$loss</code>
|
||||
LR: <code>$CURRENT_LR</code>
|
||||
Gnorm: <code>$CURRENT_GNORM</code>
|
||||
Tok/s: <code>$tokps</code>
|
||||
Mem: <code>$CURRENT_MEM</code>
|
||||
|
||||
<b>ETA</b>
|
||||
Steps left: <code>$STEPS_LEFT</code>
|
||||
Remaining: <code>~$ETA_HOURS h</code>
|
||||
Est. done: <code>$ETA_DATETIME</code>
|
||||
|
||||
<b>GPU</b>
|
||||
Avg util: <code>$GPU_AVG_UTIL</code>
|
||||
Total mem: <code>$GPU_TOTAL_MEM</code>
|
||||
|
||||
<b>Checkpoints</b>
|
||||
Last saved: <code>$LAST_CKPT</code>
|
||||
Total: <code>$CKPT_COUNT</code> checkpoints
|
||||
|
||||
<b>Disk</b>
|
||||
<code>$DISK_INFO</code>
|
||||
EOF
|
||||
)"
|
||||
|
||||
log "Sending hourly status report (step $step)..."
|
||||
$NOTIFY "$msg" || {
|
||||
log "ERROR: Failed to send Telegram message."
|
||||
return 1
|
||||
}
|
||||
log "Status report sent."
|
||||
}
|
||||
|
||||
# ─── Main ────────────────────────────────────────────────────────────────────
|
||||
main() {
|
||||
log "=== Hourly status START ==="
|
||||
|
||||
parse_log || {
|
||||
log "Cannot parse log — sending minimal status."
|
||||
$NOTIFY "<b>FRANKENSTALLM 3B</b> — Status check at $(ts)
|
||||
|
||||
<b>WARNING:</b> Cannot read training log at:
|
||||
<code>$LOG_FILE</code>
|
||||
|
||||
Process status: $(cat "$PID_FILE" 2>/dev/null && echo "(PID found)" || echo "(no PID file)")" || true
|
||||
return 0
|
||||
}
|
||||
|
||||
compute_eta "$CURRENT_STEP" "$CURRENT_TOKPS"
|
||||
get_gpu_summary
|
||||
get_disk_info
|
||||
get_process_status
|
||||
build_and_send
|
||||
|
||||
log "=== Hourly status END ==="
|
||||
}
|
||||
|
||||
main "$@"
|
||||
Reference in New Issue
Block a user