初始化项目，由ModelHub XC社区提供模型
Model: pathcosmos/frankenstallm Source: Original Platform
This commit is contained in:
ModelHub XC
595
source/eval/parallel_eval_3b.py
Normal file
595
source/eval/parallel_eval_3b.py
Normal file
@@ -0,0 +1,595 @@
|
||||
"""
|
||||
FRANKENSTALLM 3B — 6-GPU 병렬 종합 평가 스크립트.
|
||||
|
||||
GPU 배분:
|
||||
cuda:0 PPL — 3b_val.bin (145MB)
|
||||
cuda:1 PPL — korean_c4_val.bin (29MB)
|
||||
cuda:2 PPL — korean_namuwiki_val.bin (4.2MB) + korean_wiki_val.bin (1.1MB)
|
||||
cuda:3 Calibration (top-1/5/10 accuracy, entropy) on 3b_val.bin
|
||||
cuda:4 생성 품질 (10 프롬프트 × 3 온도)
|
||||
cuda:5 반복률 파라미터 그리드 탐색
|
||||
|
||||
Usage:
|
||||
cd /PROJECT/0325120031_A/ghong/taketimes/llm-bang
|
||||
python eval/parallel_eval_3b.py
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import math
|
||||
import sys
|
||||
import time
|
||||
from collections import Counter
|
||||
from concurrent.futures import ProcessPoolExecutor, as_completed
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.multiprocessing as mp
|
||||
import torch.nn.functional as F
|
||||
from torch.utils.data import DataLoader, Dataset
|
||||
|
||||
_PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
||||
if str(_PROJECT_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(_PROJECT_ROOT))
|
||||
|
||||
CHECKPOINT = str(_PROJECT_ROOT / "checkpoints" / "korean_3b_fp8_run1" / "checkpoint-0057000")
|
||||
TOKENIZER_PATH = str(_PROJECT_ROOT / "tokenizer" / "korean_sp" / "tokenizer.json")
|
||||
DATA_DIR = _PROJECT_ROOT / "data"
|
||||
OUTPUT_DIR = _PROJECT_ROOT / "eval" / "outputs"
|
||||
OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
SEQ_LEN = 2048
|
||||
STRIDE = 512
|
||||
BATCH_SIZE = 32 # 183GB VRAM이므로 충분
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Shared utilities
|
||||
# ===========================================================================
|
||||
|
||||
class SlidingWindowDataset(Dataset):
|
||||
def __init__(self, tokens: np.ndarray, seq_len: int, stride: int):
|
||||
self.tokens = tokens
|
||||
self.seq_len = seq_len
|
||||
self.stride = stride
|
||||
self.n_windows = max(0, (len(tokens) - seq_len + stride - 1) // stride)
|
||||
|
||||
def __len__(self):
|
||||
return self.n_windows
|
||||
|
||||
def __getitem__(self, idx):
|
||||
start = idx * self.stride
|
||||
end = start + self.seq_len
|
||||
actual_end = min(end, len(self.tokens))
|
||||
chunk_len = actual_end - start
|
||||
|
||||
input_ids = torch.zeros(self.seq_len, dtype=torch.long)
|
||||
targets = torch.full((self.seq_len,), fill_value=-100, dtype=torch.long)
|
||||
loss_mask = torch.zeros(self.seq_len, dtype=torch.bool)
|
||||
|
||||
if chunk_len > 1:
|
||||
toks = torch.from_numpy(self.tokens[start:actual_end].astype(np.int64))
|
||||
input_ids[:chunk_len] = toks
|
||||
targets[:chunk_len - 1] = toks[1:]
|
||||
|
||||
new_start = 0 if idx == 0 else self.stride
|
||||
if chunk_len > 1:
|
||||
for pos in range(new_start, chunk_len - 1):
|
||||
loss_mask[pos] = True
|
||||
|
||||
return input_ids, targets, loss_mask
|
||||
|
||||
|
||||
def load_model(device: str):
|
||||
from model.transformer import LLM
|
||||
model = LLM.from_pretrained(CHECKPOINT)
|
||||
model = model.to(device=device, dtype=torch.bfloat16)
|
||||
model.eval()
|
||||
return model
|
||||
|
||||
|
||||
def load_tokenizer():
|
||||
from tokenizers import Tokenizer
|
||||
return Tokenizer.from_file(TOKENIZER_PATH)
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Task 1: Perplexity (runs on cuda:0, cuda:1, cuda:2)
|
||||
# ===========================================================================
|
||||
|
||||
def eval_ppl(val_file: str, device: str) -> dict:
|
||||
"""Compute sliding-window PPL for one val set."""
|
||||
torch.cuda.set_device(int(device.split(":")[-1]))
|
||||
data_path = DATA_DIR / val_file
|
||||
name = val_file.replace("_val.bin", "").replace(".bin", "")
|
||||
|
||||
print(f"[PPL {device}] Loading model for {name}...")
|
||||
model = load_model(device)
|
||||
tokens = np.fromfile(str(data_path), dtype=np.uint16)
|
||||
n_tokens = len(tokens)
|
||||
print(f"[PPL {device}] {name}: {n_tokens:,} tokens, {n_tokens*2/1e6:.1f}MB")
|
||||
|
||||
ds = SlidingWindowDataset(tokens, SEQ_LEN, STRIDE)
|
||||
dl = DataLoader(ds, batch_size=BATCH_SIZE, shuffle=False, num_workers=4, pin_memory=True)
|
||||
|
||||
total_nll = 0.0
|
||||
total_count = 0
|
||||
t0 = time.time()
|
||||
|
||||
with torch.inference_mode():
|
||||
for batch_idx, (inp, tgt, mask) in enumerate(dl):
|
||||
inp = inp.to(device)
|
||||
tgt = tgt.to(device)
|
||||
mask = mask.to(device)
|
||||
|
||||
logits, _ = model(inp)
|
||||
loss_flat = F.cross_entropy(
|
||||
logits.view(-1, logits.size(-1)),
|
||||
tgt.view(-1),
|
||||
reduction="none",
|
||||
)
|
||||
loss_flat = loss_flat.view(mask.shape)
|
||||
nll = (loss_flat * mask.float()).sum().item()
|
||||
cnt = mask.sum().item()
|
||||
total_nll += nll
|
||||
total_count += cnt
|
||||
|
||||
if (batch_idx + 1) % 50 == 0:
|
||||
running_ppl = math.exp(total_nll / total_count) if total_count > 0 else float("inf")
|
||||
elapsed = time.time() - t0
|
||||
print(f"[PPL {device}] {name}: batch {batch_idx+1}/{len(dl)}, "
|
||||
f"running PPL={running_ppl:.4f}, {elapsed:.0f}s")
|
||||
|
||||
avg_nll = total_nll / total_count if total_count > 0 else 0
|
||||
ppl = math.exp(avg_nll)
|
||||
bpt = avg_nll / math.log(2)
|
||||
elapsed = time.time() - t0
|
||||
|
||||
result = {
|
||||
"name": name,
|
||||
"file": val_file,
|
||||
"n_tokens": int(n_tokens),
|
||||
"n_eval_tokens": int(total_count),
|
||||
"ppl": round(ppl, 4),
|
||||
"bits_per_token": round(bpt, 4),
|
||||
"avg_nll": round(avg_nll, 6),
|
||||
"elapsed_sec": round(elapsed, 1),
|
||||
"device": device,
|
||||
}
|
||||
print(f"[PPL {device}] ✓ {name}: PPL={ppl:.4f}, BPT={bpt:.4f}, {elapsed:.1f}s")
|
||||
return result
|
||||
|
||||
|
||||
def eval_ppl_multi(val_files: list[str], device: str) -> list[dict]:
|
||||
"""Compute PPL for multiple small val sets on one GPU."""
|
||||
results = []
|
||||
for f in val_files:
|
||||
results.append(eval_ppl(f, device))
|
||||
return results
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Task 2: Calibration (cuda:3)
|
||||
# ===========================================================================
|
||||
|
||||
def eval_calibration(device: str = "cuda:3", n_tokens: int = 50000) -> dict:
|
||||
"""Top-k accuracy and entropy calibration."""
|
||||
torch.cuda.set_device(int(device.split(":")[-1]))
|
||||
print(f"[CALIB {device}] Loading model...")
|
||||
model = load_model(device)
|
||||
tokenizer = load_tokenizer()
|
||||
|
||||
tokens = np.fromfile(str(DATA_DIR / "3b_val.bin"), dtype=np.uint16)
|
||||
tokens = tokens[:min(n_tokens, len(tokens))]
|
||||
|
||||
ds = SlidingWindowDataset(tokens, SEQ_LEN, STRIDE)
|
||||
dl = DataLoader(ds, batch_size=BATCH_SIZE, shuffle=False, num_workers=2, pin_memory=True)
|
||||
|
||||
top1_correct = 0
|
||||
top5_correct = 0
|
||||
top10_correct = 0
|
||||
total_entropy = 0.0
|
||||
total_prob = 0.0
|
||||
total_count = 0
|
||||
t0 = time.time()
|
||||
|
||||
with torch.inference_mode():
|
||||
for inp, tgt, mask in dl:
|
||||
inp = inp.to(device)
|
||||
tgt = tgt.to(device)
|
||||
mask = mask.to(device)
|
||||
|
||||
logits, _ = model(inp)
|
||||
probs = F.softmax(logits, dim=-1)
|
||||
|
||||
valid = mask & (tgt != -100)
|
||||
if valid.sum() == 0:
|
||||
continue
|
||||
|
||||
flat_logits = logits[valid]
|
||||
flat_tgt = tgt[valid]
|
||||
flat_probs = probs[valid]
|
||||
|
||||
# Top-k accuracy
|
||||
_, top1_pred = flat_logits.topk(1, dim=-1)
|
||||
_, top5_pred = flat_logits.topk(5, dim=-1)
|
||||
_, top10_pred = flat_logits.topk(10, dim=-1)
|
||||
|
||||
top1_correct += (top1_pred.squeeze(-1) == flat_tgt).sum().item()
|
||||
top5_correct += (top5_pred == flat_tgt.unsqueeze(-1)).any(dim=-1).sum().item()
|
||||
top10_correct += (top10_pred == flat_tgt.unsqueeze(-1)).any(dim=-1).sum().item()
|
||||
|
||||
# Mean probability of correct token
|
||||
correct_probs = flat_probs[torch.arange(len(flat_tgt)), flat_tgt]
|
||||
total_prob += correct_probs.sum().item()
|
||||
|
||||
# Entropy
|
||||
log_probs = torch.log(flat_probs + 1e-10)
|
||||
entropy = -(flat_probs * log_probs).sum(dim=-1)
|
||||
total_entropy += entropy.sum().item()
|
||||
|
||||
total_count += valid.sum().item()
|
||||
|
||||
elapsed = time.time() - t0
|
||||
result = {
|
||||
"n_eval_tokens": int(total_count),
|
||||
"top1_accuracy": round(top1_correct / total_count, 4) if total_count > 0 else 0,
|
||||
"top5_accuracy": round(top5_correct / total_count, 4) if total_count > 0 else 0,
|
||||
"top10_accuracy": round(top10_correct / total_count, 4) if total_count > 0 else 0,
|
||||
"mean_correct_prob": round(total_prob / total_count, 4) if total_count > 0 else 0,
|
||||
"mean_entropy": round(total_entropy / total_count, 4) if total_count > 0 else 0,
|
||||
"elapsed_sec": round(elapsed, 1),
|
||||
}
|
||||
print(f"[CALIB {device}] ✓ top1={result['top1_accuracy']:.4f}, "
|
||||
f"top5={result['top5_accuracy']:.4f}, entropy={result['mean_entropy']:.4f}, {elapsed:.1f}s")
|
||||
return result
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Task 3: Generation quality (cuda:4)
|
||||
# ===========================================================================
|
||||
|
||||
PROMPTS = [
|
||||
"대한민국의 수도는",
|
||||
"인공지능이란",
|
||||
"한국의 전통 음식 중에서",
|
||||
"지구 온난화의 주요 원인은",
|
||||
"프로그래밍을 배우려면",
|
||||
"조선시대에는",
|
||||
"물리학에서 에너지란",
|
||||
"한국어는 세계에서",
|
||||
"경제 성장을 위해서는",
|
||||
"우주 탐사의 역사를 보면",
|
||||
]
|
||||
|
||||
TEMPERATURES = [0.0, 0.7, 1.0]
|
||||
|
||||
|
||||
def top_p_filtering(logits, top_p=0.9, top_k=0):
|
||||
if logits.dim() == 1:
|
||||
logits = logits.unsqueeze(0)
|
||||
squeeze = True
|
||||
else:
|
||||
squeeze = False
|
||||
if top_k > 0:
|
||||
k = min(top_k, logits.size(-1))
|
||||
kth = torch.topk(logits, k, dim=-1).values[:, -1, None]
|
||||
logits = logits.masked_fill(logits < kth, float("-inf"))
|
||||
if 0.0 < top_p < 1.0:
|
||||
sorted_logits, sorted_idx = torch.sort(logits, dim=-1, descending=True)
|
||||
cum_probs = torch.cumsum(F.softmax(sorted_logits, dim=-1), dim=-1)
|
||||
remove = cum_probs - F.softmax(sorted_logits, dim=-1) >= top_p
|
||||
sorted_logits[remove] = float("-inf")
|
||||
logits = torch.zeros_like(logits).scatter_(-1, sorted_idx, sorted_logits)
|
||||
if squeeze:
|
||||
logits = logits.squeeze(0)
|
||||
return logits
|
||||
|
||||
|
||||
def generate_one(model, tokenizer, prompt, temperature, top_p=0.9, top_k=50,
|
||||
max_new_tokens=256, device="cuda:4", repetition_penalty=1.0):
|
||||
input_ids = torch.tensor([tokenizer.encode(prompt).ids], dtype=torch.long, device=device)
|
||||
eos_id = tokenizer.token_to_id("</s>")
|
||||
generated = input_ids
|
||||
new_ids = []
|
||||
hit_eos = False
|
||||
|
||||
for _ in range(max_new_tokens):
|
||||
logits_all, _ = model(generated)
|
||||
logits = logits_all[:, -1, :].clone()
|
||||
|
||||
if repetition_penalty != 1.0:
|
||||
for tid in set(generated[0].tolist()):
|
||||
if logits[0, tid] > 0:
|
||||
logits[0, tid] /= repetition_penalty
|
||||
else:
|
||||
logits[0, tid] *= repetition_penalty
|
||||
|
||||
if temperature == 0.0:
|
||||
next_id = logits.argmax(dim=-1, keepdim=True)
|
||||
else:
|
||||
logits = logits / max(temperature, 1e-8)
|
||||
logits = top_p_filtering(logits, top_p=top_p, top_k=top_k)
|
||||
probs = F.softmax(logits, dim=-1)
|
||||
next_id = torch.multinomial(probs, num_samples=1)
|
||||
|
||||
generated = torch.cat([generated, next_id], dim=-1)
|
||||
new_ids.append(next_id.item())
|
||||
|
||||
if eos_id is not None and next_id.item() == eos_id:
|
||||
hit_eos = True
|
||||
break
|
||||
|
||||
text = tokenizer.decode(new_ids)
|
||||
return text, len(new_ids), hit_eos
|
||||
|
||||
|
||||
def compute_ngram_rep(text: str, n: int) -> float:
|
||||
tokens = text.split()
|
||||
if len(tokens) < n:
|
||||
return 0.0
|
||||
ngrams = [tuple(tokens[i:i+n]) for i in range(len(tokens) - n + 1)]
|
||||
if not ngrams:
|
||||
return 0.0
|
||||
return 1.0 - len(set(ngrams)) / len(ngrams)
|
||||
|
||||
|
||||
def eval_generation(device: str = "cuda:4") -> dict:
|
||||
"""Generate text with 10 prompts × 3 temperatures."""
|
||||
torch.cuda.set_device(int(device.split(":")[-1]))
|
||||
print(f"[GEN {device}] Loading model...")
|
||||
model = load_model(device)
|
||||
tokenizer = load_tokenizer()
|
||||
t0 = time.time()
|
||||
|
||||
results = []
|
||||
for prompt in PROMPTS:
|
||||
for temp in TEMPERATURES:
|
||||
with torch.inference_mode():
|
||||
text, n_tokens, hit_eos = generate_one(
|
||||
model, tokenizer, prompt, temp, device=device
|
||||
)
|
||||
rep1 = compute_ngram_rep(text, 1)
|
||||
rep2 = compute_ngram_rep(text, 2)
|
||||
rep3 = compute_ngram_rep(text, 3)
|
||||
rep4 = compute_ngram_rep(text, 4)
|
||||
|
||||
entry = {
|
||||
"prompt": prompt,
|
||||
"temperature": temp,
|
||||
"generated_tokens": n_tokens,
|
||||
"hit_eos": hit_eos,
|
||||
"1gram_rep": round(rep1, 4),
|
||||
"2gram_rep": round(rep2, 4),
|
||||
"3gram_rep": round(rep3, 4),
|
||||
"4gram_rep": round(rep4, 4),
|
||||
"text": text[:500], # truncate for readability
|
||||
}
|
||||
results.append(entry)
|
||||
label = "greedy" if temp == 0.0 else f"t={temp}"
|
||||
print(f"[GEN {device}] {prompt[:10]}... ({label}): "
|
||||
f"{n_tokens}tok, 3gram_rep={rep3:.2%}, eos={hit_eos}")
|
||||
|
||||
elapsed = time.time() - t0
|
||||
|
||||
# Aggregate stats
|
||||
greedy = [r for r in results if r["temperature"] == 0.0]
|
||||
sampled = [r for r in results if r["temperature"] > 0.0]
|
||||
|
||||
summary = {
|
||||
"total_generations": len(results),
|
||||
"greedy_avg_3gram_rep": round(np.mean([r["3gram_rep"] for r in greedy]), 4) if greedy else 0,
|
||||
"greedy_eos_rate": round(np.mean([r["hit_eos"] for r in greedy]), 4) if greedy else 0,
|
||||
"sampled_avg_3gram_rep": round(np.mean([r["3gram_rep"] for r in sampled]), 4) if sampled else 0,
|
||||
"sampled_eos_rate": round(np.mean([r["hit_eos"] for r in sampled]), 4) if sampled else 0,
|
||||
"greedy_avg_tokens": round(np.mean([r["generated_tokens"] for r in greedy]), 1) if greedy else 0,
|
||||
"elapsed_sec": round(elapsed, 1),
|
||||
}
|
||||
print(f"[GEN {device}] ✓ greedy 3gram_rep={summary['greedy_avg_3gram_rep']:.4f}, "
|
||||
f"eos_rate={summary['greedy_eos_rate']:.2%}, {elapsed:.1f}s")
|
||||
|
||||
return {"summary": summary, "samples": results}
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Task 4: Repetition parameter grid (cuda:5)
|
||||
# ===========================================================================
|
||||
|
||||
REP_GRID = [
|
||||
{"name": "greedy", "temperature": 0.0, "repetition_penalty": 1.0},
|
||||
{"name": "t0.7", "temperature": 0.7, "repetition_penalty": 1.0},
|
||||
{"name": "t0.7_rep1.1", "temperature": 0.7, "repetition_penalty": 1.1},
|
||||
{"name": "t0.7_rep1.2", "temperature": 0.7, "repetition_penalty": 1.2},
|
||||
{"name": "t0.7_rep1.3", "temperature": 0.7, "repetition_penalty": 1.3},
|
||||
{"name": "t0.9", "temperature": 0.9, "repetition_penalty": 1.0},
|
||||
{"name": "t0.9_rep1.1", "temperature": 0.9, "repetition_penalty": 1.1},
|
||||
{"name": "t0.9_rep1.2", "temperature": 0.9, "repetition_penalty": 1.2},
|
||||
{"name": "t1.0", "temperature": 1.0, "repetition_penalty": 1.0},
|
||||
{"name": "t1.0_rep1.1", "temperature": 1.0, "repetition_penalty": 1.1},
|
||||
]
|
||||
|
||||
REP_PROMPTS = [
|
||||
"대한민국의 수도는",
|
||||
"인공지능이란",
|
||||
"한국의 전통 음식 중에서",
|
||||
"지구 온난화의 주요 원인은",
|
||||
"프로그래밍을 배우려면",
|
||||
]
|
||||
|
||||
|
||||
def eval_repetition_grid(device: str = "cuda:5") -> dict:
|
||||
"""Grid search over generation parameters to find lowest repetition."""
|
||||
torch.cuda.set_device(int(device.split(":")[-1]))
|
||||
print(f"[REP {device}] Loading model...")
|
||||
model = load_model(device)
|
||||
tokenizer = load_tokenizer()
|
||||
t0 = time.time()
|
||||
|
||||
results = []
|
||||
for params in REP_GRID:
|
||||
combo_results = []
|
||||
for prompt in REP_PROMPTS:
|
||||
with torch.inference_mode():
|
||||
text, n_tokens, hit_eos = generate_one(
|
||||
model, tokenizer, prompt,
|
||||
temperature=params["temperature"],
|
||||
repetition_penalty=params["repetition_penalty"],
|
||||
device=device, max_new_tokens=256,
|
||||
)
|
||||
combo_results.append({
|
||||
"prompt": prompt,
|
||||
"n_tokens": n_tokens,
|
||||
"hit_eos": hit_eos,
|
||||
"3gram_rep": compute_ngram_rep(text, 3),
|
||||
"4gram_rep": compute_ngram_rep(text, 4),
|
||||
})
|
||||
|
||||
avg_3gram = np.mean([r["3gram_rep"] for r in combo_results])
|
||||
avg_4gram = np.mean([r["4gram_rep"] for r in combo_results])
|
||||
eos_rate = np.mean([r["hit_eos"] for r in combo_results])
|
||||
avg_tokens = np.mean([r["n_tokens"] for r in combo_results])
|
||||
|
||||
entry = {
|
||||
"params": params["name"],
|
||||
"temperature": params["temperature"],
|
||||
"repetition_penalty": params["repetition_penalty"],
|
||||
"avg_3gram_rep": round(avg_3gram, 4),
|
||||
"avg_4gram_rep": round(avg_4gram, 4),
|
||||
"eos_rate": round(eos_rate, 4),
|
||||
"avg_tokens": round(avg_tokens, 1),
|
||||
}
|
||||
results.append(entry)
|
||||
print(f"[REP {device}] {params['name']}: 3gram={avg_3gram:.2%}, "
|
||||
f"4gram={avg_4gram:.2%}, eos={eos_rate:.0%}, {avg_tokens:.0f}tok")
|
||||
|
||||
elapsed = time.time() - t0
|
||||
|
||||
# Find best combo
|
||||
best = min(results, key=lambda r: r["avg_3gram_rep"])
|
||||
print(f"[REP {device}] ✓ Best: {best['params']} (3gram={best['avg_3gram_rep']:.2%}), {elapsed:.1f}s")
|
||||
|
||||
return {"grid_results": results, "best": best, "elapsed_sec": round(elapsed, 1)}
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Main: parallel orchestration
|
||||
# ===========================================================================
|
||||
|
||||
def run_ppl_0():
|
||||
return eval_ppl("3b_val.bin", "cuda:0")
|
||||
|
||||
def run_ppl_1():
|
||||
return eval_ppl("korean_c4_val.bin", "cuda:1")
|
||||
|
||||
def run_ppl_2():
|
||||
return eval_ppl_multi(["korean_namuwiki_val.bin", "korean_wiki_val.bin"], "cuda:2")
|
||||
|
||||
def run_calib():
|
||||
return eval_calibration("cuda:3")
|
||||
|
||||
def run_gen():
|
||||
return eval_generation("cuda:4")
|
||||
|
||||
def run_rep():
|
||||
return eval_repetition_grid("cuda:5")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
mp.set_start_method("spawn", force=True)
|
||||
print("=" * 70)
|
||||
print("FRANKENSTALLM 3B — 6-GPU 병렬 종합 평가")
|
||||
print(f"Checkpoint: {CHECKPOINT}")
|
||||
print(f"Batch size: {BATCH_SIZE}, Seq len: {SEQ_LEN}, Stride: {STRIDE}")
|
||||
print("=" * 70)
|
||||
|
||||
t_start = time.time()
|
||||
all_results = {}
|
||||
|
||||
with ProcessPoolExecutor(max_workers=6) as executor:
|
||||
futures = {
|
||||
executor.submit(run_ppl_0): "ppl_3b_val",
|
||||
executor.submit(run_ppl_1): "ppl_c4_ko",
|
||||
executor.submit(run_ppl_2): "ppl_namuwiki_wiki",
|
||||
executor.submit(run_calib): "calibration",
|
||||
executor.submit(run_gen): "generation",
|
||||
executor.submit(run_rep): "repetition",
|
||||
}
|
||||
|
||||
for future in as_completed(futures):
|
||||
key = futures[future]
|
||||
try:
|
||||
result = future.result()
|
||||
all_results[key] = result
|
||||
print(f"\n{'='*50}")
|
||||
print(f"✓ {key} COMPLETED")
|
||||
print(f"{'='*50}\n")
|
||||
except Exception as e:
|
||||
print(f"\n✗ {key} FAILED: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
all_results[key] = {"error": str(e)}
|
||||
|
||||
total_elapsed = time.time() - t_start
|
||||
|
||||
# Assemble final output
|
||||
output = {
|
||||
"model": "FRANKENSTALLM 3B",
|
||||
"checkpoint": "checkpoint-0057000",
|
||||
"total_elapsed_sec": round(total_elapsed, 1),
|
||||
"perplexity": {},
|
||||
"calibration": all_results.get("calibration", {}),
|
||||
"generation": all_results.get("generation", {}),
|
||||
"repetition": all_results.get("repetition", {}),
|
||||
}
|
||||
|
||||
# Merge PPL results
|
||||
if "ppl_3b_val" in all_results and not isinstance(all_results["ppl_3b_val"], list):
|
||||
output["perplexity"]["3b_val"] = all_results["ppl_3b_val"]
|
||||
if "ppl_c4_ko" in all_results and not isinstance(all_results["ppl_c4_ko"], list):
|
||||
output["perplexity"]["korean_c4"] = all_results["ppl_c4_ko"]
|
||||
if "ppl_namuwiki_wiki" in all_results:
|
||||
for item in (all_results["ppl_namuwiki_wiki"] if isinstance(all_results["ppl_namuwiki_wiki"], list) else [all_results["ppl_namuwiki_wiki"]]):
|
||||
if isinstance(item, dict) and "name" in item:
|
||||
output["perplexity"][item["name"]] = item
|
||||
|
||||
# Save
|
||||
out_path = OUTPUT_DIR / "3b_parallel_eval_results.json"
|
||||
with open(out_path, "w", encoding="utf-8") as f:
|
||||
json.dump(output, f, ensure_ascii=False, indent=2)
|
||||
|
||||
# Print summary
|
||||
print("\n" + "=" * 70)
|
||||
print("FRANKENSTALLM 3B 종합 평가 결과 요약")
|
||||
print("=" * 70)
|
||||
print(f"총 소요 시간: {total_elapsed:.1f}s ({total_elapsed/60:.1f}min)")
|
||||
|
||||
print("\n--- Perplexity ---")
|
||||
for name, data in output["perplexity"].items():
|
||||
if isinstance(data, dict) and "ppl" in data:
|
||||
print(f" {name}: PPL={data['ppl']:.4f}, BPT={data['bits_per_token']:.4f}")
|
||||
|
||||
calib = output.get("calibration", {})
|
||||
if "top1_accuracy" in calib:
|
||||
print(f"\n--- Calibration ---")
|
||||
print(f" Top-1 Acc: {calib['top1_accuracy']:.4f}")
|
||||
print(f" Top-5 Acc: {calib['top5_accuracy']:.4f}")
|
||||
print(f" Top-10 Acc: {calib['top10_accuracy']:.4f}")
|
||||
print(f" Mean Entropy: {calib['mean_entropy']:.4f}")
|
||||
|
||||
gen = output.get("generation", {}).get("summary", {})
|
||||
if gen:
|
||||
print(f"\n--- Generation Quality ---")
|
||||
print(f" Greedy 3-gram rep: {gen.get('greedy_avg_3gram_rep', 0):.2%}")
|
||||
print(f" Greedy EOS rate: {gen.get('greedy_eos_rate', 0):.2%}")
|
||||
print(f" Sampled 3-gram rep: {gen.get('sampled_avg_3gram_rep', 0):.2%}")
|
||||
print(f" Sampled EOS rate: {gen.get('sampled_eos_rate', 0):.2%}")
|
||||
|
||||
rep = output.get("repetition", {}).get("best", {})
|
||||
if rep:
|
||||
print(f"\n--- Best Repetition Params ---")
|
||||
print(f" Config: {rep.get('params', 'N/A')}")
|
||||
print(f" 3-gram rep: {rep.get('avg_3gram_rep', 0):.2%}")
|
||||
|
||||
print(f"\n결과 저장: {out_path}")
|
||||
print("=" * 70)
|
||||
Reference in New Issue
Block a user