初始化项目，由ModelHub XC社区提供模型
Model: iamrahulreddy/Quintus Source: Original Platform
This commit is contained in:
ModelHub XC
186
configs/__init__.py
Normal file
186
configs/__init__.py
Normal file
@@ -0,0 +1,186 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from datetime import timezone, timedelta
|
||||
from pathlib import Path
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
from omegaconf import OmegaConf
|
||||
|
||||
_THIS_DIR = Path(__file__).resolve().parent
|
||||
_YAML_PATH = _THIS_DIR / "config.yaml"
|
||||
|
||||
def _load_cfg():
|
||||
return OmegaConf.load(_YAML_PATH)
|
||||
|
||||
cfg = _load_cfg()
|
||||
|
||||
_LOG_TZ_NAME = os.environ.get("QUINTUS_LOG_TZ", "Asia/Kolkata")
|
||||
try:
|
||||
_LOG_TZ = ZoneInfo(_LOG_TZ_NAME)
|
||||
except Exception:
|
||||
_LOG_TZ = timezone(timedelta(hours=5, minutes=30))
|
||||
_LOG_TZ_NAME = "Asia/Kolkata"
|
||||
|
||||
os.environ["TZ"] = _LOG_TZ_NAME
|
||||
if hasattr(time, "tzset"):
|
||||
time.tzset()
|
||||
_LOG_TZ_LABEL = "IST" if _LOG_TZ_NAME == "Asia/Kolkata" else _LOG_TZ_NAME
|
||||
|
||||
def _read_bool_env(name: str) -> bool | None:
|
||||
raw = os.environ.get(name)
|
||||
if raw is None:
|
||||
return None
|
||||
|
||||
normalised = raw.strip().lower()
|
||||
if normalised in {"1", "true", "yes", "on"}:
|
||||
return True
|
||||
if normalised in {"0", "false", "no", "off"}:
|
||||
return False
|
||||
raise ValueError(
|
||||
f"Invalid boolean value for {name}: {raw!r}. "
|
||||
"Use 1/0, true/false, yes/no, or on/off."
|
||||
)
|
||||
|
||||
# Environment variable overrides used by the wrapper.
|
||||
if os.environ.get("QUINTUS_TEACHER_MODEL"):
|
||||
cfg.model.teacher = os.environ["QUINTUS_TEACHER_MODEL"]
|
||||
if os.environ.get("QUINTUS_TEACHER_REVISION"):
|
||||
cfg.model.teacher_revision = os.environ["QUINTUS_TEACHER_REVISION"]
|
||||
if os.environ.get("QUINTUS_STUDENT_MODEL"):
|
||||
cfg.model.student = os.environ["QUINTUS_STUDENT_MODEL"]
|
||||
if os.environ.get("QUINTUS_STUDENT_REVISION"):
|
||||
cfg.model.student_revision = os.environ["QUINTUS_STUDENT_REVISION"]
|
||||
if os.environ.get("QUINTUS_TOKENIZER_MODEL"):
|
||||
cfg.model.tokenizer = os.environ["QUINTUS_TOKENIZER_MODEL"]
|
||||
if os.environ.get("QUINTUS_TOKENIZER_REVISION"):
|
||||
cfg.model.tokenizer_revision = os.environ["QUINTUS_TOKENIZER_REVISION"]
|
||||
if os.environ.get("QUINTUS_STUDENT_DIR"):
|
||||
cfg.paths.student_dir = os.environ["QUINTUS_STUDENT_DIR"]
|
||||
if os.environ.get("QUINTUS_TOKENIZER_DIR"):
|
||||
cfg.paths.tokenizer_dir = os.environ["QUINTUS_TOKENIZER_DIR"]
|
||||
if os.environ.get("NUM_SAMPLES"):
|
||||
cfg.data.num_samples = int(os.environ["NUM_SAMPLES"])
|
||||
if os.environ.get("TRAIN_NUM_EPOCHS"):
|
||||
cfg.training.num_epochs = int(os.environ["TRAIN_NUM_EPOCHS"])
|
||||
if os.environ.get("TRAIN_LEARNING_RATE"):
|
||||
cfg.training.learning_rate = float(os.environ["TRAIN_LEARNING_RATE"])
|
||||
if os.environ.get("TRAIN_ALPHA"):
|
||||
cfg.training.alpha = float(os.environ["TRAIN_ALPHA"])
|
||||
if os.environ.get("TRAIN_TEMPERATURE"):
|
||||
cfg.training.temperature = float(os.environ["TRAIN_TEMPERATURE"])
|
||||
if os.environ.get("TRAIN_TOP_K"):
|
||||
cfg.training.top_k = int(os.environ["TRAIN_TOP_K"])
|
||||
if os.environ.get("QUINTUS_ONLINE_KD_TOKEN_CHUNK_SIZE"):
|
||||
cfg.training.online_kd_token_chunk_size = int(os.environ["QUINTUS_ONLINE_KD_TOKEN_CHUNK_SIZE"])
|
||||
if os.environ.get("TRAIN_MICRO_BATCH_SIZE"):
|
||||
cfg.training.micro_batch_size = int(os.environ["TRAIN_MICRO_BATCH_SIZE"])
|
||||
if os.environ.get("TRAIN_GRAD_ACCUM_STEPS"):
|
||||
cfg.training.grad_accum_steps = int(os.environ["TRAIN_GRAD_ACCUM_STEPS"])
|
||||
if os.environ.get("TRAIN_DATALOADER_WORKERS"):
|
||||
cfg.training.dataloader_workers = int(os.environ["TRAIN_DATALOADER_WORKERS"])
|
||||
if os.environ.get("TRAIN_PREFETCH_FACTOR"):
|
||||
cfg.training.prefetch_factor = int(os.environ["TRAIN_PREFETCH_FACTOR"])
|
||||
sequence_packing_override = _read_bool_env("QUINTUS_SEQUENCE_PACKING")
|
||||
if sequence_packing_override is not None:
|
||||
cfg.training.sequence_packing.enabled = sequence_packing_override
|
||||
if os.environ.get("QUINTUS_PACK_LENGTH"):
|
||||
cfg.training.sequence_packing.pack_length = int(os.environ["QUINTUS_PACK_LENGTH"])
|
||||
compile_override = _read_bool_env("QUINTUS_COMPILE_MODEL")
|
||||
if compile_override is not None:
|
||||
cfg.training.compile_model = compile_override
|
||||
fused_adamw_override = _read_bool_env("TRAIN_FUSED_ADAMW")
|
||||
if fused_adamw_override is not None:
|
||||
cfg.training.fused_adamw = fused_adamw_override
|
||||
if os.environ.get("QUINTUS_DISTILLED_DIR"):
|
||||
cfg.paths.distilled_dir = os.environ["QUINTUS_DISTILLED_DIR"]
|
||||
if os.environ.get("DATA_STREAM_SHUFFLE_BUFFER_SIZE"):
|
||||
cfg.data.stream_shuffle_buffer_size = int(os.environ["DATA_STREAM_SHUFFLE_BUFFER_SIZE"])
|
||||
if os.environ.get("DATA_STREAM_SHUFFLE_SEED"):
|
||||
cfg.data.stream_shuffle_seed = int(os.environ["DATA_STREAM_SHUFFLE_SEED"])
|
||||
remote_code_override = _read_bool_env("QUINTUS_ALLOW_REMOTE_CODE")
|
||||
if remote_code_override is not None:
|
||||
cfg.model.allow_remote_code = remote_code_override
|
||||
|
||||
class _TagFormatter(logging.Formatter):
|
||||
def __init__(self, tag: str, fmt: str, datefmt: str | None = None):
|
||||
super().__init__(fmt=fmt, datefmt=datefmt)
|
||||
self.tag = tag
|
||||
|
||||
def formatTime(self, record: logging.LogRecord, datefmt: str | None = None) -> str:
|
||||
dt = datetime_from_timestamp(record.created)
|
||||
if datefmt:
|
||||
return dt.strftime(datefmt)
|
||||
return dt.isoformat(timespec="seconds")
|
||||
|
||||
def format(self, record: logging.LogRecord) -> str:
|
||||
record.tag = self.tag # type: ignore[attr-defined]
|
||||
return super().format(record)
|
||||
|
||||
def datetime_from_timestamp(timestamp: float):
|
||||
from datetime import datetime
|
||||
|
||||
return datetime.fromtimestamp(timestamp, tz=_LOG_TZ)
|
||||
|
||||
def setup_logger(module_tag: str, rank: int = -1) -> logging.Logger:
|
||||
name = f"quintus.{module_tag}"
|
||||
logger = logging.getLogger(name)
|
||||
|
||||
if logger.handlers:
|
||||
return logger
|
||||
|
||||
logger.setLevel(logging.DEBUG)
|
||||
logger.propagate = False
|
||||
|
||||
# Suppress duplicate output from non-primary ranks.
|
||||
if rank not in (-1, 0):
|
||||
logger.addHandler(logging.NullHandler())
|
||||
return logger
|
||||
|
||||
# Plain text file handler.
|
||||
file_fmt = _TagFormatter(
|
||||
tag=module_tag,
|
||||
fmt=f"[%(asctime)s {_LOG_TZ_LABEL}] [%(levelname)-5s] [%(tag)-8s] %(message)s",
|
||||
datefmt="%Y-%m-%d %H:%M:%S",
|
||||
)
|
||||
log_dir = os.path.dirname(cfg.paths.log_file)
|
||||
if log_dir:
|
||||
os.makedirs(log_dir, exist_ok=True)
|
||||
file_handler = logging.FileHandler(cfg.paths.log_file, mode="a", encoding="utf-8")
|
||||
file_handler.setLevel(logging.DEBUG)
|
||||
file_handler.setFormatter(file_fmt)
|
||||
logger.addHandler(file_handler)
|
||||
|
||||
# Plain text console handler. Keep the runtime logs stable across terminals,
|
||||
# notebooks and log files.
|
||||
console_handler = logging.StreamHandler(sys.stdout)
|
||||
console_handler.setLevel(logging.INFO)
|
||||
console_handler.setFormatter(file_fmt)
|
||||
logger.addHandler(console_handler)
|
||||
|
||||
return logger
|
||||
|
||||
def emit_log_spacing(logger: logging.Logger, count: int = 2) -> None:
|
||||
if count <= 0:
|
||||
return
|
||||
|
||||
blank_block = "\n" * count
|
||||
for handler in logger.handlers:
|
||||
if isinstance(handler, logging.NullHandler):
|
||||
continue
|
||||
|
||||
stream = getattr(handler, "stream", None)
|
||||
if stream is not None and hasattr(stream, "write"):
|
||||
stream.write(blank_block)
|
||||
flush = getattr(stream, "flush", None)
|
||||
if callable(flush):
|
||||
flush()
|
||||
continue
|
||||
|
||||
console = getattr(handler, "console", None)
|
||||
if console is not None:
|
||||
console.print(blank_block, end="")
|
||||
|
||||
77
configs/config.yaml
Normal file
77
configs/config.yaml
Normal file
@@ -0,0 +1,77 @@
|
||||
# Quintus Distillation Pipeline
|
||||
# Run profile: online full-vocabulary KD, 8B teacher -> 1.7B-Base student.
|
||||
# Data: ~90K English-only samples from DistilQwen_100k.
|
||||
|
||||
data:
|
||||
dataset_path: "<REDACTED_ON_PURPOSE>"
|
||||
num_samples: 90234
|
||||
max_seq_len: 4096
|
||||
stream_shuffle_buffer_size: 20000
|
||||
stream_shuffle_seed: 25
|
||||
|
||||
model:
|
||||
teacher: "Qwen/Qwen3-8B"
|
||||
student: "Qwen/Qwen3-1.7B-Base"
|
||||
|
||||
# The instruct tokenizer carries the chat template used to format the base
|
||||
# student into assistant-style training examples.
|
||||
tokenizer: "Qwen/Qwen3-1.7B"
|
||||
|
||||
teacher_revision: "main"
|
||||
student_revision: "main"
|
||||
tokenizer_revision: "main"
|
||||
allow_remote_code: false
|
||||
|
||||
training:
|
||||
# Schedule
|
||||
num_epochs: 1
|
||||
validation_ratio: 0.02
|
||||
split_seed: 25
|
||||
|
||||
# Optimizer
|
||||
learning_rate: 5.0e-6
|
||||
weight_decay: 0.1
|
||||
warmup_ratio: 0.05
|
||||
|
||||
# Loss mix used by src/losses.py:
|
||||
# total = alpha * CE + (1 - alpha) * KD
|
||||
alpha: 0.3
|
||||
temperature: 2.0
|
||||
|
||||
# Online KD streams full-vocabulary teacher logits. top_k is retained for
|
||||
# offline-KD compatibility/provenance checks.
|
||||
top_k: 8
|
||||
online_kd_token_chunk_size: 2048
|
||||
|
||||
# Conservative B200 profile. Effective batch = 4 * 2 = 8.
|
||||
# If VRAM headroom is comfortable and Liger is installed, try 8 * 1.
|
||||
micro_batch_size: 4
|
||||
grad_accum_steps: 2
|
||||
gradient_checkpointing: false
|
||||
compile_model: false
|
||||
fused_adamw: true
|
||||
|
||||
dataloader_workers: 8
|
||||
prefetch_factor: 2
|
||||
|
||||
sequence_packing:
|
||||
enabled: true
|
||||
pack_length: 4096
|
||||
mask_first_token_after_separator: true
|
||||
|
||||
hub:
|
||||
# Prefer HF_TOKEN or huggingface-cli login for real runs.
|
||||
token: null
|
||||
username: "<REDACTED_ON_PURPOSE>"
|
||||
repo_name: "<REDACTED_ON_PURPOSE>"
|
||||
|
||||
paths:
|
||||
teacher_dir: "<REDACTED_ON_PURPOSE>"
|
||||
student_dir: "<REDACTED_ON_PURPOSE>"
|
||||
tokenizer_dir: "<REDACTED_ON_PURPOSE>"
|
||||
tokenized_dir: "<REDACTED_ON_PURPOSE>"
|
||||
logits_dir: "<REDACTED_ON_PURPOSE>"
|
||||
distilled_dir: "<REDACTED_ON_PURPOSE>"
|
||||
log_file: "<REDACTED_ON_PURPOSE>"
|
||||
system_info: "<REDACTED_ON_PURPOSE>"
|
||||
loss_csv: "<REDACTED_ON_PURPOSE>"
|
||||
20
configs/ds_zero2.json
Normal file
20
configs/ds_zero2.json
Normal file
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"zero_optimization": {
|
||||
"stage": 2,
|
||||
"allgather_partitions": true,
|
||||
"allgather_bucket_size": 500000000,
|
||||
"reduce_scatter": true,
|
||||
"reduce_bucket_size": 500000000,
|
||||
"overlap_comm": true,
|
||||
"contiguous_gradients": true
|
||||
},
|
||||
"bf16": {
|
||||
"enabled": true
|
||||
},
|
||||
"gradient_clipping": 1.0,
|
||||
"steps_per_print": 50,
|
||||
"wall_clock_breakdown": false,
|
||||
"comms_logger": {
|
||||
"enabled": false
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user