初始化项目,由ModelHub XC社区提供模型
Model: Delentia/delentia-slm-jitna-v0.4 Source: Original Platform
This commit is contained in:
96
training_config/slm_jitna_v0.2.yaml
Normal file
96
training_config/slm_jitna_v0.2.yaml
Normal file
@@ -0,0 +1,96 @@
|
||||
# Delentia SLM — JITNA v0.2 TOON Fine-tuning Configuration
|
||||
# Base: Llama 3.1 8B (Apache 2.0, Thai-capable)
|
||||
# Method: Unsloth QLoRA (4-bit) — optimized for T4 16GB / A100 40GB
|
||||
# Format: TOON (Token-Oriented Object Notation) — ALGO-42
|
||||
#
|
||||
# Delta from v0.1:
|
||||
# - Dataset uses TOON-formatted completions (jitna_pairs_toon.jsonl)
|
||||
# - Chat template includes TOON format instruction
|
||||
# - LoRA rank increased to 32 for better TOON structure learning
|
||||
# - Epochs increased to 5 for format stability convergence
|
||||
# - Learning rate lowered to 1e-4 for smoother gradient steps
|
||||
|
||||
model:
|
||||
base_model: "unsloth/Meta-Llama-3.1-8B-bnb-4bit"
|
||||
tokenizer: "unsloth/Meta-Llama-3.1-8B-bnb-4bit"
|
||||
max_seq_length: 4096 # covers JITNA v3 packets + TOON context
|
||||
dtype: null # auto-detect: bfloat16 on A100, float16 on T4
|
||||
load_in_4bit: true
|
||||
|
||||
lora:
|
||||
r: 32 # increased from 16 — TOON structure needs more rank
|
||||
lora_alpha: 64 # 2×r for stability
|
||||
lora_dropout: 0
|
||||
bias: "none"
|
||||
use_rslora: true # Rank-Stabilized LoRA — critical for TOON convergence
|
||||
target_modules:
|
||||
- "q_proj"
|
||||
- "k_proj"
|
||||
- "v_proj"
|
||||
- "o_proj"
|
||||
- "gate_proj"
|
||||
- "up_proj"
|
||||
- "down_proj"
|
||||
task_type: "CAUSAL_LM"
|
||||
|
||||
training:
|
||||
# Dataset — TOON v0.2 format
|
||||
dataset_path: "datasets/processed/jitna_pairs_toon.jsonl"
|
||||
dataset_split: "train"
|
||||
validation_split: 0.05 # 5% held out for validation
|
||||
max_samples: null # null = use all available
|
||||
|
||||
# Batch & gradient
|
||||
per_device_train_batch_size: 1
|
||||
gradient_accumulation_steps: 8
|
||||
# Effective batch = 1 × 8 = 8
|
||||
|
||||
# Learning rate — lower for smoother TOON format convergence
|
||||
learning_rate: 1.0e-4
|
||||
lr_scheduler_type: "cosine"
|
||||
warmup_ratio: 0.05
|
||||
num_train_epochs: 5 # more epochs for TOON structure stability
|
||||
|
||||
# Precision & optimizer
|
||||
bf16: true # set false if T4 (use fp16 instead)
|
||||
fp16: false
|
||||
optim: "adamw_8bit"
|
||||
weight_decay: 0.01
|
||||
max_grad_norm: 0.3
|
||||
|
||||
# Saving
|
||||
output_dir: "models/checkpoints/v0.2_toon"
|
||||
save_strategy: "epoch"
|
||||
save_total_limit: 3
|
||||
logging_steps: 10
|
||||
|
||||
# Evaluation
|
||||
evaluation_strategy: "epoch"
|
||||
load_best_model_at_end: true
|
||||
metric_for_best_model: "eval_loss"
|
||||
|
||||
# Chat template — TOON v0.2 format with special tokens
|
||||
chat_template: |
|
||||
<|system|>
|
||||
You are Delentia OS v0.2 — a constitutional AI operating under RCT v5 governance.
|
||||
You process intents through the JITNA v3 protocol.
|
||||
You respond in TOON format (Token-Oriented Object Notation) for token efficiency.
|
||||
Your responses must be factual, safe, and PDPA-compliant.
|
||||
Always provide FDIA scores when applicable (F = D^I × A).
|
||||
<|user|>
|
||||
{{ user_intent }}
|
||||
<|assistant|>
|
||||
|
||||
# MLflow experiment tracking
|
||||
mlflow:
|
||||
experiment_name: "delentia-slm-jitna-v0.2-toon"
|
||||
tracking_uri: "https://delentia-delentia-agent-monitor.hf.space"
|
||||
log_model: true
|
||||
|
||||
# Target metrics (gates for acceptance) — v0.2 with TOON compliance
|
||||
target_metrics:
|
||||
jitna_compliance: 0.98 # >= 98% JITNA v3 schema compliance
|
||||
toon_compliance: 0.95 # >= 95% TOON format compliance
|
||||
fdia_avg: 0.895 # avg F score >= 0.895
|
||||
hallucination_rate: 0.0028 # <= 0.28% factual errors (SignedAI consensus)
|
||||
token_savings_pct: 8.0 # >= 8.0% token savings vs JSON (character-based metric for flat paragraphs)
|
||||
Reference in New Issue
Block a user