初始化项目，由ModelHub XC社区提供模型

Model: Tencent-Hunyuan/Hy-MT2-7B-GGUF
Source: Original Platform

train/llama_factory_support/convert_zero_to_hf.sh

@@ -0,0 +1,175 @@
+#!/bin/bash
+# 将 DeepSpeed ZeRO 格式的 checkpoint 转换为 HuggingFace 格式
+# 使用 zero_to_fp32.py 转换权重，然后保存为 HF 格式
+
+set -e  # 遇到错误立即退出
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+PROJECT_ROOT="$(dirname "$(dirname "$SCRIPT_DIR")")"
+
+# 激活 conda 环境
+source "$(conda info --base)/etc/profile.d/conda.sh"
+conda activate llama_factory
+
+# 设置环境变量
+export LD_LIBRARY_PATH=$CONDA_PREFIX/lib:$LD_LIBRARY_PATH
+export DISABLE_VERSION_CHECK=1
+export CUDA_VISIBLE_DEVICES=""  # 使用 CPU 进行转换，避免显存不足
+
+CHECKPOINT_DIR="$SCRIPT_DIR/saves/hy_v3/full/sft/checkpoint-39"
+OUTPUT_DIR="$SCRIPT_DIR/saves/hy_v3/full/sft/checkpoint-39/hf_converted"
+TEMP_WEIGHTS_DIR="$OUTPUT_DIR/zero_fp32_output"  # 分片输出目录
+
+echo "=========================================="
+echo "Converting DeepSpeed ZeRO checkpoint to HF format"
+echo "Input:  $CHECKPOINT_DIR"
+echo "Output: $OUTPUT_DIR"
+echo "=========================================="
+
+# 创建输出目录
+mkdir -p "$OUTPUT_DIR"
+rm -rf "$TEMP_WEIGHTS_DIR"
+mkdir -p "$TEMP_WEIGHTS_DIR"
+
+# Step 1: 使用 zero_to_fp32.py 转换权重
+echo ""
+echo "[Step 1/3] Converting weights from ZeRO format to FP32..."
+cd "$CHECKPOINT_DIR"
+python3 zero_to_fp32.py . "$TEMP_WEIGHTS_DIR"
+
+# 检查输出 - zero_to_fp32.py 可能输出单个文件或多个分片
+if [ -d "$TEMP_WEIGHTS_DIR" ] && [ "$(ls -A "$TEMP_WEIGHTS_DIR" 2>/dev/null)" ]; then
+    echo "Weight conversion completed! Output in: $TEMP_WEIGHTS_DIR"
+    echo "Files: $(ls "$TEMP_WEIGHTS_DIR" | wc -l) files"
+else
+    echo "ERROR: Weight conversion failed!"
+    exit 1
+fi
+
+# Step 2: 复制配置文件
+echo ""
+echo "[Step 2/3] Copying config files..."
+BASE_MODEL_DIR="$PROJECT_ROOT/a3b_ckpt"
+
+# 从基座模型复制完整的配置文件（checkpoint 中的 tokenizer_config 不完整）
+cp "$BASE_MODEL_DIR/config.json" "$OUTPUT_DIR/" 2>/dev/null || true
+cp "$BASE_MODEL_DIR/tokenizer_config.json" "$OUTPUT_DIR/" 2>/dev/null || true
+cp "$BASE_MODEL_DIR/tokenizer.json" "$OUTPUT_DIR/" 2>/dev/null || true
+cp "$BASE_MODEL_DIR/special_tokens_map.json" "$OUTPUT_DIR/" 2>/dev/null || true
+cp "$BASE_MODEL_DIR/chat_template.jinja" "$OUTPUT_DIR/" 2>/dev/null || true
+cp "$CHECKPOINT_DIR/generation_config.json" "$OUTPUT_DIR/" 2>/dev/null || true
+
+echo "Config files copied from base model."
+
+# Step 3: 加载权重并保存为 HF 格式
+echo ""
+echo "[Step 3/3] Converting to HuggingFace format..."
+
+cat > /tmp/convert_to_hf.py << 'PYEOF'
+import torch
+import json
+import os
+import sys
+import glob
+sys.path.insert(0, os.environ.get("PROJECT_ROOT", "."))
+
+# 设置目录
+output_dir = os.environ["OUTPUT_DIR"]
+checkpoint_dir = os.environ["CHECKPOINT_DIR"]
+base_model_dir = os.environ["BASE_MODEL_DIR"]
+temp_weights_dir = os.path.join(output_dir, "zero_fp32_output")
+
+# 加载 tokenizer
+from transformers import AutoTokenizer
+print("Loading tokenizer...")
+tokenizer = AutoTokenizer.from_pretrained(base_model_dir, trust_remote_code=True)
+tokenizer.save_pretrained(output_dir)
+
+# 从 base model 加载配置和模型结构
+from transformers import AutoConfig, AutoModelForCausalLM
+print("Loading config...")
+config = AutoConfig.from_pretrained(base_model_dir, trust_remote_code=True)
+
+# 创建模型（从基座模型加载结构和权重，然后用训练后的权重覆盖）
+print("Loading base model...")
+with torch.no_grad():
+    model = AutoModelForCausalLM.from_pretrained(
+        base_model_dir,
+        torch_dtype=torch.bfloat16,
+        trust_remote_code=True,
+    )
+
+# 加载转换后的权重（可能分片）
+print(f"Loading weights from {temp_weights_dir} ...")
+weight_files = sorted(glob.glob(os.path.join(temp_weights_dir, "*.bin")) + 
+                      glob.glob(os.path.join(temp_weights_dir, "*.safetensors")))
+
+if not weight_files:
+    print(f"ERROR: No weight files found in {temp_weights_dir}")
+    sys.exit(1)
+
+print(f"Found {len(weight_files)} weight files")
+state_dict = {}
+for wf in weight_files:
+    print(f"  Loading {wf} ...")
+    if wf.endswith('.safetensors'):
+        from safetensors.torch import load_file
+        state_dict.update(load_file(wf, device="cpu"))
+    else:
+        state_dict.update(torch.load(wf, map_location="cpu"))
+
+# 加载权重到模型
+print("Loading converted weights into model...")
+model_state_dict = model.state_dict()
+filtered_state_dict = {}
+skipped = 0
+matched = 0
+for k, v in state_dict.items():
+    if k in model_state_dict:
+        # 转换 dtype
+        if v.dtype != model_state_dict[k].dtype:
+            v = v.to(model_state_dict[k].dtype)
+        filtered_state_dict[k] = v
+        matched += 1
+    else:
+        skipped += 1
+        if skipped <= 10:  # 只打印前10个跳过的key
+            print(f"  Skipping key: {k}")
+
+print(f"Matched {matched} tensors, skipped {skipped}")
+
+if matched == 0:
+    print("ERROR: No weights matched! Something is wrong with the conversion.")
+    sys.exit(1)
+
+missing, unexpected = model.load_state_dict(filtered_state_dict, strict=False)
+if missing:
+    print(f"WARNING: {len(missing)} keys missing in converted weights (using base model weights)")
+    for k in missing[:10]:
+        print(f"  Missing: {k}")
+    if len(missing) > 10:
+        print(f"  ... and {len(missing) - 10} more")
+
+# 保存为 HF 格式（使用 safetensors，更安全、更快）
+print(f"Saving model to {output_dir}...")
+model.save_pretrained(output_dir, safe_serialization=True)
+print("Done!")
+
+# 清理临时文件
+print("Cleaning up temp files...")
+import shutil
+shutil.rmtree(temp_weights_dir)
+PYEOF
+
+export OUTPUT_DIR="$OUTPUT_DIR"
+export CHECKPOINT_DIR="$CHECKPOINT_DIR"
+export BASE_MODEL_DIR="$PROJECT_ROOT/a3b_ckpt"
+export PROJECT_ROOT="$PROJECT_ROOT"
+
+python3 /tmp/convert_to_hf.py
+
+echo ""
+echo "=========================================="
+echo "Conversion completed!"
+echo "HF format model saved to: $OUTPUT_DIR"
+echo "=========================================="

train/llama_factory_support/dataset_info.json

@@ -0,0 +1,44 @@
+{
+  "hy_v3_demo": {
+    "file_name": "../example_data.jsonl",
+    "formatting": "sharegpt",
+    "columns": {
+      "messages": "messages"
+    },
+    "tags": {
+      "role_tag": "role",
+      "content_tag": "content",
+      "user_tag": "user",
+      "assistant_tag": "assistant",
+      "system_tag": "system"
+    }
+  },
+  "hy_v3_translation": {
+    "file_name": "../data/example_data.jsonl",
+    "formatting": "sharegpt",
+    "columns": {
+      "messages": "messages"
+    },
+    "tags": {
+      "role_tag": "role",
+      "content_tag": "content",
+      "user_tag": "user",
+      "assistant_tag": "assistant",
+      "system_tag": "system"
+    }
+  },
+  "hy_dense_demo": {
+    "file_name": "../data/example_data.jsonl",
+    "formatting": "sharegpt",
+    "columns": {
+      "messages": "messages"
+    },
+    "tags": {
+      "role_tag": "role",
+      "content_tag": "content",
+      "user_tag": "user",
+      "assistant_tag": "assistant",
+      "system_tag": "system"
+    }
+  }
+}

train/llama_factory_support/ds_zero2_offload.json

@@ -0,0 +1,37 @@
+{
+    "fp16": {
+        "enabled": false,
+        "loss_scale": 0,
+        "loss_scale_window": 1000,
+        "initial_scale_power": 16,
+        "hysteresis": 2,
+        "min_loss_scale": 1
+    },
+    "bf16": {
+        "enabled": true
+    },
+
+    "zero_optimization": {
+        "stage": 2,
+        "offload_optimizer": {
+            "device": "cpu",
+            "pin_memory": true
+        },
+        "offload_param": {
+            "device": "cpu",
+            "pin_memory": true
+        },
+        "allgather_partitions": true,
+        "allgather_bucket_size": 5e8,
+        "overlap_comm": true,
+        "reduce_scatter": true,
+        "reduce_bucket_size": 5e8,
+        "contiguous_gradients": true
+    },
+
+    "gradient_accumulation_steps": "auto",
+    "gradient_clipping": "auto",
+    "steps_per_print": 10,
+    "train_micro_batch_size_per_gpu": "auto",
+    "wall_clock_breakdown": false
+}

train/llama_factory_support/ds_zero2_offload_lora.json

@@ -0,0 +1,33 @@
+{
+    "fp16": {
+        "enabled": false,
+        "loss_scale": 0,
+        "loss_scale_window": 1000,
+        "initial_scale_power": 16,
+        "hysteresis": 2,
+        "min_loss_scale": 1
+    },
+    "bf16": {
+        "enabled": true
+    },
+
+    "zero_optimization": {
+        "stage": 2,
+        "offload_optimizer": {
+            "device": "cpu",
+            "pin_memory": true
+        },
+        "allgather_partitions": true,
+        "allgather_bucket_size": 5e8,
+        "overlap_comm": true,
+        "reduce_scatter": true,
+        "reduce_bucket_size": 5e8,
+        "contiguous_gradients": true
+    },
+
+    "gradient_accumulation_steps": "auto",
+    "gradient_clipping": "auto",
+    "steps_per_print": 10,
+    "train_micro_batch_size_per_gpu": "auto",
+    "wall_clock_breakdown": false
+}

train/llama_factory_support/ds_zero3_offload.json

@@ -0,0 +1,36 @@
+{
+    "fp16": {
+        "enabled": false,
+        "loss_scale": 0,
+        "loss_scale_window": 1000,
+        "initial_scale_power": 16,
+        "hysteresis": 2,
+        "min_loss_scale": 1
+    },
+    "bf16": {
+        "enabled": true
+    },
+
+    "zero_optimization": {
+        "stage": 3,
+        "offload_optimizer": {
+            "device": "cpu",
+            "pin_memory": true
+        },
+        "overlap_comm": true,
+        "contiguous_gradients": true,
+        "sub_group_size": 1e9,
+        "reduce_bucket_size": 1e8,
+        "stage3_prefetch_bucket_size": 1e8,
+        "stage3_param_persistence_threshold": 0,
+        "stage3_max_live_parameters": 1e8,
+        "stage3_max_reuse_distance": 1e8,
+        "stage3_gather_16bit_weights_on_model_save": true
+    },
+
+    "gradient_accumulation_steps": "auto",
+    "gradient_clipping": "auto",
+    "steps_per_print": 10,
+    "train_micro_batch_size_per_gpu": "auto",
+    "wall_clock_breakdown": false
+}

train/llama_factory_support/hy_dense_1_8b_full_sft.yaml

@@ -0,0 +1,50 @@
+### model
+model_name_or_path: path_to_dense_1_8b_model
+trust_remote_code: true
+
+### method
+stage: sft
+do_train: true
+finetuning_type: full
+deepspeed: ds_zero2_offload.json
+
+### dataset
+dataset_dir: .
+dataset: hy_dense_demo
+template: hy_dense_1_8b
+cutoff_len: 4096
+max_samples: 1000
+overwrite_cache: true
+
+### output
+output_dir: saves/hy_dense_1_8b/full/sft
+logging_steps: 1
+save_steps: 30
+plot_loss: true
+overwrite_output_dir: true
+save_only_model: false
+report_to: none  # choices: [none, wandb, tensorboard, swanlab, mlflow]
+
+### train
+per_device_train_batch_size: 1
+gradient_accumulation_steps: 1
+learning_rate: 1.0e-5
+num_train_epochs: 1.0
+lr_scheduler_type: cosine_with_min_lr
+lr_scheduler_kwargs:
+  min_lr_rate: 0.1  # min_lr / learning_rate = 1e-6 / 1e-5 = 0.1
+warmup_ratio: 0.1
+bf16: true
+gradient_checkpointing: true
+gradient_checkpointing_kwargs:
+  use_reentrant: true
+ddp_timeout: 180000000
+flash_attn: fa2
+resume_from_checkpoint: null
+
+### eval
+# eval_dataset: alpaca_en_demo
+# val_size: 0.1
+# per_device_eval_batch_size: 1
+# eval_strategy: steps
+# eval_steps: 500

train/llama_factory_support/hy_dense_1_8b_lora_sft.yaml

@@ -0,0 +1,56 @@
+### model
+model_name_or_path: path_to_dense_1_8b_model
+trust_remote_code: true
+
+### method
+stage: sft
+do_train: true
+finetuning_type: lora
+deepspeed: ds_zero2_offload_lora.json
+
+### LoRA parameters
+lora_rank: 64
+lora_alpha: 128
+lora_dropout: 0.05
+lora_target: q_proj,k_proj,v_proj,o_proj
+
+### dataset
+dataset_dir: .
+dataset: hy_dense_demo
+template: hy_dense_1_8b
+cutoff_len: 4096
+max_samples: 1000
+overwrite_cache: true
+
+### output
+output_dir: saves/hy_dense_1_8b/lora/sft
+logging_steps: 1
+save_steps: 10
+plot_loss: true
+overwrite_output_dir: true
+save_only_model: false
+report_to: none  # choices: [none, wandb, tensorboard, swanlab, mlflow]
+
+### train
+per_device_train_batch_size: 1
+gradient_accumulation_steps: 1
+learning_rate: 2.0e-4
+num_train_epochs: 1.0
+lr_scheduler_type: cosine_with_min_lr
+lr_scheduler_kwargs:
+  min_lr_rate: 0.1  # min_lr / learning_rate = 2e-5 / 2e-4 = 0.1
+warmup_ratio: 0.1
+bf16: true
+gradient_checkpointing: true
+gradient_checkpointing_kwargs:
+  use_reentrant: true
+ddp_timeout: 180000000
+flash_attn: fa2
+resume_from_checkpoint: null
+
+### eval
+# eval_dataset: alpaca_en_demo
+# val_size: 0.1
+# per_device_eval_batch_size: 1
+# eval_strategy: steps
+# eval_steps: 500

train/llama_factory_support/hy_dense_7b_full_sft.yaml

@@ -0,0 +1,50 @@
+### model
+model_name_or_path: path_to_dense_7b_model
+trust_remote_code: true
+
+### method
+stage: sft
+do_train: true
+finetuning_type: full
+deepspeed: ds_zero3_offload.json
+
+### dataset
+dataset_dir: .
+dataset: hy_dense_demo
+template: hy_dense_7b
+cutoff_len: 4096
+max_samples: 1000
+overwrite_cache: true
+
+### output
+output_dir: saves/hy_dense_7b/full/sft
+logging_steps: 1
+save_steps: 10
+plot_loss: true
+overwrite_output_dir: true
+save_only_model: false
+report_to: none  # choices: [none, wandb, tensorboard, swanlab, mlflow]
+
+### train
+per_device_train_batch_size: 1
+gradient_accumulation_steps: 1
+learning_rate: 1.0e-5
+num_train_epochs: 1.0
+lr_scheduler_type: cosine_with_min_lr
+lr_scheduler_kwargs:
+  min_lr_rate: 0.1  # min_lr / learning_rate = 1e-6 / 1e-5 = 0.1
+warmup_ratio: 0.1
+bf16: true
+gradient_checkpointing: true
+gradient_checkpointing_kwargs:
+  use_reentrant: true
+ddp_timeout: 180000000
+flash_attn: fa2
+resume_from_checkpoint: null
+
+### eval
+# eval_dataset: alpaca_en_demo
+# val_size: 0.1
+# per_device_eval_batch_size: 1
+# eval_strategy: steps
+# eval_steps: 500

train/llama_factory_support/hy_dense_7b_lora_sft.yaml

@@ -0,0 +1,56 @@
+### model
+model_name_or_path: path_to_dense_7b_model
+trust_remote_code: true
+
+### method
+stage: sft
+do_train: true
+finetuning_type: lora
+deepspeed: ds_zero2_offload_lora.json
+
+### LoRA parameters
+lora_rank: 64
+lora_alpha: 128
+lora_dropout: 0.05
+lora_target: q_proj,k_proj,v_proj,o_proj
+
+### dataset
+dataset_dir: .
+dataset: hy_dense_demo
+template: hy_dense_7b
+cutoff_len: 4096
+max_samples: 1000
+overwrite_cache: true
+
+### output
+output_dir: saves/hy_dense_7b/lora/sft
+logging_steps: 1
+save_steps: 30
+plot_loss: true
+overwrite_output_dir: true
+save_only_model: false
+report_to: none  # choices: [none, wandb, tensorboard, swanlab, mlflow]
+
+### train
+per_device_train_batch_size: 1
+gradient_accumulation_steps: 1
+learning_rate: 2.0e-4
+num_train_epochs: 1.0
+lr_scheduler_type: cosine_with_min_lr
+lr_scheduler_kwargs:
+  min_lr_rate: 0.1  # min_lr / learning_rate = 2e-5 / 2e-4 = 0.1
+warmup_ratio: 0.1
+bf16: true
+gradient_checkpointing: true
+gradient_checkpointing_kwargs:
+  use_reentrant: true
+ddp_timeout: 180000000
+flash_attn: fa2
+resume_from_checkpoint: null
+
+### eval
+# eval_dataset: alpaca_en_demo
+# val_size: 0.1
+# per_device_eval_batch_size: 1
+# eval_strategy: steps
+# eval_steps: 500

train/llama_factory_support/hy_dense_template.py

@@ -0,0 +1,82 @@
+"""
+HunYuan Dense model chat template registration for LLaMA Factory.
+
+Registers two templates:
+  - hy_dense_1_8b: for HunYuan Dense 1.8B model (and 0.5B/4B)
+  - hy_dense_7b:   for HunYuan Dense 7B model
+
+Usage:
+    1. Copy this file's register_template blocks into LLaMA Factory's
+       src/llamafactory/data/template.py  (for upstream MR).
+    2. Or import this module before training to register at runtime:
+       import hy_dense_template
+
+Note:
+    The existing LLaMA Factory built-in templates `hunyuan` and `hunyuan_small`
+    have subtle differences from the official chat_template.jinja files shipped
+    with the models. These new templates are designed to match the official
+    jinja templates exactly.
+"""
+
+from llamafactory.data.template import register_template
+from llamafactory.data.formatter import EmptyFormatter, StringFormatter
+
+# ---------------------------------------------------------------------------
+# Dense 1.8B chat template (also applies to 0.5B/4B)
+#
+# Token format (from dense_1_8b_0508/global_step_560/chat_template.jinja):
+#   BOS:        <｜hy_begin▁of▁sentence｜>
+#   System:     {system_content}<｜hy_place▁holder▁no▁3｜>
+#   User:       <｜hy_User｜>{user_content}
+#   Assistant:  <｜hy_Assistant｜>{assistant_content}<｜hy_place▁holder▁no▁2｜>
+#   Stop:       <｜hy_place▁holder▁no▁2｜>
+#
+# Key differences from LF built-in `hunyuan_small`:
+#   - User format: NO trailing <｜hy_place▁holder▁no▁8｜> after user content
+#   - Assistant format: HAS <｜hy_Assistant｜> prefix before assistant content
+#
+# The eos_token in tokenizer_config.json is <｜hy_place▁holder▁no▁2｜>,
+# so we use efficient_eos=True to let LF append it via {eos_token} slot.
+# ---------------------------------------------------------------------------
+
+register_template(
+    name="hy_dense_1_8b",
+    format_user=StringFormatter(slots=["<｜hy_User｜>{{content}}"]),
+    format_assistant=StringFormatter(slots=["<｜hy_Assistant｜>{{content}}", {"eos_token"}]),
+    format_system=StringFormatter(slots=["{{content}}<｜hy_place▁holder▁no▁3｜>"]),
+    format_prefix=EmptyFormatter(slots=[{"bos_token"}]),
+    stop_words=["<｜hy_place▁holder▁no▁2｜>"],
+    efficient_eos=True,
+)
+
+
+# ---------------------------------------------------------------------------
+# Dense 7B chat template
+#
+# Token format (from dense_7b_0509/global_step_560/chat_template.jinja):
+#   BOS:        <|startoftext|>
+#   System:     {system_content}<|extra_4|>
+#   User:       {user_content}<|extra_0|>
+#   Assistant:  {assistant_content}<|eos|>
+#   Stop:       <|eos|>
+#
+# Key differences from LF built-in `hunyuan`:
+#   - Uses {bos_token} and {eos_token} slots for portability
+#   - efficient_eos=True to use tokenizer's eos_token
+#
+# Note on multi-turn: The official jinja adds <|startoftext|> before each
+# user message (except the first one when system is present). LLaMA Factory's
+# format_prefix only adds BOS once at the beginning. For single-turn training
+# this is correct. For multi-turn, there is a minor discrepancy (missing
+# <|startoftext|> before 2nd+ user turns), which is acceptable for fine-tuning.
+# ---------------------------------------------------------------------------
+
+register_template(
+    name="hy_dense_7b",
+    format_user=StringFormatter(slots=["{{content}}<|extra_0|>"]),
+    format_assistant=StringFormatter(slots=["{{content}}", {"eos_token"}]),
+    format_system=StringFormatter(slots=["{{content}}<|extra_4|>"]),
+    format_prefix=EmptyFormatter(slots=[{"bos_token"}]),
+    stop_words=["<|eos|>"],
+    efficient_eos=True,
+)

train/llama_factory_support/hy_v3_full_sft.yaml

@@ -0,0 +1,50 @@
+### model
+model_name_or_path: ../hf
+trust_remote_code: true
+
+### method
+stage: sft
+do_train: true
+finetuning_type: full
+deepspeed: ds_zero3_offload.json
+
+### dataset
+dataset_dir: .
+dataset: hy_v3_demo
+template: hy_v3
+cutoff_len: 4096  # HYV3 supports 262k context length
+max_samples: 1000
+overwrite_cache: true
+
+### output
+output_dir: saves/hy_v3/full/sft
+logging_steps: 1
+save_steps: 10
+plot_loss: true
+overwrite_output_dir: true
+save_only_model: false
+report_to: none  # choices: [none, wandb, tensorboard, swanlab, mlflow]
+
+### train
+per_device_train_batch_size: 1
+gradient_accumulation_steps: 1
+learning_rate: 1.0e-5
+num_train_epochs: 3.0
+lr_scheduler_type: cosine_with_min_lr
+lr_scheduler_kwargs:
+  min_lr_rate: 0.1  # min_lr / learning_rate = 1e-6 / 1e-5 = 0.1
+warmup_ratio: 0.1
+bf16: true
+gradient_checkpointing: true
+gradient_checkpointing_kwargs:
+  use_reentrant: true
+ddp_timeout: 180000000
+flash_attn: fa2
+resume_from_checkpoint: null
+
+### eval
+# eval_dataset: alpaca_en_demo
+# val_size: 0.1
+# per_device_eval_batch_size: 1
+# eval_strategy: steps
+# eval_steps: 500

train/llama_factory_support/hy_v3_lora_sft.yaml

@@ -0,0 +1,56 @@
+### model
+model_name_or_path: ../hf
+trust_remote_code: true
+
+### method
+stage: sft
+do_train: true
+finetuning_type: lora
+deepspeed: ds_zero2_offload_lora.json
+
+### LoRA parameters
+lora_rank: 64
+lora_alpha: 128
+lora_dropout: 0.05
+lora_target: q_proj,k_proj,v_proj,o_proj
+
+### dataset
+dataset_dir: .
+dataset: hy_v3_translation
+template: hy_v3
+cutoff_len: 4096  # Use shorter context for LoRA to save memory; increase if needed
+max_samples: 1000
+overwrite_cache: true
+
+### output
+output_dir: saves/hy_v3/lora/sft
+logging_steps: 10
+save_steps: 500
+plot_loss: true
+overwrite_output_dir: true
+save_only_model: false
+report_to: none  # choices: [none, wandb, tensorboard, swanlab, mlflow]
+
+### train
+per_device_train_batch_size: 1
+gradient_accumulation_steps: 1
+learning_rate: 2.0e-4
+num_train_epochs: 3.0
+lr_scheduler_type: cosine_with_min_lr
+lr_scheduler_kwargs:
+  min_lr_rate: 0.1  # min_lr / learning_rate = 2e-5 / 2e-4 = 0.1
+warmup_ratio: 0.1
+bf16: true
+gradient_checkpointing: true
+gradient_checkpointing_kwargs:
+  use_reentrant: true
+ddp_timeout: 180000000
+flash_attn: fa2
+resume_from_checkpoint: null
+
+### eval
+# eval_dataset: alpaca_en_demo
+# val_size: 0.1
+# per_device_eval_batch_size: 1
+# eval_strategy: steps
+# eval_steps: 500

train/llama_factory_support/hy_v3_patches.py

@@ -0,0 +1,215 @@
+"""
+HYV3 monkey-patches for LLaMA Factory + DeepSpeed training.
+
+This module applies all necessary runtime patches so that HYV3 (MoE)
+can be trained correctly under LLaMA Factory with DeepSpeed.
+
+Usage:
+    Import this module **before** calling `llamafactory-cli train`:
+
+        import hy_v3_patches          # applies patches on import
+        # ... then start training
+
+    Or add to the LLaMA Factory YAML via a custom entry-point wrapper.
+
+Patches applied:
+    1. (Removed) -- transformers 5.8.1+ has built-in conversion_mapping for
+       hy_v3 that handles key renaming + expert fusing automatically.
+    2. Router forward dtype fix (MoE router gate dtype alignment for ZeRO-3)
+    3. gradient_checkpointing   (use_reentrant=True for ZeRO-3)
+    4. Tokenizer file copy      (CustomSaveCallback)
+    5. (Removed) -- was per-expert ModuleList, now using native 3D Parameters
+    6. (Removed) -- transformers 5.8.1+ has built-in revert_weight_conversion
+       in save_pretrained that handles outer->inner format automatically.
+"""
+
+import os
+import logging
+import shutil
+from typing import Optional
+
+import torch
+import torch.nn.functional as _F
+
+logger = logging.getLogger(__name__)
+
+# ============================================================================
+# Patch 2: Router forward dtype alignment for ZeRO-3
+#
+# The HYV3 MoE HYV3TopKRouter.forward() calls F.linear with .float().
+# Under DeepSpeed ZeRO-3, F.linear is replaced by zero3_linear_wrap which
+# internally does input.matmul(weight.t()) WITHOUT aligning dtypes.
+# When ZeRO-3 stores the gate weight in bf16, the fp32 input causes a
+# dtype mismatch RuntimeError.
+#
+# Fix: monkey-patch HYV3TopKRouter.forward to cast input to
+# self.weight.dtype before F.linear, then cast the output back to float32.
+# ============================================================================
+
+_router_patch_applied = False
+
+def _apply_router_dtype_patch():
+    """Monkey-patch HYV3TopKRouter.forward to align gate input dtype with weight dtype."""
+    global _router_patch_applied
+    if _router_patch_applied:
+        return
+
+    try:
+        from transformers.models.hy_v3.modeling_hy_v3 import HYV3TopKRouter
+    except ImportError:
+        try:
+            from transformers.hy_v3.modeling_hy_v3 import HYV3TopKRouter
+        except ImportError:
+            logger.warning(
+                "Could not import HYV3TopKRouter; "
+                "router dtype patch NOT applied."
+            )
+            return
+
+    def _patched_router_forward(
+        self,
+        hidden_states: torch.Tensor,
+        e_score_correction_bias: torch.Tensor,
+    ) -> tuple:
+        hidden_states = hidden_states.reshape(-1, self.hidden_dim)
+        # Cast input to match weight dtype (bf16 under ZeRO-3)
+        # instead of hard-coding float32, to avoid matmul dtype mismatch.
+        weight_dtype = self.weight.dtype
+        router_logits = _F.linear(hidden_states.to(weight_dtype), self.weight.to(weight_dtype))
+        # Cast back to float32 for numerically stable sigmoid
+        router_logits = router_logits.to(torch.float32)
+        routing_weights = torch.sigmoid(router_logits)
+
+        scores_for_choice = routing_weights + e_score_correction_bias
+        _, top_k_index = torch.topk(scores_for_choice, self.top_k, dim=-1, sorted=False)
+        top_k_weights = routing_weights.gather(1, top_k_index)
+
+        top_k_weights = top_k_weights / (top_k_weights.sum(dim=-1, keepdim=True) + 1e-20)
+        top_k_weights = top_k_weights * self.router_scaling_factor
+
+        return router_logits, top_k_weights, top_k_index
+
+    HYV3TopKRouter.forward = _patched_router_forward
+    _router_patch_applied = True
+    logger.info("HYV3 patch applied: HYV3TopKRouter.forward dtype alignment for ZeRO-3.")
+
+# ============================================================================
+# Patch 3: gradient_checkpointing use_reentrant=True
+#
+# PyTorch's torch.utils.checkpoint with use_reentrant=False (the default
+# in transformers) performs strict metadata checks on recomputed tensors.
+# Under ZeRO-3, parameters are all-gathered during the first forward pass
+# but may be partitioned back when the checkpoint recomputes, causing a
+# CheckpointError.  Setting use_reentrant=True avoids this.
+#
+# This is applied via a Trainer callback that modifies training_args
+# before training starts.
+# ============================================================================
+
+# ============================================================================
+# Patch 4: Tokenizer file copy callback
+#
+# Ensures each checkpoint directory is self-contained for inference by
+# copying all tokenizer-related files from the original tokenizer path.
+# ============================================================================
+
+# Tokenizer files that should be copied to each checkpoint
+_TOKENIZER_FILES = [
+    "generation_config.json",
+    "hy.tiktoken",
+    "tokenizer_config.json",
+    "tokenization_hy.py",
+    "tokenizer.json",
+    "special_tokens_map.json",
+    "chat_template.jinja",
+]
+
+def _copy_tokenizer_to_checkpoint(tokenizer_dir: str, checkpoint_dir: str):
+    """Copy tokenizer files from tokenizer_dir to checkpoint_dir."""
+    for fname in _TOKENIZER_FILES:
+        src = os.path.join(tokenizer_dir, fname)
+        if os.path.isfile(src):
+            shutil.copy(src, os.path.join(checkpoint_dir, fname))
+
+# ============================================================================
+# LLaMA Factory Callback: integrates patches 3, 4 into the training loop
+# ============================================================================
+
+try:
+    from transformers import TrainerCallback
+    from transformers.trainer_utils import PREFIX_CHECKPOINT_DIR
+
+    class HYV3PatchCallback(TrainerCallback):
+        """
+        LLaMA Factory compatible callback that applies HYV3-specific patches.
+
+        Add to your YAML or pass to Trainer:
+            callbacks: [hy_v3_patches.HYV3PatchCallback]
+        """
+
+        def __init__(self, tokenizer_dir: Optional[str] = None):
+            """
+            Args:
+                tokenizer_dir: Path to the original tokenizer directory.
+                    If None, will try to use model_name_or_path from training args.
+            """
+            self._tokenizer_dir = tokenizer_dir
+
+        def on_train_begin(self, args, state, control, **kwargs):
+            # --- Patch 3: gradient_checkpointing use_reentrant ---
+            if getattr(args, "gradient_checkpointing", False) and getattr(args, "deepspeed", None):
+                if not hasattr(args, "gradient_checkpointing_kwargs") or not args.gradient_checkpointing_kwargs:
+                    args.gradient_checkpointing_kwargs = {"use_reentrant": True}
+                elif "use_reentrant" not in args.gradient_checkpointing_kwargs:
+                    args.gradient_checkpointing_kwargs["use_reentrant"] = True
+                logger.info("HYV3 patch applied: gradient_checkpointing use_reentrant=True.")
+
+            return control
+
+        def on_save(self, args, state, control, **kwargs):
+            # --- Patch 4: Copy tokenizer files ---
+            if torch.distributed.is_initialized() and torch.distributed.get_rank() != 0:
+                return control
+
+            checkpoint_dir = os.path.join(
+                args.output_dir,
+                f"{PREFIX_CHECKPOINT_DIR}-{state.global_step}",
+            )
+
+            # Determine tokenizer directory
+            tokenizer_dir = self._tokenizer_dir
+            if tokenizer_dir is None:
+                # Try common locations
+                tokenizer_dir = getattr(args, "tokenizer_name_or_path", None)
+                if tokenizer_dir is None:
+                    tokenizer_dir = getattr(args, "model_name_or_path", None)
+
+            if tokenizer_dir and os.path.isdir(tokenizer_dir):
+                _copy_tokenizer_to_checkpoint(tokenizer_dir, checkpoint_dir)
+                logger.info(
+                    "HYV3: Copied tokenizer files from %s to %s",
+                    tokenizer_dir, checkpoint_dir
+                )
+
+            return control
+
+except ImportError:
+    logger.warning(
+        "transformers not available; HYV3PatchCallback not defined."
+    )
+
+# ============================================================================
+# Auto-apply patches on import
+# ============================================================================
+
+# Patch 2: Router dtype fix
+_apply_router_dtype_patch()
+
+# Patches 3, 4 are applied via HYV3PatchCallback during training.
+# Users should add HYV3PatchCallback to their Trainer callbacks.
+
+logger.info(
+    "HYV3 patches module loaded. Patch 2 (Router dtype fix) applied. "
+    "Remember to add HYV3PatchCallback to your Trainer callbacks "
+    "for gradient_checkpointing and tokenizer copy support."
+)

train/llama_factory_support/hy_v3_template.py

@@ -0,0 +1,46 @@
+"""
+HYV3 chat template registration for LLaMA Factory.
+
+Usage:
+    1. Copy this file's register_template block into LLaMA Factory's
+       src/llamafactory/data/template.py  (for upstream MR).
+    2. Or import this module before training to register at runtime:
+       import hy_v3_template
+"""
+
+from llamafactory.data.template import ReasoningTemplate, register_template
+from llamafactory.data.formatter import EmptyFormatter, StringFormatter
+
+
+# ---------------------------------------------------------------------------
+# HYV3 (MoE, pure text) chat template
+#
+# Token format (from chat_template.jinja & tokenizer_config.json):
+#   BOS:        <｜hy_begin▁of▁sentence｜>
+#   System:    {system_content}                (directly after BOS, no role tag)
+#   User:       <｜hy_User｜>{user_content}
+#   Assistant:  <｜hy_Assistant｜>{assistant_content}<｜hy_eos｜>
+#   EOS:       <｜hy_eos｜>
+#
+# Loss mask: only compute loss on assistant content (including <｜hy_eos｜>).
+#
+# Note: The system message has NO explicit role token -- it is placed right
+# after BOS.  The eos_token is <｜hy_eos｜>.
+#
+# Reasoning: Supports think tags via ReasoningTemplate.
+#   - thought_words: ("<think>", "</think>") matching jinja template
+#   - enable_thinking: set globally via data_args.enable_thinking (default True)
+#   - Training data always includes think tags (empty or with content)
+# ---------------------------------------------------------------------------
+
+register_template(
+    name="hy_v3",
+    template_class=ReasoningTemplate,
+    format_user=StringFormatter(slots=["<｜hy_User｜>{{content}}"]),
+    format_assistant=StringFormatter(slots=["<｜hy_Assistant｜>{{content}}", {"eos_token"}]),
+    format_system=StringFormatter(slots=["{{content}}"]),
+    format_prefix=EmptyFormatter(slots=[{"bos_token"}]),
+    thought_words=("<think>", "</think>"),
+    stop_words=["<｜hy_eos｜>"],
+    efficient_eos=True,
+)

train/llama_factory_support/train_hy_dense.py

@@ -0,0 +1,81 @@
+"""
+LLaMA Factory training entry-point wrapper for HunYuan Dense models.
+
+This script:
+  1. Registers the hy_dense_1_8b and hy_dense_7b chat templates
+  2. Injects a lightweight PatchCallback (tokenizer copy + gradient checkpointing fix)
+  3. Calls run_exp() to start LLaMA Factory training
+
+How it works:
+  - train_lf_dense.sh launches this script via torchrun directly:
+        torchrun ... train_hy_dense.py hy_dense_1_8b_full_sft.yaml
+  - Each torchrun worker executes this script, so all patches are applied
+    in every worker process before training begins.
+  - We call run_exp() directly (not the CLI launcher) to avoid the
+    launcher re-spawning workers and losing our patches.
+
+Note:
+  Dense models do NOT need MoE-specific patches (router dtype fix, expert
+  key rename, etc.). Only the tokenizer copy callback and gradient
+  checkpointing fix are needed.
+
+Usage:
+    # Via launch script (recommended):
+    bash train_lf_dense.sh
+
+    # Direct single-node (1 GPU, 1.8B model):
+    torchrun --nproc_per_node 1 train_hy_dense.py hy_dense_1_8b_full_sft.yaml
+
+    # Direct single-node (2 GPUs, 7B model):
+    torchrun --nproc_per_node 2 train_hy_dense.py hy_dense_7b_full_sft.yaml
+"""
+
+import sys
+import os
+
+# Add current directory to path so templates can be imported
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+# Step 1: Register Dense model templates (must be before training starts)
+import hy_dense_template  # noqa: F401
+
+# Step 2: Import the patch callback (reuse HYV3PatchCallback for tokenizer copy)
+# The MoE router patch will be silently skipped since Dense models don't have
+# HYV3TopKRouter. Only Patch 3 (gradient_checkpointing) and Patch 4 (tokenizer
+# copy) will be effective.
+import hy_v3_patches  # noqa: F401
+
+# Step 3: Inject PatchCallback into LLaMA Factory's training flow
+from llamafactory.train.sft.workflow import run_sft as _orig_run_sft
+
+
+def _patched_run_sft(model_args, data_args, training_args, finetuning_args, generating_args, callbacks=None):
+    """Wrap run_sft to inject HYV3PatchCallback for tokenizer copy."""
+    if callbacks is None:
+        callbacks = []
+
+    # Determine tokenizer directory for the save callback
+    tokenizer_dir = getattr(model_args, "model_name_or_path", None)
+    callbacks.append(hy_v3_patches.HYV3PatchCallback(tokenizer_dir=tokenizer_dir))
+
+    return _orig_run_sft(model_args, data_args, training_args, finetuning_args, generating_args, callbacks=callbacks)
+
+
+# Monkey-patch the SFT workflow
+import llamafactory.train.sft.workflow as _sft_wf
+_sft_wf.run_sft = _patched_run_sft
+
+
+def main():
+    """Entry point: called by torchrun in each worker process.
+
+    Since train_lf_dense.sh launches us via torchrun directly, all patches
+    (template registration, tokenizer copy callback injection) are already
+    applied in this process. We just call run_exp() to start training.
+    """
+    from llamafactory.train.tuner import run_exp
+    run_exp()
+
+
+if __name__ == "__main__":
+    main()

train/llama_factory_support/train_hy_v3.py

@@ -0,0 +1,72 @@
+"""
+LLaMA Factory training entry-point wrapper for HYV3.
+
+This script:
+  1. Registers the hy_v3 chat template
+  2. Applies all HYV3 monkey-patches (checkpoint key rename, dtype fix, etc.)
+  3. Injects HYV3PatchCallback into the training loop
+  4. Calls run_exp() to start LLaMA Factory training
+
+How it works:
+  - train_lf.sh launches this script via torchrun directly:
+        torchrun ... train_hy_v3.py hy_v3_full_sft.yaml
+  - Each torchrun worker executes this script, so all patches are applied
+    in every worker process before training begins.
+  - We call run_exp() directly (not the CLI launcher) to avoid the
+    launcher re-spawning workers and losing our patches.
+
+Usage:
+    # Via launch script (recommended):
+    bash train_lf.sh
+
+    # Direct single-node (8 GPUs):
+    torchrun --nproc_per_node 8 train_hy_v3.py hy_v3_full_sft.yaml
+"""
+
+import sys
+import os
+
+# Add current directory to path so patches can be imported
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+# Step 1: Register HYV3 template (must be before training starts)
+import hy_v3_template  # noqa: F401
+
+# Step 2: Apply checkpoint key rename patch (must be before model loading)
+import hy_v3_patches  # noqa: F401
+
+# Step 3: Inject HYV3PatchCallback into LLaMA Factory's training flow
+from llamafactory.train.sft.workflow import run_sft as _orig_run_sft
+
+
+def _patched_run_sft(model_args, data_args, training_args, finetuning_args, generating_args, callbacks=None):
+    """Wrap run_sft to inject HYV3PatchCallback."""
+    if callbacks is None:
+        callbacks = []
+
+    # Determine tokenizer directory for the save callback
+    tokenizer_dir = getattr(model_args, "model_name_or_path", None)
+    callbacks.append(hy_v3_patches.HYV3PatchCallback(tokenizer_dir=tokenizer_dir))
+
+    return _orig_run_sft(model_args, data_args, training_args, finetuning_args, generating_args, callbacks=callbacks)
+
+
+# Monkey-patch the SFT workflow
+import llamafactory.train.sft.workflow as _sft_wf
+_sft_wf.run_sft = _patched_run_sft
+
+
+def main():
+    """Entry point: called by torchrun in each worker process.
+
+    Since train_lf.sh launches us via torchrun directly, all patches
+    (template registration, checkpoint key rename, SFT callback injection)
+    are already applied in this process.  We just call run_exp() to start
+    training — no need to go through the CLI launcher.
+    """
+    from llamafactory.train.tuner import run_exp
+    run_exp()
+
+
+if __name__ == "__main__":
+    main()

train/llama_factory_support/train_lf.sh

@@ -0,0 +1,114 @@
+#!/bin/bash
+# ============================================================================
+# LLaMA Factory training launch script for HYV3
+#
+# This script sets up the environment and launches training via torchrun.
+#
+# We use train_hy_v3.py as the entry point (not llamafactory-cli)
+# because we need to inject HYV3-specific monkey-patches and register
+# the hy_v3 chat template BEFORE LLaMA Factory starts.
+# train_hy_v3.py directly calls run_exp() in each torchrun worker,
+# ensuring all patches are active.
+#
+# Usage:
+#   Single node:  bash train_lf.sh
+#   Multi-node:   Run this script on EACH node with the same IP_LIST.
+#                 IP_LIST="10.0.0.1,10.0.0.2" bash train_lf.sh
+# ============================================================================
+
+set -euo pipefail
+
+# -------------------- Network Configuration --------------------
+NET_TYPE="high"
+export NCCL_DEBUG=WARN
+export NCCL_P2P_LEVEL=NVL
+export NCCL_IB_TIMEOUT=24
+export NCCL_NVLS_ENABLE=0
+export NCCL_MPI_PROFILE_PRIMS_ENABLE=0
+export CUDA_DEVICE_MAX_CONNECTIONS=1
+export TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC=3600
+if [[ "${NET_TYPE}" = "low" ]]; then
+    export NCCL_SOCKET_IFNAME=eth1
+    export NCCL_IB_GID_INDEX=3
+    export NCCL_IB_HCA=mlx5_2:1
+    export NCCL_IB_SL=3
+    export NCCL_CHECK_DISABLE=1
+    export NCCL_P2P_DISABLE=0
+    export NCCL_LL_THRESHOLD=16384
+    export NCCL_IB_CUDA_SUPPORT=1
+else
+    export NCCL_IB_GID_INDEX=3
+    export NCCL_IB_SL=3
+    export NCCL_CHECK_DISABLE=1
+    export NCCL_P2P_DISABLE=0
+    export NCCL_IB_DISABLE=0
+    export NCCL_LL_THRESHOLD=16384
+    export NCCL_IB_CUDA_SUPPORT=1
+    export NCCL_SOCKET_IFNAME=bond1
+    export UCX_NET_DEVICES=bond1
+    export NCCL_IB_HCA=mlx5_bond_1,mlx5_bond_5,mlx5_bond_3,mlx5_bond_7,mlx5_bond_4,mlx5_bond_8,mlx5_bond_2,mlx5_bond_6
+    export NCCL_COLLNET_ENABLE=0
+    export SHARP_COLL_ENABLE_SAT=0
+    export NCCL_NET_GDR_LEVEL=2
+    export NCCL_IB_QPS_PER_CONNECTION=4
+    export NCCL_IB_TC=160
+    export NCCL_PXN_DISABLE=1
+fi
+
+# Skip LLaMA Factory version check (we use a newer transformers branch)
+export DISABLE_VERSION_CHECK=1
+
+# -------------------- Node Configuration --------------------
+export HOST_GPU_NUM=8
+# IP list, comma separated. e.g. "10.0.0.1,10.0.0.2" or single node "127.0.0.1"
+export IP_LIST=${IP_LIST:-"127.0.0.1"}
+
+MASTER_PORT=${MASTER_PORT:-29500}
+
+IFS=',' read -ra IP_ARRAY <<< "$IP_LIST"
+NODES=${#IP_ARRAY[@]}
+MASTER_ADDR=${IP_ARRAY[0]}
+
+# -------------------- Paths --------------------
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+YAML_FILE="${YAML_FILE:-${SCRIPT_DIR}/hy_v3_full_sft.yaml}"
+ENTRY_SCRIPT="${SCRIPT_DIR}/train_hy_v3.py"
+
+# -------------------- Distributed Environment --------------------
+export MASTER_ADDR="${MASTER_ADDR}"
+export MASTER_PORT="${MASTER_PORT}"
+export NNODES="${NODES}"
+
+if [ ${NODES} -gt 1 ]; then
+    # Determine local node rank by matching local IP against IP_LIST
+    LOCAL_IP=$(hostname -i | awk '{print $1}')
+    NODE_RANK=0
+    for i in "${!IP_ARRAY[@]}"; do
+        if [[ "${IP_ARRAY[$i]}" == "${LOCAL_IP}" ]]; then
+            NODE_RANK=$i
+            break
+        fi
+    done
+    export RANK="${NODE_RANK}"
+else
+    export RANK=0
+fi
+
+echo "============================================"
+echo "  HYV3 LLaMA Factory Training"
+echo "  Nodes: ${NNODES}, Rank: ${RANK}"
+echo "  Master: ${MASTER_ADDR}:${MASTER_PORT}"
+echo "  GPUs per node: ${HOST_GPU_NUM}"
+echo "  Total GPUs: $((NODES * HOST_GPU_NUM))"
+echo "============================================"
+
+# -------------------- Launch --------------------
+# We launch torchrun directly (instead of FORCE_TORCHRUN) so that each
+# worker process runs train_hy_v3.py with all HYV3 patches applied.
+torchrun \
+    --nnodes "${NNODES}" \
+    --node_rank "${RANK}" \
+    --nproc_per_node "${HOST_GPU_NUM}" \
+    --master_addr "${MASTER_ADDR}" \
+    --master_port "${MASTER_PORT}" \
+    "${ENTRY_SCRIPT}" "${YAML_FILE}"

train/llama_factory_support/train_lf_dense.sh

@@ -0,0 +1,124 @@
+#!/bin/bash
+# ============================================================================
+# LLaMA Factory training launch script for HunYuan Dense models (1.8B / 7B)
+#
+# This script sets up the environment and launches training via torchrun.
+#
+# We use train_hy_dense.py as the entry point (not llamafactory-cli)
+# because we need to register the hy_dense chat templates and inject
+# the PatchCallback BEFORE LLaMA Factory starts.
+# train_hy_dense.py directly calls run_exp() in each torchrun worker,
+# ensuring all patches are active.
+#
+# Usage:
+#   Single node (1.8B, default):
+#     bash train_lf_dense.sh
+#
+#   Single node (7B):
+#     YAML_FILE=hy_dense_7b_full_sft.yaml bash train_lf_dense.sh
+#
+#   Single node (LoRA 1.8B):
+#     YAML_FILE=hy_dense_1_8b_lora_sft.yaml bash train_lf_dense.sh
+#
+#   Multi-node:
+#     Run this script on EACH node with the same IP_LIST.
+#     IP_LIST="10.0.0.1,10.0.0.2" bash train_lf_dense.sh
+# ============================================================================
+
+set -euo pipefail
+
+# -------------------- Network Configuration --------------------
+NET_TYPE="high"
+export NCCL_DEBUG=WARN
+export NCCL_P2P_LEVEL=NVL
+export NCCL_IB_TIMEOUT=24
+export NCCL_NVLS_ENABLE=0
+export NCCL_MPI_PROFILE_PRIMS_ENABLE=0
+export CUDA_DEVICE_MAX_CONNECTIONS=1
+export TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC=3600
+if [[ "${NET_TYPE}" = "low" ]]; then
+    export NCCL_SOCKET_IFNAME=eth1
+    export NCCL_IB_GID_INDEX=3
+    export NCCL_IB_HCA=mlx5_2:1
+    export NCCL_IB_SL=3
+    export NCCL_CHECK_DISABLE=1
+    export NCCL_P2P_DISABLE=0
+    export NCCL_LL_THRESHOLD=16384
+    export NCCL_IB_CUDA_SUPPORT=1
+else
+    export NCCL_IB_GID_INDEX=3
+    export NCCL_IB_SL=3
+    export NCCL_CHECK_DISABLE=1
+    export NCCL_P2P_DISABLE=0
+    export NCCL_IB_DISABLE=0
+    export NCCL_LL_THRESHOLD=16384
+    export NCCL_IB_CUDA_SUPPORT=1
+    export NCCL_SOCKET_IFNAME=bond1
+    export UCX_NET_DEVICES=bond1
+    export NCCL_IB_HCA=mlx5_bond_1,mlx5_bond_5,mlx5_bond_3,mlx5_bond_7,mlx5_bond_4,mlx5_bond_8,mlx5_bond_2,mlx5_bond_6
+    export NCCL_COLLNET_ENABLE=0
+    export SHARP_COLL_ENABLE_SAT=0
+    export NCCL_NET_GDR_LEVEL=2
+    export NCCL_IB_QPS_PER_CONNECTION=4
+    export NCCL_IB_TC=160
+    export NCCL_PXN_DISABLE=1
+fi
+
+# Skip LLaMA Factory version check (we use a newer transformers branch)
+export DISABLE_VERSION_CHECK=1
+
+# -------------------- Node Configuration --------------------
+export HOST_GPU_NUM=8
+# IP list, comma separated. e.g. "10.0.0.1,10.0.0.2" or single node "127.0.0.1"
+export IP_LIST=${IP_LIST:-"127.0.0.1"}
+
+MASTER_PORT=${MASTER_PORT:-29500}
+
+IFS=',' read -ra IP_ARRAY <<< "$IP_LIST"
+NODES=${#IP_ARRAY[@]}
+MASTER_ADDR=${IP_ARRAY[0]}
+
+# -------------------- Paths --------------------
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+YAML_FILE="${YAML_FILE:-${SCRIPT_DIR}/hy_dense_1_8b_full_sft.yaml}"
+ENTRY_SCRIPT="${SCRIPT_DIR}/train_hy_dense.py"
+
+# -------------------- Distributed Environment --------------------
+export MASTER_ADDR="${MASTER_ADDR}"
+export MASTER_PORT="${MASTER_PORT}"
+export NNODES="${NODES}"
+
+if [ ${NODES} -gt 1 ]; then
+    # Determine local node rank by matching local IP against IP_LIST
+    LOCAL_IP=$(hostname -i | awk '{print $1}')
+    NODE_RANK=0
+    for i in "${!IP_ARRAY[@]}"; do
+        if [[ "${IP_ARRAY[$i]}" == "${LOCAL_IP}" ]]; then
+            NODE_RANK=$i
+            break
+        fi
+    done
+    export RANK="${NODE_RANK}"
+else
+    export RANK=0
+fi
+
+echo "============================================"
+echo "  HunYuan Dense LLaMA Factory Training"
+echo "  Nodes: ${NNODES}, Rank: ${RANK}"
+echo "  Master: ${MASTER_ADDR}:${MASTER_PORT}"
+echo "  GPUs per node: ${HOST_GPU_NUM}"
+echo "  Total GPUs: $((NODES * HOST_GPU_NUM))"
+echo "  YAML: ${YAML_FILE}"
+echo "============================================"
+
+# -------------------- Launch --------------------
+# We launch torchrun directly (instead of FORCE_TORCHRUN) so that each
+# worker process runs train_hy_dense.py with all Dense patches applied.
+torchrun \
+    --nnodes "${NNODES}" \
+    --node_rank "${RANK}" \
+    --nproc_per_node "${HOST_GPU_NUM}" \
+    --master_addr "${MASTER_ADDR}" \
+    --master_port "${MASTER_PORT}" \
+    "${ENTRY_SCRIPT}" "${YAML_FILE}"