初始化项目，由ModelHub XC社区提供模型

Model: jiamingshan/AHA-L2A-Qwen3-1.7B-repro Source: Original Platform
2026-07-21 11:06:13 +08:00
commit c13d63b439
48 changed files with 165513 additions and 0 deletions
--- a/recipe/scripts/eval_cell.sh
+++ b/recipe/scripts/eval_cell.sh
@@ -0,0 +1,82 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [[ $# -ne 4 ]]; then
+  echo "Usage: $0 METHOD MODEL_PATH GPU BENCHMARK" >&2
+  exit 2
+fi
+
+METHOD="$1"
+MODEL_PATH="$2"
+GPU="$3"
+BENCHMARK="$4"
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+RECIPE="$(cd "$SCRIPT_DIR/.." && pwd)"
+REPO="$(cd "$RECIPE/.." && pwd)"
+EVAL_ROOT="${EVAL_ROOT:-$REPO/outputs/eval}"
+OUT="$EVAL_ROOT/$METHOD"
+LOG="$EVAL_ROOT/logs/$METHOD"
+TOKENIZER="${VANILLA_DIR:-$REPO}"
+HELMET_ROWS="$RECIPE/data/eval_inputs/helmet_icl_8k_n50_per_config.jsonl"
+MRCR_ROWS="$RECIPE/data/eval_inputs/mrcr_8k_2_4_8needle_n10_per_config.jsonl"
+RULER_LIMIT="${RULER_LIMIT:-20}"
+BABILONG_LIMIT="${BABILONG_LIMIT:-50}"
+AHA_GATE_HARD_THRESHOLD="${AHA_GATE_HARD_THRESHOLD:-0.5}"
+mkdir -p "$OUT" "$LOG"
+
+if [[ -f "$OUT/${BENCHMARK}.DONE" ]]; then
+  exit 0
+fi
+
+export CUDA_VISIBLE_DEVICES="$GPU"
+export TOKENIZERS_PARALLELISM=false
+export PYTHONUNBUFFERED=1
+export AHA_SPARSITY_STATS_PATH="$OUT/sparsity_${BENCHMARK}.json"
+export AHA_GATE_HARD_THRESHOLD
+export AHA_FORCE_FULL_DECODE=0
+export AHA_FORCE_FULL_PREFILL_TAIL=0
+export AHA_FORCE_FULL_HEADS=
+export AHA_FORCE_LOW_ALPHA_FULL_HEADS=0
+export AHA_DUO_PREFILL_FULL=0
+
+model_args="pretrained=${MODEL_PATH},trust_remote_code=True,dtype=bfloat16,max_length=16384,attn_implementation=sdpa"
+split_a="niah_single_1,niah_single_3,niah_multikey_2,niah_multiquery,ruler_vt,ruler_fwe,ruler_qa_hotpot"
+split_b="niah_single_2,niah_multikey_1,niah_multikey_3,niah_multivalue,ruler_cwe,ruler_qa_squad"
+
+case "$BENCHMARK" in
+  ruler_a|ruler_b)
+    tasks="$split_a"
+    [[ "$BENCHMARK" == ruler_b ]] && tasks="$split_b"
+    python -m lm_eval \
+      --model hf --model_args "$model_args" \
+      --tasks "$tasks" --metadata '{"max_seq_lengths":[8192]}' \
+      --batch_size 1 --limit "$RULER_LIMIT" --log_samples \
+      --output_path "$OUT/ruler8k_splits/${BENCHMARK#ruler_}/lm_eval" \
+      >"$LOG/${BENCHMARK}.log" 2>&1
+    ;;
+  babilong)
+    python -m lm_eval \
+      --model hf --model_args "$model_args" \
+      --tasks babilong_longctx --metadata '{"max_seq_lengths":"8k"}' \
+      --num_fewshot 2 --batch_size 1 --limit "$BABILONG_LIMIT" --log_samples \
+      --output_path "$OUT/babilong8k_qa1_qa5_n50" \
+      >"$LOG/babilong.log" 2>&1
+    ;;
+  helmet)
+    python "$SCRIPT_DIR/eval_qwen_external_longctx_pilot.py" \
+      --benchmark helmet_icl --model-path "$MODEL_PATH" --tokenizer-path "$TOKENIZER" \
+      --attn-implementation sdpa --rows "$HELMET_ROWS" \
+      --method "$METHOD" --output "$OUT/helmet_icl8k_n50.jsonl" \
+      >"$LOG/helmet.log" 2>&1
+    ;;
+  mrcr)
+    python "$SCRIPT_DIR/eval_qwen_external_longctx_pilot.py" \
+      --benchmark mrcr --model-path "$MODEL_PATH" --tokenizer-path "$TOKENIZER" \
+      --attn-implementation sdpa --rows "$MRCR_ROWS" \
+      --method "$METHOD" --output "$OUT/mrcr_8k_2_4_8needle_n10.jsonl" \
+      >"$LOG/mrcr.log" 2>&1
+    ;;
+  *) echo "Unknown benchmark: $BENCHMARK" >&2; exit 2 ;;
+esac
+
+touch "$OUT/${BENCHMARK}.DONE"