Refactor exporting NeMo models (#2362)

Refactors and extends model export support to include new NeMo Parakeet TDT int8 variants for English and Japanese, updating the Kotlin API, export scripts, test runners, and CI workflows.

- Added support for two new int8 model types in OfflineRecognizer.kt.
- Enhanced Python export scripts to perform dynamic quantization and metadata injection.
- Updated shell scripts and GitHub workflows to package, test, and publish int8 model artifacts.

scripts/nemo/fast-conformer-hybrid-transducer-ctc/README.md

@@ -23,5 +23,6 @@ This folder contains scripts for exporting models from
   - https://catalog.ngc.nvidia.com/orgs/nvidia/teams/nemo/models/stt_multilingual_fastconformer_hybrid_large_pc
 
   - https://catalog.ngc.nvidia.com/orgs/nvidia/teams/nemo/models/parakeet-tdt_ctc-110m
+  - https://huggingface.co/nvidia/parakeet-tdt_ctc-0.6b-ja
 
 to `sherpa-onnx`.

scripts/nemo/fast-conformer-hybrid-transducer-ctc/export-onnx-ctc-non-streaming.py

@@ -6,6 +6,7 @@ from typing import Dict
 import nemo.collections.asr as nemo_asr
 import onnx
 import torch
+from onnxruntime.quantization import QuantType, quantize_dynamic
 
 
 def get_args():
@@ -86,6 +87,12 @@ def main():
     }
     add_meta_data(filename, meta_data)
 
+    quantize_dynamic(
+        model_input="./model.onnx",
+        model_output="./model.int8.onnx",
+        weight_type=QuantType.QUInt8,
+    )
+
     print("preprocessor", asr_model.cfg.preprocessor)
     print(meta_data)
 

scripts/nemo/fast-conformer-hybrid-transducer-ctc/export-onnx-ctc.py

@@ -6,6 +6,7 @@ from typing import Dict
 import nemo.collections.asr as nemo_asr
 import onnx
 import torch
+from onnxruntime.quantization import QuantType, quantize_dynamic
 
 
 def get_args():
@@ -114,6 +115,11 @@ def main():
         "comment": "Only the CTC branch is exported",
     }
     add_meta_data(filename, meta_data)
+    quantize_dynamic(
+        model_input="./model.onnx",
+        model_output="./model.int8.onnx",
+        weight_type=QuantType.QUInt8,
+    )
 
     print(meta_data)
 

scripts/nemo/fast-conformer-hybrid-transducer-ctc/export-onnx-transducer-non-streaming.py

@@ -6,6 +6,7 @@ from typing import Dict
 import nemo.collections.asr as nemo_asr
 import onnx
 import torch
+from onnxruntime.quantization import QuantType, quantize_dynamic
 
 
 def get_args():
@@ -90,6 +91,13 @@ def main():
     }
     add_meta_data("encoder.onnx", meta_data)
 
+    for m in ["encoder", "decoder", "joiner"]:
+        quantize_dynamic(
+            model_input=f"{m}.onnx",
+            model_output=f"{m}.int8.onnx",
+            weight_type=QuantType.QUInt8,
+        )
+
     print(meta_data)
 
 

scripts/nemo/fast-conformer-hybrid-transducer-ctc/export-onnx-transducer.py

@@ -6,6 +6,7 @@ from typing import Dict
 import nemo.collections.asr as nemo_asr
 import onnx
 import torch
+from onnxruntime.quantization import QuantType, quantize_dynamic
 
 
 def get_args():
@@ -122,6 +123,13 @@ def main():
     }
     add_meta_data("encoder.onnx", meta_data)
 
+    for m in ["encoder", "decoder", "joiner"]:
+        quantize_dynamic(
+            model_input=f"{m}.onnx",
+            model_output=f"{m}.int8.onnx",
+            weight_type=QuantType.QUInt8,
+        )
+
     print(meta_data)
 
 

scripts/nemo/fast-conformer-hybrid-transducer-ctc/run-ctc-non-streaming.sh

@@ -19,6 +19,12 @@ log "Process $name at $url"
 d=sherpa-onnx-nemo-parakeet_tdt_ctc_110m-en-36000
 mkdir -p $d
 mv -v model.onnx $d/
+cp -v tokens.txt $d/
+ls -lh $d
+
+d=sherpa-onnx-nemo-parakeet_tdt_ctc_110m-en-36000-int8
+mkdir -p $d
+mv -v model.int8.onnx $d/
 mv -v tokens.txt $d/
 ls -lh $d
 
@@ -33,6 +39,12 @@ log "Process $name at $url"
 d=sherpa-onnx-nemo-fast-conformer-ctc-en-24500
 mkdir -p $d
 mv -v model.onnx $d/
+cp -v tokens.txt $d/
+ls -lh $d
+
+d=sherpa-onnx-nemo-fast-conformer-ctc-en-24500-int8
+mkdir -p $d
+mv -v model.int8.onnx $d/
 mv -v tokens.txt $d/
 ls -lh $d
 
@@ -45,6 +57,12 @@ doc="This collection contains the Spanish FastConformer Hybrid (CTC and Transduc
 d=sherpa-onnx-nemo-fast-conformer-ctc-es-1424
 mkdir -p $d
 mv -v model.onnx $d/
+cp -v tokens.txt $d/
+ls -lh $d
+
+d=sherpa-onnx-nemo-fast-conformer-ctc-es-1424-int8
+mkdir -p $d
+mv -v model.int8.onnx $d/
 mv -v tokens.txt $d/
 ls -lh $d
 
@@ -57,6 +75,12 @@ doc="This collection contains the Multilingual FastConformer Hybrid (Transducer
 d=sherpa-onnx-nemo-fast-conformer-ctc-en-de-es-fr-14288
 mkdir -p $d
 mv -v model.onnx $d/
+cp -v tokens.txt $d/
+ls -lh $d
+
+d=sherpa-onnx-nemo-fast-conformer-ctc-en-de-es-fr-14288-int8
+mkdir -p $d
+mv -v model.int8.onnx $d/
 mv -v tokens.txt $d/
 ls -lh $d
 
@@ -69,6 +93,12 @@ doc="This collection contains the Multilingual FastConformer Hybrid (Transducer
 d=sherpa-onnx-nemo-fast-conformer-ctc-be-de-en-es-fr-hr-it-pl-ru-uk-20k
 mkdir -p $d
 mv -v model.onnx $d/
+cp -v tokens.txt $d/
+ls -lh $d
+
+d=sherpa-onnx-nemo-fast-conformer-ctc-be-de-en-es-fr-hr-it-pl-ru-uk-20k-int8
+mkdir -p $d
+mv -v model.int8.onnx $d/
 mv -v tokens.txt $d/
 ls -lh $d
 
@@ -92,6 +122,16 @@ mkdir -p $d/test_wavs
 cp en.wav $d/test_wavs/0.wav
 cp -v $data/en-english.wav $d/test_wavs/1.wav
 
+d=sherpa-onnx-nemo-parakeet_tdt_ctc_110m-en-36000-int8
+python3 ./test-onnx-ctc-non-streaming.py \
+  --model $d/model.int8.onnx \
+  --tokens $d/tokens.txt \
+  --wav $data/en-english.wav
+mkdir -p $d/test_wavs
+
+cp en.wav $d/test_wavs/0.wav
+cp -v $data/en-english.wav $d/test_wavs/1.wav
+
 d=sherpa-onnx-nemo-fast-conformer-ctc-en-24500
 python3 ./test-onnx-ctc-non-streaming.py \
   --model $d/model.onnx \
@@ -101,6 +141,15 @@ mkdir -p $d/test_wavs
 cp en.wav $d/test_wavs/0.wav
 cp -v $data/en-english.wav $d/test_wavs
 
+d=sherpa-onnx-nemo-fast-conformer-ctc-en-24500-int8
+python3 ./test-onnx-ctc-non-streaming.py \
+  --model $d/model.int8.onnx \
+  --tokens $d/tokens.txt \
+  --wav $data/en-english.wav
+mkdir -p $d/test_wavs
+cp en.wav $d/test_wavs/0.wav
+cp -v $data/en-english.wav $d/test_wavs
+
 d=sherpa-onnx-nemo-fast-conformer-ctc-es-1424
 python3 ./test-onnx-ctc-non-streaming.py \
   --model $d/model.onnx \
@@ -109,6 +158,14 @@ python3 ./test-onnx-ctc-non-streaming.py \
 mkdir -p $d/test_wavs
 cp -v $data/es-spanish.wav $d/test_wavs
 
+d=sherpa-onnx-nemo-fast-conformer-ctc-es-1424-int8
+python3 ./test-onnx-ctc-non-streaming.py \
+  --model $d/model.int8.onnx \
+  --tokens $d/tokens.txt \
+  --wav $data/es-spanish.wav
+mkdir -p $d/test_wavs
+cp -v $data/es-spanish.wav $d/test_wavs
+
 d=sherpa-onnx-nemo-fast-conformer-ctc-en-de-es-fr-14288
 mkdir -p $d/test_wavs
 for w in en-english.wav de-german.wav es-spanish.wav fr-french.wav; do
@@ -119,6 +176,16 @@ for w in en-english.wav de-german.wav es-spanish.wav fr-french.wav; do
   cp -v $data/$w $d/test_wavs
 done
 
+d=sherpa-onnx-nemo-fast-conformer-ctc-en-de-es-fr-14288-int8
+mkdir -p $d/test_wavs
+for w in en-english.wav de-german.wav es-spanish.wav fr-french.wav; do
+  python3 ./test-onnx-ctc-non-streaming.py \
+    --model $d/model.int8.onnx \
+    --tokens $d/tokens.txt \
+    --wav $data/$w
+  cp -v $data/$w $d/test_wavs
+done
+
 d=sherpa-onnx-nemo-fast-conformer-ctc-be-de-en-es-fr-hr-it-pl-ru-uk-20k
 mkdir -p $d/test_wavs
 for w in en-english.wav de-german.wav es-spanish.wav fr-french.wav hr-croatian.wav it-italian.wav po-polish.wav ru-russian.wav uk-ukrainian.wav; do
@@ -128,3 +195,13 @@ for w in en-english.wav de-german.wav es-spanish.wav fr-french.wav hr-croatian.w
     --wav $data/$w
   cp -v $data/$w $d/test_wavs
 done
+
+d=sherpa-onnx-nemo-fast-conformer-ctc-be-de-en-es-fr-hr-it-pl-ru-uk-20k-int8
+mkdir -p $d/test_wavs
+for w in en-english.wav de-german.wav es-spanish.wav fr-french.wav hr-croatian.wav it-italian.wav po-polish.wav ru-russian.wav uk-ukrainian.wav; do
+  python3 ./test-onnx-ctc-non-streaming.py \
+    --model $d/model.int8.onnx \
+    --tokens $d/tokens.txt \
+    --wav $data/$w
+  cp -v $data/$w $d/test_wavs
+done

scripts/nemo/fast-conformer-hybrid-transducer-ctc/run-ctc.sh

@@ -17,11 +17,22 @@ ms=(
 for m in ${ms[@]}; do
   ./export-onnx-ctc.py --model $m
   d=sherpa-onnx-nemo-streaming-fast-conformer-ctc-en-${m}ms
+
+  d_int8=sherpa-onnx-nemo-streaming-fast-conformer-ctc-en-${m}ms-int8
+
   if [ ! -f $d/model.onnx ]; then
-    mkdir -p $d
+    mkdir -p $d $d_int8
     mv -v model.onnx $d/
-    mv -v tokens.txt $d/
+    cp -v tokens.txt $d/
+
+    mv -v model.int8.onnx $d_int8/
+    mv -v tokens.txt $d_int8/
+
+    echo "---$d---"
     ls -lh $d
+
+    echo "---$d_int8---"
+    ls -lh $d_int8
   fi
 done
 
@@ -29,8 +40,16 @@ done
 
 for m in ${ms[@]}; do
   d=sherpa-onnx-nemo-streaming-fast-conformer-ctc-en-${m}ms
+  echo "---$d---"
   python3 ./test-onnx-ctc.py \
     --model $d/model.onnx \
     --tokens $d/tokens.txt \
     --wav ./0.wav
+
+  d=sherpa-onnx-nemo-streaming-fast-conformer-ctc-en-${m}ms-int8
+  echo "---$d---"
+  python3 ./test-onnx-ctc.py \
+    --model $d/model.int8.onnx \
+    --tokens $d/tokens.txt \
+    --wav ./0.wav
 done

scripts/nemo/fast-conformer-hybrid-transducer-ctc/run-transducer-non-streaming.sh

@@ -18,7 +18,17 @@ log "Process $name at $url"
 ./export-onnx-transducer-non-streaming.py --model $name --doc "$doc"
 d=sherpa-onnx-nemo-parakeet_tdt_transducer_110m-en-36000
 mkdir -p $d
-mv -v *.onnx $d/
+mv -v encoder.onnx $d/
+mv -v decoder.onnx $d/
+mv -v joiner.onnx $d/
+cp -v tokens.txt $d/
+ls -lh $d
+
+d=sherpa-onnx-nemo-parakeet_tdt_transducer_110m-en-36000-int8
+mkdir -p $d
+mv -v encoder.int8.onnx $d/
+mv -v decoder.int8.onnx $d/
+mv -v joiner.int8.onnx $d/
 mv -v tokens.txt $d/
 ls -lh $d
 
@@ -32,7 +42,17 @@ log "Process $name at $url"
 
 d=sherpa-onnx-nemo-fast-conformer-transducer-en-24500
 mkdir -p $d
-mv -v *.onnx $d/
+mv -v encoder.onnx $d/
+mv -v decoder.onnx $d/
+mv -v joiner.onnx $d/
+cp -v tokens.txt $d/
+ls -lh $d
+
+d=sherpa-onnx-nemo-fast-conformer-transducer-en-24500-int8
+mkdir -p $d
+mv -v encoder.int8.onnx $d/
+mv -v decoder.int8.onnx $d/
+mv -v joiner.int8.onnx $d/
 mv -v tokens.txt $d/
 ls -lh $d
 
@@ -44,7 +64,17 @@ doc="This collection contains the Spanish FastConformer Hybrid (CTC and Transduc
 
 d=sherpa-onnx-nemo-fast-conformer-transducer-es-1424
 mkdir -p $d
-mv -v *.onnx $d/
+mv -v encoder.onnx $d/
+mv -v decoder.onnx $d/
+mv -v joiner.onnx $d/
+cp -v tokens.txt $d/
+ls -lh $d
+
+d=sherpa-onnx-nemo-fast-conformer-transducer-es-1424-int8
+mkdir -p $d
+mv -v encoder.int8.onnx $d/
+mv -v decoder.int8.onnx $d/
+mv -v joiner.int8.onnx $d/
 mv -v tokens.txt $d/
 ls -lh $d
 
@@ -56,7 +86,17 @@ doc="This collection contains the Multilingual FastConformer Hybrid (Transducer
 
 d=sherpa-onnx-nemo-fast-conformer-transducer-en-de-es-fr-14288
 mkdir -p $d
-mv -v *.onnx $d/
+mv -v encoder.onnx $d/
+mv -v decoder.onnx $d/
+mv -v joiner.onnx $d/
+cp -v tokens.txt $d/
+ls -lh $d
+
+d=sherpa-onnx-nemo-fast-conformer-transducer-en-de-es-fr-14288-int8
+mkdir -p $d
+mv -v encoder.int8.onnx $d/
+mv -v decoder.int8.onnx $d/
+mv -v joiner.int8.onnx $d/
 mv -v tokens.txt $d/
 ls -lh $d
 
@@ -68,7 +108,17 @@ doc="This collection contains the Multilingual FastConformer Hybrid (Transducer
 
 d=sherpa-onnx-nemo-fast-conformer-transducer-be-de-en-es-fr-hr-it-pl-ru-uk-20k
 mkdir -p $d
-mv -v *.onnx $d/
+mv -v encoder.onnx $d/
+mv -v decoder.onnx $d/
+mv -v joiner.onnx $d/
+cp -v tokens.txt $d/
+ls -lh $d
+
+d=sherpa-onnx-nemo-fast-conformer-transducer-be-de-en-es-fr-hr-it-pl-ru-uk-20k-int8
+mkdir -p $d
+mv -v encoder.int8.onnx $d/
+mv -v decoder.int8.onnx $d/
+mv -v joiner.int8.onnx $d/
 mv -v tokens.txt $d/
 ls -lh $d
 
@@ -101,6 +151,25 @@ mkdir -p $d/test_wavs
 cp en.wav $d/test_wavs/0.wav
 cp -v $data/en-english.wav $d/test_wavs
 
+d=sherpa-onnx-nemo-parakeet_tdt_transducer_110m-en-36000-int8
+python3 ./test-onnx-transducer-non-streaming.py \
+  --encoder $d/encoder.int8.onnx \
+  --decoder $d/decoder.int8.onnx \
+  --joiner $d/joiner.int8.onnx \
+  --tokens $d/tokens.txt \
+  --wav $data/en-english.wav
+
+python3 ./test-onnx-transducer-non-streaming.py \
+  --encoder $d/encoder.int8.onnx \
+  --decoder $d/decoder.int8.onnx \
+  --joiner $d/joiner.int8.onnx \
+  --tokens $d/tokens.txt \
+  --wav ./en.wav
+
+mkdir -p $d/test_wavs
+cp en.wav $d/test_wavs/0.wav
+cp -v $data/en-english.wav $d/test_wavs
+
 d=sherpa-onnx-nemo-fast-conformer-transducer-en-24500
 python3 ./test-onnx-transducer-non-streaming.py \
   --encoder $d/encoder.onnx \
@@ -112,6 +181,17 @@ mkdir -p $d/test_wavs
 cp en.wav $d/test_wavs/0.wav
 cp -v $data/en-english.wav $d/test_wavs
 
+d=sherpa-onnx-nemo-fast-conformer-transducer-en-24500-int8
+python3 ./test-onnx-transducer-non-streaming.py \
+  --encoder $d/encoder.int8.onnx \
+  --decoder $d/decoder.int8.onnx \
+  --joiner $d/joiner.int8.onnx \
+  --tokens $d/tokens.txt \
+  --wav $data/en-english.wav
+mkdir -p $d/test_wavs
+cp en.wav $d/test_wavs/0.wav
+cp -v $data/en-english.wav $d/test_wavs
+
 d=sherpa-onnx-nemo-fast-conformer-transducer-es-1424
 python3 ./test-onnx-transducer-non-streaming.py \
   --encoder $d/encoder.onnx \
@@ -122,6 +202,16 @@ python3 ./test-onnx-transducer-non-streaming.py \
 mkdir -p $d/test_wavs
 cp -v $data/es-spanish.wav $d/test_wavs
 
+d=sherpa-onnx-nemo-fast-conformer-transducer-es-1424-int8
+python3 ./test-onnx-transducer-non-streaming.py \
+  --encoder $d/encoder.int8.onnx \
+  --decoder $d/decoder.int8.onnx \
+  --joiner $d/joiner.int8.onnx \
+  --tokens $d/tokens.txt \
+  --wav $data/es-spanish.wav
+mkdir -p $d/test_wavs
+cp -v $data/es-spanish.wav $d/test_wavs
+
 d=sherpa-onnx-nemo-fast-conformer-transducer-en-de-es-fr-14288
 mkdir -p $d/test_wavs
 for w in en-english.wav de-german.wav es-spanish.wav fr-french.wav; do
@@ -134,6 +224,18 @@ for w in en-english.wav de-german.wav es-spanish.wav fr-french.wav; do
   cp -v $data/$w $d/test_wavs
 done
 
+d=sherpa-onnx-nemo-fast-conformer-transducer-en-de-es-fr-14288-int8
+mkdir -p $d/test_wavs
+for w in en-english.wav de-german.wav es-spanish.wav fr-french.wav; do
+  python3 ./test-onnx-transducer-non-streaming.py \
+    --encoder $d/encoder.int8.onnx \
+    --decoder $d/decoder.int8.onnx \
+    --joiner $d/joiner.int8.onnx \
+    --tokens $d/tokens.txt \
+    --wav $data/$w
+  cp -v $data/$w $d/test_wavs
+done
+
 d=sherpa-onnx-nemo-fast-conformer-transducer-be-de-en-es-fr-hr-it-pl-ru-uk-20k
 mkdir -p $d/test_wavs
 for w in en-english.wav de-german.wav es-spanish.wav fr-french.wav hr-croatian.wav it-italian.wav po-polish.wav ru-russian.wav uk-ukrainian.wav; do
@@ -145,3 +247,15 @@ for w in en-english.wav de-german.wav es-spanish.wav fr-french.wav hr-croatian.w
     --wav $data/$w
   cp -v $data/$w $d/test_wavs
 done
+
+d=sherpa-onnx-nemo-fast-conformer-transducer-be-de-en-es-fr-hr-it-pl-ru-uk-20k-int8
+mkdir -p $d/test_wavs
+for w in en-english.wav de-german.wav es-spanish.wav fr-french.wav hr-croatian.wav it-italian.wav po-polish.wav ru-russian.wav uk-ukrainian.wav; do
+  python3 ./test-onnx-transducer-non-streaming.py \
+    --encoder $d/encoder.int8.onnx \
+    --decoder $d/decoder.int8.onnx \
+    --joiner $d/joiner.int8.onnx \
+    --tokens $d/tokens.txt \
+    --wav $data/$w
+  cp -v $data/$w $d/test_wavs
+done

scripts/nemo/fast-conformer-hybrid-transducer-ctc/run-transducer.sh

@@ -17,13 +17,24 @@ ms=(
 for m in ${ms[@]}; do
   ./export-onnx-transducer.py --model $m
   d=sherpa-onnx-nemo-streaming-fast-conformer-transducer-en-${m}ms
+  d_int8=sherpa-onnx-nemo-streaming-fast-conformer-transducer-en-${m}ms-int8
   if [ ! -f $d/encoder.onnx ]; then
-    mkdir -p $d
+    mkdir -p $d $d_int8
     mv -v encoder.onnx $d/
     mv -v decoder.onnx $d/
     mv -v joiner.onnx $d/
-    mv -v tokens.txt $d/
+    cp -v tokens.txt $d/
+
+    mv -v encoder.int8.onnx $d_int8/
+    mv -v decoder.int8.onnx $d_int8/
+    mv -v joiner.int8.onnx $d_int8/
+    mv -v tokens.txt $d_int8/
+
+    echo "---$d---"
     ls -lh $d
+
+    echo "---$d_int8---"
+    ls -lh $d_int8
   fi
 done
 
@@ -37,4 +48,12 @@ for m in ${ms[@]}; do
     --joiner $d/joiner.onnx \
     --tokens $d/tokens.txt \
     --wav ./0.wav
+
+  d=sherpa-onnx-nemo-streaming-fast-conformer-transducer-en-${m}ms-int8
+  python3 ./test-onnx-transducer.py \
+    --encoder $d/encoder.int8.onnx \
+    --decoder $d/decoder.int8.onnx \
+    --joiner $d/joiner.int8.onnx \
+    --tokens $d/tokens.txt \
+    --wav ./0.wav
 done