Add Java and Kotlin API for NeMo Canary models (#2359)

Add support for the NeMo Canary model in both Java and Kotlin APIs, wiring it through
JNI and updating examples and CI.

- Introduce OfflineCanaryModelConfig in Kotlin and Java with builder patterns
- Extend OfflineRecognizer to accept and apply the new canary config via setConfig
- Update JNI binding (GetOfflineConfig) and getOfflineModelConfig mapping (type 32), 
   plus examples and CI workflows

.github/workflows/run-java-test.yaml

@@ -117,6 +117,13 @@ jobs:
           cd ./java-api-examples
           ./run-version-test.sh
 
+      - name:  Run java test (Nemo Canary)
+        shell: bash
+        run: |
+          cd ./java-api-examples
+          ./run-non-streaming-decode-file-nemo-canary.sh
+          rm -rf sherpa-onnx-nemo-*
+
       - name:  Run java test (Non-streaming SenseVoice with homophone replacer)
         shell: bash
         run: |

java-api-examples/NonStreamingDecodeFileNemoCanary.java

@@ -0,0 +1,56 @@
+// Copyright 2024 Xiaomi Corporation
+
+// This file shows how to use an offline NeMo Canary model, i.e.,
+// non-streaming NeMo Canary model, to decode files.
+import com.k2fsa.sherpa.onnx.*;
+
+public class NonStreamingDecodeFileNemoCanary {
+  public static void main(String[] args) {
+    // please refer to
+    // https://k2-fsa.github.io/sherpa/onnx/nemo/canary.html
+    // to download model files
+    String encoder = "./sherpa-onnx-nemo-canary-180m-flash-en-es-de-fr-int8/encoder.int8.onnx";
+    String decoder = "./sherpa-onnx-nemo-canary-180m-flash-en-es-de-fr-int8/decoder.int8.onnx";
+    String tokens = "./sherpa-onnx-nemo-canary-180m-flash-en-es-de-fr-int8/tokens.txt";
+
+    String waveFilename = "./sherpa-onnx-nemo-canary-180m-flash-en-es-de-fr-int8/test_wavs/en.wav";
+
+    WaveReader reader = new WaveReader(waveFilename);
+
+    OfflineCanaryModelConfig canary =
+        OfflineCanaryModelConfig.builder()
+            .setEncoder(encoder)
+            .setDecoder(decoder)
+            .setSrcLang("en")
+            .setTgtLang("en")
+            .setUsePnc(true)
+            .build();
+
+    OfflineModelConfig modelConfig =
+        OfflineModelConfig.builder()
+            .setCanary(canary)
+            .setTokens(tokens)
+            .setNumThreads(1)
+            .setDebug(true)
+            .build();
+
+    OfflineRecognizerConfig config =
+        OfflineRecognizerConfig.builder()
+            .setOfflineModelConfig(modelConfig)
+            .setDecodingMethod("greedy_search")
+            .build();
+
+    OfflineRecognizer recognizer = new OfflineRecognizer(config);
+    OfflineStream stream = recognizer.createStream();
+    stream.acceptWaveform(reader.getSamples(), reader.getSampleRate());
+
+    recognizer.decode(stream);
+
+    String text = recognizer.getResult(stream).getText();
+
+    System.out.printf("filename:%s\nresult(English):%s\n", waveFilename, text);
+
+    stream.release();
+    recognizer.release();
+  }
+}

java-api-examples/README.md

@@ -24,11 +24,18 @@ This directory contains examples for the JAVA API of sherpa-onnx.
 
 ```bash
 ./run-non-streaming-decode-file-dolphin-ctc.sh
+./run-non-streaming-decode-file-fire-red-asr.sh
+./run-non-streaming-decode-file-moonshine.sh
+./run-non-streaming-decode-file-nemo-canary.sh
+./run-non-streaming-decode-file-nemo.sh
 ./run-non-streaming-decode-file-paraformer.sh
 ./run-non-streaming-decode-file-sense-voice.sh
+./run-non-streaming-decode-file-tele-speech-ctc.sh
+./run-non-streaming-decode-file-transducer-hotwords.sh
 ./run-non-streaming-decode-file-transducer.sh
+./run-non-streaming-decode-file-whisper-multiple.sh
 ./run-non-streaming-decode-file-whisper.sh
-./run-non-streaming-decode-file-nemo.sh
+./run-non-streaming-decode-file-zipformer-ctc.sh
 ```
 
 ## Non-Streaming Speech recognition with homophone replacer

java-api-examples/run-non-streaming-decode-file-nemo-canary.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+
+set -ex
+
+if [[ ! -f ../build/lib/libsherpa-onnx-jni.dylib  && ! -f ../build/lib/libsherpa-onnx-jni.so ]]; then
+  mkdir -p ../build
+  pushd ../build
+  cmake \
+    -DSHERPA_ONNX_ENABLE_PYTHON=OFF \
+    -DSHERPA_ONNX_ENABLE_TESTS=OFF \
+    -DSHERPA_ONNX_ENABLE_CHECK=OFF \
+    -DBUILD_SHARED_LIBS=ON \
+    -DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF \
+    -DSHERPA_ONNX_ENABLE_JNI=ON \
+    ..
+
+  make -j4
+  ls -lh lib
+  popd
+fi
+
+if [ ! -f ../sherpa-onnx/java-api/build/sherpa-onnx.jar ]; then
+  pushd ../sherpa-onnx/java-api
+  make
+  popd
+fi
+
+if [ ! -f sherpa-onnx-nemo-canary-180m-flash-en-es-de-fr-int8/encoder.int8.onnx ]; then
+  curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-canary-180m-flash-en-es-de-fr-int8.tar.bz2
+  tar xvf sherpa-onnx-nemo-canary-180m-flash-en-es-de-fr-int8.tar.bz2
+  rm sherpa-onnx-nemo-canary-180m-flash-en-es-de-fr-int8.tar.bz2
+fi
+
+java \
+  -Djava.library.path=$PWD/../build/lib \
+  -cp ../sherpa-onnx/java-api/build/sherpa-onnx.jar \
+  NonStreamingDecodeFileNemoCanary.java

kotlin-api-examples/run.sh

@@ -455,8 +455,31 @@ function testOfflineSenseVoiceWithHr() {
   ls -lh $out_filename
   java -Djava.library.path=../build/lib -jar $out_filename
 }
-testVersion
 
+function testOfflineNeMoCanary() {
+  if [ ! -f sherpa-onnx-nemo-canary-180m-flash-en-es-de-fr-int8/encoder.int8.onnx ]; then
+    curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-canary-180m-flash-en-es-de-fr-int8.tar.bz2
+    tar xvf sherpa-onnx-nemo-canary-180m-flash-en-es-de-fr-int8.tar.bz2
+    rm sherpa-onnx-nemo-canary-180m-flash-en-es-de-fr-int8.tar.bz2
+  fi
+
+  out_filename=test_offline_nemo_canary.jar
+  kotlinc-jvm -include-runtime -d $out_filename \
+    test_offline_nemo_canary.kt \
+    FeatureConfig.kt \
+    HomophoneReplacerConfig.kt \
+    OfflineRecognizer.kt \
+    OfflineStream.kt \
+    WaveReader.kt \
+    faked-asset-manager.kt
+
+  ls -lh $out_filename
+  java -Djava.library.path=../build/lib -jar $out_filename
+}
+
+# testVersion
+
+testOfflineNeMoCanary
 testOfflineSenseVoiceWithHr
 testOfflineSpeechDenoiser
 testOfflineSpeakerDiarization

kotlin-api-examples/test_offline_nemo_canary.kt

@@ -0,0 +1,48 @@
+package com.k2fsa.sherpa.onnx
+
+fun main() {
+  val recognizer = createOfflineRecognizer()
+  val waveFilename = "./sherpa-onnx-nemo-canary-180m-flash-en-es-de-fr-int8/test_wavs/en.wav"
+
+  val objArray = WaveReader.readWaveFromFile(
+      filename = waveFilename,
+  )
+  val samples: FloatArray = objArray[0] as FloatArray
+  val sampleRate: Int = objArray[1] as Int
+
+  var stream = recognizer.createStream()
+  stream.acceptWaveform(samples, sampleRate=sampleRate)
+  recognizer.decode(stream)
+
+  var result = recognizer.getResult(stream)
+  println("English: $result")
+
+  stream.release()
+
+  // now output text in German
+  val config = recognizer.config.copy(modelConfig=recognizer.config.modelConfig.copy(
+    canary=recognizer.config.modelConfig.canary.copy(
+      tgtLang="de"
+    )
+  ))
+  recognizer.setConfig(config)
+
+  stream = recognizer.createStream()
+  stream.acceptWaveform(samples, sampleRate=sampleRate)
+  recognizer.decode(stream)
+
+  result = recognizer.getResult(stream)
+  println("German: $result")
+
+  stream.release()
+  recognizer.release()
+}
+
+
+fun createOfflineRecognizer(): OfflineRecognizer {
+  val config = OfflineRecognizerConfig(
+      modelConfig = getOfflineModelConfig(type = 32)!!,
+  )
+
+  return OfflineRecognizer(config = config)
+}

sherpa-onnx/java-api/Makefile

@@ -34,6 +34,7 @@ java_files += OfflineFireRedAsrModelConfig.java
 java_files += OfflineMoonshineModelConfig.java
 java_files += OfflineNemoEncDecCtcModelConfig.java
 java_files += OfflineZipformerCtcModelConfig.java
+java_files += OfflineCanaryModelConfig.java
 java_files += OfflineSenseVoiceModelConfig.java
 java_files += OfflineDolphinModelConfig.java
 java_files += OfflineModelConfig.java

sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineCanaryModelConfig.java

@@ -0,0 +1,80 @@
+// Copyright 2025 Xiaomi Corporation
+
+package com.k2fsa.sherpa.onnx;
+
+public class OfflineCanaryModelConfig {
+    private final String encoder;
+    private final String decoder;
+    private final String srcLang;
+    private final String tgtLang;
+    private final boolean usePnc;
+
+    private OfflineCanaryModelConfig(Builder builder) {
+        this.encoder = builder.encoder;
+        this.decoder = builder.decoder;
+        this.srcLang = builder.srcLang;
+        this.tgtLang = builder.tgtLang;
+        this.usePnc = builder.usePnc;
+    }
+
+    public static Builder builder() {
+        return new Builder();
+    }
+
+    public String getEncoder() {
+        return encoder;
+    }
+
+    public String getDecoder() {
+        return decoder;
+    }
+
+    public String getSrcLang() {
+        return srcLang;
+    }
+
+    public String getTgtLang() {
+        return tgtLang;
+    }
+
+    public boolean isUsePnc() {
+        return usePnc;
+    }
+
+    public static class Builder {
+        private String encoder = "";
+        private String decoder = "";
+        private String srcLang = "en";
+        private String tgtLang = "en";
+        private boolean usePnc = true;
+
+        public OfflineCanaryModelConfig build() {
+            return new OfflineCanaryModelConfig(this);
+        }
+
+        public Builder setEncoder(String encoder) {
+            this.encoder = encoder;
+            return this;
+        }
+
+        public Builder setDecoder(String decoder) {
+            this.decoder = decoder;
+            return this;
+        }
+
+        public Builder setSrcLang(String srcLang) {
+            this.srcLang = srcLang;
+            return this;
+        }
+
+        public Builder setTgtLang(String tgtLang) {
+            this.tgtLang = tgtLang;
+            return this;
+        }
+
+        public Builder setUsePnc(boolean usePnc) {
+            this.usePnc = usePnc;
+            return this;
+        }
+    }
+}

sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineModelConfig.java

@@ -12,6 +12,7 @@ public class OfflineModelConfig {
     private final OfflineSenseVoiceModelConfig senseVoice;
     private final OfflineDolphinModelConfig dolphin;
     private final OfflineZipformerCtcModelConfig zipformerCtc;
+    private final OfflineCanaryModelConfig canary;
     private final String teleSpeech;
     private final String tokens;
     private final int numThreads;
@@ -30,6 +31,7 @@ public class OfflineModelConfig {
         this.moonshine = builder.moonshine;
         this.nemo = builder.nemo;
         this.zipformerCtc = builder.zipformerCtc;
+        this.canary = builder.canary;
         this.senseVoice = builder.senseVoice;
         this.dolphin = builder.dolphin;
         this.teleSpeech = builder.teleSpeech;
@@ -78,6 +80,10 @@ public class OfflineModelConfig {
         return zipformerCtc;
     }
 
+    public OfflineCanaryModelConfig getCanary() {
+        return canary;
+    }
+
     public String getTokens() {
         return tokens;
     }
@@ -120,6 +126,7 @@ public class OfflineModelConfig {
         private OfflineSenseVoiceModelConfig senseVoice = OfflineSenseVoiceModelConfig.builder().build();
         private OfflineDolphinModelConfig dolphin = OfflineDolphinModelConfig.builder().build();
         private OfflineZipformerCtcModelConfig zipformerCtc = OfflineZipformerCtcModelConfig.builder().build();
+        private OfflineCanaryModelConfig canary = OfflineCanaryModelConfig.builder().build();
         private String teleSpeech = "";
         private String tokens = "";
         private int numThreads = 1;
@@ -158,6 +165,11 @@ public class OfflineModelConfig {
             return this;
         }
 
+        public Builder setCanary(OfflineCanaryModelConfig canary) {
+            this.canary = canary;
+            return this;
+        }
+
         public Builder setTeleSpeech(String teleSpeech) {
             this.teleSpeech = teleSpeech;
             return this;

sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineRecognizer.java

@@ -4,10 +4,22 @@ package com.k2fsa.sherpa.onnx;
 
 public class OfflineRecognizer {
     private long ptr = 0;
+    private final OfflineRecognizerConfig config;
 
     public OfflineRecognizer(OfflineRecognizerConfig config) {
         LibraryLoader.maybeLoad();
         ptr = newFromFile(config);
+
+        this.config = config;
+    }
+
+    public void setConfig(OfflineRecognizerConfig config) {
+        setConfig(ptr, config);
+        // we don't update this.config
+    }
+
+    public OfflineRecognizerConfig getConfig() {
+        return config;
     }
 
     public void decode(OfflineStream s) {
@@ -60,6 +72,8 @@ public class OfflineRecognizer {
 
     private native void decode(long ptr, long streamPtr);
 
+    private native void setConfig(long ptr, OfflineRecognizerConfig config);
+
     private native void decodeStreams(long ptr, long[] streamPtrs);
 
     private native Object[] getResult(long streamPtr);

sherpa-onnx/jni/offline-recognizer.cc

@@ -284,6 +284,39 @@ static OfflineRecognizerConfig GetOfflineConfig(JNIEnv *env, jobject config) {
   ans.model_config.zipformer_ctc.model = p;
   env->ReleaseStringUTFChars(s, p);
 
+  // canary
+  fid = env->GetFieldID(model_config_cls, "canary",
+                        "Lcom/k2fsa/sherpa/onnx/OfflineCanaryModelConfig;");
+  jobject canary_config = env->GetObjectField(model_config, fid);
+  jclass canary_config_cls = env->GetObjectClass(canary_config);
+
+  fid = env->GetFieldID(canary_config_cls, "encoder", "Ljava/lang/String;");
+  s = (jstring)env->GetObjectField(canary_config, fid);
+  p = env->GetStringUTFChars(s, nullptr);
+  ans.model_config.canary.encoder = p;
+  env->ReleaseStringUTFChars(s, p);
+
+  fid = env->GetFieldID(canary_config_cls, "decoder", "Ljava/lang/String;");
+  s = (jstring)env->GetObjectField(canary_config, fid);
+  p = env->GetStringUTFChars(s, nullptr);
+  ans.model_config.canary.decoder = p;
+  env->ReleaseStringUTFChars(s, p);
+
+  fid = env->GetFieldID(canary_config_cls, "srcLang", "Ljava/lang/String;");
+  s = (jstring)env->GetObjectField(canary_config, fid);
+  p = env->GetStringUTFChars(s, nullptr);
+  ans.model_config.canary.src_lang = p;
+  env->ReleaseStringUTFChars(s, p);
+
+  fid = env->GetFieldID(canary_config_cls, "tgtLang", "Ljava/lang/String;");
+  s = (jstring)env->GetObjectField(canary_config, fid);
+  p = env->GetStringUTFChars(s, nullptr);
+  ans.model_config.canary.tgt_lang = p;
+  env->ReleaseStringUTFChars(s, p);
+
+  fid = env->GetFieldID(canary_config_cls, "usePnc", "Z");
+  ans.model_config.canary.use_pnc = env->GetBooleanField(canary_config, fid);
+
   // dolphin
   fid = env->GetFieldID(model_config_cls, "dolphin",
                         "Lcom/k2fsa/sherpa/onnx/OfflineDolphinModelConfig;");
@@ -347,11 +380,13 @@ Java_com_k2fsa_sherpa_onnx_OfflineRecognizer_newFromAsset(JNIEnv *env,
 #endif
   auto config = sherpa_onnx::GetOfflineConfig(env, _config);
 
+  if (config.model_config.debug) {
     // logcat truncates long strings, so we split the string into chunks
     auto str_vec = sherpa_onnx::SplitString(config.ToString(), 128);
     for (const auto &s : str_vec) {
       SHERPA_ONNX_LOGE("%s", s.c_str());
     }
+  }
 
   auto model = new sherpa_onnx::OfflineRecognizer(
 #if __ANDROID_API__ >= 9
@@ -369,10 +404,12 @@ Java_com_k2fsa_sherpa_onnx_OfflineRecognizer_newFromFile(JNIEnv *env,
                                                          jobject _config) {
   auto config = sherpa_onnx::GetOfflineConfig(env, _config);
 
+  if (config.model_config.debug) {
     auto str_vec = sherpa_onnx::SplitString(config.ToString(), 128);
     for (const auto &s : str_vec) {
       SHERPA_ONNX_LOGE("%s", s.c_str());
     }
+  }
 
   if (!config.Validate()) {
     SHERPA_ONNX_LOGE("Errors found in config!");
@@ -388,7 +425,10 @@ SHERPA_ONNX_EXTERN_C
 JNIEXPORT void JNICALL Java_com_k2fsa_sherpa_onnx_OfflineRecognizer_setConfig(
     JNIEnv *env, jobject /*obj*/, jlong ptr, jobject _config) {
   auto config = sherpa_onnx::GetOfflineConfig(env, _config);
+
+  if (config.model_config.debug) {
     SHERPA_ONNX_LOGE("config:\n%s", config.ToString().c_str());
+  }
 
   auto recognizer = reinterpret_cast<sherpa_onnx::OfflineRecognizer *>(ptr);
   recognizer->SetConfig(config);

sherpa-onnx/kotlin-api/OfflineRecognizer.kt

@@ -41,6 +41,14 @@ data class OfflineWhisperModelConfig(
     var tailPaddings: Int = 1000, // Padding added at the end of the samples
 )
 
+data class OfflineCanaryModelConfig(
+    var encoder: String = "",
+    var decoder: String = "",
+    var srcLang: String = "en",
+    var tgtLang: String = "en",
+    var usePnc: Boolean = true,
+)
+
 data class OfflineFireRedAsrModelConfig(
     var encoder: String = "",
     var decoder: String = "",
@@ -69,6 +77,7 @@ data class OfflineModelConfig(
     var senseVoice: OfflineSenseVoiceModelConfig = OfflineSenseVoiceModelConfig(),
     var dolphin: OfflineDolphinModelConfig = OfflineDolphinModelConfig(),
     var zipformerCtc: OfflineZipformerCtcModelConfig = OfflineZipformerCtcModelConfig(),
+    var canary: OfflineCanaryModelConfig = OfflineCanaryModelConfig(),
     var teleSpeech: String = "",
     var numThreads: Int = 1,
     var debug: Boolean = false,
@@ -95,7 +104,7 @@ data class OfflineRecognizerConfig(
 
 class OfflineRecognizer(
     assetManager: AssetManager? = null,
-    config: OfflineRecognizerConfig,
+    val config: OfflineRecognizerConfig,
 ) {
     private var ptr: Long
 
@@ -142,10 +151,14 @@ class OfflineRecognizer(
 
     fun decode(stream: OfflineStream) = decode(ptr, stream.ptr)
 
+    fun setConfig(config: OfflineRecognizerConfig) = setConfig(ptr, config)
+
     private external fun delete(ptr: Long)
 
     private external fun createStream(ptr: Long): Long
 
+    private external fun setConfig(ptr: Long, config: OfflineRecognizerConfig)
+
     private external fun newFromAsset(
         assetManager: AssetManager,
         config: OfflineRecognizerConfig,
@@ -574,6 +587,20 @@ fun getOfflineModelConfig(type: Int): OfflineModelConfig? {
                 tokens = "$modelDir/tokens.txt",
             )
         }
+
+        32 -> {
+            val modelDir = "sherpa-onnx-nemo-canary-180m-flash-en-es-de-fr-int8"
+            return OfflineModelConfig(
+                canary = OfflineCanaryModelConfig(
+                    encoder = "$modelDir/encoder.int8.onnx",
+                    decoder = "$modelDir/decoder.int8.onnx",
+                    srcLang = "en",
+                    tgtLang = "en",
+                    usePnc = true,
+                ),
+                tokens = "$modelDir/tokens.txt",
+            )
+        }
     }
     return null
 }