From 103e93d9f648344ffc32520c7e93153874b3fb3d Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Tue, 8 Jul 2025 13:45:26 +0800 Subject: [PATCH] Add Java and Kotlin API for NeMo Canary models (#2359) Add support for the NeMo Canary model in both Java and Kotlin APIs, wiring it through JNI and updating examples and CI. - Introduce OfflineCanaryModelConfig in Kotlin and Java with builder patterns - Extend OfflineRecognizer to accept and apply the new canary config via setConfig - Update JNI binding (GetOfflineConfig) and getOfflineModelConfig mapping (type 32), plus examples and CI workflows --- .github/workflows/run-java-test.yaml | 7 ++ .../NonStreamingDecodeFileNemoCanary.java | 56 +++++++++++++ java-api-examples/README.md | 9 ++- ...n-non-streaming-decode-file-nemo-canary.sh | 37 +++++++++ kotlin-api-examples/run.sh | 25 +++++- .../test_offline_nemo_canary.kt | 48 +++++++++++ sherpa-onnx/java-api/Makefile | 1 + .../sherpa/onnx/OfflineCanaryModelConfig.java | 80 +++++++++++++++++++ .../k2fsa/sherpa/onnx/OfflineModelConfig.java | 12 +++ .../k2fsa/sherpa/onnx/OfflineRecognizer.java | 14 ++++ sherpa-onnx/jni/offline-recognizer.cc | 56 +++++++++++-- sherpa-onnx/kotlin-api/OfflineRecognizer.kt | 29 ++++++- 12 files changed, 363 insertions(+), 11 deletions(-) create mode 100644 java-api-examples/NonStreamingDecodeFileNemoCanary.java create mode 100755 java-api-examples/run-non-streaming-decode-file-nemo-canary.sh create mode 100644 kotlin-api-examples/test_offline_nemo_canary.kt create mode 100644 sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineCanaryModelConfig.java diff --git a/.github/workflows/run-java-test.yaml b/.github/workflows/run-java-test.yaml index fd2494d7..fc2be629 100644 --- a/.github/workflows/run-java-test.yaml +++ b/.github/workflows/run-java-test.yaml @@ -117,6 +117,13 @@ jobs: cd ./java-api-examples ./run-version-test.sh + - name: Run java test (Nemo Canary) + shell: bash + run: | + cd ./java-api-examples + ./run-non-streaming-decode-file-nemo-canary.sh + rm -rf sherpa-onnx-nemo-* + - name: Run java test (Non-streaming SenseVoice with homophone replacer) shell: bash run: | diff --git a/java-api-examples/NonStreamingDecodeFileNemoCanary.java b/java-api-examples/NonStreamingDecodeFileNemoCanary.java new file mode 100644 index 00000000..0c06c4ee --- /dev/null +++ b/java-api-examples/NonStreamingDecodeFileNemoCanary.java @@ -0,0 +1,56 @@ +// Copyright 2024 Xiaomi Corporation + +// This file shows how to use an offline NeMo Canary model, i.e., +// non-streaming NeMo Canary model, to decode files. +import com.k2fsa.sherpa.onnx.*; + +public class NonStreamingDecodeFileNemoCanary { + public static void main(String[] args) { + // please refer to + // https://k2-fsa.github.io/sherpa/onnx/nemo/canary.html + // to download model files + String encoder = "./sherpa-onnx-nemo-canary-180m-flash-en-es-de-fr-int8/encoder.int8.onnx"; + String decoder = "./sherpa-onnx-nemo-canary-180m-flash-en-es-de-fr-int8/decoder.int8.onnx"; + String tokens = "./sherpa-onnx-nemo-canary-180m-flash-en-es-de-fr-int8/tokens.txt"; + + String waveFilename = "./sherpa-onnx-nemo-canary-180m-flash-en-es-de-fr-int8/test_wavs/en.wav"; + + WaveReader reader = new WaveReader(waveFilename); + + OfflineCanaryModelConfig canary = + OfflineCanaryModelConfig.builder() + .setEncoder(encoder) + .setDecoder(decoder) + .setSrcLang("en") + .setTgtLang("en") + .setUsePnc(true) + .build(); + + OfflineModelConfig modelConfig = + OfflineModelConfig.builder() + .setCanary(canary) + .setTokens(tokens) + .setNumThreads(1) + .setDebug(true) + .build(); + + OfflineRecognizerConfig config = + OfflineRecognizerConfig.builder() + .setOfflineModelConfig(modelConfig) + .setDecodingMethod("greedy_search") + .build(); + + OfflineRecognizer recognizer = new OfflineRecognizer(config); + OfflineStream stream = recognizer.createStream(); + stream.acceptWaveform(reader.getSamples(), reader.getSampleRate()); + + recognizer.decode(stream); + + String text = recognizer.getResult(stream).getText(); + + System.out.printf("filename:%s\nresult(English):%s\n", waveFilename, text); + + stream.release(); + recognizer.release(); + } +} diff --git a/java-api-examples/README.md b/java-api-examples/README.md index 1dd7d814..38147e30 100755 --- a/java-api-examples/README.md +++ b/java-api-examples/README.md @@ -24,11 +24,18 @@ This directory contains examples for the JAVA API of sherpa-onnx. ```bash ./run-non-streaming-decode-file-dolphin-ctc.sh +./run-non-streaming-decode-file-fire-red-asr.sh +./run-non-streaming-decode-file-moonshine.sh +./run-non-streaming-decode-file-nemo-canary.sh +./run-non-streaming-decode-file-nemo.sh ./run-non-streaming-decode-file-paraformer.sh ./run-non-streaming-decode-file-sense-voice.sh +./run-non-streaming-decode-file-tele-speech-ctc.sh +./run-non-streaming-decode-file-transducer-hotwords.sh ./run-non-streaming-decode-file-transducer.sh +./run-non-streaming-decode-file-whisper-multiple.sh ./run-non-streaming-decode-file-whisper.sh -./run-non-streaming-decode-file-nemo.sh +./run-non-streaming-decode-file-zipformer-ctc.sh ``` ## Non-Streaming Speech recognition with homophone replacer diff --git a/java-api-examples/run-non-streaming-decode-file-nemo-canary.sh b/java-api-examples/run-non-streaming-decode-file-nemo-canary.sh new file mode 100755 index 00000000..c18a8ceb --- /dev/null +++ b/java-api-examples/run-non-streaming-decode-file-nemo-canary.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +set -ex + +if [[ ! -f ../build/lib/libsherpa-onnx-jni.dylib && ! -f ../build/lib/libsherpa-onnx-jni.so ]]; then + mkdir -p ../build + pushd ../build + cmake \ + -DSHERPA_ONNX_ENABLE_PYTHON=OFF \ + -DSHERPA_ONNX_ENABLE_TESTS=OFF \ + -DSHERPA_ONNX_ENABLE_CHECK=OFF \ + -DBUILD_SHARED_LIBS=ON \ + -DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF \ + -DSHERPA_ONNX_ENABLE_JNI=ON \ + .. + + make -j4 + ls -lh lib + popd +fi + +if [ ! -f ../sherpa-onnx/java-api/build/sherpa-onnx.jar ]; then + pushd ../sherpa-onnx/java-api + make + popd +fi + +if [ ! -f sherpa-onnx-nemo-canary-180m-flash-en-es-de-fr-int8/encoder.int8.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-canary-180m-flash-en-es-de-fr-int8.tar.bz2 + tar xvf sherpa-onnx-nemo-canary-180m-flash-en-es-de-fr-int8.tar.bz2 + rm sherpa-onnx-nemo-canary-180m-flash-en-es-de-fr-int8.tar.bz2 +fi + +java \ + -Djava.library.path=$PWD/../build/lib \ + -cp ../sherpa-onnx/java-api/build/sherpa-onnx.jar \ + NonStreamingDecodeFileNemoCanary.java diff --git a/kotlin-api-examples/run.sh b/kotlin-api-examples/run.sh index ef3fb091..659f1632 100755 --- a/kotlin-api-examples/run.sh +++ b/kotlin-api-examples/run.sh @@ -455,8 +455,31 @@ function testOfflineSenseVoiceWithHr() { ls -lh $out_filename java -Djava.library.path=../build/lib -jar $out_filename } -testVersion +function testOfflineNeMoCanary() { + if [ ! -f sherpa-onnx-nemo-canary-180m-flash-en-es-de-fr-int8/encoder.int8.onnx ]; then + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-canary-180m-flash-en-es-de-fr-int8.tar.bz2 + tar xvf sherpa-onnx-nemo-canary-180m-flash-en-es-de-fr-int8.tar.bz2 + rm sherpa-onnx-nemo-canary-180m-flash-en-es-de-fr-int8.tar.bz2 + fi + + out_filename=test_offline_nemo_canary.jar + kotlinc-jvm -include-runtime -d $out_filename \ + test_offline_nemo_canary.kt \ + FeatureConfig.kt \ + HomophoneReplacerConfig.kt \ + OfflineRecognizer.kt \ + OfflineStream.kt \ + WaveReader.kt \ + faked-asset-manager.kt + + ls -lh $out_filename + java -Djava.library.path=../build/lib -jar $out_filename +} + +# testVersion + +testOfflineNeMoCanary testOfflineSenseVoiceWithHr testOfflineSpeechDenoiser testOfflineSpeakerDiarization diff --git a/kotlin-api-examples/test_offline_nemo_canary.kt b/kotlin-api-examples/test_offline_nemo_canary.kt new file mode 100644 index 00000000..59311841 --- /dev/null +++ b/kotlin-api-examples/test_offline_nemo_canary.kt @@ -0,0 +1,48 @@ +package com.k2fsa.sherpa.onnx + +fun main() { + val recognizer = createOfflineRecognizer() + val waveFilename = "./sherpa-onnx-nemo-canary-180m-flash-en-es-de-fr-int8/test_wavs/en.wav" + + val objArray = WaveReader.readWaveFromFile( + filename = waveFilename, + ) + val samples: FloatArray = objArray[0] as FloatArray + val sampleRate: Int = objArray[1] as Int + + var stream = recognizer.createStream() + stream.acceptWaveform(samples, sampleRate=sampleRate) + recognizer.decode(stream) + + var result = recognizer.getResult(stream) + println("English: $result") + + stream.release() + + // now output text in German + val config = recognizer.config.copy(modelConfig=recognizer.config.modelConfig.copy( + canary=recognizer.config.modelConfig.canary.copy( + tgtLang="de" + ) + )) + recognizer.setConfig(config) + + stream = recognizer.createStream() + stream.acceptWaveform(samples, sampleRate=sampleRate) + recognizer.decode(stream) + + result = recognizer.getResult(stream) + println("German: $result") + + stream.release() + recognizer.release() +} + + +fun createOfflineRecognizer(): OfflineRecognizer { + val config = OfflineRecognizerConfig( + modelConfig = getOfflineModelConfig(type = 32)!!, + ) + + return OfflineRecognizer(config = config) +} diff --git a/sherpa-onnx/java-api/Makefile b/sherpa-onnx/java-api/Makefile index fd699e88..6b17983a 100644 --- a/sherpa-onnx/java-api/Makefile +++ b/sherpa-onnx/java-api/Makefile @@ -34,6 +34,7 @@ java_files += OfflineFireRedAsrModelConfig.java java_files += OfflineMoonshineModelConfig.java java_files += OfflineNemoEncDecCtcModelConfig.java java_files += OfflineZipformerCtcModelConfig.java +java_files += OfflineCanaryModelConfig.java java_files += OfflineSenseVoiceModelConfig.java java_files += OfflineDolphinModelConfig.java java_files += OfflineModelConfig.java diff --git a/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineCanaryModelConfig.java b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineCanaryModelConfig.java new file mode 100644 index 00000000..ad082b1f --- /dev/null +++ b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineCanaryModelConfig.java @@ -0,0 +1,80 @@ +// Copyright 2025 Xiaomi Corporation + +package com.k2fsa.sherpa.onnx; + +public class OfflineCanaryModelConfig { + private final String encoder; + private final String decoder; + private final String srcLang; + private final String tgtLang; + private final boolean usePnc; + + private OfflineCanaryModelConfig(Builder builder) { + this.encoder = builder.encoder; + this.decoder = builder.decoder; + this.srcLang = builder.srcLang; + this.tgtLang = builder.tgtLang; + this.usePnc = builder.usePnc; + } + + public static Builder builder() { + return new Builder(); + } + + public String getEncoder() { + return encoder; + } + + public String getDecoder() { + return decoder; + } + + public String getSrcLang() { + return srcLang; + } + + public String getTgtLang() { + return tgtLang; + } + + public boolean isUsePnc() { + return usePnc; + } + + public static class Builder { + private String encoder = ""; + private String decoder = ""; + private String srcLang = "en"; + private String tgtLang = "en"; + private boolean usePnc = true; + + public OfflineCanaryModelConfig build() { + return new OfflineCanaryModelConfig(this); + } + + public Builder setEncoder(String encoder) { + this.encoder = encoder; + return this; + } + + public Builder setDecoder(String decoder) { + this.decoder = decoder; + return this; + } + + public Builder setSrcLang(String srcLang) { + this.srcLang = srcLang; + return this; + } + + public Builder setTgtLang(String tgtLang) { + this.tgtLang = tgtLang; + return this; + } + + public Builder setUsePnc(boolean usePnc) { + this.usePnc = usePnc; + return this; + } + } +} diff --git a/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineModelConfig.java b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineModelConfig.java index a0cdfaef..ea3f35a0 100644 --- a/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineModelConfig.java +++ b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineModelConfig.java @@ -12,6 +12,7 @@ public class OfflineModelConfig { private final OfflineSenseVoiceModelConfig senseVoice; private final OfflineDolphinModelConfig dolphin; private final OfflineZipformerCtcModelConfig zipformerCtc; + private final OfflineCanaryModelConfig canary; private final String teleSpeech; private final String tokens; private final int numThreads; @@ -30,6 +31,7 @@ public class OfflineModelConfig { this.moonshine = builder.moonshine; this.nemo = builder.nemo; this.zipformerCtc = builder.zipformerCtc; + this.canary = builder.canary; this.senseVoice = builder.senseVoice; this.dolphin = builder.dolphin; this.teleSpeech = builder.teleSpeech; @@ -78,6 +80,10 @@ public class OfflineModelConfig { return zipformerCtc; } + public OfflineCanaryModelConfig getCanary() { + return canary; + } + public String getTokens() { return tokens; } @@ -120,6 +126,7 @@ public class OfflineModelConfig { private OfflineSenseVoiceModelConfig senseVoice = OfflineSenseVoiceModelConfig.builder().build(); private OfflineDolphinModelConfig dolphin = OfflineDolphinModelConfig.builder().build(); private OfflineZipformerCtcModelConfig zipformerCtc = OfflineZipformerCtcModelConfig.builder().build(); + private OfflineCanaryModelConfig canary = OfflineCanaryModelConfig.builder().build(); private String teleSpeech = ""; private String tokens = ""; private int numThreads = 1; @@ -158,6 +165,11 @@ public class OfflineModelConfig { return this; } + public Builder setCanary(OfflineCanaryModelConfig canary) { + this.canary = canary; + return this; + } + public Builder setTeleSpeech(String teleSpeech) { this.teleSpeech = teleSpeech; return this; diff --git a/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineRecognizer.java b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineRecognizer.java index e500bbf7..b78746b2 100644 --- a/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineRecognizer.java +++ b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineRecognizer.java @@ -4,10 +4,22 @@ package com.k2fsa.sherpa.onnx; public class OfflineRecognizer { private long ptr = 0; + private final OfflineRecognizerConfig config; public OfflineRecognizer(OfflineRecognizerConfig config) { LibraryLoader.maybeLoad(); ptr = newFromFile(config); + + this.config = config; + } + + public void setConfig(OfflineRecognizerConfig config) { + setConfig(ptr, config); + // we don't update this.config + } + + public OfflineRecognizerConfig getConfig() { + return config; } public void decode(OfflineStream s) { @@ -60,6 +72,8 @@ public class OfflineRecognizer { private native void decode(long ptr, long streamPtr); + private native void setConfig(long ptr, OfflineRecognizerConfig config); + private native void decodeStreams(long ptr, long[] streamPtrs); private native Object[] getResult(long streamPtr); diff --git a/sherpa-onnx/jni/offline-recognizer.cc b/sherpa-onnx/jni/offline-recognizer.cc index 7652c398..08ad92a8 100644 --- a/sherpa-onnx/jni/offline-recognizer.cc +++ b/sherpa-onnx/jni/offline-recognizer.cc @@ -284,6 +284,39 @@ static OfflineRecognizerConfig GetOfflineConfig(JNIEnv *env, jobject config) { ans.model_config.zipformer_ctc.model = p; env->ReleaseStringUTFChars(s, p); + // canary + fid = env->GetFieldID(model_config_cls, "canary", + "Lcom/k2fsa/sherpa/onnx/OfflineCanaryModelConfig;"); + jobject canary_config = env->GetObjectField(model_config, fid); + jclass canary_config_cls = env->GetObjectClass(canary_config); + + fid = env->GetFieldID(canary_config_cls, "encoder", "Ljava/lang/String;"); + s = (jstring)env->GetObjectField(canary_config, fid); + p = env->GetStringUTFChars(s, nullptr); + ans.model_config.canary.encoder = p; + env->ReleaseStringUTFChars(s, p); + + fid = env->GetFieldID(canary_config_cls, "decoder", "Ljava/lang/String;"); + s = (jstring)env->GetObjectField(canary_config, fid); + p = env->GetStringUTFChars(s, nullptr); + ans.model_config.canary.decoder = p; + env->ReleaseStringUTFChars(s, p); + + fid = env->GetFieldID(canary_config_cls, "srcLang", "Ljava/lang/String;"); + s = (jstring)env->GetObjectField(canary_config, fid); + p = env->GetStringUTFChars(s, nullptr); + ans.model_config.canary.src_lang = p; + env->ReleaseStringUTFChars(s, p); + + fid = env->GetFieldID(canary_config_cls, "tgtLang", "Ljava/lang/String;"); + s = (jstring)env->GetObjectField(canary_config, fid); + p = env->GetStringUTFChars(s, nullptr); + ans.model_config.canary.tgt_lang = p; + env->ReleaseStringUTFChars(s, p); + + fid = env->GetFieldID(canary_config_cls, "usePnc", "Z"); + ans.model_config.canary.use_pnc = env->GetBooleanField(canary_config, fid); + // dolphin fid = env->GetFieldID(model_config_cls, "dolphin", "Lcom/k2fsa/sherpa/onnx/OfflineDolphinModelConfig;"); @@ -347,10 +380,12 @@ Java_com_k2fsa_sherpa_onnx_OfflineRecognizer_newFromAsset(JNIEnv *env, #endif auto config = sherpa_onnx::GetOfflineConfig(env, _config); - // logcat truncates long strings, so we split the string into chunks - auto str_vec = sherpa_onnx::SplitString(config.ToString(), 128); - for (const auto &s : str_vec) { - SHERPA_ONNX_LOGE("%s", s.c_str()); + if (config.model_config.debug) { + // logcat truncates long strings, so we split the string into chunks + auto str_vec = sherpa_onnx::SplitString(config.ToString(), 128); + for (const auto &s : str_vec) { + SHERPA_ONNX_LOGE("%s", s.c_str()); + } } auto model = new sherpa_onnx::OfflineRecognizer( @@ -369,9 +404,11 @@ Java_com_k2fsa_sherpa_onnx_OfflineRecognizer_newFromFile(JNIEnv *env, jobject _config) { auto config = sherpa_onnx::GetOfflineConfig(env, _config); - auto str_vec = sherpa_onnx::SplitString(config.ToString(), 128); - for (const auto &s : str_vec) { - SHERPA_ONNX_LOGE("%s", s.c_str()); + if (config.model_config.debug) { + auto str_vec = sherpa_onnx::SplitString(config.ToString(), 128); + for (const auto &s : str_vec) { + SHERPA_ONNX_LOGE("%s", s.c_str()); + } } if (!config.Validate()) { @@ -388,7 +425,10 @@ SHERPA_ONNX_EXTERN_C JNIEXPORT void JNICALL Java_com_k2fsa_sherpa_onnx_OfflineRecognizer_setConfig( JNIEnv *env, jobject /*obj*/, jlong ptr, jobject _config) { auto config = sherpa_onnx::GetOfflineConfig(env, _config); - SHERPA_ONNX_LOGE("config:\n%s", config.ToString().c_str()); + + if (config.model_config.debug) { + SHERPA_ONNX_LOGE("config:\n%s", config.ToString().c_str()); + } auto recognizer = reinterpret_cast(ptr); recognizer->SetConfig(config); diff --git a/sherpa-onnx/kotlin-api/OfflineRecognizer.kt b/sherpa-onnx/kotlin-api/OfflineRecognizer.kt index 6a6f19db..0b6ac25a 100644 --- a/sherpa-onnx/kotlin-api/OfflineRecognizer.kt +++ b/sherpa-onnx/kotlin-api/OfflineRecognizer.kt @@ -41,6 +41,14 @@ data class OfflineWhisperModelConfig( var tailPaddings: Int = 1000, // Padding added at the end of the samples ) +data class OfflineCanaryModelConfig( + var encoder: String = "", + var decoder: String = "", + var srcLang: String = "en", + var tgtLang: String = "en", + var usePnc: Boolean = true, +) + data class OfflineFireRedAsrModelConfig( var encoder: String = "", var decoder: String = "", @@ -69,6 +77,7 @@ data class OfflineModelConfig( var senseVoice: OfflineSenseVoiceModelConfig = OfflineSenseVoiceModelConfig(), var dolphin: OfflineDolphinModelConfig = OfflineDolphinModelConfig(), var zipformerCtc: OfflineZipformerCtcModelConfig = OfflineZipformerCtcModelConfig(), + var canary: OfflineCanaryModelConfig = OfflineCanaryModelConfig(), var teleSpeech: String = "", var numThreads: Int = 1, var debug: Boolean = false, @@ -95,7 +104,7 @@ data class OfflineRecognizerConfig( class OfflineRecognizer( assetManager: AssetManager? = null, - config: OfflineRecognizerConfig, + val config: OfflineRecognizerConfig, ) { private var ptr: Long @@ -142,10 +151,14 @@ class OfflineRecognizer( fun decode(stream: OfflineStream) = decode(ptr, stream.ptr) + fun setConfig(config: OfflineRecognizerConfig) = setConfig(ptr, config) + private external fun delete(ptr: Long) private external fun createStream(ptr: Long): Long + private external fun setConfig(ptr: Long, config: OfflineRecognizerConfig) + private external fun newFromAsset( assetManager: AssetManager, config: OfflineRecognizerConfig, @@ -574,6 +587,20 @@ fun getOfflineModelConfig(type: Int): OfflineModelConfig? { tokens = "$modelDir/tokens.txt", ) } + + 32 -> { + val modelDir = "sherpa-onnx-nemo-canary-180m-flash-en-es-de-fr-int8" + return OfflineModelConfig( + canary = OfflineCanaryModelConfig( + encoder = "$modelDir/encoder.int8.onnx", + decoder = "$modelDir/decoder.int8.onnx", + srcLang = "en", + tgtLang = "en", + usePnc = true, + ), + tokens = "$modelDir/tokens.txt", + ) + } } return null }