EngineX-Iluvatar/enginex_bi_series-sherpa-onnx
Archived
16
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

Add Java and Kotlin API for NeMo Canary models (#2359)

Browse Source 
Add support for the NeMo Canary model in both Java and Kotlin APIs, wiring it through
JNI and updating examples and CI.

- Introduce OfflineCanaryModelConfig in Kotlin and Java with builder patterns
- Extend OfflineRecognizer to accept and apply the new canary config via setConfig
- Update JNI binding (GetOfflineConfig) and getOfflineModelConfig mapping (type 32), 
   plus examples and CI workflows
This commit is contained in:
Fangjun Kuang
2025-07-08 13:45:26 +08:00
committed by GitHub
parent df4615ca1d
commit 103e93d9f6
12 changed files with 363 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
sherpa-onnx/java-api/Makefile
View File

@@ -34,6 +34,7 @@ java_files += OfflineFireRedAsrModelConfig.java
java_files += OfflineMoonshineModelConfig.java
java_files += OfflineNemoEncDecCtcModelConfig.java
java_files += OfflineZipformerCtcModelConfig.java
java_files += OfflineCanaryModelConfig.java
java_files += OfflineSenseVoiceModelConfig.java
java_files += OfflineDolphinModelConfig.java
java_files += OfflineModelConfig.java

80
sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineCanaryModelConfig.java Normal file
View File

@@ -0,0 +1,80 @@
// Copyright 2025 Xiaomi Corporation
package com.k2fsa.sherpa.onnx;
public class OfflineCanaryModelConfig {
private final String encoder;
private final String decoder;
private final String srcLang;
private final String tgtLang;
private final boolean usePnc;
private OfflineCanaryModelConfig(Builder builder) {
this.encoder = builder.encoder;
this.decoder = builder.decoder;
this.srcLang = builder.srcLang;
this.tgtLang = builder.tgtLang;
this.usePnc = builder.usePnc;
}
public static Builder builder() {
return new Builder();
}
public String getEncoder() {
return encoder;
}
public String getDecoder() {
return decoder;
}
public String getSrcLang() {
return srcLang;
}
public String getTgtLang() {
return tgtLang;
}
public boolean isUsePnc() {
return usePnc;
}
public static class Builder {
private String encoder = "";
private String decoder = "";
private String srcLang = "en";
private String tgtLang = "en";
private boolean usePnc = true;
public OfflineCanaryModelConfig build() {
return new OfflineCanaryModelConfig(this);
}
public Builder setEncoder(String encoder) {
this.encoder = encoder;
return this;
}
public Builder setDecoder(String decoder) {
this.decoder = decoder;
return this;
}
public Builder setSrcLang(String srcLang) {
this.srcLang = srcLang;
return this;
}
public Builder setTgtLang(String tgtLang) {
this.tgtLang = tgtLang;
return this;
}
public Builder setUsePnc(boolean usePnc) {
this.usePnc = usePnc;
return this;
}
}
}

12
sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineModelConfig.java
View File

@@ -12,6 +12,7 @@ public class OfflineModelConfig {
private final OfflineSenseVoiceModelConfig senseVoice;
private final OfflineDolphinModelConfig dolphin;
private final OfflineZipformerCtcModelConfig zipformerCtc;
private final OfflineCanaryModelConfig canary;
private final String teleSpeech;
private final String tokens;
private final int numThreads;
@@ -30,6 +31,7 @@ public class OfflineModelConfig {
this.moonshine = builder.moonshine;
this.nemo = builder.nemo;
this.zipformerCtc = builder.zipformerCtc;
this.canary = builder.canary;
this.senseVoice = builder.senseVoice;
this.dolphin = builder.dolphin;
this.teleSpeech = builder.teleSpeech;
@@ -78,6 +80,10 @@ public class OfflineModelConfig {
return zipformerCtc;
}
public OfflineCanaryModelConfig getCanary() {
return canary;
}
public String getTokens() {
return tokens;
}
@@ -120,6 +126,7 @@ public class OfflineModelConfig {
private OfflineSenseVoiceModelConfig senseVoice = OfflineSenseVoiceModelConfig.builder().build();
private OfflineDolphinModelConfig dolphin = OfflineDolphinModelConfig.builder().build();
private OfflineZipformerCtcModelConfig zipformerCtc = OfflineZipformerCtcModelConfig.builder().build();
private OfflineCanaryModelConfig canary = OfflineCanaryModelConfig.builder().build();
private String teleSpeech = "";
private String tokens = "";
private int numThreads = 1;
@@ -158,6 +165,11 @@ public class OfflineModelConfig {
return this;
}
public Builder setCanary(OfflineCanaryModelConfig canary) {
this.canary = canary;
return this;
}
public Builder setTeleSpeech(String teleSpeech) {
this.teleSpeech = teleSpeech;
return this;

14
sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineRecognizer.java
View File

@@ -4,10 +4,22 @@ package com.k2fsa.sherpa.onnx;
public class OfflineRecognizer {
private long ptr = 0;
private final OfflineRecognizerConfig config;
public OfflineRecognizer(OfflineRecognizerConfig config) {
LibraryLoader.maybeLoad();
ptr = newFromFile(config);
this.config = config;
}
public void setConfig(OfflineRecognizerConfig config) {
setConfig(ptr, config);
// we don't update this.config
}
public OfflineRecognizerConfig getConfig() {
return config;
}
public void decode(OfflineStream s) {
@@ -60,6 +72,8 @@ public class OfflineRecognizer {
private native void decode(long ptr, long streamPtr);
private native void setConfig(long ptr, OfflineRecognizerConfig config);
private native void decodeStreams(long ptr, long[] streamPtrs);
private native Object[] getResult(long streamPtr);

56
sherpa-onnx/jni/offline-recognizer.cc
View File

@@ -284,6 +284,39 @@ static OfflineRecognizerConfig GetOfflineConfig(JNIEnv *env, jobject config) {
ans.model_config.zipformer_ctc.model = p;
env->ReleaseStringUTFChars(s, p);
// canary
fid = env->GetFieldID(model_config_cls, "canary",
"Lcom/k2fsa/sherpa/onnx/OfflineCanaryModelConfig;");
jobject canary_config = env->GetObjectField(model_config, fid);
jclass canary_config_cls = env->GetObjectClass(canary_config);
fid = env->GetFieldID(canary_config_cls, "encoder", "Ljava/lang/String;");
s = (jstring)env->GetObjectField(canary_config, fid);
p = env->GetStringUTFChars(s, nullptr);
ans.model_config.canary.encoder = p;
env->ReleaseStringUTFChars(s, p);
fid = env->GetFieldID(canary_config_cls, "decoder", "Ljava/lang/String;");
s = (jstring)env->GetObjectField(canary_config, fid);
p = env->GetStringUTFChars(s, nullptr);
ans.model_config.canary.decoder = p;
env->ReleaseStringUTFChars(s, p);
fid = env->GetFieldID(canary_config_cls, "srcLang", "Ljava/lang/String;");
s = (jstring)env->GetObjectField(canary_config, fid);
p = env->GetStringUTFChars(s, nullptr);
ans.model_config.canary.src_lang = p;
env->ReleaseStringUTFChars(s, p);
fid = env->GetFieldID(canary_config_cls, "tgtLang", "Ljava/lang/String;");
s = (jstring)env->GetObjectField(canary_config, fid);
p = env->GetStringUTFChars(s, nullptr);
ans.model_config.canary.tgt_lang = p;
env->ReleaseStringUTFChars(s, p);
fid = env->GetFieldID(canary_config_cls, "usePnc", "Z");
ans.model_config.canary.use_pnc = env->GetBooleanField(canary_config, fid);
// dolphin
fid = env->GetFieldID(model_config_cls, "dolphin",
"Lcom/k2fsa/sherpa/onnx/OfflineDolphinModelConfig;");
@@ -347,10 +380,12 @@ Java_com_k2fsa_sherpa_onnx_OfflineRecognizer_newFromAsset(JNIEnv *env,
#endif
auto config = sherpa_onnx::GetOfflineConfig(env, _config);
// logcat truncates long strings, so we split the string into chunks
auto str_vec = sherpa_onnx::SplitString(config.ToString(), 128);
for (const auto &s : str_vec) {
SHERPA_ONNX_LOGE("%s", s.c_str());
if (config.model_config.debug) {
// logcat truncates long strings, so we split the string into chunks
auto str_vec = sherpa_onnx::SplitString(config.ToString(), 128);
for (const auto &s : str_vec) {
SHERPA_ONNX_LOGE("%s", s.c_str());
}
}
auto model = new sherpa_onnx::OfflineRecognizer(
@@ -369,9 +404,11 @@ Java_com_k2fsa_sherpa_onnx_OfflineRecognizer_newFromFile(JNIEnv *env,
jobject _config) {
auto config = sherpa_onnx::GetOfflineConfig(env, _config);
auto str_vec = sherpa_onnx::SplitString(config.ToString(), 128);
for (const auto &s : str_vec) {
SHERPA_ONNX_LOGE("%s", s.c_str());
if (config.model_config.debug) {
auto str_vec = sherpa_onnx::SplitString(config.ToString(), 128);
for (const auto &s : str_vec) {
SHERPA_ONNX_LOGE("%s", s.c_str());
}
}
if (!config.Validate()) {
@@ -388,7 +425,10 @@ SHERPA_ONNX_EXTERN_C
JNIEXPORT void JNICALL Java_com_k2fsa_sherpa_onnx_OfflineRecognizer_setConfig(
JNIEnv *env, jobject /*obj*/, jlong ptr, jobject _config) {
auto config = sherpa_onnx::GetOfflineConfig(env, _config);
SHERPA_ONNX_LOGE("config:\n%s", config.ToString().c_str());
if (config.model_config.debug) {
SHERPA_ONNX_LOGE("config:\n%s", config.ToString().c_str());
}
auto recognizer = reinterpret_cast<sherpa_onnx::OfflineRecognizer *>(ptr);
recognizer->SetConfig(config);

29
sherpa-onnx/kotlin-api/OfflineRecognizer.kt
View File

@@ -41,6 +41,14 @@ data class OfflineWhisperModelConfig(
var tailPaddings: Int = 1000, // Padding added at the end of the samples
)
data class OfflineCanaryModelConfig(
var encoder: String = "",
var decoder: String = "",
var srcLang: String = "en",
var tgtLang: String = "en",
var usePnc: Boolean = true,
)
data class OfflineFireRedAsrModelConfig(
var encoder: String = "",
var decoder: String = "",
@@ -69,6 +77,7 @@ data class OfflineModelConfig(
var senseVoice: OfflineSenseVoiceModelConfig = OfflineSenseVoiceModelConfig(),
var dolphin: OfflineDolphinModelConfig = OfflineDolphinModelConfig(),
var zipformerCtc: OfflineZipformerCtcModelConfig = OfflineZipformerCtcModelConfig(),
var canary: OfflineCanaryModelConfig = OfflineCanaryModelConfig(),
var teleSpeech: String = "",
var numThreads: Int = 1,
var debug: Boolean = false,
@@ -95,7 +104,7 @@ data class OfflineRecognizerConfig(
class OfflineRecognizer(
assetManager: AssetManager? = null,
config: OfflineRecognizerConfig,
val config: OfflineRecognizerConfig,
) {
private var ptr: Long
@@ -142,10 +151,14 @@ class OfflineRecognizer(
fun decode(stream: OfflineStream) = decode(ptr, stream.ptr)
fun setConfig(config: OfflineRecognizerConfig) = setConfig(ptr, config)
private external fun delete(ptr: Long)
private external fun createStream(ptr: Long): Long
private external fun setConfig(ptr: Long, config: OfflineRecognizerConfig)
private external fun newFromAsset(
assetManager: AssetManager,
config: OfflineRecognizerConfig,
@@ -574,6 +587,20 @@ fun getOfflineModelConfig(type: Int): OfflineModelConfig? {
tokens = "$modelDir/tokens.txt",
)
}
32 -> {
val modelDir = "sherpa-onnx-nemo-canary-180m-flash-en-es-de-fr-int8"
return OfflineModelConfig(
canary = OfflineCanaryModelConfig(
encoder = "$modelDir/encoder.int8.onnx",
decoder = "$modelDir/decoder.int8.onnx",
srcLang = "en",
tgtLang = "en",
usePnc = true,
),
tokens = "$modelDir/tokens.txt",
)
}
}
return null
}