EngineX-Iluvatar/enginex_bi_series-sherpa-onnx
Archived
16
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

Add Kotlin and Java API for Moonshine models (#1474)

Browse Source
This commit is contained in:
Fangjun Kuang
2024-10-26 22:30:29 +08:00
committed by GitHub
parent 669f5ef441
commit bd4b223920
15 changed files with 480 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
sherpa-onnx/java-api/Makefile
View File

@@ -26,6 +26,7 @@ java_files += OnlineRecognizer.java
java_files += OfflineTransducerModelConfig.java
java_files += OfflineParaformerModelConfig.java
java_files += OfflineWhisperModelConfig.java
java_files += OfflineMoonshineModelConfig.java
java_files += OfflineNemoEncDecCtcModelConfig.java
java_files += OfflineSenseVoiceModelConfig.java
java_files += OfflineModelConfig.java

12
sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineModelConfig.java
View File

@@ -6,6 +6,7 @@ public class OfflineModelConfig {
private final OfflineTransducerModelConfig transducer;
private final OfflineParaformerModelConfig paraformer;
private final OfflineWhisperModelConfig whisper;
private final OfflineMoonshineModelConfig moonshine;
private final OfflineNemoEncDecCtcModelConfig nemo;
private final OfflineSenseVoiceModelConfig senseVoice;
private final String teleSpeech;
@@ -22,6 +23,7 @@ public class OfflineModelConfig {
this.transducer = builder.transducer;
this.paraformer = builder.paraformer;
this.whisper = builder.whisper;
this.moonshine = builder.moonshine;
this.nemo = builder.nemo;
this.senseVoice = builder.senseVoice;
this.teleSpeech = builder.teleSpeech;
@@ -50,6 +52,10 @@ public class OfflineModelConfig {
return whisper;
}
public OfflineMoonshineModelConfig getMoonshine() {
return moonshine;
}
public OfflineSenseVoiceModelConfig getSenseVoice() {
return senseVoice;
}
@@ -90,6 +96,7 @@ public class OfflineModelConfig {
private OfflineParaformerModelConfig paraformer = OfflineParaformerModelConfig.builder().build();
private OfflineTransducerModelConfig transducer = OfflineTransducerModelConfig.builder().build();
private OfflineWhisperModelConfig whisper = OfflineWhisperModelConfig.builder().build();
private OfflineMoonshineModelConfig moonshine = OfflineMoonshineModelConfig.builder().build();
private OfflineNemoEncDecCtcModelConfig nemo = OfflineNemoEncDecCtcModelConfig.builder().build();
private OfflineSenseVoiceModelConfig senseVoice = OfflineSenseVoiceModelConfig.builder().build();
private String teleSpeech = "";
@@ -135,6 +142,11 @@ public class OfflineModelConfig {
return this;
}
public Builder setMoonshine(OfflineMoonshineModelConfig moonshine) {
this.moonshine = moonshine;
return this;
}
public Builder setTokens(String tokens) {
this.tokens = tokens;
return this;

70
sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineMoonshineModelConfig.java Normal file
View File

@@ -0,0 +1,70 @@
// Copyright 2024 Xiaomi Corporation
package com.k2fsa.sherpa.onnx;
public class OfflineMoonshineModelConfig {
private final String preprocessor;
private final String encoder;
private final String uncachedDecoder;
private final String cachedDecoder;
private OfflineMoonshineModelConfig(Builder builder) {
this.preprocessor = builder.preprocessor;
this.encoder = builder.encoder;
this.uncachedDecoder = builder.uncachedDecoder;
this.cachedDecoder = builder.cachedDecoder;
}
public static Builder builder() {
return new Builder();
}
public String getPreprocessor() {
return preprocessor;
}
public String getEncoder() {
return encoder;
}
public String getUncachedDecoder() {
return uncachedDecoder;
}
public String getCachedDecoder() {
return cachedDecoder;
}
public static class Builder {
private String preprocessor = "";
private String encoder = "";
private String uncachedDecoder = "";
private String cachedDecoder = "";
public OfflineMoonshineModelConfig build() {
return new OfflineMoonshineModelConfig(this);
}
public Builder setPreprocessor(String preprocessor) {
this.preprocessor = preprocessor;
return this;
}
public Builder setEncoder(String encoder) {
this.encoder = encoder;
return this;
}
public Builder setUncachedDecoder(String uncachedDecoder) {
this.uncachedDecoder = uncachedDecoder;
return this;
}
public Builder setCachedDecoder(String cachedDecoder) {
this.cachedDecoder = cachedDecoder;
return this;
}
}
}

33
sherpa-onnx/jni/offline-recognizer.cc
View File

@@ -174,6 +174,39 @@ static OfflineRecognizerConfig GetOfflineConfig(JNIEnv *env, jobject config) {
ans.model_config.whisper.tail_paddings =
env->GetIntField(whisper_config, fid);
// moonshine
fid = env->GetFieldID(model_config_cls, "moonshine",
"Lcom/k2fsa/sherpa/onnx/OfflineMoonshineModelConfig;");
jobject moonshine_config = env->GetObjectField(model_config, fid);
jclass moonshine_config_cls = env->GetObjectClass(moonshine_config);
fid = env->GetFieldID(moonshine_config_cls, "preprocessor",
"Ljava/lang/String;");
s = (jstring)env->GetObjectField(moonshine_config, fid);
p = env->GetStringUTFChars(s, nullptr);
ans.model_config.moonshine.preprocessor = p;
env->ReleaseStringUTFChars(s, p);
fid = env->GetFieldID(moonshine_config_cls, "encoder", "Ljava/lang/String;");
s = (jstring)env->GetObjectField(moonshine_config, fid);
p = env->GetStringUTFChars(s, nullptr);
ans.model_config.moonshine.encoder = p;
env->ReleaseStringUTFChars(s, p);
fid = env->GetFieldID(moonshine_config_cls, "uncachedDecoder",
"Ljava/lang/String;");
s = (jstring)env->GetObjectField(moonshine_config, fid);
p = env->GetStringUTFChars(s, nullptr);
ans.model_config.moonshine.uncached_decoder = p;
env->ReleaseStringUTFChars(s, p);
fid = env->GetFieldID(moonshine_config_cls, "cachedDecoder",
"Ljava/lang/String;");
s = (jstring)env->GetObjectField(moonshine_config, fid);
p = env->GetStringUTFChars(s, nullptr);
ans.model_config.moonshine.cached_decoder = p;
env->ReleaseStringUTFChars(s, p);
// sense voice
fid = env->GetFieldID(model_config_cls, "senseVoice",
"Lcom/k2fsa/sherpa/onnx/OfflineSenseVoiceModelConfig;");

21
sherpa-onnx/kotlin-api/OfflineRecognizer.kt
View File

@@ -33,6 +33,13 @@ data class OfflineWhisperModelConfig(
var tailPaddings: Int = 1000, // Padding added at the end of the samples
)
data class OfflineMoonshineModelConfig(
var preprocessor: String = "",
var encoder: String = "",
var uncachedDecoder: String = "",
var cachedDecoder: String = "",
)
data class OfflineSenseVoiceModelConfig(
var model: String = "",
var language: String = "",
@@ -43,6 +50,7 @@ data class OfflineModelConfig(
var transducer: OfflineTransducerModelConfig = OfflineTransducerModelConfig(),
var paraformer: OfflineParaformerModelConfig = OfflineParaformerModelConfig(),
var whisper: OfflineWhisperModelConfig = OfflineWhisperModelConfig(),
var moonshine: OfflineMoonshineModelConfig = OfflineMoonshineModelConfig(),
var nemo: OfflineNemoEncDecCtcModelConfig = OfflineNemoEncDecCtcModelConfig(),
var senseVoice: OfflineSenseVoiceModelConfig = OfflineSenseVoiceModelConfig(),
var teleSpeech: String = "",
@@ -417,6 +425,19 @@ fun getOfflineModelConfig(type: Int): OfflineModelConfig? {
modelType = "nemo_transducer",
)
}
21 -> {
val modelDir = "sherpa-onnx-moonshine-tiny-en-int8"
return OfflineModelConfig(
moonshine = OfflineMoonshineModelConfig(
preprocessor = "$modelDir/preprocess.onnx",
encoder = "$modelDir/encode.int8.onnx",
uncachedDecoder = "$modelDir/uncached_decode.int8.onnx",
cachedDecoder = "$modelDir/cached_decode.int8.onnx",
),
tokens = "$modelDir/tokens.txt",
)
}
}
return null
}