Refactor the JNI interface to make it more modular and maintainable (#802)
This commit is contained in:
Fangjun Kuang
186
sherpa-onnx/kotlin-api/AudioTagging.kt
Normal file
186
sherpa-onnx/kotlin-api/AudioTagging.kt
Normal file
@@ -0,0 +1,186 @@
|
||||
package com.k2fsa.sherpa.onnx
|
||||
|
||||
import android.content.res.AssetManager
|
||||
|
||||
data class OfflineZipformerAudioTaggingModelConfig(
|
||||
var model: String = "",
|
||||
)
|
||||
|
||||
data class AudioTaggingModelConfig(
|
||||
var zipformer: OfflineZipformerAudioTaggingModelConfig = OfflineZipformerAudioTaggingModelConfig(),
|
||||
var ced: String = "",
|
||||
var numThreads: Int = 1,
|
||||
var debug: Boolean = false,
|
||||
var provider: String = "cpu",
|
||||
)
|
||||
|
||||
data class AudioTaggingConfig(
|
||||
var model: AudioTaggingModelConfig,
|
||||
var labels: String,
|
||||
var topK: Int = 5,
|
||||
)
|
||||
|
||||
data class AudioEvent(
|
||||
val name: String,
|
||||
val index: Int,
|
||||
val prob: Float,
|
||||
)
|
||||
|
||||
class AudioTagging(
|
||||
assetManager: AssetManager? = null,
|
||||
config: AudioTaggingConfig,
|
||||
) {
|
||||
private var ptr: Long
|
||||
|
||||
init {
|
||||
ptr = if (assetManager != null) {
|
||||
newFromAsset(assetManager, config)
|
||||
} else {
|
||||
newFromFile(config)
|
||||
}
|
||||
}
|
||||
|
||||
protected fun finalize() {
|
||||
if (ptr != 0L) {
|
||||
delete(ptr)
|
||||
ptr = 0
|
||||
}
|
||||
}
|
||||
|
||||
fun release() = finalize()
|
||||
|
||||
fun createStream(): OfflineStream {
|
||||
val p = createStream(ptr)
|
||||
return OfflineStream(p)
|
||||
}
|
||||
|
||||
@Suppress("UNCHECKED_CAST")
|
||||
fun compute(stream: OfflineStream, topK: Int = -1): ArrayList<AudioEvent> {
|
||||
val events: Array<Any> = compute(ptr, stream.ptr, topK)
|
||||
val ans = ArrayList<AudioEvent>()
|
||||
|
||||
for (e in events) {
|
||||
val p: Array<Any> = e as Array<Any>
|
||||
ans.add(
|
||||
AudioEvent(
|
||||
name = p[0] as String,
|
||||
index = p[1] as Int,
|
||||
prob = p[2] as Float,
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
return ans
|
||||
}
|
||||
|
||||
private external fun newFromAsset(
|
||||
assetManager: AssetManager,
|
||||
config: AudioTaggingConfig,
|
||||
): Long
|
||||
|
||||
private external fun newFromFile(
|
||||
config: AudioTaggingConfig,
|
||||
): Long
|
||||
|
||||
private external fun delete(ptr: Long)
|
||||
|
||||
private external fun createStream(ptr: Long): Long
|
||||
|
||||
private external fun compute(ptr: Long, streamPtr: Long, topK: Int): Array<Any>
|
||||
|
||||
companion object {
|
||||
init {
|
||||
System.loadLibrary("sherpa-onnx-jni")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// please refer to
|
||||
// https://github.com/k2-fsa/sherpa-onnx/releases/tag/audio-tagging-models
|
||||
// to download more models
|
||||
//
|
||||
// See also
|
||||
// https://k2-fsa.github.io/sherpa/onnx/audio-tagging/
|
||||
fun getAudioTaggingConfig(type: Int, numThreads: Int = 1): AudioTaggingConfig? {
|
||||
when (type) {
|
||||
0 -> {
|
||||
val modelDir = "sherpa-onnx-zipformer-small-audio-tagging-2024-04-15"
|
||||
return AudioTaggingConfig(
|
||||
model = AudioTaggingModelConfig(
|
||||
zipformer = OfflineZipformerAudioTaggingModelConfig(model = "$modelDir/model.int8.onnx"),
|
||||
numThreads = numThreads,
|
||||
debug = true,
|
||||
),
|
||||
labels = "$modelDir/class_labels_indices.csv",
|
||||
topK = 3,
|
||||
)
|
||||
}
|
||||
|
||||
1 -> {
|
||||
val modelDir = "sherpa-onnx-zipformer-audio-tagging-2024-04-09"
|
||||
return AudioTaggingConfig(
|
||||
model = AudioTaggingModelConfig(
|
||||
zipformer = OfflineZipformerAudioTaggingModelConfig(model = "$modelDir/model.int8.onnx"),
|
||||
numThreads = numThreads,
|
||||
debug = true,
|
||||
),
|
||||
labels = "$modelDir/class_labels_indices.csv",
|
||||
topK = 3,
|
||||
)
|
||||
}
|
||||
|
||||
2 -> {
|
||||
val modelDir = "sherpa-onnx-ced-tiny-audio-tagging-2024-04-19"
|
||||
return AudioTaggingConfig(
|
||||
model = AudioTaggingModelConfig(
|
||||
ced = "$modelDir/model.int8.onnx",
|
||||
numThreads = numThreads,
|
||||
debug = true,
|
||||
),
|
||||
labels = "$modelDir/class_labels_indices.csv",
|
||||
topK = 3,
|
||||
)
|
||||
}
|
||||
|
||||
3 -> {
|
||||
val modelDir = "sherpa-onnx-ced-mini-audio-tagging-2024-04-19"
|
||||
return AudioTaggingConfig(
|
||||
model = AudioTaggingModelConfig(
|
||||
ced = "$modelDir/model.int8.onnx",
|
||||
numThreads = numThreads,
|
||||
debug = true,
|
||||
),
|
||||
labels = "$modelDir/class_labels_indices.csv",
|
||||
topK = 3,
|
||||
)
|
||||
}
|
||||
|
||||
4 -> {
|
||||
val modelDir = "sherpa-onnx-ced-small-audio-tagging-2024-04-19"
|
||||
return AudioTaggingConfig(
|
||||
model = AudioTaggingModelConfig(
|
||||
ced = "$modelDir/model.int8.onnx",
|
||||
numThreads = numThreads,
|
||||
debug = true,
|
||||
),
|
||||
labels = "$modelDir/class_labels_indices.csv",
|
||||
topK = 3,
|
||||
)
|
||||
}
|
||||
|
||||
5 -> {
|
||||
val modelDir = "sherpa-onnx-ced-base-audio-tagging-2024-04-19"
|
||||
return AudioTaggingConfig(
|
||||
model = AudioTaggingModelConfig(
|
||||
ced = "$modelDir/model.int8.onnx",
|
||||
numThreads = numThreads,
|
||||
debug = true,
|
||||
),
|
||||
labels = "$modelDir/class_labels_indices.csv",
|
||||
topK = 3,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
return null
|
||||
}
|
||||
10
sherpa-onnx/kotlin-api/FeatureConfig.kt
Normal file
10
sherpa-onnx/kotlin-api/FeatureConfig.kt
Normal file
@@ -0,0 +1,10 @@
|
||||
package com.k2fsa.sherpa.onnx
|
||||
|
||||
data class FeatureConfig(
|
||||
var sampleRate: Int = 16000,
|
||||
var featureDim: Int = 80,
|
||||
)
|
||||
|
||||
fun getFeatureConfig(sampleRate: Int, featureDim: Int): FeatureConfig {
|
||||
return FeatureConfig(sampleRate = sampleRate, featureDim = featureDim)
|
||||
}
|
||||
151
sherpa-onnx/kotlin-api/KeywordSpotter.kt
Normal file
151
sherpa-onnx/kotlin-api/KeywordSpotter.kt
Normal file
@@ -0,0 +1,151 @@
|
||||
// Copyright (c) 2024 Xiaomi Corporation
|
||||
package com.k2fsa.sherpa.onnx
|
||||
|
||||
import android.content.res.AssetManager
|
||||
|
||||
data class KeywordSpotterConfig(
|
||||
var featConfig: FeatureConfig = FeatureConfig(),
|
||||
var modelConfig: OnlineModelConfig,
|
||||
var maxActivePaths: Int = 4,
|
||||
var keywordsFile: String = "keywords.txt",
|
||||
var keywordsScore: Float = 1.5f,
|
||||
var keywordsThreshold: Float = 0.25f,
|
||||
var numTrailingBlanks: Int = 2,
|
||||
)
|
||||
|
||||
data class KeywordSpotterResult(
|
||||
val keyword: String,
|
||||
val tokens: Array<String>,
|
||||
val timestamps: FloatArray,
|
||||
// TODO(fangjun): Add more fields
|
||||
)
|
||||
|
||||
class KeywordSpotter(
|
||||
assetManager: AssetManager? = null,
|
||||
val config: KeywordSpotterConfig,
|
||||
) {
|
||||
private val ptr: Long
|
||||
|
||||
init {
|
||||
ptr = if (assetManager != null) {
|
||||
newFromAsset(assetManager, config)
|
||||
} else {
|
||||
newFromFile(config)
|
||||
}
|
||||
}
|
||||
|
||||
protected fun finalize() {
|
||||
delete(ptr)
|
||||
}
|
||||
|
||||
fun release() = finalize()
|
||||
|
||||
fun createStream(keywords: String = ""): OnlineStream {
|
||||
val p = createStream(ptr, keywords)
|
||||
return OnlineStream(p)
|
||||
}
|
||||
|
||||
fun decode(stream: OnlineStream) = decode(ptr, stream.ptr)
|
||||
fun isReady(stream: OnlineStream) = isReady(ptr, stream.ptr)
|
||||
fun getResult(stream: OnlineStream): KeywordSpotterResult {
|
||||
val objArray = getResult(ptr, stream.ptr)
|
||||
|
||||
val keyword = objArray[0] as String
|
||||
val tokens = objArray[1] as Array<String>
|
||||
val timestamps = objArray[2] as FloatArray
|
||||
|
||||
return KeywordSpotterResult(keyword = keyword, tokens = tokens, timestamps = timestamps)
|
||||
}
|
||||
|
||||
private external fun delete(ptr: Long)
|
||||
|
||||
private external fun newFromAsset(
|
||||
assetManager: AssetManager,
|
||||
config: KeywordSpotterConfig,
|
||||
): Long
|
||||
|
||||
private external fun newFromFile(
|
||||
config: KeywordSpotterConfig,
|
||||
): Long
|
||||
|
||||
private external fun createStream(ptr: Long, keywords: String): Long
|
||||
private external fun isReady(ptr: Long, streamPtr: Long): Boolean
|
||||
private external fun decode(ptr: Long, streamPtr: Long)
|
||||
private external fun getResult(ptr: Long, streamPtr: Long): Array<Any>
|
||||
|
||||
companion object {
|
||||
init {
|
||||
System.loadLibrary("sherpa-onnx-jni")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
Please see
|
||||
https://k2-fsa.github.io/sherpa/onnx/kws/pretrained_models/index.html
|
||||
for a list of pre-trained models.
|
||||
|
||||
We only add a few here. Please change the following code
|
||||
to add your own. (It should be straightforward to add a new model
|
||||
by following the code)
|
||||
|
||||
@param type
|
||||
0 - sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01 (Chinese)
|
||||
https://www.modelscope.cn/models/pkufool/sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01/summary
|
||||
|
||||
1 - sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01 (English)
|
||||
https://www.modelscope.cn/models/pkufool/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01/summary
|
||||
|
||||
*/
|
||||
fun getKwsModelConfig(type: Int): OnlineModelConfig? {
|
||||
when (type) {
|
||||
0 -> {
|
||||
val modelDir = "sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01"
|
||||
return OnlineModelConfig(
|
||||
transducer = OnlineTransducerModelConfig(
|
||||
encoder = "$modelDir/encoder-epoch-12-avg-2-chunk-16-left-64.onnx",
|
||||
decoder = "$modelDir/decoder-epoch-12-avg-2-chunk-16-left-64.onnx",
|
||||
joiner = "$modelDir/joiner-epoch-12-avg-2-chunk-16-left-64.onnx",
|
||||
),
|
||||
tokens = "$modelDir/tokens.txt",
|
||||
modelType = "zipformer2",
|
||||
)
|
||||
}
|
||||
|
||||
1 -> {
|
||||
val modelDir = "sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01"
|
||||
return OnlineModelConfig(
|
||||
transducer = OnlineTransducerModelConfig(
|
||||
encoder = "$modelDir/encoder-epoch-12-avg-2-chunk-16-left-64.onnx",
|
||||
decoder = "$modelDir/decoder-epoch-12-avg-2-chunk-16-left-64.onnx",
|
||||
joiner = "$modelDir/joiner-epoch-12-avg-2-chunk-16-left-64.onnx",
|
||||
),
|
||||
tokens = "$modelDir/tokens.txt",
|
||||
modelType = "zipformer2",
|
||||
)
|
||||
}
|
||||
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
/*
|
||||
* Get the default keywords for each model.
|
||||
* Caution: The types and modelDir should be the same as those in getModelConfig
|
||||
* function above.
|
||||
*/
|
||||
fun getKeywordsFile(type: Int): String {
|
||||
when (type) {
|
||||
0 -> {
|
||||
val modelDir = "sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01"
|
||||
return "$modelDir/keywords.txt"
|
||||
}
|
||||
|
||||
1 -> {
|
||||
val modelDir = "sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01"
|
||||
return "$modelDir/keywords.txt"
|
||||
}
|
||||
|
||||
}
|
||||
return ""
|
||||
}
|
||||
221
sherpa-onnx/kotlin-api/OfflineRecognizer.kt
Normal file
221
sherpa-onnx/kotlin-api/OfflineRecognizer.kt
Normal file
@@ -0,0 +1,221 @@
|
||||
package com.k2fsa.sherpa.onnx
|
||||
|
||||
import android.content.res.AssetManager
|
||||
|
||||
data class OfflineRecognizerResult(
|
||||
val text: String,
|
||||
val tokens: Array<String>,
|
||||
val timestamps: FloatArray,
|
||||
)
|
||||
|
||||
data class OfflineTransducerModelConfig(
|
||||
var encoder: String = "",
|
||||
var decoder: String = "",
|
||||
var joiner: String = "",
|
||||
)
|
||||
|
||||
data class OfflineParaformerModelConfig(
|
||||
var model: String = "",
|
||||
)
|
||||
|
||||
data class OfflineWhisperModelConfig(
|
||||
var encoder: String = "",
|
||||
var decoder: String = "",
|
||||
var language: String = "en", // Used with multilingual model
|
||||
var task: String = "transcribe", // transcribe or translate
|
||||
var tailPaddings: Int = 1000, // Padding added at the end of the samples
|
||||
)
|
||||
|
||||
data class OfflineModelConfig(
|
||||
var transducer: OfflineTransducerModelConfig = OfflineTransducerModelConfig(),
|
||||
var paraformer: OfflineParaformerModelConfig = OfflineParaformerModelConfig(),
|
||||
var whisper: OfflineWhisperModelConfig = OfflineWhisperModelConfig(),
|
||||
var numThreads: Int = 1,
|
||||
var debug: Boolean = false,
|
||||
var provider: String = "cpu",
|
||||
var modelType: String = "",
|
||||
var tokens: String,
|
||||
)
|
||||
|
||||
data class OfflineRecognizerConfig(
|
||||
var featConfig: FeatureConfig = FeatureConfig(),
|
||||
var modelConfig: OfflineModelConfig,
|
||||
// var lmConfig: OfflineLMConfig(), // TODO(fangjun): enable it
|
||||
var decodingMethod: String = "greedy_search",
|
||||
var maxActivePaths: Int = 4,
|
||||
var hotwordsFile: String = "",
|
||||
var hotwordsScore: Float = 1.5f,
|
||||
)
|
||||
|
||||
class OfflineRecognizer(
|
||||
assetManager: AssetManager? = null,
|
||||
config: OfflineRecognizerConfig,
|
||||
) {
|
||||
private val ptr: Long
|
||||
|
||||
init {
|
||||
ptr = if (assetManager != null) {
|
||||
newFromAsset(assetManager, config)
|
||||
} else {
|
||||
newFromFile(config)
|
||||
}
|
||||
}
|
||||
|
||||
protected fun finalize() {
|
||||
delete(ptr)
|
||||
}
|
||||
|
||||
fun release() = finalize()
|
||||
|
||||
fun createStream(): OfflineStream {
|
||||
val p = createStream(ptr)
|
||||
return OfflineStream(p)
|
||||
}
|
||||
|
||||
fun getResult(stream: OfflineStream): OfflineRecognizerResult {
|
||||
val objArray = getResult(stream.ptr)
|
||||
|
||||
val text = objArray[0] as String
|
||||
val tokens = objArray[1] as Array<String>
|
||||
val timestamps = objArray[2] as FloatArray
|
||||
return OfflineRecognizerResult(text = text, tokens = tokens, timestamps = timestamps)
|
||||
}
|
||||
|
||||
fun decode(stream: OfflineStream) = decode(ptr, stream.ptr)
|
||||
|
||||
private external fun delete(ptr: Long)
|
||||
|
||||
private external fun createStream(ptr: Long): Long
|
||||
|
||||
private external fun newFromAsset(
|
||||
assetManager: AssetManager,
|
||||
config: OfflineRecognizerConfig,
|
||||
): Long
|
||||
|
||||
private external fun newFromFile(
|
||||
config: OfflineRecognizerConfig,
|
||||
): Long
|
||||
|
||||
private external fun decode(ptr: Long, streamPtr: Long)
|
||||
|
||||
private external fun getResult(streamPtr: Long): Array<Any>
|
||||
|
||||
companion object {
|
||||
init {
|
||||
System.loadLibrary("sherpa-onnx-jni")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
Please see
|
||||
https://k2-fsa.github.io/sherpa/onnx/pretrained_models/index.html
|
||||
for a list of pre-trained models.
|
||||
|
||||
We only add a few here. Please change the following code
|
||||
to add your own. (It should be straightforward to add a new model
|
||||
by following the code)
|
||||
|
||||
@param type
|
||||
|
||||
0 - csukuangfj/sherpa-onnx-paraformer-zh-2023-03-28 (Chinese)
|
||||
https://k2-fsa.github.io/sherpa/onnx/pretrained_models/offline-paraformer/paraformer-models.html#csukuangfj-sherpa-onnx-paraformer-zh-2023-03-28-chinese
|
||||
int8
|
||||
|
||||
1 - icefall-asr-multidataset-pruned_transducer_stateless7-2023-05-04 (English)
|
||||
https://k2-fsa.github.io/sherpa/onnx/pretrained_models/offline-transducer/zipformer-transducer-models.html#icefall-asr-multidataset-pruned-transducer-stateless7-2023-05-04-english
|
||||
encoder int8, decoder/joiner float32
|
||||
|
||||
2 - sherpa-onnx-whisper-tiny.en
|
||||
https://k2-fsa.github.io/sherpa/onnx/pretrained_models/whisper/tiny.en.html#tiny-en
|
||||
encoder int8, decoder int8
|
||||
|
||||
3 - sherpa-onnx-whisper-base.en
|
||||
https://k2-fsa.github.io/sherpa/onnx/pretrained_models/whisper/tiny.en.html#tiny-en
|
||||
encoder int8, decoder int8
|
||||
|
||||
4 - pkufool/icefall-asr-zipformer-wenetspeech-20230615 (Chinese)
|
||||
https://k2-fsa.github.io/sherpa/onnx/pretrained_models/offline-transducer/zipformer-transducer-models.html#pkufool-icefall-asr-zipformer-wenetspeech-20230615-chinese
|
||||
encoder/joiner int8, decoder fp32
|
||||
|
||||
*/
|
||||
fun getOfflineModelConfig(type: Int): OfflineModelConfig? {
|
||||
when (type) {
|
||||
0 -> {
|
||||
val modelDir = "sherpa-onnx-paraformer-zh-2023-03-28"
|
||||
return OfflineModelConfig(
|
||||
paraformer = OfflineParaformerModelConfig(
|
||||
model = "$modelDir/model.int8.onnx",
|
||||
),
|
||||
tokens = "$modelDir/tokens.txt",
|
||||
modelType = "paraformer",
|
||||
)
|
||||
}
|
||||
|
||||
1 -> {
|
||||
val modelDir = "icefall-asr-multidataset-pruned_transducer_stateless7-2023-05-04"
|
||||
return OfflineModelConfig(
|
||||
transducer = OfflineTransducerModelConfig(
|
||||
encoder = "$modelDir/encoder-epoch-30-avg-4.int8.onnx",
|
||||
decoder = "$modelDir/decoder-epoch-30-avg-4.onnx",
|
||||
joiner = "$modelDir/joiner-epoch-30-avg-4.onnx",
|
||||
),
|
||||
tokens = "$modelDir/tokens.txt",
|
||||
modelType = "zipformer",
|
||||
)
|
||||
}
|
||||
|
||||
2 -> {
|
||||
val modelDir = "sherpa-onnx-whisper-tiny.en"
|
||||
return OfflineModelConfig(
|
||||
whisper = OfflineWhisperModelConfig(
|
||||
encoder = "$modelDir/tiny.en-encoder.int8.onnx",
|
||||
decoder = "$modelDir/tiny.en-decoder.int8.onnx",
|
||||
),
|
||||
tokens = "$modelDir/tiny.en-tokens.txt",
|
||||
modelType = "whisper",
|
||||
)
|
||||
}
|
||||
|
||||
3 -> {
|
||||
val modelDir = "sherpa-onnx-whisper-base.en"
|
||||
return OfflineModelConfig(
|
||||
whisper = OfflineWhisperModelConfig(
|
||||
encoder = "$modelDir/base.en-encoder.int8.onnx",
|
||||
decoder = "$modelDir/base.en-decoder.int8.onnx",
|
||||
),
|
||||
tokens = "$modelDir/base.en-tokens.txt",
|
||||
modelType = "whisper",
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
4 -> {
|
||||
val modelDir = "icefall-asr-zipformer-wenetspeech-20230615"
|
||||
return OfflineModelConfig(
|
||||
transducer = OfflineTransducerModelConfig(
|
||||
encoder = "$modelDir/encoder-epoch-12-avg-4.int8.onnx",
|
||||
decoder = "$modelDir/decoder-epoch-12-avg-4.onnx",
|
||||
joiner = "$modelDir/joiner-epoch-12-avg-4.int8.onnx",
|
||||
),
|
||||
tokens = "$modelDir/tokens.txt",
|
||||
modelType = "zipformer",
|
||||
)
|
||||
}
|
||||
|
||||
5 -> {
|
||||
val modelDir = "sherpa-onnx-zipformer-multi-zh-hans-2023-9-2"
|
||||
return OfflineModelConfig(
|
||||
transducer = OfflineTransducerModelConfig(
|
||||
encoder = "$modelDir/encoder-epoch-20-avg-1.int8.onnx",
|
||||
decoder = "$modelDir/decoder-epoch-20-avg-1.onnx",
|
||||
joiner = "$modelDir/joiner-epoch-20-avg-1.int8.onnx",
|
||||
),
|
||||
tokens = "$modelDir/tokens.txt",
|
||||
modelType = "zipformer2",
|
||||
)
|
||||
}
|
||||
|
||||
}
|
||||
return null
|
||||
}
|
||||
24
sherpa-onnx/kotlin-api/OfflineStream.kt
Normal file
24
sherpa-onnx/kotlin-api/OfflineStream.kt
Normal file
@@ -0,0 +1,24 @@
|
||||
package com.k2fsa.sherpa.onnx
|
||||
|
||||
class OfflineStream(var ptr: Long) {
|
||||
fun acceptWaveform(samples: FloatArray, sampleRate: Int) =
|
||||
acceptWaveform(ptr, samples, sampleRate)
|
||||
|
||||
protected fun finalize() {
|
||||
if (ptr != 0L) {
|
||||
delete(ptr)
|
||||
ptr = 0
|
||||
}
|
||||
}
|
||||
|
||||
fun release() = finalize()
|
||||
|
||||
private external fun acceptWaveform(ptr: Long, samples: FloatArray, sampleRate: Int)
|
||||
private external fun delete(ptr: Long)
|
||||
|
||||
companion object {
|
||||
init {
|
||||
System.loadLibrary("sherpa-onnx-jni")
|
||||
}
|
||||
}
|
||||
}
|
||||
352
sherpa-onnx/kotlin-api/OnlineRecognizer.kt
Normal file
352
sherpa-onnx/kotlin-api/OnlineRecognizer.kt
Normal file
@@ -0,0 +1,352 @@
|
||||
package com.k2fsa.sherpa.onnx
|
||||
|
||||
import android.content.res.AssetManager
|
||||
|
||||
data class EndpointRule(
|
||||
var mustContainNonSilence: Boolean,
|
||||
var minTrailingSilence: Float,
|
||||
var minUtteranceLength: Float,
|
||||
)
|
||||
|
||||
data class EndpointConfig(
|
||||
var rule1: EndpointRule = EndpointRule(false, 2.4f, 0.0f),
|
||||
var rule2: EndpointRule = EndpointRule(true, 1.4f, 0.0f),
|
||||
var rule3: EndpointRule = EndpointRule(false, 0.0f, 20.0f)
|
||||
)
|
||||
|
||||
data class OnlineTransducerModelConfig(
|
||||
var encoder: String = "",
|
||||
var decoder: String = "",
|
||||
var joiner: String = "",
|
||||
)
|
||||
|
||||
data class OnlineParaformerModelConfig(
|
||||
var encoder: String = "",
|
||||
var decoder: String = "",
|
||||
)
|
||||
|
||||
data class OnlineZipformer2CtcModelConfig(
|
||||
var model: String = "",
|
||||
)
|
||||
|
||||
data class OnlineModelConfig(
|
||||
var transducer: OnlineTransducerModelConfig = OnlineTransducerModelConfig(),
|
||||
var paraformer: OnlineParaformerModelConfig = OnlineParaformerModelConfig(),
|
||||
var zipformer2Ctc: OnlineZipformer2CtcModelConfig = OnlineZipformer2CtcModelConfig(),
|
||||
var tokens: String,
|
||||
var numThreads: Int = 1,
|
||||
var debug: Boolean = false,
|
||||
var provider: String = "cpu",
|
||||
var modelType: String = "",
|
||||
)
|
||||
|
||||
data class OnlineLMConfig(
|
||||
var model: String = "",
|
||||
var scale: Float = 0.5f,
|
||||
)
|
||||
|
||||
|
||||
data class OnlineRecognizerConfig(
|
||||
var featConfig: FeatureConfig = FeatureConfig(),
|
||||
var modelConfig: OnlineModelConfig,
|
||||
var lmConfig: OnlineLMConfig = OnlineLMConfig(),
|
||||
var endpointConfig: EndpointConfig = EndpointConfig(),
|
||||
var enableEndpoint: Boolean = true,
|
||||
var decodingMethod: String = "greedy_search",
|
||||
var maxActivePaths: Int = 4,
|
||||
var hotwordsFile: String = "",
|
||||
var hotwordsScore: Float = 1.5f,
|
||||
)
|
||||
|
||||
data class OnlineRecognizerResult(
|
||||
val text: String,
|
||||
val tokens: Array<String>,
|
||||
val timestamps: FloatArray,
|
||||
// TODO(fangjun): Add more fields
|
||||
)
|
||||
|
||||
class OnlineRecognizer(
|
||||
assetManager: AssetManager? = null,
|
||||
val config: OnlineRecognizerConfig,
|
||||
) {
|
||||
private val ptr: Long
|
||||
|
||||
init {
|
||||
ptr = if (assetManager != null) {
|
||||
newFromAsset(assetManager, config)
|
||||
} else {
|
||||
newFromFile(config)
|
||||
}
|
||||
}
|
||||
|
||||
protected fun finalize() {
|
||||
delete(ptr)
|
||||
}
|
||||
|
||||
fun release() = finalize()
|
||||
|
||||
fun createStream(hotwords: String = ""): OnlineStream {
|
||||
val p = createStream(ptr, hotwords)
|
||||
return OnlineStream(p)
|
||||
}
|
||||
|
||||
fun reset(stream: OnlineStream) = reset(ptr, stream.ptr)
|
||||
fun decode(stream: OnlineStream) = decode(ptr, stream.ptr)
|
||||
fun isEndpoint(stream: OnlineStream) = isEndpoint(ptr, stream.ptr)
|
||||
fun isReady(stream: OnlineStream) = isReady(ptr, stream.ptr)
|
||||
fun getResult(stream: OnlineStream): OnlineRecognizerResult {
|
||||
val objArray = getResult(ptr, stream.ptr)
|
||||
|
||||
val text = objArray[0] as String
|
||||
val tokens = objArray[1] as Array<String>
|
||||
val timestamps = objArray[2] as FloatArray
|
||||
|
||||
return OnlineRecognizerResult(text = text, tokens = tokens, timestamps = timestamps)
|
||||
}
|
||||
|
||||
private external fun delete(ptr: Long)
|
||||
|
||||
private external fun newFromAsset(
|
||||
assetManager: AssetManager,
|
||||
config: OnlineRecognizerConfig,
|
||||
): Long
|
||||
|
||||
private external fun newFromFile(
|
||||
config: OnlineRecognizerConfig,
|
||||
): Long
|
||||
|
||||
private external fun createStream(ptr: Long, hotwords: String): Long
|
||||
private external fun reset(ptr: Long, streamPtr: Long)
|
||||
private external fun decode(ptr: Long, streamPtr: Long)
|
||||
private external fun isEndpoint(ptr: Long, streamPtr: Long): Boolean
|
||||
private external fun isReady(ptr: Long, streamPtr: Long): Boolean
|
||||
private external fun getResult(ptr: Long, streamPtr: Long): Array<Any>
|
||||
|
||||
companion object {
|
||||
init {
|
||||
System.loadLibrary("sherpa-onnx-jni")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
Please see
|
||||
https://k2-fsa.github.io/sherpa/onnx/pretrained_models/index.html
|
||||
for a list of pre-trained models.
|
||||
|
||||
We only add a few here. Please change the following code
|
||||
to add your own. (It should be straightforward to add a new model
|
||||
by following the code)
|
||||
|
||||
@param type
|
||||
0 - sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20 (Bilingual, Chinese + English)
|
||||
https://k2-fsa.github.io/sherpa/onnx/pretrained_models/zipformer-transducer-models.html#sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20-bilingual-chinese-english
|
||||
|
||||
1 - csukuangfj/sherpa-onnx-lstm-zh-2023-02-20 (Chinese)
|
||||
|
||||
https://k2-fsa.github.io/sherpa/onnx/pretrained_models/lstm-transducer-models.html#csukuangfj-sherpa-onnx-lstm-zh-2023-02-20-chinese
|
||||
|
||||
2 - csukuangfj/sherpa-onnx-lstm-en-2023-02-17 (English)
|
||||
https://k2-fsa.github.io/sherpa/onnx/pretrained_models/lstm-transducer-models.html#csukuangfj-sherpa-onnx-lstm-en-2023-02-17-english
|
||||
|
||||
3,4 - pkufool/icefall-asr-zipformer-streaming-wenetspeech-20230615
|
||||
https://huggingface.co/pkufool/icefall-asr-zipformer-streaming-wenetspeech-20230615
|
||||
3 - int8 encoder
|
||||
4 - float32 encoder
|
||||
|
||||
5 - csukuangfj/sherpa-onnx-streaming-paraformer-bilingual-zh-en
|
||||
https://huggingface.co/csukuangfj/sherpa-onnx-streaming-paraformer-bilingual-zh-en
|
||||
|
||||
6 - sherpa-onnx-streaming-zipformer-en-2023-06-26
|
||||
https://huggingface.co/csukuangfj/sherpa-onnx-streaming-zipformer-en-2023-06-26
|
||||
|
||||
7 - shaojieli/sherpa-onnx-streaming-zipformer-fr-2023-04-14 (French)
|
||||
https://huggingface.co/shaojieli/sherpa-onnx-streaming-zipformer-fr-2023-04-14
|
||||
|
||||
8 - csukuangfj/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20 (Bilingual, Chinese + English)
|
||||
https://huggingface.co/csukuangfj/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20
|
||||
encoder int8, decoder/joiner float32
|
||||
|
||||
*/
|
||||
fun getModelConfig(type: Int): OnlineModelConfig? {
|
||||
when (type) {
|
||||
0 -> {
|
||||
val modelDir = "sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20"
|
||||
return OnlineModelConfig(
|
||||
transducer = OnlineTransducerModelConfig(
|
||||
encoder = "$modelDir/encoder-epoch-99-avg-1.onnx",
|
||||
decoder = "$modelDir/decoder-epoch-99-avg-1.onnx",
|
||||
joiner = "$modelDir/joiner-epoch-99-avg-1.onnx",
|
||||
),
|
||||
tokens = "$modelDir/tokens.txt",
|
||||
modelType = "zipformer",
|
||||
)
|
||||
}
|
||||
|
||||
1 -> {
|
||||
val modelDir = "sherpa-onnx-lstm-zh-2023-02-20"
|
||||
return OnlineModelConfig(
|
||||
transducer = OnlineTransducerModelConfig(
|
||||
encoder = "$modelDir/encoder-epoch-11-avg-1.onnx",
|
||||
decoder = "$modelDir/decoder-epoch-11-avg-1.onnx",
|
||||
joiner = "$modelDir/joiner-epoch-11-avg-1.onnx",
|
||||
),
|
||||
tokens = "$modelDir/tokens.txt",
|
||||
modelType = "lstm",
|
||||
)
|
||||
}
|
||||
|
||||
2 -> {
|
||||
val modelDir = "sherpa-onnx-lstm-en-2023-02-17"
|
||||
return OnlineModelConfig(
|
||||
transducer = OnlineTransducerModelConfig(
|
||||
encoder = "$modelDir/encoder-epoch-99-avg-1.onnx",
|
||||
decoder = "$modelDir/decoder-epoch-99-avg-1.onnx",
|
||||
joiner = "$modelDir/joiner-epoch-99-avg-1.onnx",
|
||||
),
|
||||
tokens = "$modelDir/tokens.txt",
|
||||
modelType = "lstm",
|
||||
)
|
||||
}
|
||||
|
||||
3 -> {
|
||||
val modelDir = "icefall-asr-zipformer-streaming-wenetspeech-20230615"
|
||||
return OnlineModelConfig(
|
||||
transducer = OnlineTransducerModelConfig(
|
||||
encoder = "$modelDir/exp/encoder-epoch-12-avg-4-chunk-16-left-128.int8.onnx",
|
||||
decoder = "$modelDir/exp/decoder-epoch-12-avg-4-chunk-16-left-128.onnx",
|
||||
joiner = "$modelDir/exp/joiner-epoch-12-avg-4-chunk-16-left-128.onnx",
|
||||
),
|
||||
tokens = "$modelDir/data/lang_char/tokens.txt",
|
||||
modelType = "zipformer2",
|
||||
)
|
||||
}
|
||||
|
||||
4 -> {
|
||||
val modelDir = "icefall-asr-zipformer-streaming-wenetspeech-20230615"
|
||||
return OnlineModelConfig(
|
||||
transducer = OnlineTransducerModelConfig(
|
||||
encoder = "$modelDir/exp/encoder-epoch-12-avg-4-chunk-16-left-128.onnx",
|
||||
decoder = "$modelDir/exp/decoder-epoch-12-avg-4-chunk-16-left-128.onnx",
|
||||
joiner = "$modelDir/exp/joiner-epoch-12-avg-4-chunk-16-left-128.onnx",
|
||||
),
|
||||
tokens = "$modelDir/data/lang_char/tokens.txt",
|
||||
modelType = "zipformer2",
|
||||
)
|
||||
}
|
||||
|
||||
5 -> {
|
||||
val modelDir = "sherpa-onnx-streaming-paraformer-bilingual-zh-en"
|
||||
return OnlineModelConfig(
|
||||
paraformer = OnlineParaformerModelConfig(
|
||||
encoder = "$modelDir/encoder.int8.onnx",
|
||||
decoder = "$modelDir/decoder.int8.onnx",
|
||||
),
|
||||
tokens = "$modelDir/tokens.txt",
|
||||
modelType = "paraformer",
|
||||
)
|
||||
}
|
||||
|
||||
6 -> {
|
||||
val modelDir = "sherpa-onnx-streaming-zipformer-en-2023-06-26"
|
||||
return OnlineModelConfig(
|
||||
transducer = OnlineTransducerModelConfig(
|
||||
encoder = "$modelDir/encoder-epoch-99-avg-1-chunk-16-left-128.int8.onnx",
|
||||
decoder = "$modelDir/decoder-epoch-99-avg-1-chunk-16-left-128.onnx",
|
||||
joiner = "$modelDir/joiner-epoch-99-avg-1-chunk-16-left-128.onnx",
|
||||
),
|
||||
tokens = "$modelDir/tokens.txt",
|
||||
modelType = "zipformer2",
|
||||
)
|
||||
}
|
||||
|
||||
7 -> {
|
||||
val modelDir = "sherpa-onnx-streaming-zipformer-fr-2023-04-14"
|
||||
return OnlineModelConfig(
|
||||
transducer = OnlineTransducerModelConfig(
|
||||
encoder = "$modelDir/encoder-epoch-29-avg-9-with-averaged-model.int8.onnx",
|
||||
decoder = "$modelDir/decoder-epoch-29-avg-9-with-averaged-model.onnx",
|
||||
joiner = "$modelDir/joiner-epoch-29-avg-9-with-averaged-model.onnx",
|
||||
),
|
||||
tokens = "$modelDir/tokens.txt",
|
||||
modelType = "zipformer",
|
||||
)
|
||||
}
|
||||
|
||||
8 -> {
|
||||
val modelDir = "sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20"
|
||||
return OnlineModelConfig(
|
||||
transducer = OnlineTransducerModelConfig(
|
||||
encoder = "$modelDir/encoder-epoch-99-avg-1.int8.onnx",
|
||||
decoder = "$modelDir/decoder-epoch-99-avg-1.onnx",
|
||||
joiner = "$modelDir/joiner-epoch-99-avg-1.int8.onnx",
|
||||
),
|
||||
tokens = "$modelDir/tokens.txt",
|
||||
modelType = "zipformer",
|
||||
)
|
||||
}
|
||||
|
||||
9 -> {
|
||||
val modelDir = "sherpa-onnx-streaming-zipformer-zh-14M-2023-02-23"
|
||||
return OnlineModelConfig(
|
||||
transducer = OnlineTransducerModelConfig(
|
||||
encoder = "$modelDir/encoder-epoch-99-avg-1.int8.onnx",
|
||||
decoder = "$modelDir/decoder-epoch-99-avg-1.onnx",
|
||||
joiner = "$modelDir/joiner-epoch-99-avg-1.int8.onnx",
|
||||
),
|
||||
tokens = "$modelDir/tokens.txt",
|
||||
modelType = "zipformer",
|
||||
)
|
||||
}
|
||||
|
||||
10 -> {
|
||||
val modelDir = "sherpa-onnx-streaming-zipformer-en-20M-2023-02-17"
|
||||
return OnlineModelConfig(
|
||||
transducer = OnlineTransducerModelConfig(
|
||||
encoder = "$modelDir/encoder-epoch-99-avg-1.int8.onnx",
|
||||
decoder = "$modelDir/decoder-epoch-99-avg-1.onnx",
|
||||
joiner = "$modelDir/joiner-epoch-99-avg-1.int8.onnx",
|
||||
),
|
||||
tokens = "$modelDir/tokens.txt",
|
||||
modelType = "zipformer",
|
||||
)
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
/*
|
||||
Please see
|
||||
https://k2-fsa.github.io/sherpa/onnx/pretrained_models/index.html
|
||||
for a list of pre-trained models.
|
||||
|
||||
We only add a few here. Please change the following code
|
||||
to add your own LM model. (It should be straightforward to train a new NN LM model
|
||||
by following the code, https://github.com/k2-fsa/icefall/blob/master/icefall/rnn_lm/train.py)
|
||||
|
||||
@param type
|
||||
0 - sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20 (Bilingual, Chinese + English)
|
||||
https://k2-fsa.github.io/sherpa/onnx/pretrained_models/zipformer-transducer-models.html#sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20-bilingual-chinese-english
|
||||
*/
|
||||
fun getOnlineLMConfig(type: Int): OnlineLMConfig {
|
||||
when (type) {
|
||||
0 -> {
|
||||
val modelDir = "sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20"
|
||||
return OnlineLMConfig(
|
||||
model = "$modelDir/with-state-epoch-99-avg-1.int8.onnx",
|
||||
scale = 0.5f,
|
||||
)
|
||||
}
|
||||
}
|
||||
return OnlineLMConfig()
|
||||
}
|
||||
|
||||
fun getEndpointConfig(): EndpointConfig {
|
||||
return EndpointConfig(
|
||||
rule1 = EndpointRule(false, 2.4f, 0.0f),
|
||||
rule2 = EndpointRule(true, 1.4f, 0.0f),
|
||||
rule3 = EndpointRule(false, 0.0f, 20.0f)
|
||||
)
|
||||
}
|
||||
|
||||
27
sherpa-onnx/kotlin-api/OnlineStream.kt
Normal file
27
sherpa-onnx/kotlin-api/OnlineStream.kt
Normal file
@@ -0,0 +1,27 @@
|
||||
package com.k2fsa.sherpa.onnx
|
||||
|
||||
class OnlineStream(var ptr: Long = 0) {
|
||||
fun acceptWaveform(samples: FloatArray, sampleRate: Int) =
|
||||
acceptWaveform(ptr, samples, sampleRate)
|
||||
|
||||
fun inputFinished() = inputFinished(ptr)
|
||||
|
||||
protected fun finalize() {
|
||||
if (ptr != 0L) {
|
||||
delete(ptr)
|
||||
ptr = 0
|
||||
}
|
||||
}
|
||||
|
||||
fun release() = finalize()
|
||||
|
||||
private external fun acceptWaveform(ptr: Long, samples: FloatArray, sampleRate: Int)
|
||||
private external fun inputFinished(ptr: Long)
|
||||
private external fun delete(ptr: Long)
|
||||
|
||||
companion object {
|
||||
init {
|
||||
System.loadLibrary("sherpa-onnx-jni")
|
||||
}
|
||||
}
|
||||
}
|
||||
164
sherpa-onnx/kotlin-api/Speaker.kt
Normal file
164
sherpa-onnx/kotlin-api/Speaker.kt
Normal file
@@ -0,0 +1,164 @@
|
||||
package com.k2fsa.sherpa.onnx
|
||||
|
||||
import android.content.res.AssetManager
|
||||
import android.util.Log
|
||||
|
||||
data class SpeakerEmbeddingExtractorConfig(
|
||||
val model: String,
|
||||
var numThreads: Int = 1,
|
||||
var debug: Boolean = false,
|
||||
var provider: String = "cpu",
|
||||
)
|
||||
|
||||
class SpeakerEmbeddingExtractor(
|
||||
assetManager: AssetManager? = null,
|
||||
config: SpeakerEmbeddingExtractorConfig,
|
||||
) {
|
||||
private var ptr: Long
|
||||
|
||||
init {
|
||||
ptr = if (assetManager != null) {
|
||||
newFromAsset(assetManager, config)
|
||||
} else {
|
||||
newFromFile(config)
|
||||
}
|
||||
}
|
||||
|
||||
protected fun finalize() {
|
||||
if (ptr != 0L) {
|
||||
delete(ptr)
|
||||
ptr = 0
|
||||
}
|
||||
}
|
||||
|
||||
fun release() = finalize()
|
||||
|
||||
fun createStream(): OnlineStream {
|
||||
val p = createStream(ptr)
|
||||
return OnlineStream(p)
|
||||
}
|
||||
|
||||
fun isReady(stream: OnlineStream) = isReady(ptr, stream.ptr)
|
||||
fun compute(stream: OnlineStream) = compute(ptr, stream.ptr)
|
||||
fun dim() = dim(ptr)
|
||||
|
||||
private external fun newFromAsset(
|
||||
assetManager: AssetManager,
|
||||
config: SpeakerEmbeddingExtractorConfig,
|
||||
): Long
|
||||
|
||||
private external fun newFromFile(
|
||||
config: SpeakerEmbeddingExtractorConfig,
|
||||
): Long
|
||||
|
||||
private external fun delete(ptr: Long)
|
||||
|
||||
private external fun createStream(ptr: Long): Long
|
||||
|
||||
private external fun isReady(ptr: Long, streamPtr: Long): Boolean
|
||||
|
||||
private external fun compute(ptr: Long, streamPtr: Long): FloatArray
|
||||
|
||||
private external fun dim(ptr: Long): Int
|
||||
|
||||
companion object {
|
||||
init {
|
||||
System.loadLibrary("sherpa-onnx-jni")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
class SpeakerEmbeddingManager(val dim: Int) {
|
||||
private var ptr: Long
|
||||
|
||||
init {
|
||||
ptr = create(dim)
|
||||
}
|
||||
|
||||
protected fun finalize() {
|
||||
if (ptr != 0L) {
|
||||
delete(ptr)
|
||||
ptr = 0
|
||||
}
|
||||
}
|
||||
|
||||
fun release() = finalize()
|
||||
fun add(name: String, embedding: FloatArray) = add(ptr, name, embedding)
|
||||
fun add(name: String, embedding: Array<FloatArray>) = addList(ptr, name, embedding)
|
||||
fun remove(name: String) = remove(ptr, name)
|
||||
fun search(embedding: FloatArray, threshold: Float) = search(ptr, embedding, threshold)
|
||||
fun verify(name: String, embedding: FloatArray, threshold: Float) =
|
||||
verify(ptr, name, embedding, threshold)
|
||||
|
||||
fun contains(name: String) = contains(ptr, name)
|
||||
fun numSpeakers() = numSpeakers(ptr)
|
||||
|
||||
fun allSpeakerNames() = allSpeakerNames(ptr)
|
||||
|
||||
private external fun create(dim: Int): Long
|
||||
private external fun delete(ptr: Long): Unit
|
||||
private external fun add(ptr: Long, name: String, embedding: FloatArray): Boolean
|
||||
private external fun addList(ptr: Long, name: String, embedding: Array<FloatArray>): Boolean
|
||||
private external fun remove(ptr: Long, name: String): Boolean
|
||||
private external fun search(ptr: Long, embedding: FloatArray, threshold: Float): String
|
||||
private external fun verify(
|
||||
ptr: Long,
|
||||
name: String,
|
||||
embedding: FloatArray,
|
||||
threshold: Float
|
||||
): Boolean
|
||||
|
||||
private external fun contains(ptr: Long, name: String): Boolean
|
||||
private external fun numSpeakers(ptr: Long): Int
|
||||
|
||||
private external fun allSpeakerNames(ptr: Long): Array<String>
|
||||
|
||||
companion object {
|
||||
init {
|
||||
System.loadLibrary("sherpa-onnx-jni")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Please download the model file from
|
||||
// https://github.com/k2-fsa/sherpa-onnx/releases/tag/speaker-recongition-models
|
||||
// and put it inside the assets directory.
|
||||
//
|
||||
// Please don't put it in a subdirectory of assets
|
||||
private val modelName = "3dspeaker_speech_eres2net_base_sv_zh-cn_3dspeaker_16k.onnx"
|
||||
|
||||
object SpeakerRecognition {
|
||||
var _extractor: SpeakerEmbeddingExtractor? = null
|
||||
var _manager: SpeakerEmbeddingManager? = null
|
||||
|
||||
val extractor: SpeakerEmbeddingExtractor
|
||||
get() {
|
||||
return _extractor!!
|
||||
}
|
||||
|
||||
val manager: SpeakerEmbeddingManager
|
||||
get() {
|
||||
return _manager!!
|
||||
}
|
||||
|
||||
fun initExtractor(assetManager: AssetManager? = null) {
|
||||
synchronized(this) {
|
||||
if (_extractor != null) {
|
||||
return
|
||||
}
|
||||
Log.i("sherpa-onnx", "Initializing speaker embedding extractor")
|
||||
|
||||
_extractor = SpeakerEmbeddingExtractor(
|
||||
assetManager = assetManager,
|
||||
config = SpeakerEmbeddingExtractorConfig(
|
||||
model = modelName,
|
||||
numThreads = 2,
|
||||
debug = false,
|
||||
provider = "cpu",
|
||||
)
|
||||
)
|
||||
|
||||
_manager = SpeakerEmbeddingManager(dim = _extractor!!.dim())
|
||||
}
|
||||
}
|
||||
}
|
||||
103
sherpa-onnx/kotlin-api/SpokenLanguageIdentification.kt
Normal file
103
sherpa-onnx/kotlin-api/SpokenLanguageIdentification.kt
Normal file
@@ -0,0 +1,103 @@
|
||||
package com.k2fsa.sherpa.onnx
|
||||
|
||||
import android.content.res.AssetManager
|
||||
|
||||
data class SpokenLanguageIdentificationWhisperConfig(
|
||||
var encoder: String,
|
||||
var decoder: String,
|
||||
var tailPaddings: Int = -1,
|
||||
)
|
||||
|
||||
data class SpokenLanguageIdentificationConfig(
|
||||
var whisper: SpokenLanguageIdentificationWhisperConfig,
|
||||
var numThreads: Int = 1,
|
||||
var debug: Boolean = false,
|
||||
var provider: String = "cpu",
|
||||
)
|
||||
|
||||
class SpokenLanguageIdentification(
|
||||
assetManager: AssetManager? = null,
|
||||
config: SpokenLanguageIdentificationConfig,
|
||||
) {
|
||||
private var ptr: Long
|
||||
|
||||
init {
|
||||
ptr = if (assetManager != null) {
|
||||
newFromAsset(assetManager, config)
|
||||
} else {
|
||||
newFromFile(config)
|
||||
}
|
||||
}
|
||||
|
||||
protected fun finalize() {
|
||||
if (ptr != 0L) {
|
||||
delete(ptr)
|
||||
ptr = 0
|
||||
}
|
||||
}
|
||||
|
||||
fun release() = finalize()
|
||||
|
||||
fun createStream(): OfflineStream {
|
||||
val p = createStream(ptr)
|
||||
return OfflineStream(p)
|
||||
}
|
||||
|
||||
fun compute(stream: OfflineStream) = compute(ptr, stream.ptr)
|
||||
|
||||
private external fun newFromAsset(
|
||||
assetManager: AssetManager,
|
||||
config: SpokenLanguageIdentificationConfig,
|
||||
): Long
|
||||
|
||||
private external fun newFromFile(
|
||||
config: SpokenLanguageIdentificationConfig,
|
||||
): Long
|
||||
|
||||
private external fun delete(ptr: Long)
|
||||
|
||||
private external fun createStream(ptr: Long): Long
|
||||
|
||||
private external fun compute(ptr: Long, streamPtr: Long): String
|
||||
|
||||
companion object {
|
||||
init {
|
||||
System.loadLibrary("sherpa-onnx-jni")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// please refer to
|
||||
// https://k2-fsa.github.io/sherpa/onnx/spolken-language-identification/pretrained_models.html#whisper
|
||||
// to download more models
|
||||
fun getSpokenLanguageIdentificationConfig(
|
||||
type: Int,
|
||||
numThreads: Int = 1
|
||||
): SpokenLanguageIdentificationConfig? {
|
||||
when (type) {
|
||||
0 -> {
|
||||
val modelDir = "sherpa-onnx-whisper-tiny"
|
||||
return SpokenLanguageIdentificationConfig(
|
||||
whisper = SpokenLanguageIdentificationWhisperConfig(
|
||||
encoder = "$modelDir/tiny-encoder.int8.onnx",
|
||||
decoder = "$modelDir/tiny-decoder.int8.onnx",
|
||||
),
|
||||
numThreads = numThreads,
|
||||
debug = true,
|
||||
)
|
||||
}
|
||||
|
||||
1 -> {
|
||||
val modelDir = "sherpa-onnx-whisper-base"
|
||||
return SpokenLanguageIdentificationConfig(
|
||||
whisper = SpokenLanguageIdentificationWhisperConfig(
|
||||
encoder = "$modelDir/tiny-encoder.int8.onnx",
|
||||
decoder = "$modelDir/tiny-decoder.int8.onnx",
|
||||
),
|
||||
numThreads = 1,
|
||||
debug = true,
|
||||
)
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
104
sherpa-onnx/kotlin-api/Vad.kt
Normal file
104
sherpa-onnx/kotlin-api/Vad.kt
Normal file
@@ -0,0 +1,104 @@
|
||||
// Copyright (c) 2023 Xiaomi Corporation
|
||||
package com.k2fsa.sherpa.onnx
|
||||
|
||||
import android.content.res.AssetManager
|
||||
|
||||
data class SileroVadModelConfig(
|
||||
var model: String,
|
||||
var threshold: Float = 0.5F,
|
||||
var minSilenceDuration: Float = 0.25F,
|
||||
var minSpeechDuration: Float = 0.25F,
|
||||
var windowSize: Int = 512,
|
||||
)
|
||||
|
||||
data class VadModelConfig(
|
||||
var sileroVadModelConfig: SileroVadModelConfig,
|
||||
var sampleRate: Int = 16000,
|
||||
var numThreads: Int = 1,
|
||||
var provider: String = "cpu",
|
||||
var debug: Boolean = false,
|
||||
)
|
||||
|
||||
class Vad(
|
||||
assetManager: AssetManager? = null,
|
||||
var config: VadModelConfig,
|
||||
) {
|
||||
private val ptr: Long
|
||||
|
||||
init {
|
||||
if (assetManager != null) {
|
||||
ptr = newFromAsset(assetManager, config)
|
||||
} else {
|
||||
ptr = newFromFile(config)
|
||||
}
|
||||
}
|
||||
|
||||
protected fun finalize() {
|
||||
delete(ptr)
|
||||
}
|
||||
|
||||
fun acceptWaveform(samples: FloatArray) = acceptWaveform(ptr, samples)
|
||||
|
||||
fun empty(): Boolean = empty(ptr)
|
||||
fun pop() = pop(ptr)
|
||||
|
||||
// return an array containing
|
||||
// [start: Int, samples: FloatArray]
|
||||
fun front() = front(ptr)
|
||||
|
||||
fun clear() = clear(ptr)
|
||||
|
||||
fun isSpeechDetected(): Boolean = isSpeechDetected(ptr)
|
||||
|
||||
fun reset() = reset(ptr)
|
||||
|
||||
private external fun delete(ptr: Long)
|
||||
|
||||
private external fun newFromAsset(
|
||||
assetManager: AssetManager,
|
||||
config: VadModelConfig,
|
||||
): Long
|
||||
|
||||
private external fun newFromFile(
|
||||
config: VadModelConfig,
|
||||
): Long
|
||||
|
||||
private external fun acceptWaveform(ptr: Long, samples: FloatArray)
|
||||
private external fun empty(ptr: Long): Boolean
|
||||
private external fun pop(ptr: Long)
|
||||
private external fun clear(ptr: Long)
|
||||
private external fun front(ptr: Long): Array<Any>
|
||||
private external fun isSpeechDetected(ptr: Long): Boolean
|
||||
private external fun reset(ptr: Long)
|
||||
|
||||
companion object {
|
||||
init {
|
||||
System.loadLibrary("sherpa-onnx-jni")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Please visit
|
||||
// https://github.com/snakers4/silero-vad/blob/master/files/silero_vad.onnx
|
||||
// to download silero_vad.onnx
|
||||
// and put it inside the assets/
|
||||
// directory
|
||||
fun getVadModelConfig(type: Int): VadModelConfig? {
|
||||
when (type) {
|
||||
0 -> {
|
||||
return VadModelConfig(
|
||||
sileroVadModelConfig = SileroVadModelConfig(
|
||||
model = "silero_vad.onnx",
|
||||
threshold = 0.5F,
|
||||
minSilenceDuration = 0.25F,
|
||||
minSpeechDuration = 0.25F,
|
||||
windowSize = 512,
|
||||
),
|
||||
sampleRate = 16000,
|
||||
numThreads = 1,
|
||||
provider = "cpu",
|
||||
)
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
29
sherpa-onnx/kotlin-api/WaveReader.kt
Normal file
29
sherpa-onnx/kotlin-api/WaveReader.kt
Normal file
@@ -0,0 +1,29 @@
|
||||
// Copyright (c) 2023 Xiaomi Corporation
|
||||
package com.k2fsa.sherpa.onnx
|
||||
|
||||
import android.content.res.AssetManager
|
||||
|
||||
class WaveReader {
|
||||
companion object {
|
||||
// Read a mono wave file asset
|
||||
// The returned array has two entries:
|
||||
// - the first entry contains an 1-D float array
|
||||
// - the second entry is the sample rate
|
||||
external fun readWaveFromAsset(
|
||||
assetManager: AssetManager,
|
||||
filename: String,
|
||||
): Array<Any>
|
||||
|
||||
// Read a mono wave file from disk
|
||||
// The returned array has two entries:
|
||||
// - the first entry contains an 1-D float array
|
||||
// - the second entry is the sample rate
|
||||
external fun readWaveFromFile(
|
||||
filename: String,
|
||||
): Array<Any>
|
||||
|
||||
init {
|
||||
System.loadLibrary("sherpa-onnx-jni")
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user