From 9064b3f016bec2d8416a650858703c7d515fcebe Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Fri, 24 Feb 2023 13:57:03 +0800 Subject: [PATCH] Support Android (#59) --- .github/scripts/Main.kt | 64 +++--- .github/scripts/SherpaOnnx.kt | 90 +------- .github/scripts/WaveReader.kt | 18 +- .gitignore | 6 + android/SherpaOnnx/.gitignore | 15 ++ android/SherpaOnnx/.idea/.gitignore | 3 + android/SherpaOnnx/.idea/compiler.xml | 6 + .../.idea/deploymentTargetDropDown.xml | 17 ++ android/SherpaOnnx/.idea/gradle.xml | 18 ++ android/SherpaOnnx/.idea/misc.xml | 10 + android/SherpaOnnx/.idea/vcs.xml | 6 + android/SherpaOnnx/app/.gitignore | 1 + android/SherpaOnnx/app/build.gradle | 44 ++++ android/SherpaOnnx/app/proguard-rules.pro | 21 ++ .../sherpa/onnx/ExampleInstrumentedTest.kt | 24 +++ .../app/src/main/AndroidManifest.xml | 32 +++ .../com/k2fsa/sherpa/onnx/MainActivity.kt | 204 ++++++++++++++++++ .../java/com/k2fsa/sherpa/onnx/SherpaOnnx.kt | 146 +++++++++++++ .../java/com/k2fsa/sherpa/onnx/WaveReader.kt | 17 ++ .../app/src/main/jniLibs/.gitignore | 4 + .../app/src/main/jniLibs/arm64-v8a/.gitkeep | 0 .../app/src/main/jniLibs/armeabi-v7a/.gitkeep | 0 .../app/src/main/jniLibs/x86_64/.gitkeep | 0 .../drawable-v24/ic_launcher_foreground.xml | 30 +++ .../res/drawable/ic_launcher_background.xml | 170 +++++++++++++++ .../app/src/main/res/layout/activity_main.xml | 38 ++++ .../res/mipmap-anydpi-v26/ic_launcher.xml | 5 + .../mipmap-anydpi-v26/ic_launcher_round.xml | 5 + .../src/main/res/mipmap-hdpi/ic_launcher.webp | Bin 0 -> 1404 bytes .../res/mipmap-hdpi/ic_launcher_round.webp | Bin 0 -> 2898 bytes .../src/main/res/mipmap-mdpi/ic_launcher.webp | Bin 0 -> 982 bytes .../res/mipmap-mdpi/ic_launcher_round.webp | Bin 0 -> 1772 bytes .../main/res/mipmap-xhdpi/ic_launcher.webp | Bin 0 -> 1900 bytes .../res/mipmap-xhdpi/ic_launcher_round.webp | Bin 0 -> 3918 bytes .../main/res/mipmap-xxhdpi/ic_launcher.webp | Bin 0 -> 2884 bytes .../res/mipmap-xxhdpi/ic_launcher_round.webp | Bin 0 -> 5914 bytes .../main/res/mipmap-xxxhdpi/ic_launcher.webp | Bin 0 -> 3844 bytes .../res/mipmap-xxxhdpi/ic_launcher_round.webp | Bin 0 -> 7778 bytes .../app/src/main/res/values-night/themes.xml | 16 ++ .../app/src/main/res/values/colors.xml | 10 + .../app/src/main/res/values/strings.xml | 11 + .../app/src/main/res/values/themes.xml | 16 ++ .../app/src/main/res/xml/backup_rules.xml | 13 ++ .../main/res/xml/data_extraction_rules.xml | 19 ++ .../com/k2fsa/sherpa/onnx/ExampleUnitTest.kt | 17 ++ android/SherpaOnnx/build.gradle | 6 + android/SherpaOnnx/gradle.properties | 23 ++ .../gradle/wrapper/gradle-wrapper.jar | Bin 0 -> 59203 bytes .../gradle/wrapper/gradle-wrapper.properties | 6 + android/SherpaOnnx/gradlew | 185 ++++++++++++++++ android/SherpaOnnx/gradlew.bat | 89 ++++++++ android/SherpaOnnx/settings.gradle | 16 ++ build-android-arm64-v8a.sh | 27 ++- build-android-x86-64.sh | 84 ++++++++ cmake/onnxruntime.cmake | 54 ++--- sherpa-onnx/csrc/macros.h | 17 ++ sherpa-onnx/csrc/online-recognizer.cc | 5 +- sherpa-onnx/csrc/online-recognizer.h | 3 - .../csrc/online-transducer-model-config.cc | 1 + .../csrc/online-transducer-model-config.h | 5 +- sherpa-onnx/csrc/onnx-utils.cc | 3 +- sherpa-onnx/csrc/sherpa-onnx-alsa.cc | 2 +- sherpa-onnx/csrc/sherpa-onnx-microphone.cc | 2 +- sherpa-onnx/csrc/sherpa-onnx.cc | 2 +- sherpa-onnx/jni/jni.cc | 47 ++-- sherpa-onnx/python/csrc/online-recognizer.cc | 7 +- .../csrc/online-transducer-model-config.cc | 7 +- .../python/sherpa_onnx/online_recognizer.py | 2 +- 68 files changed, 1469 insertions(+), 220 deletions(-) mode change 100644 => 120000 .github/scripts/SherpaOnnx.kt mode change 100644 => 120000 .github/scripts/WaveReader.kt create mode 100644 android/SherpaOnnx/.gitignore create mode 100644 android/SherpaOnnx/.idea/.gitignore create mode 100644 android/SherpaOnnx/.idea/compiler.xml create mode 100644 android/SherpaOnnx/.idea/deploymentTargetDropDown.xml create mode 100644 android/SherpaOnnx/.idea/gradle.xml create mode 100644 android/SherpaOnnx/.idea/misc.xml create mode 100644 android/SherpaOnnx/.idea/vcs.xml create mode 100644 android/SherpaOnnx/app/.gitignore create mode 100644 android/SherpaOnnx/app/build.gradle create mode 100644 android/SherpaOnnx/app/proguard-rules.pro create mode 100644 android/SherpaOnnx/app/src/androidTest/java/com/k2fsa/sherpa/onnx/ExampleInstrumentedTest.kt create mode 100644 android/SherpaOnnx/app/src/main/AndroidManifest.xml create mode 100644 android/SherpaOnnx/app/src/main/java/com/k2fsa/sherpa/onnx/MainActivity.kt create mode 100644 android/SherpaOnnx/app/src/main/java/com/k2fsa/sherpa/onnx/SherpaOnnx.kt create mode 100644 android/SherpaOnnx/app/src/main/java/com/k2fsa/sherpa/onnx/WaveReader.kt create mode 100644 android/SherpaOnnx/app/src/main/jniLibs/.gitignore create mode 100644 android/SherpaOnnx/app/src/main/jniLibs/arm64-v8a/.gitkeep create mode 100644 android/SherpaOnnx/app/src/main/jniLibs/armeabi-v7a/.gitkeep create mode 100644 android/SherpaOnnx/app/src/main/jniLibs/x86_64/.gitkeep create mode 100644 android/SherpaOnnx/app/src/main/res/drawable-v24/ic_launcher_foreground.xml create mode 100644 android/SherpaOnnx/app/src/main/res/drawable/ic_launcher_background.xml create mode 100644 android/SherpaOnnx/app/src/main/res/layout/activity_main.xml create mode 100644 android/SherpaOnnx/app/src/main/res/mipmap-anydpi-v26/ic_launcher.xml create mode 100644 android/SherpaOnnx/app/src/main/res/mipmap-anydpi-v26/ic_launcher_round.xml create mode 100644 android/SherpaOnnx/app/src/main/res/mipmap-hdpi/ic_launcher.webp create mode 100644 android/SherpaOnnx/app/src/main/res/mipmap-hdpi/ic_launcher_round.webp create mode 100644 android/SherpaOnnx/app/src/main/res/mipmap-mdpi/ic_launcher.webp create mode 100644 android/SherpaOnnx/app/src/main/res/mipmap-mdpi/ic_launcher_round.webp create mode 100644 android/SherpaOnnx/app/src/main/res/mipmap-xhdpi/ic_launcher.webp create mode 100644 android/SherpaOnnx/app/src/main/res/mipmap-xhdpi/ic_launcher_round.webp create mode 100644 android/SherpaOnnx/app/src/main/res/mipmap-xxhdpi/ic_launcher.webp create mode 100644 android/SherpaOnnx/app/src/main/res/mipmap-xxhdpi/ic_launcher_round.webp create mode 100644 android/SherpaOnnx/app/src/main/res/mipmap-xxxhdpi/ic_launcher.webp create mode 100644 android/SherpaOnnx/app/src/main/res/mipmap-xxxhdpi/ic_launcher_round.webp create mode 100644 android/SherpaOnnx/app/src/main/res/values-night/themes.xml create mode 100644 android/SherpaOnnx/app/src/main/res/values/colors.xml create mode 100644 android/SherpaOnnx/app/src/main/res/values/strings.xml create mode 100644 android/SherpaOnnx/app/src/main/res/values/themes.xml create mode 100644 android/SherpaOnnx/app/src/main/res/xml/backup_rules.xml create mode 100644 android/SherpaOnnx/app/src/main/res/xml/data_extraction_rules.xml create mode 100644 android/SherpaOnnx/app/src/test/java/com/k2fsa/sherpa/onnx/ExampleUnitTest.kt create mode 100644 android/SherpaOnnx/build.gradle create mode 100644 android/SherpaOnnx/gradle.properties create mode 100644 android/SherpaOnnx/gradle/wrapper/gradle-wrapper.jar create mode 100644 android/SherpaOnnx/gradle/wrapper/gradle-wrapper.properties create mode 100755 android/SherpaOnnx/gradlew create mode 100644 android/SherpaOnnx/gradlew.bat create mode 100644 android/SherpaOnnx/settings.gradle create mode 100755 build-android-x86-64.sh diff --git a/.github/scripts/Main.kt b/.github/scripts/Main.kt index 8ed0698a..8e56ee51 100644 --- a/.github/scripts/Main.kt +++ b/.github/scripts/Main.kt @@ -3,43 +3,43 @@ package com.k2fsa.sherpa.onnx import android.content.res.AssetManager fun main() { - var featConfig = FeatureConfig( - sampleRate=16000.0f, - featureDim=80, - ) + var featConfig = FeatureConfig( + sampleRate = 16000.0f, + featureDim = 80, + ) - var modelConfig = OnlineTransducerModelConfig( - encoder="./sherpa-onnx-streaming-zipformer-en-2023-02-21/encoder-epoch-99-avg-1.onnx", - decoder="./sherpa-onnx-streaming-zipformer-en-2023-02-21/decoder-epoch-99-avg-1.onnx", - joiner="./sherpa-onnx-streaming-zipformer-en-2023-02-21/joiner-epoch-99-avg-1.onnx", - numThreads=4, - debug=false, - ) + var modelConfig = OnlineTransducerModelConfig( + encoder = "./sherpa-onnx-streaming-zipformer-en-2023-02-21/encoder-epoch-99-avg-1.onnx", + decoder = "./sherpa-onnx-streaming-zipformer-en-2023-02-21/decoder-epoch-99-avg-1.onnx", + joiner = "./sherpa-onnx-streaming-zipformer-en-2023-02-21/joiner-epoch-99-avg-1.onnx", + tokens = "./sherpa-onnx-streaming-zipformer-en-2023-02-21/tokens.txt", + numThreads = 4, + debug = false, + ) - var endpointConfig = EndpointConfig() + var endpointConfig = EndpointConfig() - var config = OnlineRecognizerConfig( - modelConfig=modelConfig, - featConfig=featConfig, - endpointConfig=endpointConfig, - tokens="./sherpa-onnx-streaming-zipformer-en-2023-02-21/tokens.txt", - enableEndpoint=true, - ) + var config = OnlineRecognizerConfig( + modelConfig = modelConfig, + featConfig = featConfig, + endpointConfig = endpointConfig, + enableEndpoint = true, + ) - var model = SherpaOnnx( - assetManager = AssetManager(), - config = config, - ) - var samples = WaveReader.readWave( - assetManager = AssetManager(), - filename = "./sherpa-onnx-streaming-zipformer-en-2023-02-21/test_wavs/1089-134686-0001.wav", - ) + var model = SherpaOnnx( + assetManager = AssetManager(), + config = config, + ) + var samples = WaveReader.readWave( + assetManager = AssetManager(), + filename = "./sherpa-onnx-streaming-zipformer-en-2023-02-21/test_wavs/1089-134686-0001.wav", + ) - model.decodeSamples(samples!!) + model.decodeSamples(samples!!) - var tail_paddings = FloatArray(8000) // 0.5 seconds - model.decodeSamples(tail_paddings) + var tail_paddings = FloatArray(8000) // 0.5 seconds + model.decodeSamples(tail_paddings) - model.inputFinished() - println(model.text) + model.inputFinished() + println("results: ${model.text}") } diff --git a/.github/scripts/SherpaOnnx.kt b/.github/scripts/SherpaOnnx.kt deleted file mode 100644 index e18250bf..00000000 --- a/.github/scripts/SherpaOnnx.kt +++ /dev/null @@ -1,89 +0,0 @@ -package com.k2fsa.sherpa.onnx - -import android.content.res.AssetManager - -data class EndpointRule( - var mustContainNonSilence: Boolean, - var minTrailingSilence: Float, - var minUtteranceLength: Float, -) - -data class EndpointConfig( - var rule1: EndpointRule = EndpointRule(false, 2.4f, 0.0f), - var rule2: EndpointRule = EndpointRule(true, 1.4f, 0.0f), - var rule3: EndpointRule = EndpointRule(false, 0.0f, 20.0f) -) - -data class OnlineTransducerModelConfig( - var encoder: String, - var decoder: String, - var joiner: String, - var numThreads: Int = 4, - var debug: Boolean = false, -) - -data class FeatureConfig( - var sampleRate: Float = 16000.0f, - var featureDim: Int = 80, -) - -data class OnlineRecognizerConfig( - var featConfig: FeatureConfig = FeatureConfig(), - var modelConfig: OnlineTransducerModelConfig, - var tokens: String, - var endpointConfig: EndpointConfig = EndpointConfig(), - var enableEndpoint: Boolean, -) - -class SherpaOnnx( - assetManager: AssetManager, - var config: OnlineRecognizerConfig -) { - private val ptr: Long - - init { - ptr = new(assetManager, config) - } - - protected fun finalize() { - delete(ptr) - } - - - fun decodeSamples(samples: FloatArray) = - decodeSamples(ptr, samples, sampleRate = config.featConfig.sampleRate) - - fun inputFinished() = inputFinished(ptr) - fun reset() = reset(ptr) - fun isEndpoint(): Boolean = isEndpoint(ptr) - - val text: String - get() = getText(ptr) - - private external fun delete(ptr: Long) - - private external fun new( - assetManager: AssetManager, - config: OnlineRecognizerConfig, - ): Long - - private external fun decodeSamples(ptr: Long, samples: FloatArray, sampleRate: Float) - private external fun inputFinished(ptr: Long) - private external fun getText(ptr: Long): String - private external fun reset(ptr: Long) - private external fun isEndpoint(ptr: Long): Boolean - - companion object { - init { - System.loadLibrary("sherpa-onnx-jni") - } - } -} - -fun getFeatureConfig(): FeatureConfig { - val featConfig = FeatureConfig() - featConfig.sampleRate = 16000.0f - featConfig.featureDim = 80 - - return featConfig -} diff --git a/.github/scripts/SherpaOnnx.kt b/.github/scripts/SherpaOnnx.kt new file mode 120000 index 00000000..99dd30cd --- /dev/null +++ b/.github/scripts/SherpaOnnx.kt @@ -0,0 +1 @@ +../../android/SherpaOnnx/app/src/main/java/com/k2fsa/sherpa/onnx/SherpaOnnx.kt \ No newline at end of file diff --git a/.github/scripts/WaveReader.kt b/.github/scripts/WaveReader.kt deleted file mode 100644 index 4444ab93..00000000 --- a/.github/scripts/WaveReader.kt +++ /dev/null @@ -1,17 +0,0 @@ -package com.k2fsa.sherpa.onnx - -import android.content.res.AssetManager - -class WaveReader { - companion object { - // Read a mono wave file. - // No resampling is made. - external fun readWave( - assetManager: AssetManager, filename: String, expected_sample_rate: Float = 16000.0f - ): FloatArray? - - init { - System.loadLibrary("sherpa-onnx-jni") - } - } -} diff --git a/.github/scripts/WaveReader.kt b/.github/scripts/WaveReader.kt new file mode 120000 index 00000000..378aa32b --- /dev/null +++ b/.github/scripts/WaveReader.kt @@ -0,0 +1 @@ +../../android/SherpaOnnx/app/src/main/java/com/k2fsa/sherpa/onnx/WaveReader.kt \ No newline at end of file diff --git a/.gitignore b/.gitignore index 0f34dac8..fd87f07a 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,9 @@ build-aarch64-linux-gnu sherpa-onnx-streaming-zipformer-* sherpa-onnx-lstm-en-* sherpa-onnx-lstm-zh-* +build-android-arm64-v8a/ +build-android-x86-64/ +a.txt +run-bilingual*.sh +run-*-zipformer.sh +run-zh.sh diff --git a/android/SherpaOnnx/.gitignore b/android/SherpaOnnx/.gitignore new file mode 100644 index 00000000..aa724b77 --- /dev/null +++ b/android/SherpaOnnx/.gitignore @@ -0,0 +1,15 @@ +*.iml +.gradle +/local.properties +/.idea/caches +/.idea/libraries +/.idea/modules.xml +/.idea/workspace.xml +/.idea/navEditor.xml +/.idea/assetWizardSettings.xml +.DS_Store +/build +/captures +.externalNativeBuild +.cxx +local.properties diff --git a/android/SherpaOnnx/.idea/.gitignore b/android/SherpaOnnx/.idea/.gitignore new file mode 100644 index 00000000..26d33521 --- /dev/null +++ b/android/SherpaOnnx/.idea/.gitignore @@ -0,0 +1,3 @@ +# Default ignored files +/shelf/ +/workspace.xml diff --git a/android/SherpaOnnx/.idea/compiler.xml b/android/SherpaOnnx/.idea/compiler.xml new file mode 100644 index 00000000..fb7f4a8a --- /dev/null +++ b/android/SherpaOnnx/.idea/compiler.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/android/SherpaOnnx/.idea/deploymentTargetDropDown.xml b/android/SherpaOnnx/.idea/deploymentTargetDropDown.xml new file mode 100644 index 00000000..8f6b4f14 --- /dev/null +++ b/android/SherpaOnnx/.idea/deploymentTargetDropDown.xml @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/android/SherpaOnnx/.idea/gradle.xml b/android/SherpaOnnx/.idea/gradle.xml new file mode 100644 index 00000000..a9f4e522 --- /dev/null +++ b/android/SherpaOnnx/.idea/gradle.xml @@ -0,0 +1,18 @@ + + + + + + \ No newline at end of file diff --git a/android/SherpaOnnx/.idea/misc.xml b/android/SherpaOnnx/.idea/misc.xml new file mode 100644 index 00000000..bdd92780 --- /dev/null +++ b/android/SherpaOnnx/.idea/misc.xml @@ -0,0 +1,10 @@ + + + + + + + + + \ No newline at end of file diff --git a/android/SherpaOnnx/.idea/vcs.xml b/android/SherpaOnnx/.idea/vcs.xml new file mode 100644 index 00000000..b2bdec2d --- /dev/null +++ b/android/SherpaOnnx/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/android/SherpaOnnx/app/.gitignore b/android/SherpaOnnx/app/.gitignore new file mode 100644 index 00000000..42afabfd --- /dev/null +++ b/android/SherpaOnnx/app/.gitignore @@ -0,0 +1 @@ +/build \ No newline at end of file diff --git a/android/SherpaOnnx/app/build.gradle b/android/SherpaOnnx/app/build.gradle new file mode 100644 index 00000000..d64be807 --- /dev/null +++ b/android/SherpaOnnx/app/build.gradle @@ -0,0 +1,44 @@ +plugins { + id 'com.android.application' + id 'org.jetbrains.kotlin.android' +} + +android { + namespace 'com.k2fsa.sherpa.onnx' + compileSdk 32 + + defaultConfig { + applicationId "com.k2fsa.sherpa.onnx" + minSdk 21 + targetSdk 32 + versionCode 1 + versionName "1.0" + + testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" + } + + buildTypes { + release { + minifyEnabled false + proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro' + } + } + compileOptions { + sourceCompatibility JavaVersion.VERSION_1_8 + targetCompatibility JavaVersion.VERSION_1_8 + } + kotlinOptions { + jvmTarget = '1.8' + } +} + +dependencies { + + implementation 'androidx.core:core-ktx:1.7.0' + implementation 'androidx.appcompat:appcompat:1.5.1' + implementation 'com.google.android.material:material:1.7.0' + implementation 'androidx.constraintlayout:constraintlayout:2.1.4' + testImplementation 'junit:junit:4.13.2' + androidTestImplementation 'androidx.test.ext:junit:1.1.4' + androidTestImplementation 'androidx.test.espresso:espresso-core:3.5.0' +} \ No newline at end of file diff --git a/android/SherpaOnnx/app/proguard-rules.pro b/android/SherpaOnnx/app/proguard-rules.pro new file mode 100644 index 00000000..481bb434 --- /dev/null +++ b/android/SherpaOnnx/app/proguard-rules.pro @@ -0,0 +1,21 @@ +# Add project specific ProGuard rules here. +# You can control the set of applied configuration files using the +# proguardFiles setting in build.gradle. +# +# For more details, see +# http://developer.android.com/guide/developing/tools/proguard.html + +# If your project uses WebView with JS, uncomment the following +# and specify the fully qualified class name to the JavaScript interface +# class: +#-keepclassmembers class fqcn.of.javascript.interface.for.webview { +# public *; +#} + +# Uncomment this to preserve the line number information for +# debugging stack traces. +#-keepattributes SourceFile,LineNumberTable + +# If you keep the line number information, uncomment this to +# hide the original source file name. +#-renamesourcefileattribute SourceFile \ No newline at end of file diff --git a/android/SherpaOnnx/app/src/androidTest/java/com/k2fsa/sherpa/onnx/ExampleInstrumentedTest.kt b/android/SherpaOnnx/app/src/androidTest/java/com/k2fsa/sherpa/onnx/ExampleInstrumentedTest.kt new file mode 100644 index 00000000..18338320 --- /dev/null +++ b/android/SherpaOnnx/app/src/androidTest/java/com/k2fsa/sherpa/onnx/ExampleInstrumentedTest.kt @@ -0,0 +1,24 @@ +package com.k2fsa.sherpa.onnx + +import androidx.test.platform.app.InstrumentationRegistry +import androidx.test.ext.junit.runners.AndroidJUnit4 + +import org.junit.Test +import org.junit.runner.RunWith + +import org.junit.Assert.* + +/** + * Instrumented test, which will execute on an Android device. + * + * See [testing documentation](http://d.android.com/tools/testing). + */ +@RunWith(AndroidJUnit4::class) +class ExampleInstrumentedTest { + @Test + fun useAppContext() { + // Context of the app under test. + val appContext = InstrumentationRegistry.getInstrumentation().targetContext + assertEquals("com.k2fsa.sherpa.onnx", appContext.packageName) + } +} \ No newline at end of file diff --git a/android/SherpaOnnx/app/src/main/AndroidManifest.xml b/android/SherpaOnnx/app/src/main/AndroidManifest.xml new file mode 100644 index 00000000..935fb0e9 --- /dev/null +++ b/android/SherpaOnnx/app/src/main/AndroidManifest.xml @@ -0,0 +1,32 @@ + + + + + + + + + + + + + + + + + + diff --git a/android/SherpaOnnx/app/src/main/java/com/k2fsa/sherpa/onnx/MainActivity.kt b/android/SherpaOnnx/app/src/main/java/com/k2fsa/sherpa/onnx/MainActivity.kt new file mode 100644 index 00000000..2ca03e80 --- /dev/null +++ b/android/SherpaOnnx/app/src/main/java/com/k2fsa/sherpa/onnx/MainActivity.kt @@ -0,0 +1,204 @@ +package com.k2fsa.sherpa.onnx + +import android.Manifest +import android.content.pm.PackageManager +import android.media.AudioFormat +import android.media.AudioRecord +import android.media.MediaRecorder +import android.os.Bundle +import android.text.method.ScrollingMovementMethod +import android.util.Log +import android.widget.Button +import android.widget.TextView +import androidx.appcompat.app.AppCompatActivity +import androidx.core.app.ActivityCompat +import com.k2fsa.sherpa.onnx.* +import kotlin.concurrent.thread + +private const val TAG = "sherpa-onnx" +private const val REQUEST_RECORD_AUDIO_PERMISSION = 200 + +class MainActivity : AppCompatActivity() { + private val permissions: Array = arrayOf(Manifest.permission.RECORD_AUDIO) + + // If there is a GPU and useGPU is true, we will use GPU + // If there is no GPU and useGPU is true, we won't use GPU + private val useGPU: Boolean = true + + private lateinit var model: SherpaOnnx + private var audioRecord: AudioRecord? = null + private lateinit var recordButton: Button + private lateinit var textView: TextView + private var recordingThread: Thread? = null + + private val audioSource = MediaRecorder.AudioSource.MIC + private val sampleRateInHz = 16000 + private val channelConfig = AudioFormat.CHANNEL_IN_MONO + + // Note: We don't use AudioFormat.ENCODING_PCM_FLOAT + // since the AudioRecord.read(float[]) needs API level >= 23 + // but we are targeting API level >= 21 + private val audioFormat = AudioFormat.ENCODING_PCM_16BIT + private var idx: Int = 0 + private var lastText: String = "" + + @Volatile + private var isRecording: Boolean = false + + override fun onRequestPermissionsResult( + requestCode: Int, permissions: Array, grantResults: IntArray + ) { + super.onRequestPermissionsResult(requestCode, permissions, grantResults) + val permissionToRecordAccepted = if (requestCode == REQUEST_RECORD_AUDIO_PERMISSION) { + grantResults[0] == PackageManager.PERMISSION_GRANTED + } else { + false + } + + if (!permissionToRecordAccepted) { + Log.e(TAG, "Audio record is disallowed") + finish() + } + + Log.i(TAG, "Audio record is permitted") + } + + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + setContentView(R.layout.activity_main) + + ActivityCompat.requestPermissions(this, permissions, REQUEST_RECORD_AUDIO_PERMISSION) + + Log.i(TAG, "Start to initialize model") + initModel() + Log.i(TAG, "Finished initializing model") + + recordButton = findViewById(R.id.record_button) + recordButton.setOnClickListener { onclick() } + + textView = findViewById(R.id.my_text) + textView.movementMethod = ScrollingMovementMethod() + } + + private fun onclick() { + if (!isRecording) { + val ret = initMicrophone() + if (!ret) { + Log.e(TAG, "Failed to initialize microphone") + return + } + Log.i(TAG, "state: ${audioRecord?.state}") + audioRecord!!.startRecording() + recordButton.setText(R.string.stop) + isRecording = true + model.reset() + textView.text = "" + lastText = "" + idx = 0 + + recordingThread = thread(true) { + processSamples() + } + Log.i(TAG, "Started recording") + } else { + isRecording = false + audioRecord!!.stop() + audioRecord!!.release() + audioRecord = null + recordButton.setText(R.string.start) + Log.i(TAG, "Stopped recording") + } + } + + private fun processSamples() { + Log.i(TAG, "processing samples") + + val interval = 0.1 // i.e., 100 ms + val bufferSize = (interval * sampleRateInHz).toInt() // in samples + val buffer = ShortArray(bufferSize) + + while (isRecording) { + val ret = audioRecord?.read(buffer, 0, buffer.size) + if (ret != null && ret > 0) { + val samples = FloatArray(ret) { buffer[it] / 32768.0f } + model.decodeSamples(samples) + runOnUiThread { + val isEndpoint = model.isEndpoint() + val text = model.text + + if(text.isNotBlank()) { + if (lastText.isBlank()) { + textView.text = "${idx}: ${text}" + } else { + textView.text = "${lastText}\n${idx}: ${text}" + } + } + + if (isEndpoint) { + model.reset() + if (text.isNotBlank()) { + lastText = "${lastText}\n${idx}: ${text}" + idx += 1 + } + } + } + } + } + } + + private fun initMicrophone(): Boolean { + if (ActivityCompat.checkSelfPermission( + this, Manifest.permission.RECORD_AUDIO + ) != PackageManager.PERMISSION_GRANTED + ) { + ActivityCompat.requestPermissions(this, permissions, REQUEST_RECORD_AUDIO_PERMISSION) + return false + } + + val numBytes = AudioRecord.getMinBufferSize(sampleRateInHz, channelConfig, audioFormat) + Log.i( + TAG, "buffer size in milliseconds: ${numBytes * 1000.0f / sampleRateInHz}" + ) + + audioRecord = AudioRecord( + audioSource, + sampleRateInHz, + channelConfig, + audioFormat, + numBytes * 2 // a sample has two bytes as we are using 16-bit PCM + ) + return true + } + + private fun initModel() { + val config = OnlineRecognizerConfig( + featConfig = getFeatureConfig(sampleRate = 16000.0f, featureDim = 80), + modelConfig = getModelConfig(type = 1)!!, + endpointConfig = getEndpointConfig(), + enableEndpoint = true + ) + + model = SherpaOnnx( + assetManager = application.assets, + config = config, + ) + /* + println("reading samples") + val samples = WaveReader.readWave( + assetManager = application.assets, + // filename = "sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/test_wavs/0.wav", + filename = "sherpa-onnx-lstm-zh-2023-02-20/test_wavs/0.wav", + // filename="sherpa-onnx-lstm-en-2023-02-17/test_wavs/1089-134686-0001.wav" + ) + println("samples read done!") + + model.decodeSamples(samples!!) + + val tailPaddings = FloatArray(8000) // 0.5 seconds + model.decodeSamples(tailPaddings) + + println("result is: ${model.text}") + model.reset() + */ + } +} diff --git a/android/SherpaOnnx/app/src/main/java/com/k2fsa/sherpa/onnx/SherpaOnnx.kt b/android/SherpaOnnx/app/src/main/java/com/k2fsa/sherpa/onnx/SherpaOnnx.kt new file mode 100644 index 00000000..8e8869f0 --- /dev/null +++ b/android/SherpaOnnx/app/src/main/java/com/k2fsa/sherpa/onnx/SherpaOnnx.kt @@ -0,0 +1,146 @@ +package com.k2fsa.sherpa.onnx + +import android.content.res.AssetManager + +data class EndpointRule( + var mustContainNonSilence: Boolean, + var minTrailingSilence: Float, + var minUtteranceLength: Float, +) + +data class EndpointConfig( + var rule1: EndpointRule = EndpointRule(false, 2.4f, 0.0f), + var rule2: EndpointRule = EndpointRule(true, 1.4f, 0.0f), + var rule3: EndpointRule = EndpointRule(false, 0.0f, 20.0f) +) + +data class OnlineTransducerModelConfig( + var encoder: String, + var decoder: String, + var joiner: String, + var tokens: String, + var numThreads: Int = 2, + var debug: Boolean = false, +) + +data class FeatureConfig( + var sampleRate: Float = 16000.0f, + var featureDim: Int = 80, +) + +data class OnlineRecognizerConfig( + var featConfig: FeatureConfig = FeatureConfig(), + var modelConfig: OnlineTransducerModelConfig, + var endpointConfig: EndpointConfig = EndpointConfig(), + var enableEndpoint: Boolean, +) + +class SherpaOnnx( + assetManager: AssetManager, var config: OnlineRecognizerConfig +) { + private val ptr: Long + + init { + ptr = new(assetManager, config) + } + + protected fun finalize() { + delete(ptr) + } + + + fun decodeSamples(samples: FloatArray) = + decodeSamples(ptr, samples, sampleRate = config.featConfig.sampleRate) + + fun inputFinished() = inputFinished(ptr) + fun reset() = reset(ptr) + fun isEndpoint(): Boolean = isEndpoint(ptr) + + val text: String + get() = getText(ptr) + + private external fun delete(ptr: Long) + + private external fun new( + assetManager: AssetManager, + config: OnlineRecognizerConfig, + ): Long + + private external fun decodeSamples(ptr: Long, samples: FloatArray, sampleRate: Float) + private external fun inputFinished(ptr: Long) + private external fun getText(ptr: Long): String + private external fun reset(ptr: Long) + private external fun isEndpoint(ptr: Long): Boolean + + companion object { + init { + System.loadLibrary("sherpa-onnx-jni") + } + } +} + +fun getFeatureConfig(sampleRate: Float, featureDim: Int): FeatureConfig { + return FeatureConfig(sampleRate=sampleRate, featureDim=featureDim) +} + +/* +Please see +https://k2-fsa.github.io/sherpa/onnx/pretrained_models/index.html +for a list of pre-trained models. + +We only add a few here. Please change the following code +to add your own. (It should be straightforward to add a new model +by following the code) + +@param type +0 - sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20 (Bilingual, Chinese + English) + https://k2-fsa.github.io/sherpa/onnx/pretrained_models/zipformer-transducer-models.html#sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20-bilingual-chinese-english + +1 - csukuangfj/sherpa-onnx-lstm-zh-2023-02-20 (Chinese) + + https://k2-fsa.github.io/sherpa/onnx/pretrained_models/lstm-transducer-models.html#csukuangfj-sherpa-onnx-lstm-zh-2023-02-20-chinese + +2 - csukuangfj/sherpa-onnx-lstm-en-2023-02-17 (English) + https://k2-fsa.github.io/sherpa/onnx/pretrained_models/lstm-transducer-models.html#csukuangfj-sherpa-onnx-lstm-en-2023-02-17-english + */ +fun getModelConfig(type: Int): OnlineTransducerModelConfig? { + when (type) { + 0 -> { + val modelDir = "sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20" + return OnlineTransducerModelConfig( + encoder = "$modelDir/encoder-epoch-99-avg-1.onnx", + decoder = "$modelDir/decoder-epoch-99-avg-1.onnx", + joiner = "$modelDir/joiner-epoch-99-avg-1.onnx", + tokens = "$modelDir/tokens.txt", + ) + } + 1 -> { + val modelDir = "sherpa-onnx-lstm-zh-2023-02-20" + return OnlineTransducerModelConfig( + encoder = "$modelDir/encoder-epoch-11-avg-1.onnx", + decoder = "$modelDir/decoder-epoch-11-avg-1.onnx", + joiner = "$modelDir/joiner-epoch-11-avg-1.onnx", + tokens = "$modelDir/tokens.txt", + ) + } + + 2 -> { + val modelDir = "sherpa-onnx-lstm-en-2023-02-17" + return OnlineTransducerModelConfig( + encoder = "$modelDir/encoder-epoch-99-avg-1.onnx", + decoder = "$modelDir/decoder-epoch-99-avg-1.onnx", + joiner = "$modelDir/joiner-epoch-99-avg-1.onnx", + tokens = "$modelDir/tokens.txt", + ) + } + } + return null; +} + +fun getEndpointConfig(): EndpointConfig { + return EndpointConfig( + rule1 = EndpointRule(false, 2.4f, 0.0f), + rule2 = EndpointRule(true, 1.4f, 0.0f), + rule3 = EndpointRule(false, 0.0f, 20.0f) + ) +} diff --git a/android/SherpaOnnx/app/src/main/java/com/k2fsa/sherpa/onnx/WaveReader.kt b/android/SherpaOnnx/app/src/main/java/com/k2fsa/sherpa/onnx/WaveReader.kt new file mode 100644 index 00000000..4444ab93 --- /dev/null +++ b/android/SherpaOnnx/app/src/main/java/com/k2fsa/sherpa/onnx/WaveReader.kt @@ -0,0 +1,17 @@ +package com.k2fsa.sherpa.onnx + +import android.content.res.AssetManager + +class WaveReader { + companion object { + // Read a mono wave file. + // No resampling is made. + external fun readWave( + assetManager: AssetManager, filename: String, expected_sample_rate: Float = 16000.0f + ): FloatArray? + + init { + System.loadLibrary("sherpa-onnx-jni") + } + } +} diff --git a/android/SherpaOnnx/app/src/main/jniLibs/.gitignore b/android/SherpaOnnx/app/src/main/jniLibs/.gitignore new file mode 100644 index 00000000..949c039f --- /dev/null +++ b/android/SherpaOnnx/app/src/main/jniLibs/.gitignore @@ -0,0 +1,4 @@ +*.so +*.txt +*.onnx +*.wav diff --git a/android/SherpaOnnx/app/src/main/jniLibs/arm64-v8a/.gitkeep b/android/SherpaOnnx/app/src/main/jniLibs/arm64-v8a/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/android/SherpaOnnx/app/src/main/jniLibs/armeabi-v7a/.gitkeep b/android/SherpaOnnx/app/src/main/jniLibs/armeabi-v7a/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/android/SherpaOnnx/app/src/main/jniLibs/x86_64/.gitkeep b/android/SherpaOnnx/app/src/main/jniLibs/x86_64/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/android/SherpaOnnx/app/src/main/res/drawable-v24/ic_launcher_foreground.xml b/android/SherpaOnnx/app/src/main/res/drawable-v24/ic_launcher_foreground.xml new file mode 100644 index 00000000..2b068d11 --- /dev/null +++ b/android/SherpaOnnx/app/src/main/res/drawable-v24/ic_launcher_foreground.xml @@ -0,0 +1,30 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/android/SherpaOnnx/app/src/main/res/drawable/ic_launcher_background.xml b/android/SherpaOnnx/app/src/main/res/drawable/ic_launcher_background.xml new file mode 100644 index 00000000..07d5da9c --- /dev/null +++ b/android/SherpaOnnx/app/src/main/res/drawable/ic_launcher_background.xml @@ -0,0 +1,170 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/android/SherpaOnnx/app/src/main/res/layout/activity_main.xml b/android/SherpaOnnx/app/src/main/res/layout/activity_main.xml new file mode 100644 index 00000000..e4d61554 --- /dev/null +++ b/android/SherpaOnnx/app/src/main/res/layout/activity_main.xml @@ -0,0 +1,38 @@ + + + + + + + +