Run TTS engine service without starting the app. (#553)

CMakeLists.txt

@@ -1,7 +1,7 @@
 cmake_minimum_required(VERSION 3.13 FATAL_ERROR)
 project(sherpa-onnx)
 
-set(SHERPA_ONNX_VERSION "1.9.8")
+set(SHERPA_ONNX_VERSION "1.9.9")
 
 # Disable warning about
 #

android/SherpaOnnxTtsEngine/app/src/main/AndroidManifest.xml

@@ -1,6 +1,7 @@
 <?xml version="1.0" encoding="utf-8"?>
 <manifest xmlns:android="http://schemas.android.com/apk/res/android"
-    xmlns:tools="http://schemas.android.com/tools">
+    xmlns:tools="http://schemas.android.com/tools"
+    package="com.k2fsa.sherpa.onnx.tts.engine">
 
     <application
         android:allowBackup="true"

android/SherpaOnnxTtsEngine/app/src/main/java/com/k2fsa/sherpa/onnx/tts/engine/MainActivity.kt

@@ -9,6 +9,7 @@ import android.util.Log
 import android.widget.Toast
 import androidx.activity.ComponentActivity
 import androidx.activity.compose.setContent
+import androidx.activity.viewModels
 import androidx.compose.foundation.layout.Box
 import androidx.compose.foundation.layout.Column
 import androidx.compose.foundation.layout.Row
@@ -43,9 +44,13 @@ import java.lang.NumberFormatException
 const val TAG = "sherpa-onnx-tts-engine"
 
 class MainActivity : ComponentActivity() {
+    // TODO(fangjun): Save settings in ttsViewModel
+    private val ttsViewModel: TtsViewModel by viewModels()
+
+    private var mediaPlayer: MediaPlayer? = null
     override fun onCreate(savedInstanceState: Bundle?) {
         super.onCreate(savedInstanceState)
-        TtsEngine.createTts(this.application)
+        TtsEngine.createTts(this)
         setContent {
             SherpaOnnxTtsEngineTheme {
                 // A surface container using the 'background' color from the theme
@@ -132,11 +137,12 @@ class MainActivity : ComponentActivity() {
                                                     audio.samples.size > 0 && audio.save(filename)
 
                                                 if (ok) {
-                                                    val mediaPlayer = MediaPlayer.create(
+                                                    stopMediaPlayer()
+                                                    mediaPlayer = MediaPlayer.create(
                                                         applicationContext,
                                                         Uri.fromFile(File(filename))
                                                     )
-                                                    mediaPlayer.start()
+                                                    mediaPlayer?.start()
                                                 } else {
                                                     Log.i(TAG, "Failed to generate or save audio")
                                                 }
@@ -162,4 +168,15 @@ class MainActivity : ComponentActivity() {
             }
         }
     }
+
+    override fun onDestroy() {
+        stopMediaPlayer()
+        super.onDestroy()
+    }
+
+    private fun stopMediaPlayer() {
+        mediaPlayer?.stop()
+        mediaPlayer?.release()
+        mediaPlayer = null
+    }
 }

android/SherpaOnnxTtsEngine/app/src/main/java/com/k2fsa/sherpa/onnx/tts/engine/TtsEngine.kt

@@ -1,6 +1,6 @@
 package com.k2fsa.sherpa.onnx.tts.engine
 
-import android.app.Application
+import android.content.Context
 import android.content.res.AssetManager
 import android.util.Log
 import androidx.compose.runtime.MutableState
@@ -21,7 +21,6 @@ object TtsEngine {
     var lang: String? = null
 
 
-
     val speedState: MutableState<Float> = mutableStateOf(1.0F)
     val speakerIdState: MutableState<Int> = mutableStateOf(0)
 
@@ -44,19 +43,7 @@ object TtsEngine {
     private var dataDir: String? = null
     private var assets: AssetManager? = null
 
-    private var application: Application? = null
+    init {
-
-    fun createTts(application: Application) {
-        Log.i(TAG, "Init Next-gen Kaldi TTS")
-        if (tts == null) {
-            this.application = application
-            initTts()
-        }
-    }
-
-    private fun initTts() {
-        assets = application?.assets
-
         // The purpose of such a design is to make the CI test easier
         // Please see
         // https://github.com/k2-fsa/sherpa-onnx/blob/master/scripts/apk/generate-tts-apk-script.py
@@ -89,9 +76,21 @@ object TtsEngine {
         // ruleFsts = "vits-zh-aishell3/rule.fst"
         // lexicon = "lexicon.txt"
         // lang = "zho"
+    }
+
+
+    fun createTts(context: Context) {
+        Log.i(TAG, "Init Next-gen Kaldi TTS")
+        if (tts == null) {
+            initTts(context)
+        }
+    }
+
+    private fun initTts(context: Context) {
+        assets = context.assets
 
         if (dataDir != null) {
-            val newDir = copyDataDir(modelDir!!)
+            val newDir = copyDataDir(context, modelDir!!)
             modelDir = newDir + "/" + modelDir
             dataDir = newDir + "/" + dataDir
             assets = null
@@ -107,28 +106,28 @@ object TtsEngine {
     }
 
 
-    private fun copyDataDir(dataDir: String): String {
+    private fun copyDataDir(context: Context, dataDir: String): String {
         println("data dir is $dataDir")
-        copyAssets(dataDir)
+        copyAssets(context, dataDir)
 
-        val newDataDir = application!!.getExternalFilesDir(null)!!.absolutePath
+        val newDataDir = context.getExternalFilesDir(null)!!.absolutePath
         println("newDataDir: $newDataDir")
         return newDataDir
     }
 
-    private fun copyAssets(path: String) {
+    private fun copyAssets(context: Context, path: String) {
         val assets: Array<String>?
         try {
-            assets = application!!.assets.list(path)
+            assets = context.assets.list(path)
             if (assets!!.isEmpty()) {
-                copyFile(path)
+                copyFile(context, path)
             } else {
-                val fullPath = "${application!!.getExternalFilesDir(null)}/$path"
+                val fullPath = "${context.getExternalFilesDir(null)}/$path"
                 val dir = File(fullPath)
                 dir.mkdirs()
                 for (asset in assets.iterator()) {
                     val p: String = if (path == "") "" else path + "/"
-                    copyAssets(p + asset)
+                    copyAssets(context, p + asset)
                 }
             }
         } catch (ex: IOException) {
@@ -136,10 +135,10 @@ object TtsEngine {
         }
     }
 
-    private fun copyFile(filename: String) {
+    private fun copyFile(context: Context, filename: String) {
         try {
-            val istream = application!!.assets.open(filename)
+            val istream = context.assets.open(filename)
-            val newFilename = application!!.getExternalFilesDir(null).toString() + "/" + filename
+            val newFilename = context.getExternalFilesDir(null).toString() + "/" + filename
             val ostream = FileOutputStream(newFilename)
             // Log.i(TAG, "Copying $filename to $newFilename")
             val buffer = ByteArray(1024)

android/SherpaOnnxTtsEngine/app/src/main/java/com/k2fsa/sherpa/onnx/tts/engine/TtsService.kt

@@ -56,12 +56,18 @@ Failed to get default language from engine com.k2fsa.sherpa.chapter5
 
 class TtsService : TextToSpeechService() {
     override fun onCreate() {
+        Log.i(TAG, "onCreate tts service")
         super.onCreate()
 
         // see https://github.com/Miserlou/Android-SDK-Samples/blob/master/TtsEngine/src/com/example/android/ttsengine/RobotSpeakTtsService.java#L68
         onLoadLanguage(TtsEngine.lang, "", "")
     }
 
+    override fun onDestroy() {
+        Log.i(TAG, "onDestroy tts service")
+        super.onDestroy()
+    }
+
     // https://developer.android.com/reference/kotlin/android/speech/tts/TextToSpeechService#onislanguageavailable
     override fun onIsLanguageAvailable(_lang: String?, _country: String?, _variant: String?): Int {
         val lang = _lang ?: ""
@@ -79,12 +85,15 @@ class TtsService : TextToSpeechService() {
 
     // https://developer.android.com/reference/kotlin/android/speech/tts/TextToSpeechService#onLoadLanguage(kotlin.String,%20kotlin.String,%20kotlin.String)
     override fun onLoadLanguage(_lang: String?, _country: String?, _variant: String?): Int {
+        Log.i(TAG, "onLoadLanguage: $_lang, $_country")
         val lang = _lang ?: ""
 
         return if (lang == TtsEngine.lang) {
+            Log.i(TAG, "creating tts, lang :$lang")
             TtsEngine.createTts(application)
             TextToSpeech.LANG_AVAILABLE
         } else {
+            Log.i(TAG, "lang $lang not supported, tts engine lang: ${TtsEngine.lang}")
             TextToSpeech.LANG_NOT_SUPPORTED
         }
     }
@@ -118,7 +127,7 @@ class TtsService : TextToSpeechService() {
             return
         }
 
-        val ttsCallback = {floatSamples: FloatArray ->
+        val ttsCallback = { floatSamples: FloatArray ->
             // convert FloatArray to ByteArray
             val samples = floatArrayToByteArray(floatSamples)
             val maxBufferSize: Int = callback.maxBufferSize
@@ -136,7 +145,7 @@ class TtsService : TextToSpeechService() {
             text = text,
             sid = TtsEngine.speakerId,
             speed = TtsEngine.speed,
-            callback=ttsCallback,
+            callback = ttsCallback,
         )
 
         callback.done()

android/SherpaOnnxTtsEngine/app/src/main/java/com/k2fsa/sherpa/onnx/tts/engine/TtsViewModel.kt

@@ -0,0 +1,74 @@
+package com.k2fsa.sherpa.onnx.tts.engine
+
+import android.app.Application
+import android.os.FileUtils.ProgressListener
+import android.speech.tts.TextToSpeech
+import android.speech.tts.TextToSpeech.OnInitListener
+import android.speech.tts.UtteranceProgressListener
+import android.util.Log
+import androidx.lifecycle.ViewModel
+import java.util.Locale
+
+class TtsApp : Application() {
+    companion object {
+        lateinit var instance: TtsApp
+    }
+
+    override fun onCreate() {
+        super.onCreate()
+        instance = this
+    }
+
+}
+
+class TtsViewModel : ViewModel() {
+
+    // https://developer.android.com/reference/kotlin/android/speech/tts/TextToSpeech.OnInitListener
+    private val onInitListener = object : OnInitListener {
+        override fun onInit(status: Int) {
+            when (status) {
+                TextToSpeech.SUCCESS -> Log.i(TAG, "Init tts succeded")
+                TextToSpeech.ERROR -> Log.i(TAG, "Init tts failed")
+                else -> Log.i(TAG, "Unknown status $status")
+            }
+        }
+    }
+
+    // https://developer.android.com/reference/kotlin/android/speech/tts/UtteranceProgressListener
+    private val utteranceProgressListener = object : UtteranceProgressListener() {
+        override fun onStart(utteranceId: String?) {
+            Log.i(TAG, "onStart: $utteranceId")
+        }
+
+        override fun onStop(utteranceId: String?, interrupted: Boolean) {
+            Log.i(TAG, "onStop: $utteranceId, $interrupted")
+            super.onStop(utteranceId, interrupted)
+        }
+
+        override fun onError(utteranceId: String?, errorCode: Int) {
+            Log.i(TAG, "onError: $utteranceId, $errorCode")
+            super.onError(utteranceId, errorCode)
+        }
+
+        override fun onDone(utteranceId: String?) {
+            Log.i(TAG, "onDone: $utteranceId")
+        }
+
+        @Deprecated("Deprecated in Java")
+        override fun onError(utteranceId: String?) {
+            Log.i(TAG, "onError: $utteranceId")
+        }
+    }
+
+    val tts = TextToSpeech(TtsApp.instance, onInitListener, "com.k2fsa.sherpa.onnx.tts.engine")
+
+    init {
+        tts.setLanguage(Locale(TtsEngine.lang!!))
+        tts.setOnUtteranceProgressListener(utteranceProgressListener)
+    }
+
+    override fun onCleared() {
+        super.onCleared()
+        tts.shutdown()
+    }
+}