Add on-device tex-to-speech (TTS) demo for HarmonyOS (#1590)

harmony-os/SherpaOnnxTts/.gitignore

@@ -0,0 +1,12 @@
+/node_modules
+/oh_modules
+/local.properties
+/.idea
+**/build
+/.hvigor
+.cxx
+/.clangd
+/.clang-format
+/.clang-tidy
+**/.test
+/.appanalyzer

harmony-os/SherpaOnnxTts/AppScope/app.json5

@@ -0,0 +1,10 @@
+{
+  "app": {
+    "bundleName": "com.k2fsa.sherpa.onnx.tts",
+    "vendor": "next-gen Kaldi",
+    "versionCode": 1000000,
+    "versionName": "1.0.0",
+    "icon": "$media:app_icon",
+    "label": "$string:app_name"
+  }
+}

harmony-os/SherpaOnnxTts/AppScope/resources/base/element/string.json

@@ -0,0 +1,8 @@
+{
+  "string": [
+    {
+      "name": "app_name",
+      "value": "SherpaOnnxTts"
+    }
+  ]
+}

harmony-os/SherpaOnnxTts/build-profile.json5

@@ -0,0 +1,40 @@
+{
+  "app": {
+    "signingConfigs": [],
+    "products": [
+      {
+        "name": "default",
+        "signingConfig": "default",
+        "compatibleSdkVersion": "4.0.0(10)",
+        "runtimeOS": "HarmonyOS",
+        "buildOption": {
+          "strictMode": {
+            "caseSensitiveCheck": true,
+          }
+        }
+      }
+    ],
+    "buildModeSet": [
+      {
+        "name": "debug",
+      },
+      {
+        "name": "release"
+      }
+    ]
+  },
+  "modules": [
+    {
+      "name": "entry",
+      "srcPath": "./entry",
+      "targets": [
+        {
+          "name": "default",
+          "applyToProducts": [
+            "default"
+          ]
+        }
+      ]
+    }
+  ]
+}

harmony-os/SherpaOnnxTts/code-linter.json5

@@ -0,0 +1,20 @@
+{
+  "files": [
+    "**/*.ets"
+  ],
+  "ignore": [
+    "**/src/ohosTest/**/*",
+    "**/src/test/**/*",
+    "**/src/mock/**/*",
+    "**/node_modules/**/*",
+    "**/oh_modules/**/*",
+    "**/build/**/*",
+    "**/.preview/**/*"
+  ],
+  "ruleSet": [
+    "plugin:@performance/recommended",
+    "plugin:@typescript-eslint/recommended"
+  ],
+  "rules": {
+  }
+}

harmony-os/SherpaOnnxTts/entry/.gitignore

@@ -0,0 +1,6 @@
+/node_modules
+/oh_modules
+/.preview
+/build
+/.cxx
+/.test

harmony-os/SherpaOnnxTts/entry/build-profile.json5

@@ -0,0 +1,33 @@
+{
+  "apiType": "stageMode",
+  "buildOption": {
+    "sourceOption": {
+      "workers": [
+        "./src/main/ets/workers/NonStreamingTtsWorker.ets"
+      ]
+    }
+  },
+  "buildOptionSet": [
+    {
+      "name": "release",
+      "arkOptions": {
+        "obfuscation": {
+          "ruleOptions": {
+            "enable": false,
+            "files": [
+              "./obfuscation-rules.txt"
+            ]
+          }
+        }
+      }
+    },
+  ],
+  "targets": [
+    {
+      "name": "default"
+    },
+    {
+      "name": "ohosTest",
+    }
+  ]
+}

harmony-os/SherpaOnnxTts/entry/hvigorfile.ts

@@ -0,0 +1,6 @@
+import { hapTasks } from '@ohos/hvigor-ohos-plugin';
+
+export default {
+    system: hapTasks,  /* Built-in plugin of Hvigor. It cannot be modified. */
+    plugins:[]         /* Custom plugin to extend the functionality of Hvigor. */
+}

harmony-os/SherpaOnnxTts/entry/obfuscation-rules.txt

@@ -0,0 +1,23 @@
+# Define project specific obfuscation rules here.
+# You can include the obfuscation configuration files in the current module's build-profile.json5.
+#
+# For more details, see
+#   https://developer.huawei.com/consumer/cn/doc/harmonyos-guides-V5/source-obfuscation-V5
+
+# Obfuscation options:
+# -disable-obfuscation: disable all obfuscations
+# -enable-property-obfuscation: obfuscate the property names
+# -enable-toplevel-obfuscation: obfuscate the names in the global scope
+# -compact: remove unnecessary blank spaces and all line feeds
+# -remove-log: remove all console.* statements
+# -print-namecache: print the name cache that contains the mapping from the old names to new names
+# -apply-namecache: reuse the given cache file
+
+# Keep options:
+# -keep-property-name: specifies property names that you want to keep
+# -keep-global-name: specifies names that you want to keep in the global scope
+
+-enable-property-obfuscation
+-enable-toplevel-obfuscation
+-enable-filename-obfuscation
+-enable-export-obfuscation

harmony-os/SherpaOnnxTts/entry/oh-package-lock.json5

@@ -0,0 +1,29 @@
+{
+  "meta": {
+    "stableOrder": true
+  },
+  "lockfileVersion": 3,
+  "ATTENTION": "THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.",
+  "specifiers": {
+    "libsherpa_onnx.so@../oh_modules/.ohpm/sherpa_onnx@1.10.32/oh_modules/sherpa_onnx/src/main/cpp/types/libsherpa_onnx": "libsherpa_onnx.so@../oh_modules/.ohpm/sherpa_onnx@1.10.32/oh_modules/sherpa_onnx/src/main/cpp/types/libsherpa_onnx",
+    "sherpa_onnx@1.10.32": "sherpa_onnx@1.10.32"
+  },
+  "packages": {
+    "libsherpa_onnx.so@../oh_modules/.ohpm/sherpa_onnx@1.10.32/oh_modules/sherpa_onnx/src/main/cpp/types/libsherpa_onnx": {
+      "name": "libsherpa_onnx.so",
+      "version": "1.0.0",
+      "resolved": "../oh_modules/.ohpm/sherpa_onnx@1.10.32/oh_modules/sherpa_onnx/src/main/cpp/types/libsherpa_onnx",
+      "registryType": "local"
+    },
+    "sherpa_onnx@1.10.32": {
+      "name": "sherpa_onnx",
+      "version": "1.10.32",
+      "integrity": "sha512-yHYmWoeqhrunOqGr9gxPJJH/8+rdwcKFOW6onYByVObQVpbqypslg301IjGm9xpnc5bJEkO3S9sra2zQTpPA/w==",
+      "resolved": "https://ohpm.openharmony.cn/ohpm/sherpa_onnx/-/sherpa_onnx-1.10.32.har",
+      "registryType": "ohpm",
+      "dependencies": {
+        "libsherpa_onnx.so": "file:./src/main/cpp/types/libsherpa_onnx"
+      }
+    }
+  }
+}

harmony-os/SherpaOnnxTts/entry/oh-package.json5

@@ -0,0 +1,12 @@
+{
+  "name": "entry",
+  "version": "1.0.0",
+  "description": "Please describe the basic information.",
+  "main": "",
+  "author": "",
+  "license": "",
+  "dependencies": {
+    "sherpa_onnx": "1.10.32",
+  }
+}
+

harmony-os/SherpaOnnxTts/entry/src/main/ets/entryability/EntryAbility.ets

@@ -0,0 +1,43 @@
+import AbilityConstant from '@ohos.app.ability.AbilityConstant';
+import hilog from '@ohos.hilog';
+import UIAbility from '@ohos.app.ability.UIAbility';
+import Want from '@ohos.app.ability.Want';
+import window from '@ohos.window';
+
+export default class EntryAbility extends UIAbility {
+  onCreate(want: Want, launchParam: AbilityConstant.LaunchParam): void {
+    hilog.info(0x0000, 'testTag', '%{public}s', 'Ability onCreate');
+  }
+
+  onDestroy(): void {
+    hilog.info(0x0000, 'testTag', '%{public}s', 'Ability onDestroy');
+  }
+
+  onWindowStageCreate(windowStage: window.WindowStage): void {
+    // Main window is created, set main page for this ability
+    hilog.info(0x0000, 'testTag', '%{public}s', 'Ability onWindowStageCreate');
+
+    windowStage.loadContent('pages/Index', (err) => {
+      if (err.code) {
+        hilog.error(0x0000, 'testTag', 'Failed to load the content. Cause: %{public}s', JSON.stringify(err) ?? '');
+        return;
+      }
+      hilog.info(0x0000, 'testTag', 'Succeeded in loading the content.');
+    });
+  }
+
+  onWindowStageDestroy(): void {
+    // Main window is destroyed, release UI related resources
+    hilog.info(0x0000, 'testTag', '%{public}s', 'Ability onWindowStageDestroy');
+  }
+
+  onForeground(): void {
+    // Ability has brought to foreground
+    hilog.info(0x0000, 'testTag', '%{public}s', 'Ability onForeground');
+  }
+
+  onBackground(): void {
+    // Ability has back to background
+    hilog.info(0x0000, 'testTag', '%{public}s', 'Ability onBackground');
+  }
+}

harmony-os/SherpaOnnxTts/entry/src/main/ets/entrybackupability/EntryBackupAbility.ets

@@ -0,0 +1,12 @@
+import hilog from '@ohos.hilog';
+import BackupExtensionAbility, { BundleVersion } from '@ohos.application.BackupExtensionAbility';
+
+export default class EntryBackupAbility extends BackupExtensionAbility {
+  async onBackup() {
+    hilog.info(0x0000, 'testTag', 'onBackup ok');
+  }
+
+  async onRestore(bundleVersion: BundleVersion) {
+    hilog.info(0x0000, 'testTag', 'onRestore ok %{public}s', JSON.stringify(bundleVersion));
+  }
+}

harmony-os/SherpaOnnxTts/entry/src/main/ets/pages/Index.ets

@@ -0,0 +1,409 @@
+import { CircularBuffer } from 'sherpa_onnx';
+import worker, { MessageEvents } from '@ohos.worker';
+import { audio } from '@kit.AudioKit';
+import picker from '@ohos.file.picker';
+import fs from '@ohos.file.fs';
+import systemTime from '@ohos.systemTime';
+
+
+function savePcmToWav(filename: string, samples: Int16Array, sampleRate: number) {
+  const fp = fs.openSync(filename, fs.OpenMode.READ_WRITE | fs.OpenMode.CREATE);
+
+  const header = new ArrayBuffer(44);
+  const view = new DataView(header);
+
+  // http://soundfile.sapp.org/doc/WaveFormat/
+  //                   F F I R
+  view.setUint32(0, 0x46464952, true); // chunkID
+  view.setUint32(4, 36 + samples.length * 2, true); // chunkSize //                   E V A W
+  view.setUint32(8, 0x45564157, true); // format // //                      t m f
+  view.setUint32(12, 0x20746d66, true); // subchunk1ID
+  view.setUint32(16, 16, true); // subchunk1Size, 16 for PCM
+  view.setUint32(20, 1, true); // audioFormat, 1 for PCM
+  view.setUint16(22, 1, true); // numChannels: 1 channel
+  view.setUint32(24, sampleRate, true); // sampleRate
+  view.setUint32(28, sampleRate * 2, true); // byteRate
+  view.setUint16(32, 2, true); // blockAlign
+  view.setUint16(34, 16, true); // bitsPerSample
+  view.setUint32(36, 0x61746164, true); // Subchunk2ID
+  view.setUint32(40, samples.length * 2, true); // subchunk2Size
+
+  fs.writeSync(fp.fd, new Uint8Array(header).buffer, { length: header.byteLength });
+  fs.writeSync(fp.fd, samples.buffer, { length: samples.buffer.byteLength });
+
+  fs.closeSync(fp.fd);
+}
+
+function toInt16Samples(samples: Float32Array): Int16Array {
+  const int16Samples = new Int16Array(samples.length);
+  for (let i = 0; i < samples.length; ++i) {
+    let s = samples[i] * 32767;
+    s = s > 32767 ? 32767 : s;
+    s = s < -32768 ? -32768 : s;
+    int16Samples[i] = s;
+  }
+
+  return int16Samples;
+}
+
+
+@Entry
+@Component
+struct Index {
+  @State currentIndex: number = 0;
+  @State title: string = 'Next-gen Kaldi: Text-to-speech';
+  @State info: string = '';
+  @State btnStartCaption: string = 'Start';
+  @State btnStartEnabled: boolean = false;
+  @State btnStopCaption: string = 'Stop';
+  @State btnStopEnabled: boolean = false;
+  @State btnSaveCaption: string = 'Save';
+  @State btnSaveEnabled: boolean = false;
+  @State progress: number = 0;
+  @State sid: string = '0';
+  @State speechSpeed: string = '1.0';
+  @State isGenerating: boolean = false;
+  @State initTtsDone: boolean = false;
+  @State ttsGeneratedDone: boolean = true;
+  @State numSpeakers: number = 1;
+  @State initAudioDone: boolean = false;
+  private controller: TabsController = new TabsController();
+  private cancelled: boolean = false;
+  private sampleRate: number = 0;
+  private startTime: number = 0;
+  private stopTime: number = 0;
+  private inputText: string = '';
+  // it specifies only the initial capacity.
+  private workerInstance?: worker.ThreadWorker
+  private readonly scriptURL: string = 'entry/ets/workers/NonStreamingTtsWorker.ets'
+  // note that circular buffer can automatically resize.
+  private sampleBuffer: CircularBuffer = new CircularBuffer(16000 * 5);
+  private finalSamples: Float32Array | null = null;
+  private audioRenderer: audio.AudioRenderer | null = null;
+
+  initAudioRenderer() {
+    if (this.audioRenderer) {
+      console.log(`Audio renderer has already been created. Skip creating`);
+      return;
+    } // see // https://developer.huawei.com/consumer/cn/doc/harmonyos-guides-V5/using-audiorenderer-for-playback-V5
+    console.log('Initializing audio renderer');
+    const audioStreamInfo: audio.AudioStreamInfo = {
+      samplingRate: this.sampleRate,
+      channels: audio.AudioChannel.CHANNEL_1, // 通道
+      sampleFormat: audio.AudioSampleFormat.SAMPLE_FORMAT_S16LE,
+      encodingType: audio.AudioEncodingType.ENCODING_TYPE_RAW
+    };
+
+    const audioRendererInfo: audio.AudioRendererInfo = {
+      usage: audio.StreamUsage.STREAM_USAGE_MUSIC, rendererFlags: 0
+    };
+
+    const audioRendererOptions: audio.AudioRendererOptions = {
+      streamInfo: audioStreamInfo, rendererInfo: audioRendererInfo
+    };
+
+    audio.createAudioRenderer(audioRendererOptions, (err, renderer) => {
+      if (!err) {
+        console.log('audio renderer initialized successfully');
+        this.initAudioDone = true;
+        if (renderer) {
+          this.audioRenderer = renderer;
+          this.audioRenderer.on("writeData", this.audioPlayCallback);
+          if (this.sampleBuffer.size()) {
+            this.audioRenderer.start();
+          }
+        } else {
+          console.log(`returned audio renderer is ${renderer}`);
+        }
+      } else {
+        console.log(`Failed to initialize audio renderer. error message: ${err.message}, error code: ${err.code}`);
+      }
+    });
+  }
+
+  async aboutToAppear() {
+    this.initAudioRenderer();
+
+    this.workerInstance = new worker.ThreadWorker(this.scriptURL, {
+      name: 'NonStreaming TTS worker'
+    });
+    this.workerInstance.onmessage = (e: MessageEvents) => {
+      const msgType = e.data['msgType'] as string;
+      console.log(`received msg from worker: ${msgType}`);
+
+      if (msgType == 'init-tts-done') {
+        this.info = 'Model initialized!\nPlease enter text and press start.';
+        this.sampleRate = e.data['sampleRate'] as number;
+        this.numSpeakers = e.data['numSpeakers'] as number;
+
+        this.initTtsDone = true;
+      }
+
+      if (msgType == 'tts-generate-partial') {
+        if (this.cancelled) {
+          return;
+        }
+
+        const samples: Float32Array = e.data['samples'] as Float32Array;
+        const progress: number = e.data['progress'] as number;
+        this.progress = progress;
+
+        this.sampleBuffer.push(samples);
+
+        if (!this.initAudioDone) {
+          this.initAudioRenderer();
+        }
+
+        if (this.audioRenderer && this.audioRenderer?.state != audio.AudioState.STATE_RUNNING) {
+          this.audioRenderer.start();
+        }
+      }
+
+      if (msgType == 'tts-generate-done') {
+        this.isGenerating = false;
+        const samples: Float32Array = e.data['samples'] as Float32Array;
+
+        systemTime.getRealTime((err, data) => {
+
+          if (err) {
+            console.log(`Failed to get stop time`)
+          } else {
+            this.stopTime = data;
+
+            const audioDuration = samples.length / this.sampleRate;
+            const elapsedSeconds = (this.stopTime - this.startTime) / 1000;
+            const RTF = elapsedSeconds / audioDuration;
+
+            this.info = `Audio duration: ${audioDuration} s
+Elapsed: ${elapsedSeconds} s
+RTF = ${elapsedSeconds.toFixed(2)}/${audioDuration.toFixed(2)} = ${RTF.toFixed(3)}
+`;
+            if (this.cancelled) {
+              this.info += '\nCancelled.';
+            }
+          }
+        });
+
+        this.finalSamples = samples;
+        this.ttsGeneratedDone = true;
+        this.btnSaveEnabled = true;
+
+        this.ttsGeneratedDone = true;
+
+        if (this.audioRenderer && this.audioRenderer?.state != audio.AudioState.STATE_RUNNING &&
+          this.sampleBuffer.size() == 0) {
+          this.sampleBuffer.push(samples);
+          this.progress = 1;
+          this.audioRenderer.start();
+        }
+
+        if (!this.initAudioDone) {
+          this.btnStartEnabled = true;
+          this.btnStopEnabled = false;
+          this.info += '\nAudio renderer is not initialized. Disable playing audio.';
+        }
+      }
+    }
+
+    this.info = 'Initializing TTS model ...';
+    this.workerInstance.postMessage({ msgType: 'init-tts', context: getContext() });
+  }
+
+  @Builder
+  TabBuilder(title: string, targetIndex: number, selectedImg: Resource, normalImg: Resource) {
+    Column() {
+      Image(this.currentIndex == targetIndex ? selectedImg : normalImg).size({ width: 25, height: 25 })
+      Text(title).fontColor(this.currentIndex == targetIndex ? '#28bff1' : '#8a8a8a')
+    }.width('100%').height(50).justifyContent(FlexAlign.Center).onClick(() => {
+      this.currentIndex = targetIndex;
+      this.controller.changeIndex(this.currentIndex);
+    })
+  }
+
+  build() {
+    Column() {
+      Tabs({ barPosition: BarPosition.End, controller: this.controller }) {
+        TabContent() {
+          Column({ space: 10 }) {
+            Text(this.title).fontSize(20).fontWeight(FontWeight.Bold);
+            if (this.numSpeakers > 1) {
+              Row({ space: 10 }) {
+                Text(`Speaker ID (0-${this.numSpeakers - 1})`).width('60%')
+
+                TextInput({ text: this.sid }).onChange((text) => {
+                  this.sid = text.trim();
+                }).width('20%')
+              }.justifyContent(FlexAlign.Center)
+            }
+
+            Row() {
+              Text('Speech speed').width('60%');
+
+              TextInput({ text: this.speechSpeed }).onChange((text) => {
+                this.speechSpeed = text.trim();
+              }).width('20%')
+            }
+
+            Row({ space: 10 }) {
+              Button(this.btnStartCaption).enabled(this.btnStartEnabled).onClick(async () => {
+                let sid = parseInt(this.sid);
+                if (sid.toString() != this.sid) {
+                  this.info = 'Please input a valid speaker ID';
+                  return;
+                }
+
+                let speed = parseFloat(this.speechSpeed);
+                if (isNaN(speed)) {
+                  this.info = 'Please enter a valid speech speed';
+                  return;
+                }
+
+                if (speed <= 0) {
+                  this.info = 'Please enter a positive speech speed';
+                  return;
+                }
+
+                if (this.workerInstance && this.initTtsDone) {
+                  this.info = 'Generating...';
+                  this.cancelled = false;
+                  this.finalSamples = null;
+                  this.sampleBuffer.reset();
+                  this.ttsGeneratedDone = false;
+                  this.progress = 0;
+
+                  this.btnStartEnabled = false;
+                  this.btnStopEnabled = true;
+                  this.btnSaveEnabled = false;
+                  console.log(`sending ${this.inputText}`)
+                  this.ttsGeneratedDone = false;
+                  this.startTime = await systemTime.getRealTime();
+                  this.workerInstance?.postMessage({
+                    msgType: 'tts-generate',
+                    text: this.inputText,
+                    sid: sid,
+                    speed: speed,
+                  });
+                  this.isGenerating = true;
+                  this.info = '';
+                } else {
+                  this.info = 'Failed to initialize tts model';
+                  this.btnStartEnabled = false;
+                }
+              });
+
+              Button(this.btnStopCaption).enabled(this.btnStopEnabled).onClick(() => {
+                this.ttsGeneratedDone = true;
+                this.btnStartEnabled = true;
+                this.btnStopEnabled = false;
+                this.sampleBuffer.reset();
+                this.cancelled = true;
+                this.isGenerating = false;
+
+                if (this.workerInstance && this.initTtsDone) {
+                  this.workerInstance.postMessage({ msgType: 'tts-generate-cancel' });
+                }
+                this.audioRenderer?.stop();
+              })
+
+              Button(this.btnSaveCaption).enabled(this.btnSaveEnabled).onClick(() => {
+                if (!this.finalSamples || this.finalSamples.length == 0) {
+
+                  this.btnSaveEnabled = false;
+                  return;
+                }
+
+                let uri: string = '';
+
+                const audioOptions = new picker.AudioSaveOptions(); // audioOptions.newFileNames = ['o.wav'];
+
+                const audioViewPicker = new picker.AudioViewPicker();
+
+                audioViewPicker.save(audioOptions).then((audioSelectResult: Array<string>) => {
+                  uri = audioSelectResult[0];
+                  if (this.finalSamples) {
+                    savePcmToWav(uri, toInt16Samples(this.finalSamples), this.sampleRate);
+                    console.log(`Saved to ${uri}`);
+                    this.info += `\nSaved to ${uri}`;
+                  }
+                });
+              });
+            }
+
+            if (this.info != '') {
+              TextArea({ text: this.info }).focusable(false);
+            }
+            if (this.progress > 0) {
+              Row() {
+                Progress({ value: 0, total: 100, type: ProgressType.Capsule })
+                  .width('80%')
+                  .height(20)
+                  .value(this.progress * 100);
+
+                Text(`${(this.progress * 100).toFixed(2)}%`).width('15%')
+              }.width('100%').justifyContent(FlexAlign.Center)
+            }
+
+            TextArea({ placeholder: 'Input text for TTS and click the start button' })
+              .width('100%')
+              .height('100%')
+              .focusable(this.isGenerating == false && this.initTtsDone)
+              .onChange((text) => {
+                this.inputText = text;
+                if (text.trim() == '') {
+                  this.btnStartEnabled = false;
+                  return;
+                }
+                this.btnStartEnabled = true;
+              })
+          }.width('100%')
+
+          // see https://composeicons.com/
+        }.tabBar(this.TabBuilder('TTS', 0, $r('app.media.home'), $r('app.media.home')))
+
+        TabContent() {
+          Column({space: 10}) {
+            Text(this.title).fontSize(20).fontWeight(FontWeight.Bold);
+            TextArea({text: `
+Everyting is open-sourced.
+
+It runs locally, without accessing the network
+
+See also https://github.com/k2-fsa/sherpa-onnx
+
+新一代 Kaldi QQ 和微信交流群: 请看
+
+https://k2-fsa.github.io/sherpa/social-groups.html
+
+微信公众号: 新一代 Kaldi
+            `}).width('100%')
+              .height('100%')
+              .focusable(false)
+          }.justifyContent(FlexAlign.Start)
+        }.tabBar(this.TabBuilder('Help', 1, $r('app.media.info'), $r('app.media.info')))
+      }.scrollable(false)
+    }
+  }
+
+  private audioPlayCallback = (buffer: ArrayBuffer) => {
+    const numSamples = buffer.byteLength / 2;
+    if (this.sampleBuffer.size() >= numSamples) {
+      const samples: Float32Array = this.sampleBuffer.get(this.sampleBuffer.head(), numSamples);
+
+      const int16Samples = new Int16Array(buffer);
+      for (let i = 0; i < numSamples; ++i) {
+        let s = samples[i] * 32767;
+        s = s > 32767 ? 32767 : s;
+        s = s < -32768 ? -32768 : s;
+        int16Samples[i] = s;
+      }
+      this.sampleBuffer.pop(numSamples);
+    } else {
+      (new Int16Array(buffer)).fill(0);
+      if (this.ttsGeneratedDone) {
+        this.audioRenderer?.stop();
+        this.btnStartEnabled = true;
+        this.btnStopEnabled = false;
+      }
+    }
+  };
+}

harmony-os/SherpaOnnxTts/entry/src/main/ets/workers/NonStreamingTtsWorker.ets

@@ -0,0 +1,284 @@
+import worker, { ThreadWorkerGlobalScope, MessageEvents, ErrorEvent } from '@ohos.worker';
+
+import { fileIo as fs } from '@kit.CoreFileKit';
+
+import {OfflineTtsConfig, OfflineTts, listRawfileDir, TtsInput, TtsOutput} from 'sherpa_onnx';
+import { buffer } from '@kit.ArkTS';
+
+const workerPort: ThreadWorkerGlobalScope = worker.workerPort;
+
+let tts: OfflineTts;
+let cancelled = false;
+
+function mkdir(context: Context, parts: string[]) {
+  const path = parts.join('/');
+  if (fs.accessSync(path)) {
+    return;
+  }
+
+  const sandboxPath: string = context.getApplicationContext().filesDir;
+  let d = sandboxPath
+  for (const p of parts) {
+    d = d + '/' + p;
+
+    if (fs.accessSync(d)) {
+      continue;
+    }
+
+    fs.mkdirSync(d);
+  }
+}
+
+function copyRawFileDirToSandbox(context: Context, srcDir: string) {
+  let mgr = context.resourceManager;
+  const allFiles: string[] = listRawfileDir(mgr, srcDir);
+  for (const src of allFiles) {
+    const parts: string[] = src.split('/');
+    if (parts.length != 1) {
+      mkdir(context, parts.slice(0, -1));
+    }
+
+    copyRawFileToSandbox(context, src, src);
+  }
+}
+
+function copyRawFileToSandbox(context: Context, src: string, dst: string) {
+  // see https://blog.csdn.net/weixin_44640245/article/details/142634846
+  // https://developer.huawei.com/consumer/cn/doc/harmonyos-guides-V5/rawfile-guidelines-V5
+  let uint8Array: Uint8Array = context.resourceManager.getRawFileContentSync(src);
+
+  // https://developer.huawei.com/consumer/cn/doc/harmonyos-references-V5/js-apis-file-fs-V5#fsmkdir
+  let sandboxPath: string = context.getApplicationContext().filesDir;
+  let filepath = sandboxPath + '/'  + dst;
+
+  if (fs.accessSync(filepath)) {
+    // if the destination exists and has the expected file size,
+    // then we skip copying it
+    let stat = fs.statSync(filepath);
+    if (stat.size == uint8Array.length) {
+      return;
+    }
+  }
+
+  const fp = fs.openSync(filepath, fs.OpenMode.WRITE_ONLY | fs.OpenMode.CREATE | fs.OpenMode.TRUNC);
+  fs.writeSync(fp.fd, buffer.from(uint8Array).buffer)
+  fs.close(fp.fd);
+}
+
+function initTts(context: Context): OfflineTts {
+  // Such a design is to make it easier to build flutter APPs with
+  // github actions for a variety of tts models
+  //
+  // See https://github.com/k2-fsa/sherpa-onnx/blob/master/scripts/flutter/generate-tts.py
+  // for details
+
+  let modelDir = '';
+  let modelName = '';
+  let ruleFsts = '';
+  let ruleFars = '';
+  let lexicon = '';
+  let dataDir = '';
+  let dictDir = '';
+  // You can select an example below and change it according to match your
+  // selected tts model
+
+  // ============================================================
+  // Your change starts here
+  // ============================================================
+
+  // Example 1:
+  // modelDir = 'vits-vctk';
+  // modelName = 'vits-vctk.onnx';
+  // lexicon = 'lexicon.txt';
+
+  // Example 2:
+  // https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models
+  // https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_US-amy-low.tar.bz2
+  // modelDir = 'vits-piper-en_US-amy-low';
+  // modelName = 'en_US-amy-low.onnx';
+  // dataDir = 'espeak-ng-data';
+
+  // Example 3:
+  // https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-icefall-zh-aishell3.tar.bz2
+  // modelDir = 'vits-icefall-zh-aishell3';
+  // modelName = 'model.onnx';
+  // ruleFsts = 'phone.fst,date.fst,number.fst,new_heteronym.fst';
+  // ruleFars = 'rule.far';
+  // lexicon = 'lexicon.txt';
+
+  // Example 4:
+  // https://k2-fsa.github.io/sherpa/onnx/tts/pretrained_models/vits.html#csukuangfj-vits-zh-hf-fanchen-c-chinese-187-speakers
+  // modelDir = 'vits-zh-hf-fanchen-C';
+  // modelName = 'vits-zh-hf-fanchen-C.onnx';
+  // lexicon = 'lexicon.txt';
+  // dictDir = 'dict';
+
+  // Example 5:
+  // https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-coqui-de-css10.tar.bz2
+  // modelDir = 'vits-coqui-de-css10';
+  // modelName = 'model.onnx';
+
+  // Example 6
+  // https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models
+  // https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-piper-en_US-libritts_r-medium.tar.bz2
+  // modelDir = 'vits-piper-en_US-libritts_r-medium';
+  // modelName = 'en_US-libritts_r-medium.onnx';
+  // dataDir = 'espeak-ng-data';
+
+  // Example 7
+  // https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models
+  // https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/vits-melo-tts-zh_en.tar.bz2
+  // modelDir = 'vits-melo-tts-zh_en';
+  // modelName = 'model.onnx';
+  // lexicon = 'lexicon.txt';
+  // dictDir = 'dict';
+  // ruleFsts = `date.fst,phone.fst,number.fst`;
+
+  // ============================================================
+  // Please don't change the remaining part of this function
+  // ============================================================
+
+  if (modelName == '') {
+    throw new Error('You are supposed to select a model by changing the code before you run the app');
+  }
+
+  modelName = modelDir + '/' + modelName;
+
+  if (ruleFsts != '') {
+    let fsts = ruleFsts.split(',')
+    let tmp: string[] = [];
+    for (const f of fsts) {
+      tmp.push(modelDir + '/' + f);
+    }
+    ruleFsts = tmp.join(',');
+  }
+
+  if (ruleFars != '') {
+    let fars = ruleFars.split(',')
+    let tmp: string[] = [];
+    for (const f of fars) {
+      tmp.push(modelDir + '/' + f);
+    }
+    ruleFars = tmp.join(',');
+  }
+
+  if (lexicon != '') {
+    lexicon = modelDir + '/' + lexicon;
+  }
+
+  if (dataDir != '') {
+    copyRawFileDirToSandbox(context, modelDir + '/' + dataDir)
+    let sandboxPath: string = context.getApplicationContext().filesDir;
+    dataDir = sandboxPath + '/' + modelDir + '/'  + dataDir;
+  }
+
+  if (dictDir != '') {
+    copyRawFileDirToSandbox(context, modelDir + '/' + dictDir)
+    let sandboxPath: string = context.getApplicationContext().filesDir;
+    dictDir = sandboxPath + '/' + modelDir + '/'  + dictDir;
+  }
+
+  const tokens = modelDir + '/tokens.txt';
+
+  const config: OfflineTtsConfig = new OfflineTtsConfig();
+  config.model.vits.model = modelName;
+  config.model.vits.lexicon = lexicon;
+  config.model.vits.tokens = tokens;
+  config.model.vits.dataDir = dataDir;
+  config.model.vits.dictDir = dictDir;
+  config.model.numThreads = 2;
+  config.model.debug = true;
+  config.ruleFsts = ruleFsts;
+  config.ruleFars = ruleFars;
+
+  return new OfflineTts(config, context.resourceManager);
+}
+
+interface TtsCallbackData {
+  samples: Float32Array;
+  progress: number;
+}
+
+function callback(data: TtsCallbackData): number {
+  workerPort.postMessage({
+    'msgType': 'tts-generate-partial',
+    samples: Float32Array.from(data.samples),
+    progress: data.progress,
+  });
+
+  // 0 means to stop generating in C++
+  // 1 means to continue generating in C++
+  return cancelled? 0 : 1;
+}
+
+/**
+ * Defines the event handler to be called when the worker thread receives a message sent by the host thread.
+ * The event handler is executed in the worker thread.
+ *
+ * @param e message data
+ */
+workerPort.onmessage = (e: MessageEvents) => {
+  const msgType = e.data['msgType'] as string;
+  console.log(`msg-type: ${msgType}`);
+  if (msgType == 'init-tts' && !tts) {
+    const context = e.data['context'] as Context;
+    tts = initTts(context);
+    workerPort.postMessage({ 'msgType': 'init-tts-done',
+      sampleRate: tts.sampleRate,
+      numSpeakers: tts.numSpeakers,
+    });
+  }
+
+  if (msgType == 'tts-generate-cancel') {
+    cancelled = true;
+  }
+
+  if (msgType == 'tts-generate') {
+    const text = e.data['text'] as string;
+    console.log(`recevied text ${text}`);
+    const input: TtsInput = new TtsInput();
+    input.text = text;
+    input.sid = e.data['sid'] as number;
+    input.speed = e.data['speed'] as number;
+    input.callback = callback;
+
+    cancelled = false;
+    if (true) {
+      tts.generateAsync(input).then((ttsOutput: TtsOutput) => {
+        console.log(`sampleRate: ${ttsOutput.sampleRate}`);
+
+        workerPort.postMessage({
+          'msgType': 'tts-generate-done',
+          samples: Float32Array.from(ttsOutput.samples),
+        });
+
+      });
+    } else {
+      const ttsOutput: TtsOutput = tts.generate(input);
+      workerPort.postMessage({
+        'msgType': 'tts-generate-done',
+        samples: Float32Array.from(ttsOutput.samples),
+      });
+    }
+
+
+  }
+}
+
+/**
+ * Defines the event handler to be called when the worker receives a message that cannot be deserialized.
+ * The event handler is executed in the worker thread.
+ *
+ * @param e message data
+ */
+workerPort.onmessageerror = (e: MessageEvents) => {
+}
+
+/**
+ * Defines the event handler to be called when an exception occurs during worker execution.
+ * The event handler is executed in the worker thread.
+ *
+ * @param e error message
+ */
+workerPort.onerror = (e: ErrorEvent) => {
+}

harmony-os/SherpaOnnxTts/entry/src/main/module.json5

@@ -0,0 +1,52 @@
+{
+  "module": {
+    "name": "entry",
+    "type": "entry",
+    "description": "$string:module_desc",
+    "mainElement": "EntryAbility",
+    "deviceTypes": [
+      "phone",
+      "tablet",
+      "2in1"
+    ],
+    "deliveryWithInstall": true,
+    "installationFree": false,
+    "pages": "$profile:main_pages",
+    "abilities": [
+      {
+        "name": "EntryAbility",
+        "srcEntry": "./ets/entryability/EntryAbility.ets",
+        "description": "$string:EntryAbility_desc",
+        "icon": "$media:layered_image",
+        "label": "$string:EntryAbility_label",
+        "startWindowIcon": "$media:startIcon",
+        "startWindowBackground": "$color:start_window_background",
+        "exported": true,
+        "skills": [
+          {
+            "entities": [
+              "entity.system.home"
+            ],
+            "actions": [
+              "action.system.home"
+            ]
+          }
+        ]
+      }
+    ],
+    "extensionAbilities": [
+      {
+        "name": "EntryBackupAbility",
+        "srcEntry": "./ets/entrybackupability/EntryBackupAbility.ets",
+        "type": "backup",
+        "exported": false,
+        "metadata": [
+          {
+            "name": "ohos.extension.backup",
+            "resource": "$profile:backup_config"
+          }
+        ],
+      }
+    ]
+  }
+}

harmony-os/SherpaOnnxTts/entry/src/main/resources/base/element/color.json

@@ -0,0 +1,8 @@
+{
+  "color": [
+    {
+      "name": "start_window_background",
+      "value": "#FFFFFF"
+    }
+  ]
+}

harmony-os/SherpaOnnxTts/entry/src/main/resources/base/element/string.json

@@ -0,0 +1,16 @@
+{
+  "string": [
+    {
+      "name": "module_desc",
+      "value": "On-device text-to-speech with Next-gen Kaldi"
+    },
+    {
+      "name": "EntryAbility_desc",
+      "value": "On-device text-to-speech with Next-gen Kaldi"
+    },
+    {
+      "name": "EntryAbility_label",
+      "value": "TTS"
+    }
+  ]
+}

harmony-os/SherpaOnnxTts/entry/src/main/resources/base/media/home.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 -960 960 960"><path d="m480-840 440 330-48 64-72-54v380H160v-380l-72 54-48-64zM294-478q0 53 57 113t129 125q72-65 129-125t57-113q0-44-30-73t-72-29q-26 0-47.5 10.5T480-542q-15-17-37.5-27.5T396-580q-42 0-72 29t-30 73m426 278v-360L480-740 240-560v360zm0 0H240z"/></svg>

harmony-os/SherpaOnnxTts/entry/src/main/resources/base/media/info.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 -960 960 960"><path d="M440-280h80v-240h-80zm40-320q17 0 28.5-11.5T520-640t-11.5-28.5T480-680t-28.5 11.5T440-640t11.5 28.5T480-600m0 520q-83 0-156-31.5T197-197t-85.5-127T80-480t31.5-156T197-763t127-85.5T480-880t156 31.5T763-763t85.5 127T880-480t-31.5 156T763-197t-127 85.5T480-80m0-80q134 0 227-93t93-227-93-227-227-93-227 93-93 227 93 227 227 93m0-320"/></svg>

harmony-os/SherpaOnnxTts/entry/src/main/resources/base/media/layered_image.json

@@ -0,0 +1,7 @@
+{
+  "layered-image":
+  {
+    "background" : "$media:background",
+    "foreground" : "$media:foreground"
+  }
+}

harmony-os/SherpaOnnxTts/entry/src/main/resources/base/profile/backup_config.json

@@ -0,0 +1,3 @@
+{
+  "allowToBackupRestore": true
+}

harmony-os/SherpaOnnxTts/entry/src/main/resources/base/profile/main_pages.json

@@ -0,0 +1,5 @@
+{
+  "src": [
+    "pages/Index"
+  ]
+}

harmony-os/SherpaOnnxTts/entry/src/main/resources/en_US/element/string.json

@@ -0,0 +1,16 @@
+{
+  "string": [
+    {
+      "name": "module_desc",
+      "value": "On-device text-to-speech with Next-gen Kaldi"
+    },
+    {
+      "name": "EntryAbility_desc",
+      "value": "On-device text-to-speech with Next-gen Kaldi"
+    },
+    {
+      "name": "EntryAbility_label",
+      "value": "TTS"
+    }
+  ]
+}

harmony-os/SherpaOnnxTts/entry/src/main/resources/zh_CN/element/string.json

@@ -0,0 +1,16 @@
+{
+  "string": [
+    {
+      "name": "module_desc",
+      "value": "使用新一代Kaldi进行本地离线语音合成"
+    },
+    {
+      "name": "EntryAbility_desc",
+      "value": "使用新一代Kaldi进行本地离线语音合成"
+    },
+    {
+      "name": "EntryAbility_label",
+      "value": "本地语音合成"
+    }
+  ]
+}

harmony-os/SherpaOnnxTts/entry/src/ohosTest/ets/test/Ability.test.ets

@@ -0,0 +1,35 @@
+import hilog from '@ohos.hilog';
+import { describe, beforeAll, beforeEach, afterEach, afterAll, it, expect } from '@ohos/hypium';
+
+export default function abilityTest() {
+  describe('ActsAbilityTest', () => {
+    // Defines a test suite. Two parameters are supported: test suite name and test suite function.
+    beforeAll(() => {
+      // Presets an action, which is performed only once before all test cases of the test suite start.
+      // This API supports only one parameter: preset action function.
+    })
+    beforeEach(() => {
+      // Presets an action, which is performed before each unit test case starts.
+      // The number of execution times is the same as the number of test cases defined by **it**.
+      // This API supports only one parameter: preset action function.
+    })
+    afterEach(() => {
+      // Presets a clear action, which is performed after each unit test case ends.
+      // The number of execution times is the same as the number of test cases defined by **it**.
+      // This API supports only one parameter: clear action function.
+    })
+    afterAll(() => {
+      // Presets a clear action, which is performed after all test cases of the test suite end.
+      // This API supports only one parameter: clear action function.
+    })
+    it('assertContain', 0, () => {
+      // Defines a test case. This API supports three parameters: test case name, filter parameter, and test case function.
+      hilog.info(0x0000, 'testTag', '%{public}s', 'it begin');
+      let a = 'abc';
+      let b = 'b';
+      // Defines a variety of assertion methods, which are used to declare expected boolean conditions.
+      expect(a).assertContain(b);
+      expect(a).assertEqual(a);
+    })
+  })
+}

harmony-os/SherpaOnnxTts/entry/src/ohosTest/ets/test/List.test.ets

@@ -0,0 +1,5 @@
+import abilityTest from './Ability.test';
+
+export default function testsuite() {
+  abilityTest();
+}

harmony-os/SherpaOnnxTts/entry/src/ohosTest/module.json5

@@ -0,0 +1,13 @@
+{
+  "module": {
+    "name": "entry_test",
+    "type": "feature",
+    "deviceTypes": [
+      "phone",
+      "tablet",
+      "2in1"
+    ],
+    "deliveryWithInstall": true,
+    "installationFree": false
+  }
+}

harmony-os/SherpaOnnxTts/entry/src/test/List.test.ets

@@ -0,0 +1,5 @@
+import localUnitTest from './LocalUnit.test';
+
+export default function testsuite() {
+  localUnitTest();
+}

harmony-os/SherpaOnnxTts/entry/src/test/LocalUnit.test.ets

@@ -0,0 +1,33 @@
+import { describe, beforeAll, beforeEach, afterEach, afterAll, it, expect } from '@ohos/hypium';
+
+export default function localUnitTest() {
+  describe('localUnitTest', () => {
+    // Defines a test suite. Two parameters are supported: test suite name and test suite function.
+    beforeAll(() => {
+      // Presets an action, which is performed only once before all test cases of the test suite start.
+      // This API supports only one parameter: preset action function.
+    });
+    beforeEach(() => {
+      // Presets an action, which is performed before each unit test case starts.
+      // The number of execution times is the same as the number of test cases defined by **it**.
+      // This API supports only one parameter: preset action function.
+    });
+    afterEach(() => {
+      // Presets a clear action, which is performed after each unit test case ends.
+      // The number of execution times is the same as the number of test cases defined by **it**.
+      // This API supports only one parameter: clear action function.
+    });
+    afterAll(() => {
+      // Presets a clear action, which is performed after all test cases of the test suite end.
+      // This API supports only one parameter: clear action function.
+    });
+    it('assertContain', 0, () => {
+      // Defines a test case. This API supports three parameters: test case name, filter parameter, and test case function.
+      let a = 'abc';
+      let b = 'b';
+      // Defines a variety of assertion methods, which are used to declare expected boolean conditions.
+      expect(a).assertContain(b);
+      expect(a).assertEqual(a);
+    });
+  });
+}

harmony-os/SherpaOnnxTts/hvigor/hvigor-config.json5

@@ -0,0 +1,22 @@
+{
+  "modelVersion": "5.0.0",
+  "dependencies": {
+  },
+  "execution": {
+    // "analyze": "normal",                     /* Define the build analyze mode. Value: [ "normal" | "advanced" | false ]. Default: "normal" */
+    // "daemon": true,                          /* Enable daemon compilation. Value: [ true | false ]. Default: true */
+    // "incremental": true,                     /* Enable incremental compilation. Value: [ true | false ]. Default: true */
+    // "parallel": true,                        /* Enable parallel compilation. Value: [ true | false ]. Default: true */
+    // "typeCheck": false,                      /* Enable typeCheck. Value: [ true | false ]. Default: false */
+  },
+  "logging": {
+    // "level": "info"                          /* Define the log level. Value: [ "debug" | "info" | "warn" | "error" ]. Default: "info" */
+  },
+  "debugging": {
+    // "stacktrace": false                      /* Disable stacktrace compilation. Value: [ true | false ]. Default: false */
+  },
+  "nodeOptions": {
+    // "maxOldSpaceSize": 8192                  /* Enable nodeOptions maxOldSpaceSize compilation. Unit M. Used for the daemon process. Default: 8192*/
+    // "exposeGC": true                         /* Enable to trigger garbage collection explicitly. Default: true*/
+  }
+}

harmony-os/SherpaOnnxTts/hvigorfile.ts

@@ -0,0 +1,6 @@
+import { appTasks } from '@ohos/hvigor-ohos-plugin';
+
+export default {
+    system: appTasks,  /* Built-in plugin of Hvigor. It cannot be modified. */
+    plugins:[]         /* Custom plugin to extend the functionality of Hvigor. */
+}

harmony-os/SherpaOnnxTts/oh-package-lock.json5

@@ -0,0 +1,19 @@
+{
+  "meta": {
+    "stableOrder": true
+  },
+  "lockfileVersion": 3,
+  "ATTENTION": "THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.",
+  "specifiers": {
+    "@ohos/hypium@1.0.19": "@ohos/hypium@1.0.19"
+  },
+  "packages": {
+    "@ohos/hypium@1.0.19": {
+      "name": "@ohos/hypium",
+      "version": "1.0.19",
+      "integrity": "sha512-cEjDgLFCm3cWZDeRXk7agBUkPqjWxUo6AQeiu0gEkb3J8ESqlduQLSIXeo3cCsm8U/asL7iKjF85ZyOuufAGSQ==",
+      "resolved": "https://ohpm.openharmony.cn/ohpm/@ohos/hypium/-/hypium-1.0.19.har",
+      "registryType": "ohpm"
+    }
+  }
+}

harmony-os/SherpaOnnxTts/oh-package.json5

@@ -0,0 +1,9 @@
+{
+  "modelVersion": "5.0.0",
+  "description": "Please describe the basic information.",
+  "dependencies": {
+  },
+  "devDependencies": {
+    "@ohos/hypium": "1.0.19"
+  }
+}