Add VAD+ASR demo for HarmonyOS (#1573)

harmony-os/README.md

@@ -0,0 +1,9 @@
+# Introduction
+
+- [./SherpaOnnxHar](./SherpaOnnxHar) It is for building `sherpa_onnx.har`.
+  If you don't need to change the C++ or Typescript code of sherpa-onnx, then
+  you can download pre-built `sherpa_onnx.har` from us. Please refer to
+  our [doc](https://k2-fsa.github.io/sherpa/onnx) for how to download it.
+
+- [./SherpaOnnxVadAsr](./SherpaOnnxVadAsr) It shows how to use
+  VAD + Non-streaming ASR for speech recognition.

harmony-os/SherpaOnnxVadAsr/.gitignore

@@ -0,0 +1,12 @@
+/node_modules
+/oh_modules
+/local.properties
+/.idea
+**/build
+/.hvigor
+.cxx
+/.clangd
+/.clang-format
+/.clang-tidy
+**/.test
+/.appanalyzer

harmony-os/SherpaOnnxVadAsr/AppScope/app.json5

@@ -0,0 +1,10 @@
+{
+  "app": {
+    "bundleName": "com.k2fsa.sherpa.onnx.vad.asr",
+    "vendor": "example",
+    "versionCode": 1000000,
+    "versionName": "1.0.0",
+    "icon": "$media:app_icon",
+    "label": "$string:app_name"
+  }
+}

harmony-os/SherpaOnnxVadAsr/AppScope/resources/base/element/string.json

@@ -0,0 +1,8 @@
+{
+  "string": [
+    {
+      "name": "app_name",
+      "value": "SherpaOnnxVadAsr"
+    }
+  ]
+}

harmony-os/SherpaOnnxVadAsr/build-profile.json5

@@ -0,0 +1,40 @@
+{
+  "app": {
+    "signingConfigs": [],
+    "products": [
+      {
+        "name": "default",
+        "signingConfig": "default",
+        "compatibleSdkVersion": "4.0.0(10)",
+        "runtimeOS": "HarmonyOS",
+        "buildOption": {
+          "strictMode": {
+            "caseSensitiveCheck": true,
+          }
+        }
+      }
+    ],
+    "buildModeSet": [
+      {
+        "name": "debug",
+      },
+      {
+        "name": "release"
+      }
+    ]
+  },
+  "modules": [
+    {
+      "name": "entry",
+      "srcPath": "./entry",
+      "targets": [
+        {
+          "name": "default",
+          "applyToProducts": [
+            "default"
+          ]
+        }
+      ]
+    }
+  ]
+}

harmony-os/SherpaOnnxVadAsr/code-linter.json5

@@ -0,0 +1,20 @@
+{
+  "files": [
+    "**/*.ets"
+  ],
+  "ignore": [
+    "**/src/ohosTest/**/*",
+    "**/src/test/**/*",
+    "**/src/mock/**/*",
+    "**/node_modules/**/*",
+    "**/oh_modules/**/*",
+    "**/build/**/*",
+    "**/.preview/**/*"
+  ],
+  "ruleSet": [
+    "plugin:@performance/recommended",
+    "plugin:@typescript-eslint/recommended"
+  ],
+  "rules": {
+  }
+}

harmony-os/SherpaOnnxVadAsr/entry/.gitignore

@@ -0,0 +1,7 @@
+/node_modules
+/oh_modules
+/.preview
+/build
+/.cxx
+/.test
+*.har

harmony-os/SherpaOnnxVadAsr/entry/README.md

@@ -0,0 +1,7 @@
+# Introduction
+
+Please download ./sherpa_onnx-v1.10.32.har
+from <https://huggingface.co/csukuangfj/sherpa-onnx-harmony-os/tree/main/har>
+
+Hint: For users who have no access to huggingface, please use
+<https://hf-mirror.com/csukuangfj/sherpa-onnx-harmony-os/tree/main/har>

harmony-os/SherpaOnnxVadAsr/entry/build-profile.json5

@@ -0,0 +1,33 @@
+{
+  "apiType": "stageMode",
+  "buildOption": {
+    "sourceOption": {
+      "workers": [
+        './src/main/ets/workers/NonStreamingAsrWithVadWorker.ets'
+      ]
+    }
+  },
+  "buildOptionSet": [
+    {
+      "name": "release",
+      "arkOptions": {
+        "obfuscation": {
+          "ruleOptions": {
+            "enable": false,
+            "files": [
+              "./obfuscation-rules.txt"
+            ]
+          }
+        }
+      }
+    },
+  ],
+  "targets": [
+    {
+      "name": "default"
+    },
+    {
+      "name": "ohosTest",
+    }
+  ]
+}

harmony-os/SherpaOnnxVadAsr/entry/hvigorfile.ts

@@ -0,0 +1,6 @@
+import { hapTasks } from '@ohos/hvigor-ohos-plugin';
+
+export default {
+    system: hapTasks,  /* Built-in plugin of Hvigor. It cannot be modified. */
+    plugins:[]         /* Custom plugin to extend the functionality of Hvigor. */
+}

harmony-os/SherpaOnnxVadAsr/entry/obfuscation-rules.txt

@@ -0,0 +1,23 @@
+# Define project specific obfuscation rules here.
+# You can include the obfuscation configuration files in the current module's build-profile.json5.
+#
+# For more details, see
+#   https://developer.huawei.com/consumer/cn/doc/harmonyos-guides-V5/source-obfuscation-V5
+
+# Obfuscation options:
+# -disable-obfuscation: disable all obfuscations
+# -enable-property-obfuscation: obfuscate the property names
+# -enable-toplevel-obfuscation: obfuscate the names in the global scope
+# -compact: remove unnecessary blank spaces and all line feeds
+# -remove-log: remove all console.* statements
+# -print-namecache: print the name cache that contains the mapping from the old names to new names
+# -apply-namecache: reuse the given cache file
+
+# Keep options:
+# -keep-property-name: specifies property names that you want to keep
+# -keep-global-name: specifies names that you want to keep in the global scope
+
+-enable-property-obfuscation
+-enable-toplevel-obfuscation
+-enable-filename-obfuscation
+-enable-export-obfuscation

harmony-os/SherpaOnnxVadAsr/entry/oh-package.json5

@@ -0,0 +1,10 @@
+{
+  "name": "entry",
+  "version": "1.0.0",
+  "description": "Please describe the basic information.",
+  "main": "",
+  "author": "",
+  "license": "",
+  "dependencies": {}
+}
+

harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/entryability/EntryAbility.ets

@@ -0,0 +1,43 @@
+import AbilityConstant from '@ohos.app.ability.AbilityConstant';
+import hilog from '@ohos.hilog';
+import UIAbility from '@ohos.app.ability.UIAbility';
+import Want from '@ohos.app.ability.Want';
+import window from '@ohos.window';
+
+export default class EntryAbility extends UIAbility {
+  onCreate(want: Want, launchParam: AbilityConstant.LaunchParam): void {
+    hilog.info(0x0000, 'testTag', '%{public}s', 'Ability onCreate');
+  }
+
+  onDestroy(): void {
+    hilog.info(0x0000, 'testTag', '%{public}s', 'Ability onDestroy');
+  }
+
+  onWindowStageCreate(windowStage: window.WindowStage): void {
+    // Main window is created, set main page for this ability
+    hilog.info(0x0000, 'testTag', '%{public}s', 'Ability onWindowStageCreate');
+
+    windowStage.loadContent('pages/Index', (err) => {
+      if (err.code) {
+        hilog.error(0x0000, 'testTag', 'Failed to load the content. Cause: %{public}s', JSON.stringify(err) ?? '');
+        return;
+      }
+      hilog.info(0x0000, 'testTag', 'Succeeded in loading the content.');
+    });
+  }
+
+  onWindowStageDestroy(): void {
+    // Main window is destroyed, release UI related resources
+    hilog.info(0x0000, 'testTag', '%{public}s', 'Ability onWindowStageDestroy');
+  }
+
+  onForeground(): void {
+    // Ability has brought to foreground
+    hilog.info(0x0000, 'testTag', '%{public}s', 'Ability onForeground');
+  }
+
+  onBackground(): void {
+    // Ability has back to background
+    hilog.info(0x0000, 'testTag', '%{public}s', 'Ability onBackground');
+  }
+}

harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/entrybackupability/EntryBackupAbility.ets

@@ -0,0 +1,12 @@
+import hilog from '@ohos.hilog';
+import BackupExtensionAbility, { BundleVersion } from '@ohos.application.BackupExtensionAbility';
+
+export default class EntryBackupAbility extends BackupExtensionAbility {
+  async onBackup() {
+    hilog.info(0x0000, 'testTag', 'onBackup ok');
+  }
+
+  async onRestore(bundleVersion: BundleVersion) {
+    hilog.info(0x0000, 'testTag', 'onRestore ok %{public}s', JSON.stringify(bundleVersion));
+  }
+}

harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/pages/Index.ets

@@ -0,0 +1,173 @@
+import { LengthUnit } from '@kit.ArkUI';
+import worker, { MessageEvents } from '@ohos.worker';
+import { BusinessError } from '@kit.BasicServicesKit';
+import { picker } from '@kit.CoreFileKit';
+
+
+@Entry
+@Component
+struct Index {
+  @State currentIndex: number = 0;
+  @State resultFromFile: string = '';
+  @State progressForFile: number = 0;
+  @State selectFileBtnEnabled: boolean = false;
+  @State message: string = 'To be implemented';
+  @State lang: string = 'English';
+  private controller: TabsController = new TabsController();
+  private workerInstance?: worker.ThreadWorker
+  private readonly scriptURL: string = 'entry/ets/workers/NonStreamingAsrWithVadWorker.ets'
+
+  aboutToAppear(): void {
+    this.workerInstance = new worker.ThreadWorker(this.scriptURL, {
+      name: 'NonStreaming ASR worker'
+    });
+
+    this.workerInstance.onmessage = (e: MessageEvents) => {
+      const msgType = e.data['msgType'] as string;
+      console.log(`received data ${msgType}`);
+
+      if (msgType == 'init-non-streaming-asr-done') {
+        this.selectFileBtnEnabled = true;
+      }
+
+      if (msgType == 'non-streaming-asr-vad-decode-done') {
+        this.resultFromFile = e.data['text'] as string + '\n';
+      }
+
+      if (msgType == 'non-streaming-asr-vad-decode-partial') {
+        if (this.resultFromFile == '') {
+          this.resultFromFile = e.data['text'] as string;
+        } else {
+          this.resultFromFile += '\n\n' + e.data['text'] as string;
+        }
+      }
+
+      if (msgType == 'non-streaming-asr-vad-decode-error') {
+        this.resultFromFile = e.data['text'] as string;
+      }
+
+      if (msgType == 'non-streaming-asr-vad-decode-progress') {
+        this.progressForFile = e.data['progress'] as number;
+
+        this.selectFileBtnEnabled = this.progressForFile >= 100;
+      }
+    }
+
+    const context = getContext();
+    this.workerInstance.postMessage({ msgType: 'init-vad', context });
+    this.workerInstance.postMessage({ msgType: 'init-non-streaming-asr', context });
+  }
+
+  @Builder
+  TabBuilder(title: string, targetIndex: number, selectedImg: Resource, normalImg: Resource) {
+    Column() {
+      Image(this.currentIndex == targetIndex ? selectedImg : normalImg)
+        .size({ width: 25, height: 25 })
+      Text(title)
+        .fontColor(this.currentIndex == targetIndex ? '#28bff1' : '#8a8a8a')
+    }
+    .width('100%')
+    .height(50)
+    .justifyContent(FlexAlign.Center)
+    .onClick(() => {
+      this.currentIndex = targetIndex;
+      this.controller.changeIndex(this.currentIndex);
+    })
+  }
+
+  build() {
+    Column() {
+      Tabs({ barPosition: BarPosition.End, controller: this.controller }) {
+        TabContent() {
+          Column({ space: 10 }) {
+            Text('Next-gen Kaldi: VAD + ASR')
+              .fontColor('#182431')
+              .fontSize(25)
+              .lineHeight(41)
+              .fontWeight(500)
+
+            Button('Select .wav file ')
+              .enabled(this.selectFileBtnEnabled)
+              .fontSize(13)
+              .width(296)
+              .height(60)
+              .onClick(() => {
+                this.resultFromFile = '';
+                this.progressForFile = 0;
+
+                const documentSelectOptions = new picker.DocumentSelectOptions();
+                documentSelectOptions.maxSelectNumber = 1;
+                documentSelectOptions.fileSuffixFilters = ['.wav'];
+                const documentViewPicker = new picker.DocumentViewPicker();
+                documentViewPicker.select(documentSelectOptions).then((result: Array<string>) => {
+                  console.log(`Result: ${result}`);
+
+                  if (!result[0]) {
+                    this.resultFromFile = 'Please select a file to decode';
+                    this.selectFileBtnEnabled = true;
+                    return;
+                  }
+
+                  if (this.workerInstance) {
+                    this.workerInstance.postMessage({
+                      msgType: 'non-streaming-asr-vad-decode',
+                      filename: result[0],
+                    });
+                  } else {
+                    console.log(`this worker instance is undefined ${this.workerInstance}`);
+                  }
+                }).catch((err: BusinessError) => {
+                  console.error(`Failed to select file, code is ${err.code}, message is ${err.message}`);
+                })
+
+              })
+
+            Text(`Supported languages: ${this.lang}`)
+
+            if (this.progressForFile > 0) {
+              Row() {
+                Progress({ value: 0, total: 100, type: ProgressType.Capsule })
+                  .width('80%')
+                  .height(20)
+                  .value(this.progressForFile);
+
+                Text(`${this.progressForFile.toFixed(2)}%`).width('15%')
+              }.width('100%').justifyContent(FlexAlign.Center)
+            }
+
+            TextArea({ text: this.resultFromFile }).width('100%').lineSpacing({ value: 10, unit: LengthUnit.VP });
+
+          }
+          .alignItems(HorizontalAlign.Center)
+          .justifyContent(FlexAlign.Start)
+        }.tabBar(this.TabBuilder('From file', 0, $r('app.media.icon_doc'), $r('app.media.icon_doc_default')))
+
+        TabContent() {
+          Column() {
+            Text(this.message)
+              .fontSize(50)
+              .fontWeight(FontWeight.Bold);
+          }
+        }
+        .tabBar(this.TabBuilder('From mic', 1, $r('app.media.ic_public_input_voice'),
+          $r('app.media.ic_public_input_voice_default')))
+
+        TabContent() {
+          Column() {
+            Text("Everything is open-sourced");
+            Divider();
+            Text("It runs locally, without accessing the network");
+            Divider();
+            Text("See also https://github.com/k2-fsa/sherpa-onnx");
+            Divider();
+            Text("and https://k2-fsa.github.io/sherpa/social-groups.html");
+          }.justifyContent(FlexAlign.Start)
+        }.tabBar(this.TabBuilder('Help', 2, $r('app.media.info_circle'),
+          $r('app.media.info_circle_default')))
+
+      }.scrollable(false)
+    }
+    .width('100%')
+    .justifyContent(FlexAlign.Start)
+  }
+}

harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/pages/NonStreamingAsrModels.ets

@@ -0,0 +1,237 @@
+// Please keep in sync with
+// https://github.com/k2-fsa/sherpa-onnx/blob/master/sherpa-onnx/kotlin-api/OfflineRecognizer.kt#L184
+
+import { OfflineModelConfig } from 'sherpa_onnx';
+
+export function getOfflineModelConfig(type: number): OfflineModelConfig {
+  const c = new OfflineModelConfig();
+  switch (type) {
+    case 0: {
+      const modelDir = 'sherpa-onnx-paraformer-zh-2023-09-14'
+      c.paraformer.model = `${modelDir}/model.int8.onnx`;
+      c.tokens = `${modelDir}/tokens.txt`;
+      c.modelType = "paraformer";
+
+      break;
+    }
+
+    case 1: {
+      const modelDir = 'icefall-asr-multidataset-pruned_transducer_stateless7-2023-05-04'
+      c.transducer.encoder = `$modelDir}/encoder-epoch-30-avg-4.int8.onnx`;
+      c.transducer.decoder = `${modelDir}/decoder-epoch-30-avg-4.onnx`;
+      c.transducer.encoder = `${modelDir}/joiner-epoch-30-avg-4.onnx`;
+      c.tokens = `${modelDir}/tokens.txt`;
+      c.modelType = "transducer";
+
+      break;
+    }
+
+    case 2: {
+      const modelDir = 'sherpa-onnx-whisper-tiny.en';
+      c.whisper.encoder = `${modelDir}/tiny.en-encoder.int8.onnx`;
+      c.whisper.decoder = `${modelDir}/tiny.en-decoder.int8.onnx`;
+      c.tokens = `${modelDir}/tiny.en-tokens.txt`;
+      c.modelType = "whisper";
+
+      break;
+    }
+
+    case 3: {
+      const modelDir = 'sherpa-onnx-whisper-base.en';
+      c.whisper.encoder = `${modelDir}/base.en-encoder.int8.onnx`;
+      c.whisper.decoder = `${modelDir}/base.en-decoder.int8.onnx`;
+      c.tokens = `${modelDir}/base.en-tokens.txt`;
+      c.modelType = "whisper";
+
+      break;
+    }
+
+    case 4: {
+      const modelDir = "icefall-asr-zipformer-wenetspeech-20230615";
+      c.transducer.encoder = `${modelDir}/encoder-epoch-12-avg-4.int8.onnx`;
+      c.transducer.decoder = `${modelDir}/decoder-epoch-12-avg-4.onnx`;
+      c.transducer.joiner = `${modelDir}/joiner-epoch-12-avg-4.int8.onnx`;
+      c.tokens = `${modelDir}/tokens.txt`;
+      c.modelType = "transducer";
+
+      break;
+    }
+
+    case 5: {
+      const modelDir = "sherpa-onnx-zipformer-multi-zh-hans-2023-9-2";
+      c.transducer.encoder = `${modelDir}/encoder-epoch-20-avg-1.int8.onnx`;
+      c.transducer.decoder = `${modelDir}/decoder-epoch-20-avg-1.onnx`;
+      c.transducer.joiner = `${modelDir}/joiner-epoch-20-avg-1.int8.onnx`;
+      c.tokens = `${modelDir}/tokens.txt`;
+      c.modelType = "transducer";
+
+      break;
+    }
+
+    case 6: {
+      const modelDir = "sherpa-onnx-nemo-ctc-en-citrinet-512";
+      c.nemoCtc.model = `${modelDir}/model.int8.onnx`;
+      c.tokens = `${modelDir}/tokens.txt`;
+
+      break;
+    }
+
+    case 7: {
+      const modelDir = "sherpa-onnx-nemo-fast-conformer-ctc-be-de-en-es-fr-hr-it-pl-ru-uk-20k"
+      c.nemoCtc.model = `${modelDir}/model.onnx`;
+      c.tokens = `${modelDir}/tokens.txt`;
+
+      break;
+    }
+
+    case 8: {
+      const modelDir = "sherpa-onnx-nemo-fast-conformer-ctc-en-24500"
+      c.nemoCtc.model = `${modelDir}/model.onnx`;
+      c.tokens = `${modelDir}/tokens.txt`;
+
+      break;
+    }
+
+    case 9: {
+      const modelDir = "sherpa-onnx-nemo-fast-conformer-ctc-en-de-es-fr-14288"
+      c.nemoCtc.model = `${modelDir}/model.onnx`;
+      c.tokens = `${modelDir}/tokens.txt`;
+
+      break;
+    }
+
+    case 10: {
+      const modelDir = "sherpa-onnx-nemo-fast-conformer-ctc-es-1424"
+      c.nemoCtc.model = `${modelDir}/model.onnx`;
+      c.tokens = `${modelDir}/tokens.txt`;
+
+      break;
+    }
+
+    case 11: {
+      const modelDir = "sherpa-onnx-telespeech-ctc-int8-zh-2024-06-04"
+      c.telespeechCtc = `${modelDir}/model.int8.onnx`;
+      c.tokens = `${modelDir}/tokens.txt`;
+      c.modelType = "telespeech_ctc";
+
+      break;
+    }
+
+    case 12: {
+      const modelDir = "sherpa-onnx-zipformer-thai-2024-06-20"
+      c.transducer.encoder = `${modelDir}/encoder-epoch-12-avg-5.int8.onnx`;
+      c.transducer.decoder = `${modelDir}/decoder-epoch-12-avg-5.onnx`;
+      c.transducer.joiner = `${modelDir}/joiner-epoch-12-avg-5.int8.onnx`;
+      c.tokens = `${modelDir}/tokens.txt`;
+      c.modelType = "transducer";
+
+      break;
+    }
+
+    case 13: {
+      const modelDir = "sherpa-onnx-zipformer-korean-2024-06-24";
+      c.transducer.encoder = `${modelDir}/encoder-epoch-99-avg-1.int8.onnx`;
+      c.transducer.decoder = `${modelDir}/decoder-epoch-99-avg-1.onnx`;
+      c.transducer.joiner = `${modelDir}/joiner-epoch-99-avg-1.int8.onnx`;
+      c.tokens = `${modelDir}/tokens.txt`;
+      c.modelType = "transducer";
+
+      break;
+    }
+
+    case 14: {
+      const modelDir = "sherpa-onnx-paraformer-zh-small-2024-03-09";
+      c.paraformer.model = `${modelDir}/model.int8.onnx`;
+      c.tokens = `${modelDir}/tokens.txt`;
+      c.modelType = "paraformer";
+
+      break;
+    }
+
+    case 15: {
+      const modelDir = "sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17";
+      c.senseVoice.model = `${modelDir}/model.int8.onnx`;
+      c.tokens = `${modelDir}/tokens.txt`;
+
+      break;
+    }
+
+    case 16: {
+      const modelDir = "sherpa-onnx-zipformer-ja-reazonspeech-2024-08-01";
+      c.transducer.encoder = `${modelDir}/encoder-epoch-99-avg-1.int8.onnx`;
+      c.transducer.decoder = `${modelDir}/decoder-epoch-99-avg-1.onnx`;
+      c.transducer.joiner = `${modelDir}/joiner-epoch-99-avg-1.int8.onnx`;
+      c.tokens = `${modelDir}/tokens.txt`;
+      c.modelType = "transducer";
+
+      break;
+    }
+
+    case 17: {
+      const modelDir = "sherpa-onnx-zipformer-ru-2024-09-18";
+      c.transducer.encoder = `${modelDir}/encoder.int8.onnx`;
+      c.transducer.decoder = `${modelDir}/decoder.onnx`;
+      c.transducer.joiner = `${modelDir}/joiner.int8.onnx`;
+      c.tokens = `${modelDir}/tokens.txt`;
+      c.modelType = "transducer";
+
+      break;
+    }
+
+    case 18: {
+      const modelDir = "sherpa-onnx-small-zipformer-ru-2024-09-18";
+      c.transducer.encoder = `${modelDir}/encoder.int8.onnx`;
+      c.transducer.decoder = `${modelDir}/decoder.onnx`;
+      c.transducer.joiner = `${modelDir}/joiner.int8.onnx`;
+      c.tokens = `${modelDir}/tokens.txt`;
+      c.modelType = "transducer";
+
+      break;
+    }
+
+    case 19: {
+      const modelDir = "sherpa-onnx-nemo-ctc-giga-am-russian-2024-10-24";
+      c.nemoCtc.model = `${modelDir}/model.int8.onnx`;
+      c.tokens = `${modelDir}/tokens.txt`;
+
+      break;
+    }
+
+    case 20: {
+      const modelDir = "sherpa-onnx-nemo-transducer-giga-am-russian-2024-10-24";
+      c.transducer.encoder = `${modelDir}/encoder.int8.onnx`;
+      c.transducer.decoder = `${modelDir}/decoder.onnx`;
+      c.transducer.joiner = `${modelDir}/joiner.onnx`;
+      c.tokens = `${modelDir}/tokens.txt`;
+      c.modelType = "nemo_transducer";
+
+      break;
+    }
+
+    case 21: {
+      const modelDir = "sherpa-onnx-moonshine-tiny-en-int8";
+      c.moonshine.preprocessor = `${modelDir}/preprocess.onnx`;
+      c.moonshine.encoder = `${modelDir}/encode.int8.onnx`;
+      c.moonshine.uncachedDecoder = `${modelDir}/uncached_decode.int8.onnx`;
+      c.moonshine.cachedDecoder = `${modelDir}/cached_decode.int8.onnx`;
+      c.tokens = `${modelDir}/tokens.txt`;
+
+      break;
+    }
+
+    case 22: {
+      const modelDir = "sherpa-onnx-moonshine-base-en-int8";
+      c.moonshine.preprocessor = `${modelDir}/preprocess.onnx`;
+      c.moonshine.encoder = `${modelDir}/encode.int8.onnx`;
+      c.moonshine.uncachedDecoder = `${modelDir}/uncached_decode.int8.onnx`;
+      c.moonshine.cachedDecoder = `${modelDir}/cached_decode.int8.onnx`;
+      c.tokens = `${modelDir}/tokens.txt`;
+
+      break;
+    }
+  }
+
+  console.log(`Please specify a supported type. Given type ${type}`);
+
+  return c;
+}

harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/workers/NonStreamingAsrWithVadWorker.ets

@@ -0,0 +1,177 @@
+import { ErrorEvent, MessageEvents, ThreadWorkerGlobalScope, worker } from '@kit.ArkTS';
+import {
+  OfflineRecognizer,
+  OfflineRecognizerConfig,
+  readWaveFromBinary,
+  SileroVadConfig,
+  Vad,
+  VadConfig,
+} from 'sherpa_onnx';
+import { Context } from '@kit.AbilityKit';
+import { fileIo } from '@kit.CoreFileKit';
+import { getOfflineModelConfig } from '../pages/NonStreamingAsrModels';
+
+const workerPort: ThreadWorkerGlobalScope = worker.workerPort;
+
+let recognizer: OfflineRecognizer;
+let vad: Vad; // vad for decoding files
+
+function initVad(context: Context): Vad {
+  let mgr = context.resourceManager;
+  const config = new VadConfig(
+    new SileroVadConfig(
+      'silero_vad.onnx',
+      0.5,
+      0.25,
+      0.5,
+      512,
+    ),
+    16000,
+    true,
+    1,
+  );
+
+  const bufferSizeInSeconds = 60;
+  return new Vad(config, bufferSizeInSeconds, mgr);
+}
+
+function initNonStreamingAsr(context: Context): OfflineRecognizer {
+  let mgr = context.resourceManager;
+  const config = new OfflineRecognizerConfig();
+
+  // Note that you can switch to a new model by changing type
+  //
+  // If you use type = 2, which means you will use
+  // sherpa-onnx-whisper-tiny.en
+  // we assume you have the following folder structure in you resources/rawfile
+  /*
+  (py38) fangjuns-MacBook-Pro:main fangjun$ pwd
+  /Users/fangjun/open-source/sherpa-onnx/harmony-os/SherpaOnnxVadAsr/entry/src/main
+  (py38) fangjuns-MacBook-Pro:main fangjun$ tree resources/rawfile/
+  resources/rawfile/
+  ├── sherpa-onnx-whisper-tiny.en
+  │   ├── README.md
+  │   ├── tiny.en-decoder.int8.onnx
+  │   ├── tiny.en-encoder.int8.onnx
+  │   └── tiny.en-tokens.txt
+  └── silero_vad.onnx
+
+  1 directory, 5 files
+   */
+  const type = 2;
+  config.modelConfig = getOfflineModelConfig(type);
+  config.modelConfig.debug = true;
+  return new OfflineRecognizer(config, mgr)
+}
+
+function decode(filename: string): string {
+  vad.reset();
+
+  const fp = fileIo.openSync(filename);
+  const stat = fileIo.statSync(fp.fd);
+  const arrayBuffer = new ArrayBuffer(stat.size);
+  fileIo.readSync(fp.fd, arrayBuffer);
+  const data = new Uint8Array(arrayBuffer);
+
+  const wave = readWaveFromBinary(data);
+
+  console.log(`sample rate ${wave.sampleRate}`);
+  console.log(`samples length ${wave.samples.length}`);
+  const resultList: string[] = [];
+
+  const windowSize = vad.config.sileroVad.windowSize;
+  for (let i = 0; i < wave.samples.length; i += windowSize) {
+    const thisWindow = wave.samples.subarray(i, i + windowSize)
+    vad.acceptWaveform(thisWindow);
+    if (i + windowSize >= wave.samples.length) {
+      vad.flush();
+    }
+    while (!vad.isEmpty()) {
+      const segment = vad.front();
+      const _startTime = (segment.start / wave.sampleRate);
+      const _endTime = _startTime + segment.samples.length / wave.sampleRate;
+
+      if (_endTime - _startTime < 0.2) {
+        vad.pop();
+        continue;
+      }
+
+      const startTime = _startTime.toFixed(2);
+      const endTime = _endTime.toFixed(2);
+
+      const progress = (segment.start + segment.samples.length) / wave.samples.length * 100;
+
+      workerPort.postMessage({ 'msgType': 'non-streaming-asr-vad-decode-progress', progress });
+
+      const stream = recognizer.createStream();
+      stream.acceptWaveform({ samples: segment.samples, sampleRate: wave.sampleRate });
+      recognizer.decode(stream);
+      const result = recognizer.getResult(stream);
+
+      const text = `${startTime} -- ${endTime} ${result.text}`
+      resultList.push(text);
+      console.log(`partial result ${text}`);
+
+      workerPort.postMessage({ 'msgType': 'non-streaming-asr-vad-decode-partial', text });
+
+      vad.pop();
+    }
+  }
+
+  return resultList.join('\n\n');
+}
+
+/**
+ * Defines the event handler to be called when the worker thread receives a message sent by the host thread.
+ * The event handler is executed in the worker thread.
+ *
+ * @param e message data
+ */
+workerPort.onmessage = (e: MessageEvents) => {
+  const msgType = e.data['msgType'] as string;
+  console.log(`msg-type: ${msgType}`)
+  if (msgType == 'init-vad' && !vad) {
+    const context = e.data['context'] as Context;
+    vad = initVad(context);
+    console.log('init vad done');
+    workerPort.postMessage({ 'msgType': 'init-vad-done' });
+  }
+
+  if (msgType == 'init-non-streaming-asr' && !recognizer) {
+    const context = e.data['context'] as Context;
+    recognizer = initNonStreamingAsr(context);
+    console.log('init non streaming ASR done');
+    workerPort.postMessage({ 'msgType': 'init-non-streaming-asr-done' });
+  }
+
+  if (msgType == 'non-streaming-asr-vad-decode') {
+    const filename = e.data['filename'] as string;
+    console.log(`decoding ${filename}`);
+    try {
+      const text = decode(filename);
+      workerPort.postMessage({ msgType: 'non-streaming-asr-vad-decode-done', text });
+    } catch (e) {
+      workerPort.postMessage({ msgType: 'non-streaming-asr-vad-decode-error', text: `Failed to decode ${filename}` });
+    }
+
+    workerPort.postMessage({ 'msgType': 'non-streaming-asr-vad-decode-progress', progress: 100 });
+  }
+}
+
+/**
+ * Defines the event handler to be called when the worker receives a message that cannot be deserialized.
+ * The event handler is executed in the worker thread.
+ *
+ * @param e message data
+ */
+workerPort.onmessageerror = (e: MessageEvents) => {
+}
+
+/**
+ * Defines the event handler to be called when an exception occurs during worker execution.
+ * The event handler is executed in the worker thread.
+ *
+ * @param e error message
+ */
+workerPort.onerror = (e: ErrorEvent) => {
+}

harmony-os/SherpaOnnxVadAsr/entry/src/main/module.json5

@@ -0,0 +1,52 @@
+{
+  "module": {
+    "name": "entry",
+    "type": "entry",
+    "description": "$string:module_desc",
+    "mainElement": "EntryAbility",
+    "deviceTypes": [
+      "phone",
+      "tablet",
+      "2in1"
+    ],
+    "deliveryWithInstall": true,
+    "installationFree": false,
+    "pages": "$profile:main_pages",
+    "abilities": [
+      {
+        "name": "EntryAbility",
+        "srcEntry": "./ets/entryability/EntryAbility.ets",
+        "description": "$string:EntryAbility_desc",
+        "icon": "$media:layered_image",
+        "label": "$string:EntryAbility_label",
+        "startWindowIcon": "$media:startIcon",
+        "startWindowBackground": "$color:start_window_background",
+        "exported": true,
+        "skills": [
+          {
+            "entities": [
+              "entity.system.home"
+            ],
+            "actions": [
+              "action.system.home"
+            ]
+          }
+        ]
+      }
+    ],
+    "extensionAbilities": [
+      {
+        "name": "EntryBackupAbility",
+        "srcEntry": "./ets/entrybackupability/EntryBackupAbility.ets",
+        "type": "backup",
+        "exported": false,
+        "metadata": [
+          {
+            "name": "ohos.extension.backup",
+            "resource": "$profile:backup_config"
+          }
+        ],
+      }
+    ]
+  }
+}

harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/element/color.json

@@ -0,0 +1,8 @@
+{
+  "color": [
+    {
+      "name": "start_window_background",
+      "value": "#FFFFFF"
+    }
+  ]
+}

harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/element/string.json

@@ -0,0 +1,16 @@
+{
+  "string": [
+    {
+      "name": "module_desc",
+      "value": "VAD+ASR with Next-gen Kaldi"
+    },
+    {
+      "name": "EntryAbility_desc",
+      "value": "VAD+ASR"
+    },
+    {
+      "name": "EntryAbility_label",
+      "value": "VAD_ASR"
+    }
+  ]
+}

harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/media/layered_image.json

@@ -0,0 +1,7 @@
+{
+  "layered-image":
+  {
+    "background" : "$media:background",
+    "foreground" : "$media:foreground"
+  }
+}

harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/profile/backup_config.json

@@ -0,0 +1,3 @@
+{
+  "allowToBackupRestore": true
+}

harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/base/profile/main_pages.json

@@ -0,0 +1,5 @@
+{
+  "src": [
+    "pages/Index"
+  ]
+}

harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/en_US/element/string.json

@@ -0,0 +1,16 @@
+{
+  "string": [
+    {
+      "name": "module_desc",
+      "value": "module description"
+    },
+    {
+      "name": "EntryAbility_desc",
+      "value": "description"
+    },
+    {
+      "name": "EntryAbility_label",
+      "value": "label"
+    }
+  ]
+}

harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/zh_CN/element/string.json

@@ -0,0 +1,16 @@
+{
+  "string": [
+    {
+      "name": "module_desc",
+      "value": "模块描述"
+    },
+    {
+      "name": "EntryAbility_desc",
+      "value": "description"
+    },
+    {
+      "name": "EntryAbility_label",
+      "value": "label"
+    }
+  ]
+}

harmony-os/SherpaOnnxVadAsr/entry/src/ohosTest/ets/test/Ability.test.ets

@@ -0,0 +1,35 @@
+import hilog from '@ohos.hilog';
+import { describe, beforeAll, beforeEach, afterEach, afterAll, it, expect } from '@ohos/hypium';
+
+export default function abilityTest() {
+  describe('ActsAbilityTest', () => {
+    // Defines a test suite. Two parameters are supported: test suite name and test suite function.
+    beforeAll(() => {
+      // Presets an action, which is performed only once before all test cases of the test suite start.
+      // This API supports only one parameter: preset action function.
+    })
+    beforeEach(() => {
+      // Presets an action, which is performed before each unit test case starts.
+      // The number of execution times is the same as the number of test cases defined by **it**.
+      // This API supports only one parameter: preset action function.
+    })
+    afterEach(() => {
+      // Presets a clear action, which is performed after each unit test case ends.
+      // The number of execution times is the same as the number of test cases defined by **it**.
+      // This API supports only one parameter: clear action function.
+    })
+    afterAll(() => {
+      // Presets a clear action, which is performed after all test cases of the test suite end.
+      // This API supports only one parameter: clear action function.
+    })
+    it('assertContain', 0, () => {
+      // Defines a test case. This API supports three parameters: test case name, filter parameter, and test case function.
+      hilog.info(0x0000, 'testTag', '%{public}s', 'it begin');
+      let a = 'abc';
+      let b = 'b';
+      // Defines a variety of assertion methods, which are used to declare expected boolean conditions.
+      expect(a).assertContain(b);
+      expect(a).assertEqual(a);
+    })
+  })
+}

harmony-os/SherpaOnnxVadAsr/entry/src/ohosTest/ets/test/List.test.ets

@@ -0,0 +1,5 @@
+import abilityTest from './Ability.test';
+
+export default function testsuite() {
+  abilityTest();
+}

harmony-os/SherpaOnnxVadAsr/entry/src/ohosTest/module.json5

@@ -0,0 +1,13 @@
+{
+  "module": {
+    "name": "entry_test",
+    "type": "feature",
+    "deviceTypes": [
+      "phone",
+      "tablet",
+      "2in1"
+    ],
+    "deliveryWithInstall": true,
+    "installationFree": false
+  }
+}

harmony-os/SherpaOnnxVadAsr/entry/src/test/List.test.ets

@@ -0,0 +1,5 @@
+import localUnitTest from './LocalUnit.test';
+
+export default function testsuite() {
+  localUnitTest();
+}

harmony-os/SherpaOnnxVadAsr/entry/src/test/LocalUnit.test.ets

@@ -0,0 +1,33 @@
+import { describe, beforeAll, beforeEach, afterEach, afterAll, it, expect } from '@ohos/hypium';
+
+export default function localUnitTest() {
+  describe('localUnitTest', () => {
+    // Defines a test suite. Two parameters are supported: test suite name and test suite function.
+    beforeAll(() => {
+      // Presets an action, which is performed only once before all test cases of the test suite start.
+      // This API supports only one parameter: preset action function.
+    });
+    beforeEach(() => {
+      // Presets an action, which is performed before each unit test case starts.
+      // The number of execution times is the same as the number of test cases defined by **it**.
+      // This API supports only one parameter: preset action function.
+    });
+    afterEach(() => {
+      // Presets a clear action, which is performed after each unit test case ends.
+      // The number of execution times is the same as the number of test cases defined by **it**.
+      // This API supports only one parameter: clear action function.
+    });
+    afterAll(() => {
+      // Presets a clear action, which is performed after all test cases of the test suite end.
+      // This API supports only one parameter: clear action function.
+    });
+    it('assertContain', 0, () => {
+      // Defines a test case. This API supports three parameters: test case name, filter parameter, and test case function.
+      let a = 'abc';
+      let b = 'b';
+      // Defines a variety of assertion methods, which are used to declare expected boolean conditions.
+      expect(a).assertContain(b);
+      expect(a).assertEqual(a);
+    });
+  });
+}

harmony-os/SherpaOnnxVadAsr/hvigor/hvigor-config.json5

@@ -0,0 +1,22 @@
+{
+  "modelVersion": "5.0.0",
+  "dependencies": {
+  },
+  "execution": {
+    // "analyze": "normal",                     /* Define the build analyze mode. Value: [ "normal" | "advanced" | false ]. Default: "normal" */
+    // "daemon": true,                          /* Enable daemon compilation. Value: [ true | false ]. Default: true */
+    // "incremental": true,                     /* Enable incremental compilation. Value: [ true | false ]. Default: true */
+    // "parallel": true,                        /* Enable parallel compilation. Value: [ true | false ]. Default: true */
+    // "typeCheck": false,                      /* Enable typeCheck. Value: [ true | false ]. Default: false */
+  },
+  "logging": {
+    // "level": "info"                          /* Define the log level. Value: [ "debug" | "info" | "warn" | "error" ]. Default: "info" */
+  },
+  "debugging": {
+    // "stacktrace": false                      /* Disable stacktrace compilation. Value: [ true | false ]. Default: false */
+  },
+  "nodeOptions": {
+    // "maxOldSpaceSize": 8192                  /* Enable nodeOptions maxOldSpaceSize compilation. Unit M. Used for the daemon process. Default: 8192*/
+    // "exposeGC": true                         /* Enable to trigger garbage collection explicitly. Default: true*/
+  }
+}

harmony-os/SherpaOnnxVadAsr/hvigorfile.ts

@@ -0,0 +1,6 @@
+import { appTasks } from '@ohos/hvigor-ohos-plugin';
+
+export default {
+    system: appTasks,  /* Built-in plugin of Hvigor. It cannot be modified. */
+    plugins:[]         /* Custom plugin to extend the functionality of Hvigor. */
+}

harmony-os/SherpaOnnxVadAsr/oh-package-lock.json5

@@ -0,0 +1,19 @@
+{
+  "meta": {
+    "stableOrder": true
+  },
+  "lockfileVersion": 3,
+  "ATTENTION": "THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.",
+  "specifiers": {
+    "@ohos/hypium@1.0.19": "@ohos/hypium@1.0.19"
+  },
+  "packages": {
+    "@ohos/hypium@1.0.19": {
+      "name": "@ohos/hypium",
+      "version": "1.0.19",
+      "integrity": "sha512-cEjDgLFCm3cWZDeRXk7agBUkPqjWxUo6AQeiu0gEkb3J8ESqlduQLSIXeo3cCsm8U/asL7iKjF85ZyOuufAGSQ==",
+      "resolved": "https://ohpm.openharmony.cn/ohpm/@ohos/hypium/-/hypium-1.0.19.har",
+      "registryType": "ohpm"
+    }
+  }
+}

harmony-os/SherpaOnnxVadAsr/oh-package.json5

@@ -0,0 +1,14 @@
+{
+  "modelVersion": "5.0.0",
+  "description": "Please describe the basic information.",
+  "dependencies": {
+
+    // You can download sherpa_onnx-v1.10.32.har
+    // from
+    // https://huggingface.co/csukuangfj/sherpa-onnx-harmony-os/tree/main/har
+    "sherpa_onnx": "file:./entry/sherpa_onnx-v1.10.32.har"
+  },
+  "devDependencies": {
+    "@ohos/hypium": "1.0.19"
+  }
+}