Add CI to build HAPs for HarmonyOS (#1578)

2024-11-29 21:13:01 +08:00
parent be159f943e
commit 299f2392e2
11 changed files with 376 additions and 24 deletions
--- a/scripts/apk/generate-vad-asr-apk-script.py
+++ b/scripts/apk/generate-vad-asr-apk-script.py
@@ -2,6 +2,7 @@

 import argparse
 from dataclasses import dataclass
+from pathlib import Path

 import jinja2

@@ -34,6 +35,7 @@ class Model:

    # e.g., zh, en, zh_en
    lang: str
+    lang2: str

    # e.g., whisper, paraformer, zipformer
    short_name: str = ""
@@ -51,6 +53,7 @@ def get_models():
            model_name="sherpa-onnx-whisper-tiny.en",
            idx=2,
            lang="en",
+            lang2="English",
            short_name="whisper_tiny",
            cmd="""
            pushd $model_name
@@ -71,6 +74,7 @@ def get_models():
            model_name="sherpa-onnx-paraformer-zh-2023-09-14",
            idx=0,
            lang="zh_en",
+            lang2="Chinese,English",
            short_name="paraformer",
            rule_fsts="itn_zh_number.fst",
            cmd="""
@@ -92,6 +96,7 @@ def get_models():
            model_name="sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17",
            idx=15,
            lang="zh_en_ko_ja_yue",
+            lang2="中英粤日韩",
            short_name="sense_voice",
            cmd="""
            pushd $model_name
@@ -109,6 +114,7 @@ def get_models():
            model_name="sherpa-onnx-paraformer-zh-small-2024-03-09",
            idx=14,
            lang="zh_en",
+            lang2="Chinese,English",
            short_name="small_paraformer",
            rule_fsts="itn_zh_number.fst",
            cmd="""
@@ -132,6 +138,7 @@ def get_models():
            model_name="icefall-asr-zipformer-wenetspeech-20230615",
            idx=4,
            lang="zh",
+            lang2="Chinese",
            short_name="zipformer",
            rule_fsts="itn_zh_number.fst",
            cmd="""
@@ -159,6 +166,7 @@ def get_models():
            model_name="sherpa-onnx-nemo-fast-conformer-ctc-be-de-en-es-fr-hr-it-pl-ru-uk-20k",
            idx=7,
            lang="be_de_en_es_fr_hr_it_pl_ru_uk",
+            lang2="be_de_en_es_fr_hr_it_pl_ru_uk",
            short_name="fast_conformer_ctc_20k",
            cmd="""
            pushd $model_name
@@ -174,6 +182,7 @@ def get_models():
            model_name="sherpa-onnx-nemo-fast-conformer-ctc-en-24500",
            idx=8,
            lang="en",
+            lang2="English",
            short_name="fast_conformer_ctc_24500",
            cmd="""
            pushd $model_name
@@ -188,7 +197,8 @@ def get_models():
        Model(
            model_name="sherpa-onnx-nemo-fast-conformer-ctc-en-de-es-fr-14288",
            idx=9,
-            lang="en_des_es_fr",
+            lang="en_de_es_fr",
+            lang2="English,German,Spanish,French",
            short_name="fast_conformer_ctc_14288",
            cmd="""
            pushd $model_name
@@ -204,6 +214,7 @@ def get_models():
            model_name="sherpa-onnx-nemo-fast-conformer-ctc-es-1424",
            idx=10,
            lang="es",
+            lang2="Spanish",
            short_name="fast_conformer_ctc_1424",
            cmd="""
            pushd $model_name
@@ -219,6 +230,7 @@ def get_models():
            model_name="sherpa-onnx-telespeech-ctc-int8-zh-2024-06-04",
            idx=11,
            lang="zh",
+            lang2="Chinese",
            short_name="telespeech",
            rule_fsts="itn_zh_number.fst",
            cmd="""
@@ -239,6 +251,7 @@ def get_models():
            model_name="sherpa-onnx-zipformer-thai-2024-06-20",
            idx=12,
            lang="th",
+            lang2="Thai",
            short_name="zipformer",
            cmd="""
            pushd $model_name
@@ -260,6 +273,7 @@ def get_models():
            model_name="sherpa-onnx-zipformer-korean-2024-06-24",
            idx=13,
            lang="ko",
+            lang2="Korean",
            short_name="zipformer",
            cmd="""
            pushd $model_name
@@ -281,6 +295,7 @@ def get_models():
            model_name="sherpa-onnx-zipformer-ja-reazonspeech-2024-08-01",
            idx=16,
            lang="ja",
+            lang2="Japanese",
            short_name="zipformer_reazonspeech",
            cmd="""
            pushd $model_name
@@ -300,6 +315,7 @@ def get_models():
            model_name="sherpa-onnx-zipformer-ru-2024-09-18",
            idx=17,
            lang="ru",
+            lang2="Russian",
            short_name="zipformer",
            cmd="""
            pushd $model_name
@@ -320,6 +336,7 @@ def get_models():
            model_name="sherpa-onnx-small-zipformer-ru-2024-09-18",
            idx=18,
            lang="ru",
+            lang2="Russian",
            short_name="small_zipformer",
            cmd="""
            pushd $model_name
@@ -340,6 +357,7 @@ def get_models():
            model_name="sherpa-onnx-nemo-ctc-giga-am-russian-2024-10-24",
            idx=19,
            lang="ru",
+            lang2="Russian",
            short_name="nemo_ctc_giga_am",
            cmd="""
            pushd $model_name
@@ -358,6 +376,7 @@ def get_models():
            model_name="sherpa-onnx-nemo-transducer-giga-am-russian-2024-10-24",
            idx=20,
            lang="ru",
+            lang2="Russian",
            short_name="nemo_transducer_giga_am",
            cmd="""
            pushd $model_name
@@ -376,6 +395,7 @@ def get_models():
            model_name="sherpa-onnx-moonshine-tiny-en-int8",
            idx=21,
            lang="en",
+            lang2="English",
            short_name="moonshine_tiny_int8",
            cmd="""
            pushd $model_name
@@ -391,6 +411,7 @@ def get_models():
            model_name="sherpa-onnx-moonshine-base-en-int8",
            idx=22,
            lang="en",
+            lang2="English",
            short_name="moonshine_base_int8",
            cmd="""
            pushd $model_name
@@ -436,9 +457,14 @@ def main():

    filename_list = [
        "./build-apk-vad-asr.sh",
+        "./build-hap-vad-asr.sh",
    ]
    for filename in filename_list:
        environment = jinja2.Environment()
+        if not Path(f"{filename}.in").is_file():
+            print(f"skip {filename}")
+            continue
+
        with open(f"{filename}.in") as f:
            s = f.read()
        template = environment.from_string(s)
--- a/scripts/hap/.gitignore
+++ b/scripts/hap/.gitignore
@@ -0,0 +1 @@
+!build-*.in
--- a/scripts/hap/build-hap-vad-asr.sh.in
+++ b/scripts/hap/build-hap-vad-asr.sh.in
@@ -0,0 +1,115 @@
+#!/usr/bin/env bash
+#
+# Auto generated! Please DO NOT EDIT!
+
+# Please set the environment variable COMMANDLINE_TOOLS_DIR
+# before running this script
+
+# Inside the $COMMANDLINE_TOOL_DIR directory, you can find the following:
+#
+# command-line-tools fangjun$ ls
+# LICENSE.txt NOTICE.txt  bin         codelinter  hstack      hvigor      ohpm        sdk         tool
+
+set -ex
+
+log() {
+  # This function is from espnet
+  local fname=${BASH_SOURCE[1]##*/}
+  echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*"
+}
+
+SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt  | cut -d " " -f 2  | cut -d '"' -f 2)
+
+log "Building streaming VAD + ASR Hap for sherpa-onnx v${SHERPA_ONNX_VERSION}"
+
+export SHERPA_ONNX_ENABLE_TTS=OFF
+
+if [ ! -f $COMMANDLINE_TOOLS_DIR/bin/hvigorw ]; then
+  echo "Please first download Command Line Tools for HarmonyOS"
+  echo "See https://developer.huawei.com/consumer/cn/download/"
+  echo "or"
+  echo "https://hf-mirror.com/csukuangfj/harmonyos-commandline-tools/tree/main"
+  exit 1
+fi
+
+jar=$COMMANDLINE_TOOLS_DIR/sdk/default/openharmony/toolchains/lib/hap-sign-tool.jar
+
+export PATH=$COMMANDLINE_TOOLS_DIR/bin:$PATH
+
+mkdir -p haps
+
+{% for model in model_list %}
+pushd ./harmony-os/SherpaOnnxVadAsr/entry/src/main/resources/rawfile
+model_name={{ model.model_name }}
+type={{ model.idx }}
+lang={{ model.lang }}
+lang2={{ model.lang2 }}
+short_name={{ model.short_name }}
+
+curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/${model_name}.tar.bz2
+tar xvf ${model_name}.tar.bz2
+
+{{ model.cmd }}
+
+rm -rf  *.tar.bz2
+ls -lh $model_name
+
+if [ ! -f ./silero_vad.onnx ]; then
+  curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/silero_vad.onnx
+fi
+
+popd
+# Now we are at the project root directory
+
+git checkout .
+pushd harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/workers/
+sed -i.bak s/"const type = 2/const type = $type/" ./NonStreamingAsrWithVadWorker.ets
+
+{% if model.rule_fsts %}
+  rule_fsts={{ model.rule_fsts }}
+  sed -i.bak s%"ruleFsts = ''"%"ruleFsts = \"$rule_fsts\""% ./NonStreamingAsrWithVadWorker.ets
+{% endif %}
+
+git diff
+popd
+
+pushd harmony-os/SherpaOnnxVadAsr/entry/src/main/ets/pages
+sed -i.bak s/English/$lang2/ ./Index.ets
+popd
+
+pushd harmony-os/SherpaOnnxVadAsr
+
+git diff
+
+cd entry
+ohpm install
+cd ..
+
+hvigorw clean --no-daemon
+hvigorw assembleHap --mode module -p product=default -p buildMode=release --no-daemon
+
+ls -lh ./entry/build/default/outputs/default/entry-default-unsigned.hap
+
+in_file=./entry/build/default/outputs/default/entry-default-unsigned.hap
+out_file=$PWD/entry/build/default/outputs/default/entry-default-signed.hap
+
+java -jar $jar sign-app -keyAlias "$HAP_KEY_ALIAS" -signAlg "SHA256withECDSA" -mode "localSign" \
+  -appCertFile "/tmp/sherpa_onnx.cer" -profileFile "/tmp/sherpa_onnx_profileRelease.p7b" \
+  -inFile $in_file -keystoreFile "/tmp/sherpa_onnx_ohos_key.p12" \
+  -outFile $out_file -keyPwd "$HAP_KEY_PWD" -keystorePwd "$HAP_KEY_STORE_PWD" -signCode "1"
+
+ls -l $in_file $out_file
+ls -lh $in_file $out_file
+rm $in_file
+rm -rf ./entry/src/main/resources/rawfile/$model_name
+popd
+
+mv $out_file ./haps/sherpa-onnx-${SHERPA_ONNX_VERSION}-vad_asr-$lang-$short_name.hap
+
+ls -lh haps
+
+{% endfor %}
+
+git checkout .
+
+ls -lh haps/
--- a/scripts/hap/generate-vad-asr-hap-script.py
+++ b/scripts/hap/generate-vad-asr-hap-script.py
@@ -0,0 +1 @@
+../apk/generate-vad-asr-apk-script.py