Fix CI tests (#1061)

2024-06-27 18:05:18 +08:00
parent 03ebdf3fc6
commit 598c12c4e5
14 changed files with 214 additions and 64 deletions
--- a/.github/workflows/build-wheels-win64.yaml
+++ b/.github/workflows/build-wheels-win64.yaml
@@ -22,20 +22,27 @@ jobs:
    strategy:
      fail-fast: false
      matrix:
-        os: [windows-latest]
-        python-version: ["cp37", "cp38", "cp39", "cp310", "cp311", "cp312"]
+        os: [windows-2019]
+        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]

    steps:
      - uses: actions/checkout@v4

-      # see https://cibuildwheel.readthedocs.io/en/stable/changelog/
-      # for a list of versions
+      - name: Setup Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
      - name: Build wheels
-        uses: pypa/cibuildwheel@v2.16.5
-        env:
-          CIBW_BUILD: "${{ matrix.python-version}}-* "
-          CIBW_SKIP: "cp27-* cp35-* *-win32 pp* *-musllinux*"
-          CIBW_BUILD_VERBOSITY: 3
+        shell: bash
+        run: |
+          pip install setuptools wheel
+
+          python3 setup.py bdist_wheel
+
+          ls -lh ./dist/
+
+          mv dist wheelhouse

      - name: Display wheels
        shell: bash
@@ -48,7 +55,6 @@ jobs:
          path: ./wheelhouse/*.whl

      - name: Publish to huggingface
-        if: matrix.python-version == 'cp38'
        env:
          HF_TOKEN: ${{ secrets.HF_TOKEN }}
        uses: nick-fields/retry@v3
@@ -77,36 +83,6 @@ jobs:
            git commit -m "add more wheels"
            git push https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-wheels main

-      - name: Publish to huggingface
-        if: matrix.python-version == 'cp38'
-        env:
-          HF_TOKEN: ${{ secrets.HF_TOKEN }}
-        uses: nick-fields/retry@v2
-        with:
-          max_attempts: 20
-          timeout_seconds: 200
-          shell: bash
-          command: |
-            git config --global user.email "csukuangfj@gmail.com"
-            git config --global user.name "Fangjun Kuang"
-
-            rm -rf huggingface
-            export GIT_LFS_SKIP_SMUDGE=1
-            export GIT_CLONE_PROTECTION_ACTIVE=false
-
-            git clone https://huggingface.co/csukuangfj/sherpa-onnx-wheels huggingface
-            cd huggingface
-            git fetch
-            git pull
-            git merge -m "merge remote" --ff origin main
-
-            cp -v ../wheelhouse/*.whl .
-
-            git status
-            git add .
-            git commit -m "add more wheels"
-            git push https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-wheels main
-
      - name: Publish wheels to PyPI
        env:
          TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
--- a/.github/workflows/dot-net.yaml
+++ b/.github/workflows/dot-net.yaml
@@ -11,8 +11,103 @@ permissions:
  contents: read

 jobs:
+  build-libs:
+    name: ${{ matrix.os }} ${{ matrix.arch }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [windows-2019]
+        arch: [x64, x86, arm64]
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Build sherpa-onnx
+        shell: bash
+        run: |
+          arch=${{ matrix.arch }}
+          opts=""
+          if [ $arch == x86 ]; then
+            opts="-A Win32"
+          elif [ $arch == arm64 ]; then
+            opts="-A ARM64"
+          fi
+
+          mkdir build
+          cd build
+          cmake \
+            $opts \
+            -DBUILD_SHARED_LIBS=ON \
+            -DCMAKE_INSTALL_PREFIX=./install \
+            -DCMAKE_BUILD_TYPE=Release \
+            -DSHERPA_ONNX_ENABLE_WEBSOCKET=OFF \
+            -DBUILD_ESPEAK_NG_EXE=OFF \
+            -DSHERPA_ONNX_BUILD_C_API_EXAMPLES=OFF  \
+            -DSHERPA_ONNX_ENABLE_BINARY=ON \
+            ..
+
+          cmake --build . --target install --config Release
+          rm -rf install/pkgconfig
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: windows-${{ matrix.arch }}
+          path: ./build/install/lib/
+
+      - name: Create tar file
+        shell: bash
+        run: |
+          arch=${{ matrix.arch }}
+
+          cd build
+
+          SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" ../CMakeLists.txt  | cut -d " " -f 2  | cut -d '"' -f 2)
+
+          dst=sherpa-onnx-$SHERPA_ONNX_VERSION-win-$arch
+          mv install/lib $dst
+          tar cjvf $dst.tar.bz2 $dst
+          ls -lh *.tar.bz2
+          mv *.tar.bz2 ../
+
+      # https://huggingface.co/docs/hub/spaces-github-actions
+      - name: Publish to huggingface
+        if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && (github.event_name == 'push' || github.event_name == 'workflow_dispatch')
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+        uses: nick-fields/retry@v3
+        with:
+          max_attempts: 20
+          timeout_seconds: 200
+          shell: bash
+          command: |
+            git config --global user.email "csukuangfj@gmail.com"
+            git config --global user.name "Fangjun Kuang"
+
+            rm -rf huggingface
+            export GIT_CLONE_PROTECTION_ACTIVE=false
+            GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/csukuangfj/sherpa-onnx-libs huggingface
+
+            cd huggingface
+            mkdir -p windows-for-dotnet
+
+            cp -v ../sherpa-onnx-*.tar.bz2 ./windows-for-dotnet
+
+            git status
+            git lfs track "*.bz2"
+
+            git add .
+
+            git commit -m "add more files"
+
+            git push https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-libs main
+            rm -rf huggingface
+
  release-nuget-package:
    runs-on: ${{ matrix.os }}
+    needs: [build-libs]
    strategy:
      fail-fast: false
      matrix:
@@ -30,9 +125,44 @@ jobs:
            6.0.x
            7.0.x

+      - name: Install Python dependencies
+        shell: bash
+        run: |
+          python3 -m pip install --upgrade pip Jinja2
+
+      - name: Retrieve artifact from windows x64
+        uses: actions/download-artifact@v4
+        with:
+          name: windows-x64
+          path: /tmp/windows-x64
+
+      - name: Retrieve artifact from windows x86
+        uses: actions/download-artifact@v4
+        with:
+          name: windows-x86
+          path: /tmp/windows-x86
+
+      - name: Retrieve artifact from windows arm64
+        uses: actions/download-artifact@v4
+        with:
+          name: windows-arm64
+          path: /tmp/windows-arm64
+
      - name: Check dotnet
        run: dotnet --info

+      - name: Build
+        shell: bash
+        run: |
+          sudo apt-get install -y tree
+          ls -lh /tmp/
+
+          tree /tmp/windows*
+          echo "----"
+
+          rm -v /tmp/windows*/*.lib
+          tree /tmp/windows*
+
      - name: Build
        shell: bash
        run: |
--- a/.github/workflows/run-python-test-macos.yaml
+++ b/.github/workflows/run-python-test-macos.yaml
@@ -44,17 +44,17 @@ jobs:
          - os: macos-11
            python-version: "3.7"

-          - os: macos-12
+          - os: macos-13
            python-version: "3.8"

          - os: macos-13
            python-version: "3.9"
-          - os: macos-13
+          - os: macos-14
            python-version: "3.10"
-          - os: macos-13
+          - os: macos-14
            python-version: "3.11"

-          - os: macos-14
+          - os: macos-latest
            python-version: "3.12"

    steps:
@@ -90,7 +90,7 @@ jobs:
          export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH"
          cmake --version

-          python3 -m pip install --verbose .
+          python3 setup.py install

      - name: Test sherpa-onnx
        shell: bash
--- a/.github/workflows/test-python-offline-websocket-server.yaml
+++ b/.github/workflows/test-python-offline-websocket-server.yaml
@@ -66,7 +66,7 @@ jobs:
          export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH"
          cmake --version

-          python3 -m pip install --no-deps --verbose .
+          python3 setup.py install
          python3 -m pip install websockets

      - name: Start server for transducer models
--- a/.github/workflows/test-python-online-websocket-server.yaml
+++ b/.github/workflows/test-python-online-websocket-server.yaml
@@ -66,7 +66,7 @@ jobs:
          export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH"
          cmake --version

-          python3 -m pip install --no-deps --verbose .
+          python3 setup.py install
          python3 -m pip install websockets

      - name: Start server for zipformer2 CTC models
--- a/.github/workflows/windows-x64.yaml
+++ b/.github/workflows/windows-x64.yaml
@@ -69,11 +69,16 @@ jobs:
        shell: bash
        run: |
          cd build
-          cmake --build . --config Release -- -m:2
+          cmake --build . --config Release  -- -m:2
          cmake --build . --config Release --target install -- -m:2

          ls -lh ./bin/Release/sherpa-onnx.exe

+      - uses: actions/upload-artifact@v4
+        with:
+          name: release-windows-x64-${{ matrix.shared_lib }}-${{ matrix.with_tts }}
+          path: build/install/*
+
      - name: Test offline punctuation
        shell: bash
        run: |
--- a/.github/workflows/windows-x86.yaml
+++ b/.github/workflows/windows-x86.yaml
@@ -74,6 +74,11 @@ jobs:

          ls -lh ./bin/Release/sherpa-onnx.exe

+      - uses: actions/upload-artifact@v4
+        with:
+          name: release-windows-x86-${{ matrix.shared_lib }}-${{ matrix.with_tts }}
+          path: build/install/*
+
      - name: Test offline punctuation
        shell: bash
        run: |
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -8,7 +8,7 @@ project(sherpa-onnx)
 # ./nodejs-addon-examples
 # ./dart-api-examples/
 # ./sherpa-onnx/flutter/CHANGELOG.md
-set(SHERPA_ONNX_VERSION "1.10.2")
+set(SHERPA_ONNX_VERSION "1.10.5")

 # Disable warning about
 #
@@ -37,6 +37,7 @@ option(SHERPA_ONNX_ENABLE_TTS "Whether to build TTS related code" ON)
 option(SHERPA_ONNX_LINK_LIBSTDCPP_STATICALLY "True to link libstdc++ statically. Used only when BUILD_SHARED_LIBS is OFF on Linux" ON)
 option(SHERPA_ONNX_USE_PRE_INSTALLED_ONNXRUNTIME_IF_AVAILABLE "True to use pre-installed onnxruntime if available" ON)
 option(SHERPA_ONNX_ENABLE_SANITIZER "Whether to enable ubsan and asan" OFF)
+option(SHERPA_ONNX_BUILD_C_API_EXAMPLES "Whether to enable C API examples" ON)

 set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib")
 set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib")
@@ -127,6 +128,7 @@ message(STATUS "SHERPA_ONNX_ENABLE_TTS ${SHERPA_ONNX_ENABLE_TTS}")
 message(STATUS "SHERPA_ONNX_LINK_LIBSTDCPP_STATICALLY ${SHERPA_ONNX_LINK_LIBSTDCPP_STATICALLY}")
 message(STATUS "SHERPA_ONNX_USE_PRE_INSTALLED_ONNXRUNTIME_IF_AVAILABLE ${SHERPA_ONNX_USE_PRE_INSTALLED_ONNXRUNTIME_IF_AVAILABLE}")
 message(STATUS "SHERPA_ONNX_ENABLE_SANITIZER: ${SHERPA_ONNX_ENABLE_SANITIZER}")
+message(STATUS "SHERPA_ONNX_BUILD_C_API_EXAMPLES: ${SHERPA_ONNX_BUILD_C_API_EXAMPLES}")

 if(SHERPA_ONNX_ENABLE_TTS)
  message(STATUS "TTS is enabled")
@@ -302,7 +304,7 @@ endif()

 add_subdirectory(sherpa-onnx)

-if(SHERPA_ONNX_ENABLE_C_API AND SHERPA_ONNX_ENABLE_BINARY)
+if(SHERPA_ONNX_ENABLE_C_API AND SHERPA_ONNX_ENABLE_BINARY AND SHERPA_ONNX_BUILD_C_API_EXAMPLES)
  set(SHERPA_ONNX_PKG_WITH_CARGS "-lcargs")
  add_subdirectory(c-api-examples)
 endif()
--- a/cmake/cmake_extension.py
+++ b/cmake/cmake_extension.py
@@ -146,6 +146,7 @@ class BuildExtension(build_ext):
        extra_cmake_args += " -DBUILD_ESPEAK_NG_EXE=OFF "
        extra_cmake_args += " -DBUILD_ESPEAK_NG_TESTS=OFF "

+        extra_cmake_args += " -DSHERPA_ONNX_BUILD_C_API_EXAMPLES=OFF "
        extra_cmake_args += " -DSHERPA_ONNX_ENABLE_CHECK=OFF "
        extra_cmake_args += " -DSHERPA_ONNX_ENABLE_PYTHON=ON "
        extra_cmake_args += " -DSHERPA_ONNX_ENABLE_PORTAUDIO=ON "
--- a/nodejs-addon-examples/package.json
+++ b/nodejs-addon-examples/package.json
@@ -1,5 +1,5 @@
 {
  "dependencies": {
-    "sherpa-onnx-node": "^1.10.2"
+    "sherpa-onnx-node": "^1.10.3"
  }
 }
--- a/scripts/dotnet/generate.py
+++ b/scripts/dotnet/generate.py
@@ -134,6 +134,7 @@ def main():
    process_linux(s)
    process_windows(s, "x64")
    process_windows(s, "x86")
+    process_windows(s, "arm64")

    s = read_proj_file("./sherpa-onnx.csproj.in")
    d = get_dict()
--- a/scripts/dotnet/run.sh
+++ b/scripts/dotnet/run.sh
@@ -24,7 +24,7 @@ export src_dir
 mkdir -p $src_dir
 pushd $src_dir

-mkdir -p linux macos-x64 macos-arm64 windows-x64 windows-x86
+mkdir -p linux macos-x64 macos-arm64 windows-x64 windows-x86 windows-arm64

 linux_wheel_filename=sherpa_onnx-${SHERPA_ONNX_VERSION}-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
 linux_wheel=$src_dir/$linux_wheel_filename
@@ -41,6 +41,9 @@ windows_x64_wheel=$src_dir/$windows_x64_wheel_filename
 windows_x86_wheel_filename=sherpa_onnx-${SHERPA_ONNX_VERSION}-cp38-cp38-win32.whl
 windows_x86_wheel=$src_dir/$windows_x86_wheel_filename

+windows_arm64_wheel_filename=sherpa-onnx-${SHERPA_ONNX_VERSION}-win-arm64.tar.bz2
+windows_arm64_wheel=$src_dir/$windows_arm64_wheel_filename
+
 if [ ! -f $src_dir/linux/libsherpa-onnx-core.so ]; then
  echo "---linux x86_64---"
  cd linux
@@ -54,9 +57,9 @@ if [ ! -f $src_dir/linux/libsherpa-onnx-core.so ]; then
  unzip $linux_wheel_filename
  cp -v sherpa_onnx/lib/*.so* ../
  cd ..
-  rm -v libpiper_phonemize.so libpiper_phonemize.so.1.2.0
-  rm -v libonnxruntime.so
-  rm -v libcargs.so
+  rm -fv libpiper_phonemize.so libpiper_phonemize.so.1.2.0
+  rm -fv libonnxruntime.so
+  rm -fv libcargs.so
  rm -rf wheel
  ls -lh
  cd ..
@@ -77,9 +80,9 @@ if [ ! -f $src_dir/macos-x64/libsherpa-onnx-core.dylib ]; then

  cd ..

-  rm -v libcargs.dylib
-  rm -v libonnxruntime.dylib
-  rm -v libpiper_phonemize.1.2.0.dylib libpiper_phonemize.dylib
+  rm -fv libcargs.dylib
+  rm -fv libonnxruntime.dylib
+  rm -fv libpiper_phonemize.1.2.0.dylib libpiper_phonemize.dylib
  rm -rf wheel
  ls -lh
  cd ..
@@ -100,9 +103,9 @@ if [ ! -f $src_dir/macos-arm64/libsherpa-onnx-core.dylib ]; then

  cd ..

-  rm -v libcargs.dylib
-  rm -v libonnxruntime.dylib
-  rm -v libpiper_phonemize.1.2.0.dylib libpiper_phonemize.dylib
+  rm -fv libcargs.dylib
+  rm -fv libonnxruntime.dylib
+  rm -fv libpiper_phonemize.1.2.0.dylib libpiper_phonemize.dylib
  rm -rf wheel
  ls -lh
  cd ..
@@ -146,9 +149,28 @@ if [ ! -f $src_dir/windows-x86/sherpa-onnx-core.dll ]; then
  cd ..
 fi

+if [ ! -f $src_dir/windows-arm64/sherpa-onnx-core.dll ]; then
+  echo "---windows arm64---"
+  cd windows-arm64
+  mkdir -p wheel
+  cd wheel
+  if [ -f $windows_arm64_wheel ]; then
+    cp -v $windows_arm64_wheel .
+  else
+    curl -OL https://$HF_MIRROR/csukuangfj/sherpa-onnx-libs/resolve/main/windows-for-dotnet/$windows_arm64_wheel_filename
+  fi
+  unzip $windows_arm64_wheel_filename
+  cp -v sherpa-onnx-${SHERPA_ONNX_VERSION}-win-arm64/*dll ../
+  cd ..
+
+  rm -rf wheel
+  ls -lh
+  cd ..
+fi
+
 popd

-mkdir -p macos-x64 macos-arm64 linux windows-x64 windows-x86 all
+mkdir -p macos-x64 macos-arm64 linux windows-x64 windows-x86 windows-arm64 all

 cp ./*.cs all

@@ -179,6 +201,11 @@ dotnet build -c Release
 dotnet pack -c Release -o ../packages
 popd

+pushd windows-arm64
+dotnet build -c Release
+dotnet pack -c Release -o ../packages
+popd
+
 pushd all
 dotnet build -c Release
 dotnet pack -c Release -o ../packages
--- a/scripts/dotnet/sherpa-onnx.csproj.in
+++ b/scripts/dotnet/sherpa-onnx.csproj.in
@@ -5,7 +5,7 @@
    <OutputType>Library</OutputType>
    <LangVersion>10.0</LangVersion>
    <TargetFrameworks>netstandard2.0</TargetFrameworks>
-    <RuntimeIdentifiers>linux-x64;osx-x64;osx-arm64;win-x64;win-x86</RuntimeIdentifiers>
+    <RuntimeIdentifiers>linux-x64;osx-x64;osx-arm64;win-x64;win-x86;win-arm64</RuntimeIdentifiers>
    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
    <AssemblyName>sherpa-onnx</AssemblyName>
    <Version>{{ version }}</Version>
@@ -53,6 +53,7 @@
    <PackageReference Include="org.k2fsa.sherpa.onnx.runtime.osx-arm64" Version="{{ version }}" />
    <PackageReference Include="org.k2fsa.sherpa.onnx.runtime.win-x64"   Version="{{ version }}" />
    <PackageReference Include="org.k2fsa.sherpa.onnx.runtime.win-x86"   Version="{{ version }}" />
+    <PackageReference Include="org.k2fsa.sherpa.onnx.runtime.win-arm64" Version="{{ version }}" />
  </ItemGroup>

 </Project>
--- a/sherpa-onnx/csrc/CMakeLists.txt
+++ b/sherpa-onnx/csrc/CMakeLists.txt
@@ -184,7 +184,9 @@ else()
  target_link_libraries(sherpa-onnx-core ${onnxruntime_lib_files})
 endif()

-target_link_libraries(sherpa-onnx-core -lm)
+if(NOT WIN32)
+  target_link_libraries(sherpa-onnx-core -lm)
+endif()

 if(NOT BUILD_SHARED_LIBS AND APPLE)
  target_link_libraries(sherpa-onnx-core "-framework Foundation")