From 1952772654fb7e118620bcab40183a02fa71c796 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Sat, 23 Mar 2024 18:51:56 +0800 Subject: [PATCH] Add timestamps and tokens for .Net's online models. (#690) --- .../workflows/aarch64-linux-gnu-shared.yaml | 4 +- .../workflows/aarch64-linux-gnu-static.yaml | 4 +- .github/workflows/android.yaml | 2 +- .github/workflows/arm-linux-gnueabihf.yaml | 4 +- .github/workflows/dot-net.yaml | 2 +- .github/workflows/linux-gpu.yaml | 2 +- .github/workflows/linux.yaml | 3 +- .github/workflows/macos.yaml | 3 +- .github/workflows/mfc.yaml | 4 +- .github/workflows/release-go.yaml | 4 -- .github/workflows/riscv64-linux.yaml | 4 +- .github/workflows/run-python-test.yaml | 2 +- .github/workflows/test-dot-net-nuget.yaml | 2 +- .github/workflows/test-dot-net.yaml | 2 +- .github/workflows/windows-x64-cuda.yaml | 3 +- .github/workflows/windows-x64.yaml | 4 +- .github/workflows/windows-x86.yaml | 4 +- cmake/kaldi-native-fbank.cmake | 13 ++-- .../offline-decode-files/run-hotwords.sh | 2 +- .../offline-decode-files/run-zipformer.sh | 2 +- .../online-decode-files/Program.cs | 10 ++- .../online-decode-files/run-transducer.sh | 6 +- scripts/dotnet/online.cs | 61 ++++++++++++++++++- sherpa-onnx/c-api/c-api.cc | 55 +++++++++-------- sherpa-onnx/c-api/c-api.h | 4 ++ .../csrc/online-zipformer2-transducer-model.h | 2 +- 26 files changed, 135 insertions(+), 73 deletions(-) diff --git a/.github/workflows/aarch64-linux-gnu-shared.yaml b/.github/workflows/aarch64-linux-gnu-shared.yaml index a5df2c9c..34ad56fd 100644 --- a/.github/workflows/aarch64-linux-gnu-shared.yaml +++ b/.github/workflows/aarch64-linux-gnu-shared.yaml @@ -179,7 +179,7 @@ jobs: # https://huggingface.co/docs/hub/spaces-github-actions - name: Publish to huggingface - if: github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa' + if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && (github.event_name == 'push' && github.event_name == 'workflow_dispatch') env: HF_TOKEN: ${{ secrets.HF_TOKEN }} uses: nick-fields/retry@v3 @@ -210,7 +210,7 @@ jobs: git push https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-libs main - name: Release pre-compiled binaries and libs for aarch64 linux - if: github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa' && github.event_name == 'push' && contains(github.ref, 'refs/tags/') + if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && github.event_name == 'push' && contains(github.ref, 'refs/tags/') uses: svenstaro/upload-release-action@v2 with: file_glob: true diff --git a/.github/workflows/aarch64-linux-gnu-static.yaml b/.github/workflows/aarch64-linux-gnu-static.yaml index 3c6e6b36..69916076 100644 --- a/.github/workflows/aarch64-linux-gnu-static.yaml +++ b/.github/workflows/aarch64-linux-gnu-static.yaml @@ -170,7 +170,7 @@ jobs: # https://huggingface.co/docs/hub/spaces-github-actions - name: Publish to huggingface - if: github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa' + if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && (github.event_name == 'push' && github.event_name == 'workflow_dispatch') env: HF_TOKEN: ${{ secrets.HF_TOKEN }} uses: nick-fields/retry@v3 @@ -201,7 +201,7 @@ jobs: git push https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-libs main - name: Release pre-compiled binaries and libs for aarch64 linux - if: github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa' && github.event_name == 'push' && contains(github.ref, 'refs/tags/') + if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && github.event_name == 'push' && contains(github.ref, 'refs/tags/') uses: svenstaro/upload-release-action@v2 with: file_glob: true diff --git a/.github/workflows/android.yaml b/.github/workflows/android.yaml index e4189857..f3129751 100644 --- a/.github/workflows/android.yaml +++ b/.github/workflows/android.yaml @@ -101,7 +101,7 @@ jobs: # https://huggingface.co/docs/hub/spaces-github-actions - name: Publish to huggingface - if: github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa' && github.event_name == 'push' && contains(github.ref, 'refs/tags/') + if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && (github.event_name == 'push' && github.event_name == 'workflow_dispatch') env: HF_TOKEN: ${{ secrets.HF_TOKEN }} run: | diff --git a/.github/workflows/arm-linux-gnueabihf.yaml b/.github/workflows/arm-linux-gnueabihf.yaml index ee348b94..7bc58d1c 100644 --- a/.github/workflows/arm-linux-gnueabihf.yaml +++ b/.github/workflows/arm-linux-gnueabihf.yaml @@ -193,7 +193,7 @@ jobs: # https://huggingface.co/docs/hub/spaces-github-actions - name: Publish to huggingface - if: github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa' + if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && (github.event_name == 'push' && github.event_name == 'workflow_dispatch') env: HF_TOKEN: ${{ secrets.HF_TOKEN }} uses: nick-fields/retry@v3 @@ -224,7 +224,7 @@ jobs: git push https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-libs main - name: Release pre-compiled binaries and libs for arm linux gnueabihf ${{ matrix.lib_type }} - if: github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa' && github.event_name == 'push' && contains(github.ref, 'refs/tags/') + if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && github.event_name == 'push' && contains(github.ref, 'refs/tags/') uses: svenstaro/upload-release-action@v2 with: file_glob: true diff --git a/.github/workflows/dot-net.yaml b/.github/workflows/dot-net.yaml index cbfbe622..d9e4edd7 100644 --- a/.github/workflows/dot-net.yaml +++ b/.github/workflows/dot-net.yaml @@ -24,7 +24,7 @@ jobs: fetch-depth: 0 - name: Setup .NET - uses: actions/setup-dotnet@v1 + uses: actions/setup-dotnet@v4 with: dotnet-version: | 6.0.x diff --git a/.github/workflows/linux-gpu.yaml b/.github/workflows/linux-gpu.yaml index 5aa4ecfe..d3bfd118 100644 --- a/.github/workflows/linux-gpu.yaml +++ b/.github/workflows/linux-gpu.yaml @@ -174,7 +174,7 @@ jobs: tar cjvf ${dst}.tar.bz2 $dst - name: Release pre-compiled binaries and libs for linux x64 - if: github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa' && github.event_name == 'push' && contains(github.ref, 'refs/tags/') + if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && github.event_name == 'push' && contains(github.ref, 'refs/tags/') uses: svenstaro/upload-release-action@v2 with: file_glob: true diff --git a/.github/workflows/linux.yaml b/.github/workflows/linux.yaml index 13374610..037c7977 100644 --- a/.github/workflows/linux.yaml +++ b/.github/workflows/linux.yaml @@ -210,7 +210,6 @@ jobs: .github/scripts/test-online-transducer.sh - name: Copy files - if: github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa' && github.event_name == 'push' && contains(github.ref, 'refs/tags/') shell: bash run: | SHERPA_ONNX_VERSION=v$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) @@ -227,7 +226,7 @@ jobs: tar cjvf ${dst}.tar.bz2 $dst - name: Release pre-compiled binaries and libs for linux x64 - if: github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa' && github.event_name == 'push' && contains(github.ref, 'refs/tags/') && matrix.os == 'ubuntu-20.04' && matrix.gcc_version == '7' + if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && github.event_name == 'push' && contains(github.ref, 'refs/tags/') && matrix.os == 'ubuntu-20.04' && matrix.gcc_version == '7' uses: svenstaro/upload-release-action@v2 with: file_glob: true diff --git a/.github/workflows/macos.yaml b/.github/workflows/macos.yaml index 9948d560..b48a4a00 100644 --- a/.github/workflows/macos.yaml +++ b/.github/workflows/macos.yaml @@ -175,7 +175,6 @@ jobs: .github/scripts/test-online-transducer.sh - name: Copy files - if: github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa' && github.event_name == 'push' && contains(github.ref, 'refs/tags/') shell: bash run: | SHERPA_ONNX_VERSION=v$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) @@ -193,7 +192,7 @@ jobs: tar cjvf ${dst}.tar.bz2 $dst - name: Release pre-compiled binaries and libs for macOS - if: github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa' && github.event_name == 'push' && contains(github.ref, 'refs/tags/') + if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && github.event_name == 'push' && contains(github.ref, 'refs/tags/') uses: svenstaro/upload-release-action@v2 with: file_glob: true diff --git a/.github/workflows/mfc.yaml b/.github/workflows/mfc.yaml index dadbe993..197386bb 100644 --- a/.github/workflows/mfc.yaml +++ b/.github/workflows/mfc.yaml @@ -124,7 +124,7 @@ jobs: path: ./mfc-examples/${{ matrix.arch }}/Release/sherpa-onnx-non-streaming-asr-*.exe - name: Release pre-compiled binaries and libs for Windows ${{ matrix.arch }} - if: github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa' && github.event_name == 'push' && contains(github.ref, 'refs/tags/') + if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && github.event_name == 'push' && contains(github.ref, 'refs/tags/') uses: svenstaro/upload-release-action@v2 with: file_glob: true @@ -132,7 +132,7 @@ jobs: file: ./mfc-examples/${{ matrix.arch }}/Release/sherpa-onnx-streaming-*.exe - name: Release pre-compiled binaries and libs for Windows ${{ matrix.arch }} - if: github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa' && github.event_name == 'push' && contains(github.ref, 'refs/tags/') + if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && github.event_name == 'push' && contains(github.ref, 'refs/tags/') uses: svenstaro/upload-release-action@v2 with: file_glob: true diff --git a/.github/workflows/release-go.yaml b/.github/workflows/release-go.yaml index f6b1777c..b10cf251 100644 --- a/.github/workflows/release-go.yaml +++ b/.github/workflows/release-go.yaml @@ -1,10 +1,6 @@ name: release-go on: - push: - tags: - - '*' - workflow_dispatch: concurrency: diff --git a/.github/workflows/riscv64-linux.yaml b/.github/workflows/riscv64-linux.yaml index 8d4e3c66..b887f894 100644 --- a/.github/workflows/riscv64-linux.yaml +++ b/.github/workflows/riscv64-linux.yaml @@ -225,7 +225,7 @@ jobs: # https://huggingface.co/docs/hub/spaces-github-actions - name: Publish to huggingface - if: github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa' + if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && (github.event_name == 'push' && github.event_name == 'workflow_dispatch') env: HF_TOKEN: ${{ secrets.HF_TOKEN }} uses: nick-fields/retry@v3 @@ -269,7 +269,7 @@ jobs: path: sherpa-onnx-*linux-riscv64-static.tar.bz2 - name: Release pre-compiled binaries and libs for riscv64 linux ${{ matrix.lib_type }} - if: github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa' && github.event_name == 'push' && contains(github.ref, 'refs/tags/') + if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && github.event_name == 'push' && contains(github.ref, 'refs/tags/') uses: svenstaro/upload-release-action@v2 with: file_glob: true diff --git a/.github/workflows/run-python-test.yaml b/.github/workflows/run-python-test.yaml index fcb9b192..ec6a1091 100644 --- a/.github/workflows/run-python-test.yaml +++ b/.github/workflows/run-python-test.yaml @@ -63,7 +63,7 @@ jobs: - name: Run sccache-cache if: matrix.os == 'windows-latest' - uses: mozilla-actions/sccache-action@v0.0.3 + uses: mozilla-actions/sccache-action@v0.0.4 with: version: "v0.7.4" diff --git a/.github/workflows/test-dot-net-nuget.yaml b/.github/workflows/test-dot-net-nuget.yaml index 6a40a7e8..4af976b2 100644 --- a/.github/workflows/test-dot-net-nuget.yaml +++ b/.github/workflows/test-dot-net-nuget.yaml @@ -33,7 +33,7 @@ jobs: fetch-depth: 0 - name: Setup .NET 6.0 - uses: actions/setup-dotnet@v1 + uses: actions/setup-dotnet@v4 with: dotnet-version: 6.0.x diff --git a/.github/workflows/test-dot-net.yaml b/.github/workflows/test-dot-net.yaml index b3d7d8f7..f47c838f 100644 --- a/.github/workflows/test-dot-net.yaml +++ b/.github/workflows/test-dot-net.yaml @@ -137,7 +137,7 @@ jobs: path: /tmp/windows-x86 - name: Setup .NET - uses: actions/setup-dotnet@v3 + uses: actions/setup-dotnet@v4 with: dotnet-version: | 6.0.x diff --git a/.github/workflows/windows-x64-cuda.yaml b/.github/workflows/windows-x64-cuda.yaml index 936990fe..d4ca33a7 100644 --- a/.github/workflows/windows-x64-cuda.yaml +++ b/.github/workflows/windows-x64-cuda.yaml @@ -133,7 +133,6 @@ jobs: .github/scripts/test-online-transducer.sh - name: Copy files - if: github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa' && github.event_name == 'push' && contains(github.ref, 'refs/tags/') shell: bash run: | SHERPA_ONNX_VERSION=v$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) @@ -148,7 +147,7 @@ jobs: tar cjvf ${dst}.tar.bz2 $dst - name: Release pre-compiled binaries and libs for Windows x64 CUDA - if: github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa' && github.event_name == 'push' && contains(github.ref, 'refs/tags/') + if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && github.event_name == 'push' && contains(github.ref, 'refs/tags/') uses: svenstaro/upload-release-action@v2 with: file_glob: true diff --git a/.github/workflows/windows-x64.yaml b/.github/workflows/windows-x64.yaml index 6ee2314d..5ea29c81 100644 --- a/.github/workflows/windows-x64.yaml +++ b/.github/workflows/windows-x64.yaml @@ -154,7 +154,7 @@ jobs: # https://huggingface.co/docs/hub/spaces-github-actions - name: Publish to huggingface - if: github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa' + if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && (github.event_name == 'push' && github.event_name == 'workflow_dispatch') env: HF_TOKEN: ${{ secrets.HF_TOKEN }} uses: nick-fields/retry@v3 @@ -185,7 +185,7 @@ jobs: git push https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-libs main - name: Release pre-compiled binaries and libs for Windows x64 - if: github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa' && github.event_name == 'push' && contains(github.ref, 'refs/tags/') + if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && github.event_name == 'push' && contains(github.ref, 'refs/tags/') uses: svenstaro/upload-release-action@v2 with: file_glob: true diff --git a/.github/workflows/windows-x86.yaml b/.github/workflows/windows-x86.yaml index c0cef5a3..b2bd25f3 100644 --- a/.github/workflows/windows-x86.yaml +++ b/.github/workflows/windows-x86.yaml @@ -157,7 +157,7 @@ jobs: # https://huggingface.co/docs/hub/spaces-github-actions - name: Publish to huggingface - if: github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa' + if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && (github.event_name == 'push' && github.event_name == 'workflow_dispatch') env: HF_TOKEN: ${{ secrets.HF_TOKEN }} uses: nick-fields/retry@v3 @@ -188,7 +188,7 @@ jobs: git push https://csukuangfj:$HF_TOKEN@huggingface.co/csukuangfj/sherpa-onnx-libs main - name: Release pre-compiled binaries and libs for Windows x86 - if: github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa' && github.event_name == 'push' && contains(github.ref, 'refs/tags/') + if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && github.event_name == 'push' && contains(github.ref, 'refs/tags/') uses: svenstaro/upload-release-action@v2 with: file_glob: true diff --git a/cmake/kaldi-native-fbank.cmake b/cmake/kaldi-native-fbank.cmake index 03dacef6..ae478faf 100644 --- a/cmake/kaldi-native-fbank.cmake +++ b/cmake/kaldi-native-fbank.cmake @@ -1,7 +1,8 @@ function(download_kaldi_native_fbank) include(FetchContent) - set(kaldi_native_fbank_URL "https://github.com/csukuangfj/kaldi-native-fbank/archive/refs/tags/v1.19.1.tar.gz") + set(kaldi_native_fbank_URL "https://github.com/csukuangfj/kaldi-native-fbank/archive/refs/tags/v1.19.1.tar.gz") + set(kaldi_native_fbank_URL2 "https://hub.nuaa.cf/csukuangfj/kaldi-native-fbank/archive/refs/tags/v1.19.1.tar.gz") set(kaldi_native_fbank_URL2 "https://huggingface.co/csukuangfj/sherpa-onnx-cmake-deps/resolve/main/kaldi-native-fbank-1.19.1.tar.gz") set(kaldi_native_fbank_HASH "SHA256=0cae8cbb9ea42916b214e088912f9e8f2f648f54756b305f93f552382f31f904") @@ -12,11 +13,11 @@ function(download_kaldi_native_fbank) # If you don't have access to the Internet, # please pre-download kaldi-native-fbank set(possible_file_locations - $ENV{HOME}/Downloads/kaldi-native-fbank-1.18.7.tar.gz - ${CMAKE_SOURCE_DIR}/kaldi-native-fbank-1.18.7.tar.gz - ${CMAKE_BINARY_DIR}/kaldi-native-fbank-1.18.7.tar.gz - /tmp/kaldi-native-fbank-1.18.7.tar.gz - /star-fj/fangjun/download/github/kaldi-native-fbank-1.18.7.tar.gz + $ENV{HOME}/Downloads/kaldi-native-fbank-1.19.1.tar.gz + ${CMAKE_SOURCE_DIR}/kaldi-native-fbank-1.19.1.tar.gz + ${CMAKE_BINARY_DIR}/kaldi-native-fbank-1.19.1.tar.gz + /tmp/kaldi-native-fbank-1.19.1.tar.gz + /star-fj/fangjun/download/github/kaldi-native-fbank-1.19.1.tar.gz ) foreach(f IN LISTS possible_file_locations) diff --git a/dotnet-examples/offline-decode-files/run-hotwords.sh b/dotnet-examples/offline-decode-files/run-hotwords.sh index bfa1dfc0..de17344f 100755 --- a/dotnet-examples/offline-decode-files/run-hotwords.sh +++ b/dotnet-examples/offline-decode-files/run-hotwords.sh @@ -3,7 +3,7 @@ set -ex if [ ! -d ./sherpa-onnx-zipformer-en-2023-04-01 ]; then - wget https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-zipformer-en-2023-04-01.tar.bz2 + wget -q https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-zipformer-en-2023-04-01.tar.bz2 tar xvf sherpa-onnx-zipformer-en-2023-04-01.tar.bz2 rm sherpa-onnx-zipformer-en-2023-04-01.tar.bz2 fi diff --git a/dotnet-examples/offline-decode-files/run-zipformer.sh b/dotnet-examples/offline-decode-files/run-zipformer.sh index 6844307d..4c69067a 100755 --- a/dotnet-examples/offline-decode-files/run-zipformer.sh +++ b/dotnet-examples/offline-decode-files/run-zipformer.sh @@ -3,7 +3,7 @@ set -ex if [ ! -d ./sherpa-onnx-zipformer-en-2023-04-01 ]; then - wget https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-zipformer-en-2023-04-01.tar.bz2 + wget -q https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-zipformer-en-2023-04-01.tar.bz2 tar xvf sherpa-onnx-zipformer-en-2023-04-01.tar.bz2 rm sherpa-onnx-zipformer-en-2023-04-01.tar.bz2 fi diff --git a/dotnet-examples/online-decode-files/Program.cs b/dotnet-examples/online-decode-files/Program.cs index 2d94c4d1..5103dc01 100644 --- a/dotnet-examples/online-decode-files/Program.cs +++ b/dotnet-examples/online-decode-files/Program.cs @@ -226,10 +226,16 @@ to download pre-trained streaming models. // display results for (int i = 0; i != files.Length; ++i) { - var text = recognizer.GetResult(streams[i]).Text; + OnlineRecognizerResult r = recognizer.GetResult(streams[i]); + var text = r.Text; + var tokens = r.Tokens; Console.WriteLine("--------------------"); Console.WriteLine(files[i]); - Console.WriteLine(text); + Console.WriteLine("text: {0}", text); + Console.WriteLine("tokens: [{0}]", string.Join(", ", tokens)); + Console.Write("timestamps: ["); + r.Timestamps.ToList().ForEach(i => Console.Write(String.Format("{0:0.00}", i) + ", ")); + Console.WriteLine("]"); } Console.WriteLine("--------------------"); } diff --git a/dotnet-examples/online-decode-files/run-transducer.sh b/dotnet-examples/online-decode-files/run-transducer.sh index 06c478ba..ad98af13 100755 --- a/dotnet-examples/online-decode-files/run-transducer.sh +++ b/dotnet-examples/online-decode-files/run-transducer.sh @@ -6,10 +6,8 @@ set -ex if [ ! -d ./sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20 ]; then - GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/csukuangfj/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20 - cd sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20 - git lfs pull --include "*.onnx" - cd .. + wget -q https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2 + tar xvf sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2 fi dotnet run -c Release \ diff --git a/scripts/dotnet/online.cs b/scripts/dotnet/online.cs index dd79ba6b..09b827ad 100644 --- a/scripts/dotnet/online.cs +++ b/scripts/dotnet/online.cs @@ -185,23 +185,82 @@ namespace SherpaOnnx while (*buffer != 0) { ++buffer; + length += 1; } - length = (int)(buffer - (byte*)impl.Text); } byte[] stringBuffer = new byte[length]; Marshal.Copy(impl.Text, stringBuffer, 0, length); _text = Encoding.UTF8.GetString(stringBuffer); + + _tokens = new String[impl.Count]; + + unsafe + { + byte* buf = (byte*)impl.Tokens; + for (int i = 0; i < impl.Count; i++) + { + length = 0; + byte* start = buf; + while (*buf != 0) + { + ++buf; + length += 1; + } + ++buf; + + stringBuffer = new byte[length]; + fixed (byte* pTarget = stringBuffer) + { + for (int k = 0; k < length; k++) + { + pTarget[k] = start[k]; + } + } + + _tokens[i] = Encoding.UTF8.GetString(stringBuffer); + } + } + + unsafe + { + float* t = (float*)impl.Timestamps; + if (t != null) + { + _timestamps = new float[impl.Count]; + fixed (float* pTarget = _timestamps) + { + for (int i = 0; i < impl.Count; i++) + { + pTarget[i] = t[i]; + } + } + } + else + { + _timestamps = Array.Empty(); + } + } } [StructLayout(LayoutKind.Sequential)] struct Impl { public IntPtr Text; + public IntPtr Tokens; + public IntPtr TokensArr; + public IntPtr Timestamps; + public int Count; } private String _text; public String Text => _text; + + private String[] _tokens; + public String[] Tokens => _tokens; + + private float[] _timestamps; + public float[] Timestamps => _timestamps; } public class OnlineStream : IDisposable diff --git a/sherpa-onnx/c-api/c-api.cc b/sherpa-onnx/c-api/c-api.cc index 9ef5ad25..a064604a 100644 --- a/sherpa-onnx/c-api/c-api.cc +++ b/sherpa-onnx/c-api/c-api.cc @@ -162,15 +162,17 @@ const SherpaOnnxOnlineRecognizerResult *GetOnlineStreamResult( memset(r, 0, sizeof(SherpaOnnxOnlineRecognizerResult)); // copy text - r->text = new char[text.size() + 1]; - std::copy(text.begin(), text.end(), const_cast(r->text)); - const_cast(r->text)[text.size()] = 0; + char *pText = new char[text.size() + 1]; + std::copy(text.begin(), text.end(), pText); + pText[text.size()] = 0; + r->text = pText; // copy json const auto &json = result.AsJsonString(); - r->json = new char[json.size() + 1]; - std::copy(json.begin(), json.end(), const_cast(r->json)); - const_cast(r->json)[json.size()] = 0; + char *pJson = new char[json.size() + 1]; + std::copy(json.begin(), json.end(), pJson); + pJson[json.size()] = 0; + r->json = pJson; // copy tokens auto count = result.tokens.size(); @@ -183,15 +185,12 @@ const SherpaOnnxOnlineRecognizerResult *GetOnlineStreamResult( r->count = count; // Each word ends with nullptr - r->tokens = new char[total_length]; - memset(reinterpret_cast(const_cast(r->tokens)), 0, - total_length); + char *tokens = new char[total_length]{}; char **tokens_temp = new char *[r->count]; int32_t pos = 0; for (int32_t i = 0; i < r->count; ++i) { - tokens_temp[i] = const_cast(r->tokens) + pos; - memcpy(reinterpret_cast(const_cast(r->tokens + pos)), - result.tokens[i].c_str(), result.tokens[i].size()); + tokens_temp[i] = tokens + pos; + memcpy(tokens + pos, result.tokens[i].c_str(), result.tokens[i].size()); // +1 to move past the null character pos += result.tokens[i].size() + 1; } @@ -205,6 +204,7 @@ const SherpaOnnxOnlineRecognizerResult *GetOnlineStreamResult( r->timestamps = nullptr; } + r->tokens = tokens; } else { r->count = 0; r->timestamps = nullptr; @@ -391,9 +391,10 @@ const SherpaOnnxOfflineRecognizerResult *GetOfflineStreamResult( auto r = new SherpaOnnxOfflineRecognizerResult; memset(r, 0, sizeof(SherpaOnnxOfflineRecognizerResult)); - r->text = new char[text.size() + 1]; - std::copy(text.begin(), text.end(), const_cast(r->text)); - const_cast(r->text)[text.size()] = 0; + char *pText = new char[text.size() + 1]; + std::copy(text.begin(), text.end(), pText); + pText[text.size()] = 0; + r->text = pText; if (!result.timestamps.empty()) { r->timestamps = new float[result.timestamps.size()]; @@ -530,15 +531,17 @@ const SherpaOnnxKeywordResult *GetKeywordResult( r->start_time = result.start_time; // copy keyword - r->keyword = new char[keyword.size() + 1]; - std::copy(keyword.begin(), keyword.end(), const_cast(r->keyword)); - const_cast(r->keyword)[keyword.size()] = 0; + char *pKeyword = new char[keyword.size() + 1]; + std::copy(keyword.begin(), keyword.end(), pKeyword); + pKeyword[keyword.size()] = 0; + r->keyword = pKeyword; // copy json const auto &json = result.AsJsonString(); - r->json = new char[json.size() + 1]; - std::copy(json.begin(), json.end(), const_cast(r->json)); - const_cast(r->json)[json.size()] = 0; + char *pJson = new char[json.size() + 1]; + std::copy(json.begin(), json.end(), pJson); + pJson[json.size()] = 0; + r->json = pJson; // copy tokens auto count = result.tokens.size(); @@ -551,18 +554,16 @@ const SherpaOnnxKeywordResult *GetKeywordResult( r->count = count; // Each word ends with nullptr - r->tokens = new char[total_length]; - memset(reinterpret_cast(const_cast(r->tokens)), 0, - total_length); + char *pTokens = new char[total_length]{}; char **tokens_temp = new char *[r->count]; int32_t pos = 0; for (int32_t i = 0; i < r->count; ++i) { - tokens_temp[i] = const_cast(r->tokens) + pos; - memcpy(reinterpret_cast(const_cast(r->tokens + pos)), - result.tokens[i].c_str(), result.tokens[i].size()); + tokens_temp[i] = pTokens + pos; + memcpy(pTokens + pos, result.tokens[i].c_str(), result.tokens[i].size()); // +1 to move past the null character pos += result.tokens[i].size() + 1; } + r->tokens = pTokens; r->tokens_arr = tokens_temp; if (!result.timestamps.empty()) { diff --git a/sherpa-onnx/c-api/c-api.h b/sherpa-onnx/c-api/c-api.h index e8e59ae6..5b5dd803 100644 --- a/sherpa-onnx/c-api/c-api.h +++ b/sherpa-onnx/c-api/c-api.h @@ -145,6 +145,10 @@ SHERPA_ONNX_API typedef struct SherpaOnnxOnlineRecognizerResult { const char *const *tokens_arr; // Pointer to continuous memory which holds timestamps + // + // Caution: If timestamp information is not available, this pointer is NULL. + // Please check whether it is NULL before you access it; otherwise, you would + // get segmentation fault. float *timestamps; // The number of tokens/timestamps in above pointer diff --git a/sherpa-onnx/csrc/online-zipformer2-transducer-model.h b/sherpa-onnx/csrc/online-zipformer2-transducer-model.h index acad4517..07c9e925 100644 --- a/sherpa-onnx/csrc/online-zipformer2-transducer-model.h +++ b/sherpa-onnx/csrc/online-zipformer2-transducer-model.h @@ -105,7 +105,7 @@ class OnlineZipformer2TransducerModel : public OnlineTransducerModel { int32_t context_size_ = 0; int32_t vocab_size_ = 0; - int32_t feature_dim_ = 0; + int32_t feature_dim_ = 80; }; } // namespace sherpa_onnx