From b03f6e6e8c14d1c39d15dfa458fd4e1a8eac6bed Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Mon, 17 Feb 2025 15:16:23 +0800 Subject: [PATCH] Add Swift API for FireRedAsr AED Model (#1876) --- .github/scripts/test-swift.sh | 4 ++ swift-api-examples/SherpaOnnx.swift | 16 +++++- swift-api-examples/fire-red-asr.swift | 68 ++++++++++++++++++++++++++ swift-api-examples/run-fire-red-asr.sh | 42 ++++++++++++++++ 4 files changed, 128 insertions(+), 2 deletions(-) create mode 100644 swift-api-examples/fire-red-asr.swift create mode 100755 swift-api-examples/run-fire-red-asr.sh diff --git a/.github/scripts/test-swift.sh b/.github/scripts/test-swift.sh index 65fe4588..be6d08fc 100755 --- a/.github/scripts/test-swift.sh +++ b/.github/scripts/test-swift.sh @@ -7,6 +7,10 @@ echo "pwd: $PWD" cd swift-api-examples ls -lh + +./run-fire-red-asr.sh +rm -rf sherpa-onnx-fire-red-asr-* + ./run-tts-vits.sh ls -lh rm -rf vits-piper-* diff --git a/swift-api-examples/SherpaOnnx.swift b/swift-api-examples/SherpaOnnx.swift index b4efaa1f..40b006af 100644 --- a/swift-api-examples/SherpaOnnx.swift +++ b/swift-api-examples/SherpaOnnx.swift @@ -357,6 +357,16 @@ func sherpaOnnxOfflineWhisperModelConfig( ) } +func sherpaOnnxOfflineFireRedAsrModelConfig( + encoder: String = "", + decoder: String = "" +) -> SherpaOnnxOfflineFireRedAsrModelConfig { + return SherpaOnnxOfflineFireRedAsrModelConfig( + encoder: toCPointer(encoder), + decoder: toCPointer(decoder) + ) +} + func sherpaOnnxOfflineMoonshineModelConfig( preprocessor: String = "", encoder: String = "", @@ -416,7 +426,8 @@ func sherpaOnnxOfflineModelConfig( bpeVocab: String = "", teleSpeechCtc: String = "", senseVoice: SherpaOnnxOfflineSenseVoiceModelConfig = sherpaOnnxOfflineSenseVoiceModelConfig(), - moonshine: SherpaOnnxOfflineMoonshineModelConfig = sherpaOnnxOfflineMoonshineModelConfig() + moonshine: SherpaOnnxOfflineMoonshineModelConfig = sherpaOnnxOfflineMoonshineModelConfig(), + fireRedAsr: SherpaOnnxOfflineFireRedAsrModelConfig = sherpaOnnxOfflineFireRedAsrModelConfig() ) -> SherpaOnnxOfflineModelConfig { return SherpaOnnxOfflineModelConfig( transducer: transducer, @@ -433,7 +444,8 @@ func sherpaOnnxOfflineModelConfig( bpe_vocab: toCPointer(bpeVocab), telespeech_ctc: toCPointer(teleSpeechCtc), sense_voice: senseVoice, - moonshine: moonshine + moonshine: moonshine, + fire_red_asr: fireRedAsr ) } diff --git a/swift-api-examples/fire-red-asr.swift b/swift-api-examples/fire-red-asr.swift new file mode 100644 index 00000000..f3cd3113 --- /dev/null +++ b/swift-api-examples/fire-red-asr.swift @@ -0,0 +1,68 @@ +import AVFoundation + +extension AudioBuffer { + func array() -> [Float] { + return Array(UnsafeBufferPointer(self)) + } +} + +extension AVAudioPCMBuffer { + func array() -> [Float] { + return self.audioBufferList.pointee.mBuffers.array() + } +} + +func run() { + let encoder = "./sherpa-onnx-fire-red-asr-large-zh_en-2025-02-16/encoder.int8.onnx" + let decoder = "./sherpa-onnx-fire-red-asr-large-zh_en-2025-02-16/decoder.int8.onnx" + let tokens = "./sherpa-onnx-fire-red-asr-large-zh_en-2025-02-16/tokens.txt" + + let fireRedAsr = sherpaOnnxOfflineFireRedAsrModelConfig( + encoder: encoder, + decoder: decoder + ) + + let modelConfig = sherpaOnnxOfflineModelConfig( + tokens: tokens, + debug: 0, + fireRedAsr: fireRedAsr + ) + + let featConfig = sherpaOnnxFeatureConfig( + sampleRate: 16000, + featureDim: 80 + ) + var config = sherpaOnnxOfflineRecognizerConfig( + featConfig: featConfig, + modelConfig: modelConfig + ) + + let recognizer = SherpaOnnxOfflineRecognizer(config: &config) + + let filePath = "./sherpa-onnx-fire-red-asr-large-zh_en-2025-02-16/test_wavs/0.wav" + let fileURL: NSURL = NSURL(fileURLWithPath: filePath) + let audioFile = try! AVAudioFile(forReading: fileURL as URL) + + let audioFormat = audioFile.processingFormat + assert(audioFormat.channelCount == 1) + assert(audioFormat.commonFormat == AVAudioCommonFormat.pcmFormatFloat32) + + let audioFrameCount = UInt32(audioFile.length) + let audioFileBuffer = AVAudioPCMBuffer(pcmFormat: audioFormat, frameCapacity: audioFrameCount) + + try! audioFile.read(into: audioFileBuffer!) + let array: [Float]! = audioFileBuffer?.array() + let result = recognizer.decode(samples: array, sampleRate: Int(audioFormat.sampleRate)) + print("\nresult is:\n\(result.text)") + if result.timestamps.count != 0 { + print("\ntimestamps is:\n\(result.timestamps)") + } + +} + +@main +struct App { + static func main() { + run() + } +} diff --git a/swift-api-examples/run-fire-red-asr.sh b/swift-api-examples/run-fire-red-asr.sh new file mode 100755 index 00000000..951d17c5 --- /dev/null +++ b/swift-api-examples/run-fire-red-asr.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +set -ex + +if [ ! -d ../build-swift-macos ]; then + echo "Please run ../build-swift-macos.sh first!" + exit 1 +fi + +if [ ! -f ./sherpa-onnx-fire-red-asr-large-zh_en-2025-02-16/encoder.int8.onnx ]; then + echo "Please download the pre-trained model for testing." + echo "You can refer to" + echo "" + echo "https://k2-fsa.github.io/sherpa/onnx/pretrained_models/FireRedAsr/index.html" + echo "" + echo "for help" + + curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-fire-red-asr-large-zh_en-2025-02-16.tar.bz2 + tar xvf sherpa-onnx-fire-red-asr-large-zh_en-2025-02-16.tar.bz2 + rm sherpa-onnx-fire-red-asr-large-zh_en-2025-02-16.tar.bz2 + ls -lh sherpa-onnx-fire-red-asr-large-zh_en-2025-02-16 +fi + +if [ ! -e ./fire-red-asr ]; then + # Note: We use -lc++ to link against libc++ instead of libstdc++ + swiftc \ + -lc++ \ + -I ../build-swift-macos/install/include \ + -import-objc-header ./SherpaOnnx-Bridging-Header.h \ + ./fire-red-asr.swift ./SherpaOnnx.swift \ + -L ../build-swift-macos/install/lib/ \ + -l sherpa-onnx \ + -l onnxruntime \ + -o fire-red-asr + + strip fire-red-asr +else + echo "./fire-red-asr exists - skip building" +fi + +export DYLD_LIBRARY_PATH=$PWD/../build-swift-macos/install/lib:$DYLD_LIBRARY_PATH +./fire-red-asr