diff --git a/cmake/cmake_extension.py b/cmake/cmake_extension.py index 4b9028ba..1337cf6a 100644 --- a/cmake/cmake_extension.py +++ b/cmake/cmake_extension.py @@ -71,10 +71,11 @@ def get_binaries(): binaries += [ "sherpa-onnx-alsa", "sherpa-onnx-alsa-offline", + "sherpa-onnx-alsa-offline-audio-tagging", "sherpa-onnx-alsa-offline-speaker-identification", "sherpa-onnx-offline-tts-play-alsa", "sherpa-onnx-vad-alsa", - "sherpa-onnx-alsa-offline-audio-tagging", + "sherpa-onnx-vad-alsa-offline-asr", ] if is_windows(): diff --git a/sherpa-onnx/csrc/CMakeLists.txt b/sherpa-onnx/csrc/CMakeLists.txt index 20133a35..aa622c8f 100644 --- a/sherpa-onnx/csrc/CMakeLists.txt +++ b/sherpa-onnx/csrc/CMakeLists.txt @@ -380,6 +380,7 @@ if(SHERPA_ONNX_HAS_ALSA AND SHERPA_ONNX_ENABLE_BINARY) add_executable(sherpa-onnx-alsa-offline-speaker-identification sherpa-onnx-alsa-offline-speaker-identification.cc alsa.cc) add_executable(sherpa-onnx-keyword-spotter-alsa sherpa-onnx-keyword-spotter-alsa.cc alsa.cc) add_executable(sherpa-onnx-vad-alsa sherpa-onnx-vad-alsa.cc alsa.cc) + add_executable(sherpa-onnx-vad-alsa-offline-asr sherpa-onnx-vad-alsa-offline-asr.cc alsa.cc) if(SHERPA_ONNX_ENABLE_TTS) @@ -392,6 +393,7 @@ if(SHERPA_ONNX_HAS_ALSA AND SHERPA_ONNX_ENABLE_BINARY) sherpa-onnx-alsa-offline-speaker-identification sherpa-onnx-keyword-spotter-alsa sherpa-onnx-vad-alsa + sherpa-onnx-vad-alsa-offline-asr sherpa-onnx-alsa-offline-audio-tagging ) diff --git a/sherpa-onnx/csrc/sherpa-onnx-vad-alsa-offline-asr.cc b/sherpa-onnx/csrc/sherpa-onnx-vad-alsa-offline-asr.cc new file mode 100644 index 00000000..0ee6da63 --- /dev/null +++ b/sherpa-onnx/csrc/sherpa-onnx-vad-alsa-offline-asr.cc @@ -0,0 +1,159 @@ +// sherpa-onnx/csrc/sherpa-onnx-vad-alsa-offline-asr.cc +// +// Copyright (c) 2022-2025 Xiaomi Corporation + +#include +#include +#include + +#include +#include // NOLINT + +#include "sherpa-onnx/csrc/alsa.h" +#include "sherpa-onnx/csrc/circular-buffer.h" +#include "sherpa-onnx/csrc/offline-recognizer.h" +#include "sherpa-onnx/csrc/resample.h" +#include "sherpa-onnx/csrc/voice-activity-detector.h" + +bool stop = false; +static void Handler(int32_t /*sig*/) { + stop = true; + fprintf(stderr, "\nCaught Ctrl + C. Exiting...\n"); +} + +int32_t main(int32_t argc, char *argv[]) { + signal(SIGINT, Handler); + + const char *kUsageMessage = R"usage( +This program shows how to use a streaming VAD with non-streaming ASR in +sherpa-onnx. + +Please download silero_vad.onnx from +https://github.com/snakers4/silero-vad/raw/master/src/silero_vad/data/silero_vad.onnx + +For instance, use +wget https://github.com/snakers4/silero-vad/raw/master/src/silero_vad/data/silero_vad.onnx + +Please refer to ./sherpa-onnx-microphone-offline.cc +to download models for offline ASR. + +(1) Transducer from icefall + + ./bin/sherpa-onnx-vad-microphone-offline-asr \ + --silero-vad-model=/path/to/silero_vad.onnx \ + --tokens=/path/to/tokens.txt \ + --encoder=/path/to/encoder.onnx \ + --decoder=/path/to/decoder.onnx \ + --joiner=/path/to/joiner.onnx \ + device_name + +(2) Paraformer from FunASR + + ./bin/sherpa-onnx-vad-microphone-offline-asr \ + --silero-vad-model=/path/to/silero_vad.onnx \ + --tokens=/path/to/tokens.txt \ + --paraformer=/path/to/model.onnx \ + device_name + +(3) Whisper models + + ./bin/sherpa-onnx-vad-microphone-offline-asr \ + --silero-vad-model=/path/to/silero_vad.onnx \ + --whisper-encoder=./sherpa-onnx-whisper-base.en/base.en-encoder.int8.onnx \ + --whisper-decoder=./sherpa-onnx-whisper-base.en/base.en-decoder.int8.onnx \ + --tokens=./sherpa-onnx-whisper-base.en/base.en-tokens.txt \ + device_name + +The device name specifies which microphone to use in case there are several +on your system. You can use + + arecord -l + +to find all available microphones on your computer. For instance, if it outputs + +**** List of CAPTURE Hardware Devices **** +card 3: UACDemoV10 [UACDemoV1.0], device 0: USB Audio [USB Audio] + Subdevices: 1/1 + Subdevice #0: subdevice #0 + +and if you want to select card 3 and device 0 on that card, please use: + + plughw:3,0 + +as the device_name. +)usage"; + + sherpa_onnx::ParseOptions po(kUsageMessage); + sherpa_onnx::VadModelConfig vad_config; + + sherpa_onnx::OfflineRecognizerConfig asr_config; + + vad_config.Register(&po); + asr_config.Register(&po); + + po.Read(argc, argv); + if (po.NumArgs() != 1) { + fprintf(stderr, "Please provide only 1 argument: the device name\n"); + po.PrintUsage(); + exit(EXIT_FAILURE); + } + + fprintf(stderr, "%s\n", vad_config.ToString().c_str()); + fprintf(stderr, "%s\n", asr_config.ToString().c_str()); + + if (!vad_config.Validate()) { + fprintf(stderr, "Errors in vad_config!\n"); + return -1; + } + + if (!asr_config.Validate()) { + fprintf(stderr, "Errors in asr_config!\n"); + return -1; + } + + fprintf(stderr, "Creating recognizer ...\n"); + sherpa_onnx::OfflineRecognizer recognizer(asr_config); + fprintf(stderr, "Recognizer created!\n"); + + auto vad = std::make_unique(vad_config); + + std::string device_name = po.GetArg(1); + sherpa_onnx::Alsa alsa(device_name.c_str()); + fprintf(stderr, "Use recording device: %s\n", device_name.c_str()); + + int32_t sample_rate = 16000; + + if (alsa.GetExpectedSampleRate() != sample_rate) { + fprintf(stderr, "sample rate: %d != %d\n", alsa.GetExpectedSampleRate(), + sample_rate); + exit(-1); + } + + int32_t chunk = 0.1 * alsa.GetActualSampleRate(); + + fprintf(stderr, "Started. Please speak\n"); + + int32_t window_size = vad_config.silero_vad.window_size; + int32_t index = 0; + + while (!stop) { + const std::vector &samples = alsa.Read(chunk); + vad->AcceptWaveform(samples.data(), samples.size()); + + while (!vad->Empty()) { + const auto &segment = vad->Front(); + auto s = recognizer.CreateStream(); + s->AcceptWaveform(sample_rate, segment.samples.data(), + segment.samples.size()); + recognizer.DecodeStream(s.get()); + const auto &result = s->GetResult(); + if (!result.text.empty()) { + fprintf(stderr, "%2d: %s\n", index, result.text.c_str()); + ++index; + } + vad->Pop(); + } + } + + return 0; +} diff --git a/sherpa-onnx/csrc/sherpa-onnx-vad-microphone-offline-asr.cc b/sherpa-onnx/csrc/sherpa-onnx-vad-microphone-offline-asr.cc index df3e250a..d5cd920d 100644 --- a/sherpa-onnx/csrc/sherpa-onnx-vad-microphone-offline-asr.cc +++ b/sherpa-onnx/csrc/sherpa-onnx-vad-microphone-offline-asr.cc @@ -115,11 +115,20 @@ to download models for offline ASR. PaDeviceIndex num_devices = Pa_GetDeviceCount(); fprintf(stderr, "Num devices: %d\n", num_devices); + if (num_devices == 0) { + fprintf(stderr, + " If you are using Linux, please try " + "./build/bin/sherpa-onnx-vad-alsa-offline-asr\n"); + exit(-1); + } int32_t device_index = Pa_GetDefaultInputDevice(); if (device_index == paNoDevice) { fprintf(stderr, "No default input device found\n"); + fprintf(stderr, + " If you are using Linux, please try " + "./build/bin/sherpa-onnx-vad-alsa-offline-asr\n"); exit(EXIT_FAILURE); } diff --git a/sherpa-onnx/csrc/vocoder.cc b/sherpa-onnx/csrc/vocoder.cc index e8290673..91f70a8a 100644 --- a/sherpa-onnx/csrc/vocoder.cc +++ b/sherpa-onnx/csrc/vocoder.cc @@ -87,6 +87,8 @@ std::unique_ptr Vocoder::Create(const OfflineTtsModelConfig &config) { SHERPA_ONNX_LOGE("Unknown model type in vocoder!"); return nullptr; } + + return nullptr; } template