Add Python API for keyword spotting (#576)

* Add alsa & microphone support for keyword spotting

* Add python wrapper

sherpa-onnx/csrc/CMakeLists.txt

@@ -230,12 +230,14 @@ endif()
 
 if(SHERPA_ONNX_HAS_ALSA AND SHERPA_ONNX_ENABLE_BINARY)
   add_executable(sherpa-onnx-alsa sherpa-onnx-alsa.cc alsa.cc)
+  add_executable(sherpa-onnx-keyword-spotter-alsa sherpa-onnx-keyword-spotter-alsa.cc alsa.cc)
   add_executable(sherpa-onnx-offline-tts-play-alsa sherpa-onnx-offline-tts-play-alsa.cc alsa-play.cc)
   add_executable(sherpa-onnx-alsa-offline sherpa-onnx-alsa-offline.cc alsa.cc)
   add_executable(sherpa-onnx-alsa-offline-speaker-identification sherpa-onnx-alsa-offline-speaker-identification.cc alsa.cc)
 
   set(exes
     sherpa-onnx-alsa
+    sherpa-onnx-keyword-spotter-alsa
     sherpa-onnx-alsa-offline
     sherpa-onnx-offline-tts-play-alsa
     sherpa-onnx-alsa-offline-speaker-identification
@@ -278,6 +280,11 @@ if(SHERPA_ONNX_ENABLE_PORTAUDIO AND SHERPA_ONNX_ENABLE_BINARY)
     microphone.cc
   )
 
+  add_executable(sherpa-onnx-keyword-spotter-microphone
+    sherpa-onnx-keyword-spotter-microphone.cc
+    microphone.cc
+  )
+
   add_executable(sherpa-onnx-microphone
     sherpa-onnx-microphone.cc
     microphone.cc
@@ -311,6 +318,7 @@ if(SHERPA_ONNX_ENABLE_PORTAUDIO AND SHERPA_ONNX_ENABLE_BINARY)
 
   set(exes
     sherpa-onnx-microphone
+    sherpa-onnx-keyword-spotter-microphone
     sherpa-onnx-microphone-offline
     sherpa-onnx-microphone-offline-speaker-identification
     sherpa-onnx-offline-tts-play

sherpa-onnx/csrc/sherpa-onnx-keyword-spotter-alsa.cc

@@ -0,0 +1,124 @@
+// sherpa-onnx/csrc/sherpa-onnx-keyword-spotter-alsa.cc
+//
+// Copyright (c)  2024  Xiaomi Corporation
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#include <algorithm>
+#include <cstdint>
+
+#include "sherpa-onnx/csrc/alsa.h"
+#include "sherpa-onnx/csrc/display.h"
+#include "sherpa-onnx/csrc/keyword-spotter.h"
+#include "sherpa-onnx/csrc/parse-options.h"
+
+bool stop = false;
+
+static void Handler(int sig) {
+  stop = true;
+  fprintf(stderr, "\nCaught Ctrl + C. Exiting...\n");
+}
+
+int main(int32_t argc, char *argv[]) {
+  signal(SIGINT, Handler);
+
+  const char *kUsageMessage = R"usage(
+Usage:
+  ./bin/sherpa-onnx-keyword-spotter-alsa \
+    --tokens=/path/to/tokens.txt \
+    --encoder=/path/to/encoder.onnx \
+    --decoder=/path/to/decoder.onnx \
+    --joiner=/path/to/joiner.onnx \
+    --provider=cpu \
+    --num-threads=2 \
+    --keywords-file=keywords.txt \
+    device_name
+
+Please refer to
+https://k2-fsa.github.io/sherpa/onnx/kws/pretrained_models/index.html
+for a list of pre-trained models to download.
+
+The device name specifies which microphone to use in case there are several
+on you system. You can use
+
+  arecord -l
+
+to find all available microphones on your computer. For instance, if it outputs
+
+**** List of CAPTURE Hardware Devices ****
+card 3: UACDemoV10 [UACDemoV1.0], device 0: USB Audio [USB Audio]
+  Subdevices: 1/1
+  Subdevice #0: subdevice #0
+
+and if you want to select card 3 and the device 0 on that card, please use:
+
+  hw:3,0
+
+or
+
+  plughw:3,0
+
+as the device_name.
+)usage";
+  sherpa_onnx::ParseOptions po(kUsageMessage);
+  sherpa_onnx::KeywordSpotterConfig config;
+
+  config.Register(&po);
+
+  po.Read(argc, argv);
+  if (po.NumArgs() != 1) {
+    fprintf(stderr, "Please provide only 1 argument: the device name\n");
+    po.PrintUsage();
+    exit(EXIT_FAILURE);
+  }
+
+  fprintf(stderr, "%s\n", config.ToString().c_str());
+
+  if (!config.Validate()) {
+    fprintf(stderr, "Errors in config!\n");
+    return -1;
+  }
+  sherpa_onnx::KeywordSpotter spotter(config);
+
+  int32_t expected_sample_rate = config.feat_config.sampling_rate;
+
+  std::string device_name = po.GetArg(1);
+  sherpa_onnx::Alsa alsa(device_name.c_str());
+  fprintf(stderr, "Use recording device: %s\n", device_name.c_str());
+
+  if (alsa.GetExpectedSampleRate() != expected_sample_rate) {
+    fprintf(stderr, "sample rate: %d != %d\n", alsa.GetExpectedSampleRate(),
+            expected_sample_rate);
+    exit(-1);
+  }
+
+  int32_t chunk = 0.1 * alsa.GetActualSampleRate();
+
+  std::string last_text;
+
+  auto stream = spotter.CreateStream();
+
+  sherpa_onnx::Display display;
+
+  int32_t keyword_index = 0;
+  while (!stop) {
+    const std::vector<float> &samples = alsa.Read(chunk);
+
+    stream->AcceptWaveform(expected_sample_rate, samples.data(),
+                           samples.size());
+
+    while (spotter.IsReady(stream.get())) {
+      spotter.DecodeStream(stream.get());
+    }
+
+    const auto r = spotter.GetResult(stream.get());
+    if (!r.keyword.empty()) {
+      display.Print(keyword_index, r.AsJsonString());
+      fflush(stderr);
+      keyword_index++;
+    }
+  }
+
+  return 0;
+}

sherpa-onnx/csrc/sherpa-onnx-keyword-spotter-microphone.cc

@@ -0,0 +1,148 @@
+// sherpa-onnx/csrc/sherpa-onnx-keyword-spotter-microphone.cc
+//
+// Copyright (c)  2024  Xiaomi Corporation
+
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#include <algorithm>
+
+#include "portaudio.h"  // NOLINT
+#include "sherpa-onnx/csrc/display.h"
+#include "sherpa-onnx/csrc/microphone.h"
+#include "sherpa-onnx/csrc/keyword-spotter.h"
+
+bool stop = false;
+
+static int32_t RecordCallback(const void *input_buffer,
+                              void * /*output_buffer*/,
+                              unsigned long frames_per_buffer,  // NOLINT
+                              const PaStreamCallbackTimeInfo * /*time_info*/,
+                              PaStreamCallbackFlags /*status_flags*/,
+                              void *user_data) {
+  auto stream = reinterpret_cast<sherpa_onnx::OnlineStream *>(user_data);
+
+  stream->AcceptWaveform(16000, reinterpret_cast<const float *>(input_buffer),
+                         frames_per_buffer);
+
+  return stop ? paComplete : paContinue;
+}
+
+static void Handler(int32_t sig) {
+  stop = true;
+  fprintf(stderr, "\nCaught Ctrl + C. Exiting...\n");
+}
+
+int32_t main(int32_t argc, char *argv[]) {
+  signal(SIGINT, Handler);
+
+  const char *kUsageMessage = R"usage(
+This program uses streaming models with microphone for keyword spotting.
+Usage:
+
+  ./bin/sherpa-onnx-keyword-spotter-microphone \
+    --tokens=/path/to/tokens.txt \
+    --encoder=/path/to/encoder.onnx \
+    --decoder=/path/to/decoder.onnx \
+    --joiner=/path/to/joiner.onnx \
+    --provider=cpu \
+    --num-threads=1 \
+    --keywords-file=keywords.txt
+
+Please refer to
+https://k2-fsa.github.io/sherpa/onnx/kws/pretrained_models/index.html
+for a list of pre-trained models to download.
+)usage";
+
+  sherpa_onnx::ParseOptions po(kUsageMessage);
+  sherpa_onnx::KeywordSpotterConfig config;
+
+  config.Register(&po);
+  po.Read(argc, argv);
+  if (po.NumArgs() != 0) {
+    po.PrintUsage();
+    exit(EXIT_FAILURE);
+  }
+
+  fprintf(stderr, "%s\n", config.ToString().c_str());
+
+  if (!config.Validate()) {
+    fprintf(stderr, "Errors in config!\n");
+    return -1;
+  }
+
+  sherpa_onnx::KeywordSpotter spotter(config);
+  auto s = spotter.CreateStream();
+
+  sherpa_onnx::Microphone mic;
+
+  PaDeviceIndex num_devices = Pa_GetDeviceCount();
+  fprintf(stderr, "Num devices: %d\n", num_devices);
+
+  PaStreamParameters param;
+
+  param.device = Pa_GetDefaultInputDevice();
+  if (param.device == paNoDevice) {
+    fprintf(stderr, "No default input device found\n");
+    exit(EXIT_FAILURE);
+  }
+  fprintf(stderr, "Use default device: %d\n", param.device);
+
+  const PaDeviceInfo *info = Pa_GetDeviceInfo(param.device);
+  fprintf(stderr, "  Name: %s\n", info->name);
+  fprintf(stderr, "  Max input channels: %d\n", info->maxInputChannels);
+
+  param.channelCount = 1;
+  param.sampleFormat = paFloat32;
+
+  param.suggestedLatency = info->defaultLowInputLatency;
+  param.hostApiSpecificStreamInfo = nullptr;
+  float sample_rate = 16000;
+
+  PaStream *stream;
+  PaError err =
+      Pa_OpenStream(&stream, &param, nullptr, /* &outputParameters, */
+                    sample_rate,
+                    0,          // frames per buffer
+                    paClipOff,  // we won't output out of range samples
+                                // so don't bother clipping them
+                    RecordCallback, s.get());
+  if (err != paNoError) {
+    fprintf(stderr, "portaudio error: %s\n", Pa_GetErrorText(err));
+    exit(EXIT_FAILURE);
+  }
+
+  err = Pa_StartStream(stream);
+  fprintf(stderr, "Started\n");
+
+  if (err != paNoError) {
+    fprintf(stderr, "portaudio error: %s\n", Pa_GetErrorText(err));
+    exit(EXIT_FAILURE);
+  }
+
+  int32_t keyword_index = 0;
+  sherpa_onnx::Display display;
+  while (!stop) {
+    while (spotter.IsReady(s.get())) {
+      spotter.DecodeStream(s.get());
+    }
+
+    const auto r = spotter.GetResult(s.get());
+    if (!r.keyword.empty()) {
+      display.Print(keyword_index, r.AsJsonString());
+      fflush(stderr);
+      keyword_index++;
+    }
+
+    Pa_Sleep(20);  // sleep for 20ms
+  }
+
+  err = Pa_CloseStream(stream);
+  if (err != paNoError) {
+    fprintf(stderr, "portaudio error: %s\n", Pa_GetErrorText(err));
+    exit(EXIT_FAILURE);
+  }
+
+  return 0;
+}

sherpa-onnx/csrc/sherpa-onnx-keyword-spotter.cc

@@ -12,7 +12,6 @@
 #include "sherpa-onnx/csrc/keyword-spotter.h"
 #include "sherpa-onnx/csrc/online-stream.h"
 #include "sherpa-onnx/csrc/parse-options.h"
-#include "sherpa-onnx/csrc/symbol-table.h"
 #include "sherpa-onnx/csrc/wave-reader.h"
 
 typedef struct {