Add C++ API for non-streaming ASR (#1456)

2024-10-23 16:40:12 +08:00
parent effd5ef2be
commit ceb69ebd94
31 changed files with 604 additions and 43 deletions
--- a/sherpa-onnx/c-api/c-api.cc
+++ b/sherpa-onnx/c-api/c-api.cc
@@ -168,14 +168,14 @@ void SherpaOnnxDestroyOnlineRecognizer(
  delete recognizer;
 }

-SherpaOnnxOnlineStream *SherpaOnnxCreateOnlineStream(
+const SherpaOnnxOnlineStream *SherpaOnnxCreateOnlineStream(
    const SherpaOnnxOnlineRecognizer *recognizer) {
  SherpaOnnxOnlineStream *stream =
      new SherpaOnnxOnlineStream(recognizer->impl->CreateStream());
  return stream;
 }

-SherpaOnnxOnlineStream *SherpaOnnxCreateOnlineStreamWithHotwords(
+const SherpaOnnxOnlineStream *SherpaOnnxCreateOnlineStreamWithHotwords(
    const SherpaOnnxOnlineRecognizer *recognizer, const char *hotwords) {
  SherpaOnnxOnlineStream *stream =
      new SherpaOnnxOnlineStream(recognizer->impl->CreateStream(hotwords));
@@ -351,7 +351,7 @@ struct SherpaOnnxOfflineStream {
 static sherpa_onnx::OfflineRecognizerConfig convertConfig(
    const SherpaOnnxOfflineRecognizerConfig *config);

-SherpaOnnxOfflineRecognizer *SherpaOnnxCreateOfflineRecognizer(
+const SherpaOnnxOfflineRecognizer *SherpaOnnxCreateOfflineRecognizer(
    const SherpaOnnxOfflineRecognizerConfig *config) {
  sherpa_onnx::OfflineRecognizerConfig recognizer_config =
      convertConfig(config);
@@ -490,11 +490,11 @@ void SherpaOnnxOfflineRecognizerSetConfig(
 }

 void SherpaOnnxDestroyOfflineRecognizer(
-    SherpaOnnxOfflineRecognizer *recognizer) {
+    const SherpaOnnxOfflineRecognizer *recognizer) {
  delete recognizer;
 }

-SherpaOnnxOfflineStream *SherpaOnnxCreateOfflineStream(
+const SherpaOnnxOfflineStream *SherpaOnnxCreateOfflineStream(
    const SherpaOnnxOfflineRecognizer *recognizer) {
  SherpaOnnxOfflineStream *stream =
      new SherpaOnnxOfflineStream(recognizer->impl->CreateStream());
@@ -518,8 +518,8 @@ void SherpaOnnxDecodeOfflineStream(
 }

 void SherpaOnnxDecodeMultipleOfflineStreams(
-    SherpaOnnxOfflineRecognizer *recognizer, SherpaOnnxOfflineStream **streams,
-    int32_t n) {
+    const SherpaOnnxOfflineRecognizer *recognizer,
+    const SherpaOnnxOfflineStream **streams, int32_t n) {
  std::vector<sherpa_onnx::OfflineStream *> ss(n);
  for (int32_t i = 0; i != n; ++i) {
    ss[i] = streams[i]->impl.get();
--- a/sherpa-onnx/c-api/c-api.h
+++ b/sherpa-onnx/c-api/c-api.h
@@ -220,7 +220,7 @@ SHERPA_ONNX_API void SherpaOnnxDestroyOnlineRecognizer(
 /// @param recognizer  A pointer returned by SherpaOnnxCreateOnlineRecognizer()
 /// @return Return a pointer to an OnlineStream. The user has to invoke
 ///         SherpaOnnxDestroyOnlineStream() to free it to avoid memory leak.
-SHERPA_ONNX_API SherpaOnnxOnlineStream *SherpaOnnxCreateOnlineStream(
+SHERPA_ONNX_API const SherpaOnnxOnlineStream *SherpaOnnxCreateOnlineStream(
    const SherpaOnnxOnlineRecognizer *recognizer);

 /// Create an online stream for accepting wave samples with the specified hot
@@ -229,7 +229,7 @@ SHERPA_ONNX_API SherpaOnnxOnlineStream *SherpaOnnxCreateOnlineStream(
 /// @param recognizer  A pointer returned by SherpaOnnxCreateOnlineRecognizer()
 /// @return Return a pointer to an OnlineStream. The user has to invoke
 ///         SherpaOnnxDestroyOnlineStream() to free it to avoid memory leak.
-SHERPA_ONNX_API SherpaOnnxOnlineStream *
+SHERPA_ONNX_API const SherpaOnnxOnlineStream *
 SherpaOnnxCreateOnlineStreamWithHotwords(
    const SherpaOnnxOnlineRecognizer *recognizer, const char *hotwords);

@@ -453,7 +453,8 @@ SHERPA_ONNX_API typedef struct SherpaOnnxOfflineStream SherpaOnnxOfflineStream;
 /// @return Return a pointer to the recognizer. The user has to invoke
 //          SherpaOnnxDestroyOfflineRecognizer() to free it to avoid memory
 //          leak.
-SHERPA_ONNX_API SherpaOnnxOfflineRecognizer *SherpaOnnxCreateOfflineRecognizer(
+SHERPA_ONNX_API const SherpaOnnxOfflineRecognizer *
+SherpaOnnxCreateOfflineRecognizer(
    const SherpaOnnxOfflineRecognizerConfig *config);

 /// @param config  Config for the recognizer.
@@ -465,14 +466,14 @@ SHERPA_ONNX_API void SherpaOnnxOfflineRecognizerSetConfig(
 ///
 /// @param p A pointer returned by SherpaOnnxCreateOfflineRecognizer()
 SHERPA_ONNX_API void SherpaOnnxDestroyOfflineRecognizer(
-    SherpaOnnxOfflineRecognizer *recognizer);
+    const SherpaOnnxOfflineRecognizer *recognizer);

 /// Create an offline stream for accepting wave samples.
 ///
 /// @param recognizer  A pointer returned by SherpaOnnxCreateOfflineRecognizer()
 /// @return Return a pointer to an OfflineStream. The user has to invoke
 ///         SherpaOnnxDestroyOfflineStream() to free it to avoid memory leak.
-SHERPA_ONNX_API SherpaOnnxOfflineStream *SherpaOnnxCreateOfflineStream(
+SHERPA_ONNX_API const SherpaOnnxOfflineStream *SherpaOnnxCreateOfflineStream(
    const SherpaOnnxOfflineRecognizer *recognizer);

 /// Destroy an offline stream.
@@ -518,8 +519,8 @@ SHERPA_ONNX_API void SherpaOnnxDecodeOfflineStream(
 ///                by SherpaOnnxCreateOfflineStream().
 /// @param n Number of entries in the given streams.
 SHERPA_ONNX_API void SherpaOnnxDecodeMultipleOfflineStreams(
-    SherpaOnnxOfflineRecognizer *recognizer, SherpaOnnxOfflineStream **streams,
-    int32_t n);
+    const SherpaOnnxOfflineRecognizer *recognizer,
+    const SherpaOnnxOfflineStream **streams, int32_t n);

 SHERPA_ONNX_API typedef struct SherpaOnnxOfflineRecognizerResult {
  const char *text;
--- a/sherpa-onnx/c-api/cxx-api.cc
+++ b/sherpa-onnx/c-api/cxx-api.cc
@@ -36,6 +36,10 @@ void OnlineStream::AcceptWaveform(int32_t sample_rate, const float *samples,
  SherpaOnnxOnlineStreamAcceptWaveform(p_, sample_rate, samples, n);
 }

+void OnlineStream::InputFinished() const {
+  SherpaOnnxOnlineStreamInputFinished(p_);
+}
+
 OnlineRecognizer OnlineRecognizer::Create(
    const OnlineRecognizerConfig &config) {
  struct SherpaOnnxOnlineRecognizerConfig c;
@@ -119,6 +123,14 @@ void OnlineRecognizer::Decode(const OnlineStream *s) const {
  SherpaOnnxDecodeOnlineStream(p_, s->Get());
 }

+void OnlineRecognizer::Reset(const OnlineStream *s) const {
+  SherpaOnnxOnlineStreamReset(p_, s->Get());
+}
+
+bool OnlineRecognizer::IsEndpoint(const OnlineStream *s) const {
+  return SherpaOnnxOnlineStreamIsEndpoint(p_, s->Get());
+}
+
 void OnlineRecognizer::Decode(const OnlineStream *ss, int32_t n) const {
  if (n <= 0) {
    return;
@@ -156,4 +168,138 @@ OnlineRecognizerResult OnlineRecognizer::GetResult(
  return ans;
 }

+// ============================================================================
+// Non-streaming ASR
+// ============================================================================
+OfflineStream::OfflineStream(const SherpaOnnxOfflineStream *p)
+    : MoveOnly<OfflineStream, SherpaOnnxOfflineStream>(p) {}
+
+void OfflineStream::Destroy(const SherpaOnnxOfflineStream *p) const {
+  SherpaOnnxDestroyOfflineStream(p);
+}
+
+void OfflineStream::AcceptWaveform(int32_t sample_rate, const float *samples,
+                                   int32_t n) const {
+  SherpaOnnxAcceptWaveformOffline(p_, sample_rate, samples, n);
+}
+
+OfflineRecognizer OfflineRecognizer::Create(
+    const OfflineRecognizerConfig &config) {
+  struct SherpaOnnxOfflineRecognizerConfig c;
+  memset(&c, 0, sizeof(c));
+
+  c.feat_config.sample_rate = config.feat_config.sample_rate;
+  c.feat_config.feature_dim = config.feat_config.feature_dim;
+  c.model_config.transducer.encoder =
+      config.model_config.transducer.encoder.c_str();
+  c.model_config.transducer.decoder =
+      config.model_config.transducer.decoder.c_str();
+  c.model_config.transducer.joiner =
+      config.model_config.transducer.joiner.c_str();
+
+  c.model_config.paraformer.model =
+      config.model_config.paraformer.model.c_str();
+
+  c.model_config.nemo_ctc.model = config.model_config.nemo_ctc.model.c_str();
+
+  c.model_config.whisper.encoder = config.model_config.whisper.encoder.c_str();
+  c.model_config.whisper.decoder = config.model_config.whisper.decoder.c_str();
+  c.model_config.whisper.language =
+      config.model_config.whisper.language.c_str();
+  c.model_config.whisper.task = config.model_config.whisper.task.c_str();
+  c.model_config.whisper.tail_paddings =
+      config.model_config.whisper.tail_paddings;
+
+  c.model_config.tdnn.model = config.model_config.tdnn.model.c_str();
+
+  c.model_config.tokens = config.model_config.tokens.c_str();
+  c.model_config.num_threads = config.model_config.num_threads;
+  c.model_config.debug = config.model_config.debug;
+  c.model_config.provider = config.model_config.provider.c_str();
+  c.model_config.model_type = config.model_config.model_type.c_str();
+  c.model_config.modeling_unit = config.model_config.modeling_unit.c_str();
+  c.model_config.bpe_vocab = config.model_config.bpe_vocab.c_str();
+  c.model_config.telespeech_ctc = config.model_config.telespeech_ctc.c_str();
+
+  c.model_config.sense_voice.model =
+      config.model_config.sense_voice.model.c_str();
+  c.model_config.sense_voice.language =
+      config.model_config.sense_voice.language.c_str();
+  c.model_config.sense_voice.use_itn = config.model_config.sense_voice.use_itn;
+
+  c.lm_config.model = config.lm_config.model.c_str();
+  c.lm_config.scale = config.lm_config.scale;
+
+  c.decoding_method = config.decoding_method.c_str();
+  c.max_active_paths = config.max_active_paths;
+  c.hotwords_file = config.hotwords_file.c_str();
+  c.hotwords_score = config.hotwords_score;
+
+  c.rule_fsts = config.rule_fsts.c_str();
+  c.rule_fars = config.rule_fars.c_str();
+
+  c.blank_penalty = config.blank_penalty;
+
+  auto p = SherpaOnnxCreateOfflineRecognizer(&c);
+  return OfflineRecognizer(p);
+}
+
+OfflineRecognizer::OfflineRecognizer(const SherpaOnnxOfflineRecognizer *p)
+    : MoveOnly<OfflineRecognizer, SherpaOnnxOfflineRecognizer>(p) {}
+
+void OfflineRecognizer::Destroy(const SherpaOnnxOfflineRecognizer *p) const {
+  SherpaOnnxDestroyOfflineRecognizer(p_);
+}
+
+OfflineStream OfflineRecognizer::CreateStream() const {
+  auto p = SherpaOnnxCreateOfflineStream(p_);
+  return OfflineStream{p};
+}
+
+void OfflineRecognizer::Decode(const OfflineStream *s) const {
+  SherpaOnnxDecodeOfflineStream(p_, s->Get());
+}
+
+void OfflineRecognizer::Decode(const OfflineStream *ss, int32_t n) const {
+  if (n <= 0) {
+    return;
+  }
+
+  std::vector<const SherpaOnnxOfflineStream *> streams(n);
+  for (int32_t i = 0; i != n; ++i) {
+    streams[i] = ss[i].Get();
+  }
+
+  SherpaOnnxDecodeMultipleOfflineStreams(p_, streams.data(), n);
+}
+
+OfflineRecognizerResult OfflineRecognizer::GetResult(
+    const OfflineStream *s) const {
+  auto r = SherpaOnnxGetOfflineStreamResult(s->Get());
+
+  OfflineRecognizerResult ans;
+  if (r) {
+    ans.text = r->text;
+
+    if (r->timestamps) {
+      ans.timestamps.resize(r->count);
+      std::copy(r->timestamps, r->timestamps + r->count, ans.timestamps.data());
+    }
+
+    ans.tokens.resize(r->count);
+    for (int32_t i = 0; i != r->count; ++i) {
+      ans.tokens[i] = r->tokens_arr[i];
+    }
+
+    ans.json = r->json;
+    ans.lang = r->lang ? r->lang : "";
+    ans.emotion = r->emotion ? r->emotion : "";
+    ans.event = r->event ? r->event : "";
+  }
+
+  SherpaOnnxDestroyOfflineRecognizerResult(r);
+
+  return ans;
+}
+
 }  // namespace sherpa_onnx::cxx
--- a/sherpa-onnx/c-api/cxx-api.h
+++ b/sherpa-onnx/c-api/cxx-api.h
@@ -13,6 +13,9 @@

 namespace sherpa_onnx::cxx {

+// ============================================================================
+// Streaming ASR
+// ============================================================================
 struct SHERPA_ONNX_API OnlineTransducerModelConfig {
  std::string encoder;
  std::string decoder;
@@ -148,6 +151,8 @@ class SHERPA_ONNX_API OnlineStream
  void AcceptWaveform(int32_t sample_rate, const float *samples,
                      int32_t n) const;

+  void InputFinished() const;
+
  void Destroy(const SherpaOnnxOnlineStream *p) const;
 };

@@ -170,10 +175,134 @@ class SHERPA_ONNX_API OnlineRecognizer

  OnlineRecognizerResult GetResult(const OnlineStream *s) const;

+  void Reset(const OnlineStream *s) const;
+
+  bool IsEndpoint(const OnlineStream *s) const;
+
 private:
  explicit OnlineRecognizer(const SherpaOnnxOnlineRecognizer *p);
 };

+// ============================================================================
+// Non-streaming ASR
+// ============================================================================
+struct SHERPA_ONNX_API OfflineTransducerModelConfig {
+  std::string encoder;
+  std::string decoder;
+  std::string joiner;
+};
+
+struct SHERPA_ONNX_API OfflineParaformerModelConfig {
+  std::string model;
+};
+
+struct SHERPA_ONNX_API OfflineNemoEncDecCtcModelConfig {
+  std::string model;
+};
+
+struct SHERPA_ONNX_API OfflineWhisperModelConfig {
+  std::string encoder;
+  std::string decoder;
+  std::string language;
+  std::string task = "transcribe";
+  int32_t tail_paddings = -1;
+};
+
+struct SHERPA_ONNX_API OfflineTdnnModelConfig {
+  std::string model;
+};
+
+struct SHERPA_ONNX_API SherpaOnnxOfflineLMConfig {
+  std::string model;
+  float scale = 1.0;
+};
+
+struct SHERPA_ONNX_API OfflineSenseVoiceModelConfig {
+  std::string model;
+  std::string language;
+  bool use_itn = false;
+};
+
+struct SHERPA_ONNX_API OfflineModelConfig {
+  OfflineTransducerModelConfig transducer;
+  OfflineParaformerModelConfig paraformer;
+  OfflineNemoEncDecCtcModelConfig nemo_ctc;
+  OfflineWhisperModelConfig whisper;
+  OfflineTdnnModelConfig tdnn;
+
+  std::string tokens;
+  int32_t num_threads = 1;
+  bool debug = false;
+  std::string provider = "cpu";
+  std::string model_type;
+  std::string modeling_unit = "cjkchar";
+  std::string bpe_vocab;
+  std::string telespeech_ctc;
+  OfflineSenseVoiceModelConfig sense_voice;
+};
+
+struct SHERPA_ONNX_API OfflineLMConfig {
+  std::string model;
+  float scale = 1.0;
+};
+
+struct SHERPA_ONNX_API OfflineRecognizerConfig {
+  FeatureConfig feat_config;
+  OfflineModelConfig model_config;
+  OfflineLMConfig lm_config;
+
+  std::string decoding_method = "greedy_search";
+  int32_t max_active_paths = 4;
+
+  std::string hotwords_file;
+
+  float hotwords_score = 1.5;
+  std::string rule_fsts;
+  std::string rule_fars;
+  float blank_penalty = 0;
+};
+
+struct SHERPA_ONNX_API OfflineRecognizerResult {
+  std::string text;
+  std::vector<float> timestamps;
+  std::vector<std::string> tokens;
+  std::string json;
+  std::string lang;
+  std::string emotion;
+  std::string event;
+};
+
+class SHERPA_ONNX_API OfflineStream
+    : public MoveOnly<OfflineStream, SherpaOnnxOfflineStream> {
+ public:
+  explicit OfflineStream(const SherpaOnnxOfflineStream *p);
+
+  void AcceptWaveform(int32_t sample_rate, const float *samples,
+                      int32_t n) const;
+
+  void Destroy(const SherpaOnnxOfflineStream *p) const;
+};
+
+class SHERPA_ONNX_API OfflineRecognizer
+    : public MoveOnly<OfflineRecognizer, SherpaOnnxOfflineRecognizer> {
+ public:
+  static OfflineRecognizer Create(const OfflineRecognizerConfig &config);
+
+  void Destroy(const SherpaOnnxOfflineRecognizer *p) const;
+
+  OfflineStream CreateStream() const;
+
+  void Decode(const OfflineStream *s) const;
+
+  void Decode(const OfflineStream *ss, int32_t n) const;
+
+  OfflineRecognizerResult GetResult(const OfflineStream *s) const;
+
+ private:
+  explicit OfflineRecognizer(const SherpaOnnxOfflineRecognizer *p);
+};
+
 }  // namespace sherpa_onnx::cxx

 #endif  // SHERPA_ONNX_C_API_CXX_API_H_
+        //
--- a/sherpa-onnx/csrc/online-recognizer-impl.cc
+++ b/sherpa-onnx/csrc/online-recognizer-impl.cc
@@ -30,9 +30,13 @@ std::unique_ptr<OnlineRecognizerImpl> OnlineRecognizerImpl::Create(
  if (!config.model_config.transducer.encoder.empty()) {
    Ort::Env env(ORT_LOGGING_LEVEL_ERROR);

+    Ort::SessionOptions sess_opts;
+    sess_opts.SetIntraOpNumThreads(1);
+    sess_opts.SetInterOpNumThreads(1);
+
    auto decoder_model = ReadFile(config.model_config.transducer.decoder);
-    auto sess = std::make_unique<Ort::Session>(
-        env, decoder_model.data(), decoder_model.size(), Ort::SessionOptions{});
+    auto sess = std::make_unique<Ort::Session>(env, decoder_model.data(),
+                                               decoder_model.size(), sess_opts);

    size_t node_count = sess->GetOutputCount();

@@ -63,9 +67,13 @@ std::unique_ptr<OnlineRecognizerImpl> OnlineRecognizerImpl::Create(
  if (!config.model_config.transducer.encoder.empty()) {
    Ort::Env env(ORT_LOGGING_LEVEL_ERROR);

+    Ort::SessionOptions sess_opts;
+    sess_opts.SetIntraOpNumThreads(1);
+    sess_opts.SetInterOpNumThreads(1);
+
    auto decoder_model = ReadFile(mgr, config.model_config.transducer.decoder);
-    auto sess = std::make_unique<Ort::Session>(
-        env, decoder_model.data(), decoder_model.size(), Ort::SessionOptions{});
+    auto sess = std::make_unique<Ort::Session>(env, decoder_model.data(),
+                                               decoder_model.size(), sess_opts);

    size_t node_count = sess->GetOutputCount();