Add C++ runtime for SenseVoice models (#1148)

sherpa-onnx/python/csrc/CMakeLists.txt

@@ -15,6 +15,7 @@ set(srcs
   offline-paraformer-model-config.cc
   offline-punctuation.cc
   offline-recognizer.cc
+  offline-sense-voice-model-config.cc
   offline-stream.cc
   offline-tdnn-model-config.cc
   offline-transducer-model-config.cc

sherpa-onnx/python/csrc/offline-model-config.cc

@@ -10,6 +10,7 @@
 #include "sherpa-onnx/csrc/offline-model-config.h"
 #include "sherpa-onnx/python/csrc/offline-nemo-enc-dec-ctc-model-config.h"
 #include "sherpa-onnx/python/csrc/offline-paraformer-model-config.h"
+#include "sherpa-onnx/python/csrc/offline-sense-voice-model-config.h"
 #include "sherpa-onnx/python/csrc/offline-tdnn-model-config.h"
 #include "sherpa-onnx/python/csrc/offline-transducer-model-config.h"
 #include "sherpa-onnx/python/csrc/offline-wenet-ctc-model-config.h"
@@ -26,6 +27,7 @@ void PybindOfflineModelConfig(py::module *m) {
   PybindOfflineTdnnModelConfig(m);
   PybindOfflineZipformerCtcModelConfig(m);
   PybindOfflineWenetCtcModelConfig(m);
+  PybindOfflineSenseVoiceModelConfig(m);
 
   using PyClass = OfflineModelConfig;
   py::class_<PyClass>(*m, "OfflineModelConfig")
@@ -36,7 +38,8 @@ void PybindOfflineModelConfig(py::module *m) {
               const OfflineNemoEncDecCtcModelConfig &,
               const OfflineWhisperModelConfig &, const OfflineTdnnModelConfig &,
               const OfflineZipformerCtcModelConfig &,
-              const OfflineWenetCtcModelConfig &, const std::string &,
+              const OfflineWenetCtcModelConfig &,
+              const OfflineSenseVoiceModelConfig &, const std::string &,
               const std::string &, int32_t, bool, const std::string &,
               const std::string &, const std::string &, const std::string &>(),
           py::arg("transducer") = OfflineTransducerModelConfig(),
@@ -46,6 +49,7 @@ void PybindOfflineModelConfig(py::module *m) {
           py::arg("tdnn") = OfflineTdnnModelConfig(),
           py::arg("zipformer_ctc") = OfflineZipformerCtcModelConfig(),
           py::arg("wenet_ctc") = OfflineWenetCtcModelConfig(),
+          py::arg("sense_voice") = OfflineSenseVoiceModelConfig(),
           py::arg("telespeech_ctc") = "", py::arg("tokens"),
           py::arg("num_threads"), py::arg("debug") = false,
           py::arg("provider") = "cpu", py::arg("model_type") = "",
@@ -57,6 +61,7 @@ void PybindOfflineModelConfig(py::module *m) {
       .def_readwrite("tdnn", &PyClass::tdnn)
       .def_readwrite("zipformer_ctc", &PyClass::zipformer_ctc)
       .def_readwrite("wenet_ctc", &PyClass::wenet_ctc)
+      .def_readwrite("sense_voice", &PyClass::sense_voice)
       .def_readwrite("telespeech_ctc", &PyClass::telespeech_ctc)
       .def_readwrite("tokens", &PyClass::tokens)
       .def_readwrite("num_threads", &PyClass::num_threads)

sherpa-onnx/python/csrc/offline-paraformer-model-config.cc

@@ -14,6 +14,7 @@ namespace sherpa_onnx {
 void PybindOfflineParaformerModelConfig(py::module *m) {
   using PyClass = OfflineParaformerModelConfig;
   py::class_<PyClass>(*m, "OfflineParaformerModelConfig")
+      .def(py::init<>())
       .def(py::init<const std::string &>(), py::arg("model"))
       .def_readwrite("model", &PyClass::model)
       .def("__str__", &PyClass::ToString);

sherpa-onnx/python/csrc/offline-sense-voice-model-config.cc

@@ -0,0 +1,26 @@
+// sherpa-onnx/python/csrc/offline-sense-voice-model-config.cc
+//
+// Copyright (c)  2024  Xiaomi Corporation
+
+#include "sherpa-onnx/csrc/offline-sense-voice-model-config.h"
+
+#include <string>
+#include <vector>
+
+#include "sherpa-onnx/python/csrc/offline-sense-voice-model-config.h"
+
+namespace sherpa_onnx {
+
+void PybindOfflineSenseVoiceModelConfig(py::module *m) {
+  using PyClass = OfflineSenseVoiceModelConfig;
+  py::class_<PyClass>(*m, "OfflineSenseVoiceModelConfig")
+      .def(py::init<>())
+      .def(py::init<const std::string &, const std::string &, bool>(),
+           py::arg("model"), py::arg("language"), py::arg("use_itn"))
+      .def_readwrite("model", &PyClass::model)
+      .def_readwrite("language", &PyClass::language)
+      .def_readwrite("use_itn", &PyClass::use_itn)
+      .def("__str__", &PyClass::ToString);
+}
+
+}  // namespace sherpa_onnx

sherpa-onnx/python/csrc/offline-sense-voice-model-config.h

@@ -0,0 +1,16 @@
+// sherpa-onnx/python/csrc/offline-sense-voice-model-config.h
+//
+// Copyright (c)  2024  Xiaomi Corporation
+
+#ifndef SHERPA_ONNX_PYTHON_CSRC_OFFLINE_SENSE_VOICE_MODEL_CONFIG_H_
+#define SHERPA_ONNX_PYTHON_CSRC_OFFLINE_SENSE_VOICE_MODEL_CONFIG_H_
+
+#include "sherpa-onnx/python/csrc/sherpa-onnx.h"
+
+namespace sherpa_onnx {
+
+void PybindOfflineSenseVoiceModelConfig(py::module *m);
+
+}
+
+#endif  // SHERPA_ONNX_PYTHON_CSRC_OFFLINE_SENSE_VOICE_MODEL_CONFIG_H_

sherpa-onnx/python/sherpa_onnx/offline_recognizer.py

@@ -10,6 +10,7 @@ from _sherpa_onnx import (
     OfflineModelConfig,
     OfflineNemoEncDecCtcModelConfig,
     OfflineParaformerModelConfig,
+    OfflineSenseVoiceModelConfig,
 )
 from _sherpa_onnx import OfflineRecognizer as _Recognizer
 from _sherpa_onnx import (
@@ -173,6 +174,88 @@ class OfflineRecognizer(object):
         self.config = recognizer_config
         return self
 
+    @classmethod
+    def from_sense_voice(
+        cls,
+        model: str,
+        tokens: str,
+        num_threads: int = 1,
+        sample_rate: int = 16000,
+        feature_dim: int = 80,
+        decoding_method: str = "greedy_search",
+        debug: bool = False,
+        provider: str = "cpu",
+        language: str = "",
+        use_itn: bool = False,
+        rule_fsts: str = "",
+        rule_fars: str = "",
+    ):
+        """
+        Please refer to
+        `<https://github.com/k2-fsa/sherpa-onnx/releases/tag/asr-models>`_
+        to download pre-trained models.
+
+        Args:
+          tokens:
+            Path to ``tokens.txt``. Each line in ``tokens.txt`` contains two
+            columns::
+
+                symbol integer_id
+
+          model:
+            Path to ``model.onnx``.
+          num_threads:
+            Number of threads for neural network computation.
+          sample_rate:
+            Sample rate of the training data used to train the model.
+          feature_dim:
+            Dimension of the feature used to train the model.
+          decoding_method:
+            Valid values are greedy_search.
+          debug:
+            True to show debug messages.
+          provider:
+            onnxruntime execution providers. Valid values are: cpu, cuda, coreml.
+          language:
+            If not empty, then valid values are: auto, zh, en, ja, ko, yue
+          use_itn:
+            True to enable inverse text normalization; False to disable it.
+          rule_fsts:
+            If not empty, it specifies fsts for inverse text normalization.
+            If there are multiple fsts, they are separated by a comma.
+          rule_fars:
+            If not empty, it specifies fst archives for inverse text normalization.
+            If there are multiple archives, they are separated by a comma.
+        """
+        self = cls.__new__(cls)
+        model_config = OfflineModelConfig(
+            sense_voice=OfflineSenseVoiceModelConfig(
+                model=model,
+                language=language,
+                use_itn=use_itn,
+            ),
+            tokens=tokens,
+            num_threads=num_threads,
+            debug=debug,
+            provider=provider,
+        )
+
+        feat_config = FeatureExtractorConfig(
+            sampling_rate=sample_rate,
+            feature_dim=feature_dim,
+        )
+
+        recognizer_config = OfflineRecognizerConfig(
+            feat_config=feat_config,
+            model_config=model_config,
+            decoding_method=decoding_method,
+            rule_fsts=rule_fsts,
+            rule_fars=rule_fars,
+        )
+        self.recognizer = _Recognizer(recognizer_config)
+        self.config = recognizer_config
+        return self
+
     @classmethod
     def from_paraformer(
         cls,