EngineX-Iluvatar/enginex-mr_series-sherpa-onnx
Archived
16
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

Begin to support CTC models (#119)

Browse Source 
Please see https://k2-fsa.github.io/sherpa/onnx/pretrained_models/offline-ctc/nemo/index.html for a list of pre-trained CTC models from NeMo.
This commit is contained in:
Fangjun Kuang
2023-04-07 23:11:34 +08:00
committed by GitHub
parent 9ac747248b
commit 80060c276d
40 changed files with 1244 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
sherpa-onnx/python/csrc/CMakeLists.txt
View File

@@ -5,6 +5,7 @@ pybind11_add_module(_sherpa_onnx
endpoint.cc
features.cc
offline-model-config.cc
offline-nemo-enc-dec-ctc-model-config.cc
offline-paraformer-model-config.cc
offline-recognizer.cc
offline-stream.cc

21
sherpa-onnx/python/csrc/offline-model-config.cc
View File

@@ -7,26 +7,31 @@
#include <string>
#include <vector>
#include "sherpa-onnx/python/csrc/offline-transducer-model-config.h"
#include "sherpa-onnx/python/csrc/offline-paraformer-model-config.h"
#include "sherpa-onnx/csrc/offline-model-config.h"
#include "sherpa-onnx/python/csrc/offline-nemo-enc-dec-ctc-model-config.h"
#include "sherpa-onnx/python/csrc/offline-paraformer-model-config.h"
#include "sherpa-onnx/python/csrc/offline-transducer-model-config.h"
namespace sherpa_onnx {
void PybindOfflineModelConfig(py::module *m) {
PybindOfflineTransducerModelConfig(m);
PybindOfflineParaformerModelConfig(m);
PybindOfflineNemoEncDecCtcModelConfig(m);
using PyClass = OfflineModelConfig;
py::class_<PyClass>(*m, "OfflineModelConfig")
.def(py::init<OfflineTransducerModelConfig &,
OfflineParaformerModelConfig &,
const std::string &, int32_t, bool>(),
py::arg("transducer"), py::arg("paraformer"), py::arg("tokens"),
py::arg("num_threads"), py::arg("debug") = false)
.def(py::init<const OfflineTransducerModelConfig &,
const OfflineParaformerModelConfig &,
const OfflineNemoEncDecCtcModelConfig &,
const std::string &, int32_t, bool>(),
py::arg("transducer") = OfflineTransducerModelConfig(),
py::arg("paraformer") = OfflineParaformerModelConfig(),
py::arg("nemo_ctc") = OfflineNemoEncDecCtcModelConfig(),
py::arg("tokens"), py::arg("num_threads"), py::arg("debug") = false)
.def_readwrite("transducer", &PyClass::transducer)
.def_readwrite("paraformer", &PyClass::paraformer)
.def_readwrite("nemo_ctc", &PyClass::nemo_ctc)
.def_readwrite("tokens", &PyClass::tokens)
.def_readwrite("num_threads", &PyClass::num_threads)
.def_readwrite("debug", &PyClass::debug)

22
sherpa-onnx/python/csrc/offline-nemo-enc-dec-ctc-model-config.cc Normal file
View File

@@ -0,0 +1,22 @@
// sherpa-onnx/python/csrc/offline-nemo-enc-dec-ctc-model-config.cc
//
// Copyright (c) 2023 Xiaomi Corporation
#include "sherpa-onnx/python/csrc/offline-nemo-enc-dec-ctc-model-config.h"
#include <string>
#include <vector>
#include "sherpa-onnx/csrc/offline-nemo-enc-dec-ctc-model-config.h"
namespace sherpa_onnx {
void PybindOfflineNemoEncDecCtcModelConfig(py::module *m) {
using PyClass = OfflineNemoEncDecCtcModelConfig;
py::class_<PyClass>(*m, "OfflineNemoEncDecCtcModelConfig")
.def(py::init<const std::string &>(), py::arg("model"))
.def_readwrite("model", &PyClass::model)
.def("__str__", &PyClass::ToString);
}
} // namespace sherpa_onnx

16
sherpa-onnx/python/csrc/offline-nemo-enc-dec-ctc-model-config.h Normal file
View File

@@ -0,0 +1,16 @@
// sherpa-onnx/python/csrc/offline-nemo-enc-dec-ctc-model-config.h
//
// Copyright (c) 2023 Xiaomi Corporation
#ifndef SHERPA_ONNX_PYTHON_CSRC_OFFLINE_NEMO_ENC_DEC_CTC_MODEL_CONFIG_H_
#define SHERPA_ONNX_PYTHON_CSRC_OFFLINE_NEMO_ENC_DEC_CTC_MODEL_CONFIG_H_
#include "sherpa-onnx/python/csrc/sherpa-onnx.h"
namespace sherpa_onnx {
void PybindOfflineNemoEncDecCtcModelConfig(py::module *m);
}
#endif // SHERPA_ONNX_PYTHON_CSRC_OFFLINE_NEMO_ENC_DEC_CTC_MODEL_CONFIG_H_

4
sherpa-onnx/python/csrc/offline-paraformer-model-config.cc
View File

@@ -4,7 +4,6 @@
#include "sherpa-onnx/python/csrc/offline-paraformer-model-config.h"
#include <string>
#include <vector>
@@ -15,8 +14,7 @@ namespace sherpa_onnx {
void PybindOfflineParaformerModelConfig(py::module *m) {
using PyClass = OfflineParaformerModelConfig;
py::class_<PyClass>(*m, "OfflineParaformerModelConfig")
.def(py::init<const std::string &>(),
py::arg("model"))
.def(py::init<const std::string &>(), py::arg("model"))
.def_readwrite("model", &PyClass::model)
.def("__str__", &PyClass::ToString);
}

2
sherpa-onnx/python/csrc/offline-recognizer.cc
View File

@@ -11,8 +11,6 @@
namespace sherpa_onnx {
static void PybindOfflineRecognizerConfig(py::module *m) {
using PyClass = OfflineRecognizerConfig;
py::class_<PyClass>(*m, "OfflineRecognizerConfig")

4
sherpa-onnx/python/csrc/offline-stream.cc
View File

@@ -31,7 +31,6 @@ static void PybindOfflineRecognitionResult(py::module *m) { // NOLINT
"timestamps", [](const PyClass &self) { return self.timestamps; });
}
static void PybindOfflineFeatureExtractorConfig(py::module *m) {
using PyClass = OfflineFeatureExtractorConfig;
py::class_<PyClass>(*m, "OfflineFeatureExtractorConfig")
@@ -42,7 +41,6 @@ static void PybindOfflineFeatureExtractorConfig(py::module *m) {
.def("__str__", &PyClass::ToString);
}
void PybindOfflineStream(py::module *m) {
PybindOfflineFeatureExtractorConfig(m);
PybindOfflineRecognitionResult(m);
@@ -55,7 +53,7 @@ void PybindOfflineStream(py::module *m) {
self.AcceptWaveform(sample_rate, waveform.data(), waveform.size());
},
py::arg("sample_rate"), py::arg("waveform"), kAcceptWaveformUsage)
.def_property_readonly("result", &PyClass::GetResult);
.def_property_readonly("result", &PyClass::GetResult);
}
} // namespace sherpa_onnx

9
sherpa-onnx/python/csrc/sherpa-onnx.cc
View File

@@ -7,16 +7,13 @@
#include "sherpa-onnx/python/csrc/display.h"
#include "sherpa-onnx/python/csrc/endpoint.h"
#include "sherpa-onnx/python/csrc/features.h"
#include "sherpa-onnx/python/csrc/offline-model-config.h"
#include "sherpa-onnx/python/csrc/offline-recognizer.h"
#include "sherpa-onnx/python/csrc/offline-stream.h"
#include "sherpa-onnx/python/csrc/online-recognizer.h"
#include "sherpa-onnx/python/csrc/online-stream.h"
#include "sherpa-onnx/python/csrc/online-transducer-model-config.h"
#include "sherpa-onnx/python/csrc/offline-model-config.h"
#include "sherpa-onnx/python/csrc/offline-paraformer-model-config.h"
#include "sherpa-onnx/python/csrc/offline-recognizer.h"
#include "sherpa-onnx/python/csrc/offline-stream.h"
#include "sherpa-onnx/python/csrc/offline-transducer-model-config.h"
namespace sherpa_onnx {
PYBIND11_MODULE(_sherpa_onnx, m) {

73
sherpa-onnx/python/sherpa_onnx/offline_recognizer.py
View File

@@ -4,12 +4,15 @@ from typing import List
from _sherpa_onnx import (
OfflineFeatureExtractorConfig,
OfflineRecognizer as _Recognizer,
OfflineModelConfig,
OfflineNemoEncDecCtcModelConfig,
OfflineParaformerModelConfig,
)
from _sherpa_onnx import OfflineRecognizer as _Recognizer
from _sherpa_onnx import (
OfflineRecognizerConfig,
OfflineStream,
OfflineModelConfig,
OfflineTransducerModelConfig,
OfflineParaformerModelConfig,
)
@@ -75,7 +78,6 @@ class OfflineRecognizer(object):
decoder_filename=decoder,
joiner_filename=joiner,
),
paraformer=OfflineParaformerModelConfig(model=""),
tokens=tokens,
num_threads=num_threads,
debug=debug,
@@ -119,7 +121,7 @@ class OfflineRecognizer(object):
symbol integer_id
paraformer:
Path to ``paraformer.onnx``.
Path to ``model.onnx``.
num_threads:
Number of threads for neural network computation.
sample_rate:
@@ -133,9 +135,6 @@ class OfflineRecognizer(object):
"""
self = cls.__new__(cls)
model_config = OfflineModelConfig(
transducer=OfflineTransducerModelConfig(
encoder_filename="", decoder_filename="", joiner_filename=""
),
paraformer=OfflineParaformerModelConfig(model=paraformer),
tokens=tokens,
num_threads=num_threads,
@@ -155,6 +154,64 @@ class OfflineRecognizer(object):
self.recognizer = _Recognizer(recognizer_config)
return self
@classmethod
def from_nemo_ctc(
cls,
model: str,
tokens: str,
num_threads: int,
sample_rate: int = 16000,
feature_dim: int = 80,
decoding_method: str = "greedy_search",
debug: bool = False,
):
"""
Please refer to
`<https://k2-fsa.github.io/sherpa/onnx/pretrained_models/index.html>`_
to download pre-trained models for different languages, e.g., Chinese,
English, etc.
Args:
tokens:
Path to ``tokens.txt``. Each line in ``tokens.txt`` contains two
columns::
symbol integer_id
model:
Path to ``model.onnx``.
num_threads:
Number of threads for neural network computation.
sample_rate:
Sample rate of the training data used to train the model.
feature_dim:
Dimension of the feature used to train the model.
decoding_method:
Valid values are greedy_search, modified_beam_search.
debug:
True to show debug messages.
"""
self = cls.__new__(cls)
model_config = OfflineModelConfig(
nemo_ctc=OfflineNemoEncDecCtcModelConfig(model=model),
tokens=tokens,
num_threads=num_threads,
debug=debug,
)
feat_config = OfflineFeatureExtractorConfig(
sampling_rate=sample_rate,
feature_dim=feature_dim,
)
recognizer_config = OfflineRecognizerConfig(
feat_config=feat_config,
model_config=model_config,
decoding_method=decoding_method,
)
self.recognizer = _Recognizer(recognizer_config)
return self
def create_stream(self):
return self.recognizer.create_stream()

65
sherpa-onnx/python/tests/test_offline_recognizer.py
View File

@@ -196,6 +196,71 @@ class TestOfflineRecognizer(unittest.TestCase):
print(s2.result.text)
print(s3.result.text)
def test_nemo_ctc_single_file(self):
for use_int8 in [True, False]:
if use_int8:
model = f"{d}/sherpa-onnx-nemo-ctc-en-citrinet-512/model.int8.onnx"
else:
model = f"{d}/sherpa-onnx-nemo-ctc-en-citrinet-512/model.onnx"
tokens = f"{d}/sherpa-onnx-nemo-ctc-en-citrinet-512/tokens.txt"
wave0 = f"{d}/sherpa-onnx-nemo-ctc-en-citrinet-512/test_wavs/0.wav"
if not Path(model).is_file():
print("skipping test_nemo_ctc_single_file()")
return
recognizer = sherpa_onnx.OfflineRecognizer.from_nemo_ctc(
model=model,
tokens=tokens,
num_threads=1,
)
s = recognizer.create_stream()
samples, sample_rate = read_wave(wave0)
s.accept_waveform(sample_rate, samples)
recognizer.decode_stream(s)
print(s.result.text)
def test_nemo_ctc_multiple_files(self):
for use_int8 in [True, False]:
if use_int8:
model = f"{d}/sherpa-onnx-nemo-ctc-en-citrinet-512/model.int8.onnx"
else:
model = f"{d}/sherpa-onnx-nemo-ctc-en-citrinet-512/model.onnx"
tokens = f"{d}/sherpa-onnx-nemo-ctc-en-citrinet-512/tokens.txt"
wave0 = f"{d}/sherpa-onnx-nemo-ctc-en-citrinet-512/test_wavs/0.wav"
wave1 = f"{d}/sherpa-onnx-nemo-ctc-en-citrinet-512/test_wavs/1.wav"
wave2 = f"{d}/sherpa-onnx-nemo-ctc-en-citrinet-512/test_wavs/8k.wav"
if not Path(model).is_file():
print("skipping test_nemo_ctc_multiple_files()")
return
recognizer = sherpa_onnx.OfflineRecognizer.from_nemo_ctc(
model=model,
tokens=tokens,
num_threads=1,
)
s0 = recognizer.create_stream()
samples0, sample_rate0 = read_wave(wave0)
s0.accept_waveform(sample_rate0, samples0)
s1 = recognizer.create_stream()
samples1, sample_rate1 = read_wave(wave1)
s1.accept_waveform(sample_rate1, samples1)
s2 = recognizer.create_stream()
samples2, sample_rate2 = read_wave(wave2)
s2.accept_waveform(sample_rate2, samples2)
recognizer.decode_streams([s0, s1, s2])
print(s0.result.text)
print(s1.result.text)
print(s2.result.text)
if __name__ == "__main__":
unittest.main()