EngineX-Iluvatar/enginex-mr_series-sherpa-onnx
Archived
16
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

Add config for TensorRT and CUDA execution provider (#992)

Browse Source 
Signed-off-by: manickavela1998@gmail.com <manickavela1998@gmail.com>
Signed-off-by: manickavela1998@gmail.com <manickavela.arumugam@uniphore.com>
This commit is contained in:
Manix
2024-07-05 12:48:37 +05:30
committed by GitHub
parent f5e9a162d1
commit 55decb7bee
21 changed files with 622 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
sherpa-onnx/python/csrc/CMakeLists.txt
View File

@@ -3,6 +3,7 @@ include_directories(${CMAKE_SOURCE_DIR})
set(srcs
audio-tagging.cc
circular-buffer.cc
cuda-config.cc
display.cc
endpoint.cc
features.cc
@@ -30,11 +31,13 @@ set(srcs
online-transducer-model-config.cc
online-wenet-ctc-model-config.cc
online-zipformer2-ctc-model-config.cc
provider-config.cc
sherpa-onnx.cc
silero-vad-model-config.cc
speaker-embedding-extractor.cc
speaker-embedding-manager.cc
spoken-language-identification.cc
tensorrt-config.cc
vad-model-config.cc
vad-model.cc
voice-activity-detector.cc

24
sherpa-onnx/python/csrc/cuda-config.cc Normal file
View File

@@ -0,0 +1,24 @@
// sherpa-onnx/python/csrc/cuda-config.cc
//
// Copyright (c) 2024 Uniphore (Author: Manickavela A)
#include "sherpa-onnx/python/csrc/cuda-config.h"
#include <memory>
#include <string>
#include "sherpa-onnx/csrc/provider-config.h"
namespace sherpa_onnx {
void PybindCudaConfig(py::module *m) {
using PyClass = CudaConfig;
py::class_<PyClass>(*m, "CudaConfig")
.def(py::init<>())
.def(py::init<int32_t>(),
py::arg("cudnn_conv_algo_search") = 1)
.def_readwrite("cudnn_conv_algo_search", &PyClass::cudnn_conv_algo_search)
.def("__str__", &PyClass::ToString);
}
} // namespace sherpa_onnx

16
sherpa-onnx/python/csrc/cuda-config.h Normal file
View File

@@ -0,0 +1,16 @@
// sherpa-onnx/python/csrc/cuda-config.h
//
// Copyright (c) 2024 Uniphore (Author: Manickavela A)
#ifndef SHERPA_ONNX_PYTHON_CSRC_CUDA_CONFIG_H_
#define SHERPA_ONNX_PYTHON_CSRC_CUDA_CONFIG_H_
#include "sherpa-onnx/python/csrc/sherpa-onnx.h"
namespace sherpa_onnx {
void PybindCudaConfig(py::module *m);
}
#endif // SHERPA_ONNX_PYTHON_CSRC_CUDA_CONFIG_H_

24
sherpa-onnx/python/csrc/online-model-config.cc
View File

@@ -9,11 +9,13 @@
#include "sherpa-onnx/csrc/online-model-config.h"
#include "sherpa-onnx/csrc/online-transducer-model-config.h"
#include "sherpa-onnx/csrc/provider-config.h"
#include "sherpa-onnx/python/csrc/online-nemo-ctc-model-config.h"
#include "sherpa-onnx/python/csrc/online-paraformer-model-config.h"
#include "sherpa-onnx/python/csrc/online-transducer-model-config.h"
#include "sherpa-onnx/python/csrc/online-wenet-ctc-model-config.h"
#include "sherpa-onnx/python/csrc/online-zipformer2-ctc-model-config.h"
#include "sherpa-onnx/python/csrc/provider-config.h"
namespace sherpa_onnx {
@@ -23,6 +25,7 @@ void PybindOnlineModelConfig(py::module *m) {
PybindOnlineWenetCtcModelConfig(m);
PybindOnlineZipformer2CtcModelConfig(m);
PybindOnlineNeMoCtcModelConfig(m);
PybindProviderConfig(m);
using PyClass = OnlineModelConfig;
py::class_<PyClass>(*m, "OnlineModelConfig")
@@ -30,33 +33,34 @@ void PybindOnlineModelConfig(py::module *m) {
const OnlineParaformerModelConfig &,
const OnlineWenetCtcModelConfig &,
const OnlineZipformer2CtcModelConfig &,
const OnlineNeMoCtcModelConfig &, const std::string &,
int32_t, int32_t, bool, const std::string &,
const std::string &, const std::string &,
const OnlineNeMoCtcModelConfig &,
const ProviderConfig &,
const std::string &, int32_t, int32_t,
bool, const std::string &, const std::string &,
const std::string &>(),
py::arg("transducer") = OnlineTransducerModelConfig(),
py::arg("paraformer") = OnlineParaformerModelConfig(),
py::arg("wenet_ctc") = OnlineWenetCtcModelConfig(),
py::arg("zipformer2_ctc") = OnlineZipformer2CtcModelConfig(),
py::arg("nemo_ctc") = OnlineNeMoCtcModelConfig(), py::arg("tokens"),
py::arg("num_threads"), py::arg("warm_up") = 0,
py::arg("debug") = false, py::arg("provider") = "cpu",
py::arg("model_type") = "", py::arg("modeling_unit") = "",
py::arg("bpe_vocab") = "")
py::arg("nemo_ctc") = OnlineNeMoCtcModelConfig(),
py::arg("provider_config") = ProviderConfig(),
py::arg("tokens"), py::arg("num_threads"), py::arg("warm_up") = 0,
py::arg("debug") = false, py::arg("model_type") = "",
py::arg("modeling_unit") = "", py::arg("bpe_vocab") = "")
.def_readwrite("transducer", &PyClass::transducer)
.def_readwrite("paraformer", &PyClass::paraformer)
.def_readwrite("wenet_ctc", &PyClass::wenet_ctc)
.def_readwrite("zipformer2_ctc", &PyClass::zipformer2_ctc)
.def_readwrite("nemo_ctc", &PyClass::nemo_ctc)
.def_readwrite("provider_config", &PyClass::provider_config)
.def_readwrite("tokens", &PyClass::tokens)
.def_readwrite("num_threads", &PyClass::num_threads)
.def_readwrite("warm_up", &PyClass::warm_up)
.def_readwrite("debug", &PyClass::debug)
.def_readwrite("provider", &PyClass::provider)
.def_readwrite("model_type", &PyClass::model_type)
.def_readwrite("modeling_unit", &PyClass::modeling_unit)
.def_readwrite("bpe_vocab", &PyClass::bpe_vocab)
.def("validate", &PyClass::Validate)
.def("__str__", &PyClass::ToString);
}
} // namespace sherpa_onnx

39
sherpa-onnx/python/csrc/provider-config.cc Normal file
View File

@@ -0,0 +1,39 @@
// sherpa-onnx/python/csrc/provider-config.cc
//
// Copyright (c) 2024 Uniphore (Author: Manickavela A)
#include "sherpa-onnx/python/csrc/provider-config.h"
#include <string>
#include "sherpa-onnx/csrc/provider-config.h"
#include "sherpa-onnx/python/csrc/cuda-config.h"
#include "sherpa-onnx/python/csrc/tensorrt-config.h"
namespace sherpa_onnx {
void PybindProviderConfig(py::module *m) {
PybindCudaConfig(m);
PybindTensorrtConfig(m);
using PyClass = ProviderConfig;
py::class_<PyClass>(*m, "ProviderConfig")
.def(py::init<>())
.def(py::init<const std::string &, int32_t>(),
py::arg("provider") = "cpu",
py::arg("device") = 0)
.def(py::init<const TensorrtConfig &, const CudaConfig &,
const std::string &, int32_t>(),
py::arg("trt_config") = TensorrtConfig{},
py::arg("cuda_config") = CudaConfig{},
py::arg("provider") = "cpu",
py::arg("device") = 0)
.def_readwrite("trt_config", &PyClass::trt_config)
.def_readwrite("cuda_config", &PyClass::cuda_config)
.def_readwrite("provider", &PyClass::provider)
.def_readwrite("device", &PyClass::device)
.def("__str__", &PyClass::ToString)
.def("validate", &PyClass::Validate);
}
} // namespace sherpa_onnx

16
sherpa-onnx/python/csrc/provider-config.h Normal file
View File

@@ -0,0 +1,16 @@
// sherpa-onnx/python/csrc/provider-config.h
//
// Copyright (c) 2024 Uniphore (Author: Manickavela A)
#ifndef SHERPA_ONNX_PYTHON_CSRC_PROVIDER_CONFIG_H_
#define SHERPA_ONNX_PYTHON_CSRC_PROVIDER_CONFIG_H_
#include "sherpa-onnx/python/csrc/sherpa-onnx.h"
namespace sherpa_onnx {
void PybindProviderConfig(py::module *m);
}
#endif // SHERPA_ONNX_PYTHON_CSRC_PROVIDER_CONFIG_H_

1
sherpa-onnx/python/csrc/sherpa-onnx.cc
View File

@@ -51,7 +51,6 @@ PYBIND11_MODULE(_sherpa_onnx, m) {
PybindEndpoint(&m);
PybindOnlineRecognizer(&m);
PybindKeywordSpotter(&m);
PybindDisplay(&m);
PybindOfflineStream(&m);

72
sherpa-onnx/python/csrc/tensorrt-config.cc Normal file
View File

@@ -0,0 +1,72 @@
// sherpa-onnx/python/csrc/tensorrt-config.cc
//
// Copyright (c) 2024 Uniphore (Author: Manickavela A)
#include "sherpa-onnx/python/csrc/tensorrt-config.h"
#include <string>
#include <memory>
#include "sherpa-onnx/csrc/provider-config.h"
namespace sherpa_onnx {
void PybindTensorrtConfig(py::module *m) {
using PyClass = TensorrtConfig;
py::class_<PyClass>(*m, "TensorrtConfig")
.def(py::init<>())
.def(py::init([](int32_t trt_max_workspace_size,
int32_t trt_max_partition_iterations,
int32_t trt_min_subgraph_size,
bool trt_fp16_enable,
bool trt_detailed_build_log,
bool trt_engine_cache_enable,
bool trt_timing_cache_enable,
const std::string &trt_engine_cache_path,
const std::string &trt_timing_cache_path,
bool trt_dump_subgraphs) -> std::unique_ptr<PyClass> {
auto ans = std::make_unique<PyClass>();
ans->trt_max_workspace_size = trt_max_workspace_size;
ans->trt_max_partition_iterations = trt_max_partition_iterations;
ans->trt_min_subgraph_size = trt_min_subgraph_size;
ans->trt_fp16_enable = trt_fp16_enable;
ans->trt_detailed_build_log = trt_detailed_build_log;
ans->trt_engine_cache_enable = trt_engine_cache_enable;
ans->trt_timing_cache_enable = trt_timing_cache_enable;
ans->trt_engine_cache_path = trt_engine_cache_path;
ans->trt_timing_cache_path = trt_timing_cache_path;
ans->trt_dump_subgraphs = trt_dump_subgraphs;
return ans;
}),
py::arg("trt_max_workspace_size") = 2147483647,
py::arg("trt_max_partition_iterations") = 10,
py::arg("trt_min_subgraph_size") = 5,
py::arg("trt_fp16_enable") = true,
py::arg("trt_detailed_build_log") = false,
py::arg("trt_engine_cache_enable") = true,
py::arg("trt_timing_cache_enable") = true,
py::arg("trt_engine_cache_path") = ".",
py::arg("trt_timing_cache_path") = ".",
py::arg("trt_dump_subgraphs") = false)
.def_readwrite("trt_max_workspace_size",
&PyClass::trt_max_workspace_size)
.def_readwrite("trt_max_partition_iterations",
&PyClass::trt_max_partition_iterations)
.def_readwrite("trt_min_subgraph_size", &PyClass::trt_min_subgraph_size)
.def_readwrite("trt_fp16_enable", &PyClass::trt_fp16_enable)
.def_readwrite("trt_detailed_build_log",
&PyClass::trt_detailed_build_log)
.def_readwrite("trt_engine_cache_enable",
&PyClass::trt_engine_cache_enable)
.def_readwrite("trt_timing_cache_enable",
&PyClass::trt_timing_cache_enable)
.def_readwrite("trt_engine_cache_path", &PyClass::trt_engine_cache_path)
.def_readwrite("trt_timing_cache_path", &PyClass::trt_timing_cache_path)
.def_readwrite("trt_dump_subgraphs", &PyClass::trt_dump_subgraphs)
.def("__str__", &PyClass::ToString)
.def("validate", &PyClass::Validate);
}
} // namespace sherpa_onnx

16
sherpa-onnx/python/csrc/tensorrt-config.h Normal file
View File

@@ -0,0 +1,16 @@
// sherpa-onnx/python/csrc/tensorrt-config.h
//
// Copyright (c) 2024 Uniphore (Author: Manickavela A)
#ifndef SHERPA_ONNX_PYTHON_CSRC_TENSORRT_CONFIG_H_
#define SHERPA_ONNX_PYTHON_CSRC_TENSORRT_CONFIG_H_
#include "sherpa-onnx/python/csrc/sherpa-onnx.h"
namespace sherpa_onnx {
void PybindTensorrtConfig(py::module *m);
}
#endif // SHERPA_ONNX_PYTHON_CSRC_TENSORRT_CONFIG_H_

11
sherpa-onnx/python/sherpa_onnx/keyword_spotter.py
View File

@@ -9,6 +9,7 @@ from _sherpa_onnx import (
OnlineModelConfig,
OnlineTransducerModelConfig,
OnlineStream,
ProviderConfig,
)
from _sherpa_onnx import KeywordSpotter as _KeywordSpotter
@@ -41,6 +42,7 @@ class KeywordSpotter(object):
keywords_threshold: float = 0.25,
num_trailing_blanks: int = 1,
provider: str = "cpu",
device: int = 0,
):
"""
Please refer to
@@ -85,6 +87,8 @@ class KeywordSpotter(object):
between each other.
provider:
onnxruntime execution providers. Valid values are: cpu, cuda, coreml.
device:
onnxruntime cuda device index.
"""
_assert_file_exists(tokens)
_assert_file_exists(encoder)
@@ -99,11 +103,16 @@ class KeywordSpotter(object):
joiner=joiner,
)
provider_config = ProviderConfig(
provider=provider,
device = device,
)
model_config = OnlineModelConfig(
transducer=transducer_config,
tokens=tokens,
num_threads=num_threads,
provider=provider,
provider_config=provider_config,
)
feat_config = FeatureExtractorConfig(

111
sherpa-onnx/python/sherpa_onnx/online_recognizer.py
View File

@@ -11,6 +11,9 @@ from _sherpa_onnx import (
)
from _sherpa_onnx import OnlineRecognizer as _Recognizer
from _sherpa_onnx import (
CudaConfig,
TensorrtConfig,
ProviderConfig,
OnlineRecognizerConfig,
OnlineRecognizerResult,
OnlineStream,
@@ -56,7 +59,6 @@ class OnlineRecognizer(object):
hotwords_score: float = 1.5,
blank_penalty: float = 0.0,
hotwords_file: str = "",
provider: str = "cpu",
model_type: str = "",
modeling_unit: str = "cjkchar",
bpe_vocab: str = "",
@@ -66,6 +68,19 @@ class OnlineRecognizer(object):
debug: bool = False,
rule_fsts: str = "",
rule_fars: str = "",
provider: str = "cpu",
device: int = 0,
cudnn_conv_algo_search: int = 1,
trt_max_workspace_size: int = 2147483647,
trt_max_partition_iterations: int = 10,
trt_min_subgraph_size: int = 5,
trt_fp16_enable: bool = True,
trt_detailed_build_log: bool = False,
trt_engine_cache_enable: bool = True,
trt_timing_cache_enable: bool = True,
trt_engine_cache_path: str ="",
trt_timing_cache_path: str ="",
trt_dump_subgraphs: bool = False,
):
"""
Please refer to
@@ -135,8 +150,6 @@ class OnlineRecognizer(object):
Temperature scaling for output symbol confidence estiamation.
It affects only confidence values, the decoding uses the original
logits without temperature.
provider:
onnxruntime execution providers. Valid values are: cpu, cuda, coreml.
model_type:
Online transducer model type. Valid values are: conformer, lstm,
zipformer, zipformer2. All other values lead to loading the model twice.
@@ -156,6 +169,32 @@ class OnlineRecognizer(object):
rule_fars:
If not empty, it specifies fst archives for inverse text normalization.
If there are multiple archives, they are separated by a comma.
provider:
onnxruntime execution providers. Valid values are: cpu, cuda, coreml.
device:
onnxruntime cuda device index.
cudnn_conv_algo_search:
onxrt CuDNN convolution search algorithm selection. CUDA EP
trt_max_workspace_size:
Set TensorRT EP GPU memory usage limit. TensorRT EP
trt_max_partition_iterations:
Limit partitioning iterations for model conversion. TensorRT EP
trt_min_subgraph_size:
Set minimum size for subgraphs in partitioning. TensorRT EP
trt_fp16_enable: bool = True,
Enable FP16 precision for faster performance. TensorRT EP
trt_detailed_build_log: bool = False,
Enable detailed logging of build steps. TensorRT EP
trt_engine_cache_enable: bool = True,
Enable caching of TensorRT engines. TensorRT EP
trt_timing_cache_enable: bool = True,
"Enable use of timing cache to speed up builds." TensorRT EP
trt_engine_cache_path: str ="",
"Set path to store cached TensorRT engines." TensorRT EP
trt_timing_cache_path: str ="",
"Set path for storing timing cache." TensorRT EP
trt_dump_subgraphs: bool = False,
"Dump optimized subgraphs for debugging." TensorRT EP
"""
self = cls.__new__(cls)
_assert_file_exists(tokens)
@@ -171,11 +210,35 @@ class OnlineRecognizer(object):
joiner=joiner,
)
cuda_config = CudaConfig(
cudnn_conv_algo_search=cudnn_conv_algo_search,
)
trt_config = TensorrtConfig(
trt_max_workspace_size=trt_max_workspace_size,
trt_max_partition_iterations=trt_max_partition_iterations,
trt_min_subgraph_size=trt_min_subgraph_size,
trt_fp16_enable=trt_fp16_enable,
trt_detailed_build_log=trt_detailed_build_log,
trt_engine_cache_enable=trt_engine_cache_enable,
trt_timing_cache_enable=trt_timing_cache_enable,
trt_engine_cache_path=trt_engine_cache_path,
trt_timing_cache_path=trt_timing_cache_path,
trt_dump_subgraphs=trt_dump_subgraphs,
)
provider_config = ProviderConfig(
trt_config=trt_config,
cuda_config=cuda_config,
provider=provider,
device=device,
)
model_config = OnlineModelConfig(
transducer=transducer_config,
tokens=tokens,
num_threads=num_threads,
provider=provider,
provider_config=provider_config,
model_type=model_type,
modeling_unit=modeling_unit,
bpe_vocab=bpe_vocab,
@@ -251,6 +314,7 @@ class OnlineRecognizer(object):
debug: bool = False,
rule_fsts: str = "",
rule_fars: str = "",
device: int = 0,
):
"""
Please refer to
@@ -301,6 +365,8 @@ class OnlineRecognizer(object):
rule_fars:
If not empty, it specifies fst archives for inverse text normalization.
If there are multiple archives, they are separated by a comma.
device:
onnxruntime cuda device index.
"""
self = cls.__new__(cls)
_assert_file_exists(tokens)
@@ -314,11 +380,16 @@ class OnlineRecognizer(object):
decoder=decoder,
)
provider_config = ProviderConfig(
provider=provider,
device=device,
)
model_config = OnlineModelConfig(
paraformer=paraformer_config,
tokens=tokens,
num_threads=num_threads,
provider=provider,
provider_config=provider_config,
model_type="paraformer",
debug=debug,
)
@@ -367,6 +438,7 @@ class OnlineRecognizer(object):
debug: bool = False,
rule_fsts: str = "",
rule_fars: str = "",
device: int = 0,
):
"""
Please refer to
@@ -421,6 +493,8 @@ class OnlineRecognizer(object):
rule_fars:
If not empty, it specifies fst archives for inverse text normalization.
If there are multiple archives, they are separated by a comma.
device:
onnxruntime cuda device index.
"""
self = cls.__new__(cls)
_assert_file_exists(tokens)
@@ -430,11 +504,16 @@ class OnlineRecognizer(object):
zipformer2_ctc_config = OnlineZipformer2CtcModelConfig(model=model)
provider_config = ProviderConfig(
provider=provider,
device=device,
)
model_config = OnlineModelConfig(
zipformer2_ctc=zipformer2_ctc_config,
tokens=tokens,
num_threads=num_threads,
provider=provider,
provider_config=provider_config,
debug=debug,
)
@@ -486,6 +565,7 @@ class OnlineRecognizer(object):
debug: bool = False,
rule_fsts: str = "",
rule_fars: str = "",
device: int = 0,
):
"""
Please refer to
@@ -535,6 +615,8 @@ class OnlineRecognizer(object):
rule_fars:
If not empty, it specifies fst archives for inverse text normalization.
If there are multiple archives, they are separated by a comma.
device:
onnxruntime cuda device index.
"""
self = cls.__new__(cls)
_assert_file_exists(tokens)
@@ -546,11 +628,16 @@ class OnlineRecognizer(object):
model=model,
)
provider_config = ProviderConfig(
provider=provider,
device=device,
)
model_config = OnlineModelConfig(
nemo_ctc=nemo_ctc_config,
tokens=tokens,
num_threads=num_threads,
provider=provider,
provider_config=provider_config,
debug=debug,
)
@@ -598,6 +685,7 @@ class OnlineRecognizer(object):
debug: bool = False,
rule_fsts: str = "",
rule_fars: str = "",
device: int = 0,
):
"""
Please refer to
@@ -650,6 +738,8 @@ class OnlineRecognizer(object):
rule_fars:
If not empty, it specifies fst archives for inverse text normalization.
If there are multiple archives, they are separated by a comma.
device:
onnxruntime cuda device index.
"""
self = cls.__new__(cls)
_assert_file_exists(tokens)
@@ -663,11 +753,16 @@ class OnlineRecognizer(object):
num_left_chunks=num_left_chunks,
)
provider_config = ProviderConfig(
provider=provider,
device=device,
)
model_config = OnlineModelConfig(
wenet_ctc=wenet_ctc_config,
tokens=tokens,
num_threads=num_threads,
provider=provider,
provider_config=provider_config,
debug=debug,
)