Add speaker identification APIs for HarmonyOS (#1607)

* Add speaker embedding extractor API for HarmonyOS

* Add ArkTS API for speaker identification
This commit is contained in:
Fangjun Kuang
2024-12-09 19:23:18 +08:00
committed by GitHub
parent a743a4400f
commit 314545f938
19 changed files with 374 additions and 60 deletions

View File

@@ -1328,8 +1328,8 @@ struct SherpaOnnxSpeakerEmbeddingExtractor {
std::unique_ptr<sherpa_onnx::SpeakerEmbeddingExtractor> impl;
};
const SherpaOnnxSpeakerEmbeddingExtractor *
SherpaOnnxCreateSpeakerEmbeddingExtractor(
static sherpa_onnx::SpeakerEmbeddingExtractorConfig
GetSpeakerEmbeddingExtractorConfig(
const SherpaOnnxSpeakerEmbeddingExtractorConfig *config) {
sherpa_onnx::SpeakerEmbeddingExtractorConfig c;
c.model = SHERPA_ONNX_OR(config->model, "");
@@ -1342,9 +1342,21 @@ SherpaOnnxCreateSpeakerEmbeddingExtractor(
}
if (config->debug) {
#if __OHOS__
SHERPA_ONNX_LOGE("%{public}s\n", c.ToString().c_str());
#else
SHERPA_ONNX_LOGE("%s\n", c.ToString().c_str());
#endif
}
return c;
}
const SherpaOnnxSpeakerEmbeddingExtractor *
SherpaOnnxCreateSpeakerEmbeddingExtractor(
const SherpaOnnxSpeakerEmbeddingExtractorConfig *config) {
auto c = GetSpeakerEmbeddingExtractorConfig(config);
if (!c.Validate()) {
SHERPA_ONNX_LOGE("Errors in config!");
return nullptr;
@@ -1983,6 +1995,23 @@ SherpaOnnxVoiceActivityDetector *SherpaOnnxCreateVoiceActivityDetectorOHOS(
return p;
}
const SherpaOnnxSpeakerEmbeddingExtractor *
SherpaOnnxCreateSpeakerEmbeddingExtractorOHOS(
const SherpaOnnxSpeakerEmbeddingExtractorConfig *config,
NativeResourceManager *mgr) {
if (!mgr) {
return SherpaOnnxCreateSpeakerEmbeddingExtractor(config);
}
auto c = GetSpeakerEmbeddingExtractorConfig(config);
auto p = new SherpaOnnxSpeakerEmbeddingExtractor;
p->impl = std::make_unique<sherpa_onnx::SpeakerEmbeddingExtractor>(mgr, c);
return p;
}
#if SHERPA_ONNX_ENABLE_TTS == 1
SherpaOnnxOfflineTts *SherpaOnnxCreateOfflineTtsOHOS(
const SherpaOnnxOfflineTtsConfig *config, NativeResourceManager *mgr) {

View File

@@ -1572,6 +1572,11 @@ SherpaOnnxCreateVoiceActivityDetectorOHOS(
SHERPA_ONNX_API SherpaOnnxOfflineTts *SherpaOnnxCreateOfflineTtsOHOS(
const SherpaOnnxOfflineTtsConfig *config, NativeResourceManager *mgr);
SHERPA_ONNX_API const SherpaOnnxSpeakerEmbeddingExtractor *
SherpaOnnxCreateSpeakerEmbeddingExtractorOHOS(
const SherpaOnnxSpeakerEmbeddingExtractorConfig *config,
NativeResourceManager *mgr);
#endif
#if defined(__GNUC__)

View File

@@ -62,9 +62,9 @@ class OfflineTtsVitsImpl : public OfflineTtsImpl {
for (const auto &f : files) {
if (config.model.debug) {
#if __OHOS__
SHERPA_ONNX_LOGE("rule far: %s", f.c_str());
#else
SHERPA_ONNX_LOGE("rule far: %{public}s", f.c_str());
#else
SHERPA_ONNX_LOGE("rule far: %s", f.c_str());
#endif
}
std::unique_ptr<fst::FarReader<fst::StdArc>> reader(

View File

@@ -22,11 +22,10 @@ class SpeakerEmbeddingExtractorGeneralImpl
const SpeakerEmbeddingExtractorConfig &config)
: model_(config) {}
#if __ANDROID_API__ >= 9
template <typename Manager>
SpeakerEmbeddingExtractorGeneralImpl(
AAssetManager *mgr, const SpeakerEmbeddingExtractorConfig &config)
Manager *mgr, const SpeakerEmbeddingExtractorConfig &config)
: model_(mgr, config) {}
#endif
int32_t Dim() const override { return model_.GetMetaData().output_dim; }
@@ -46,9 +45,15 @@ class SpeakerEmbeddingExtractorGeneralImpl
std::vector<float> Compute(OnlineStream *s) const override {
int32_t num_frames = s->NumFramesReady() - s->GetNumProcessedFrames();
if (num_frames <= 0) {
#if __OHOS__
SHERPA_ONNX_LOGE(
"Please make sure IsReady(s) returns true. num_frames: %{public}d",
num_frames);
#else
SHERPA_ONNX_LOGE(
"Please make sure IsReady(s) returns true. num_frames: %d",
num_frames);
#endif
return {};
}
@@ -64,8 +69,13 @@ class SpeakerEmbeddingExtractorGeneralImpl
if (meta_data.feature_normalize_type == "global-mean") {
SubtractGlobalMean(features.data(), num_frames, feat_dim);
} else {
#if __OHOS__
SHERPA_ONNX_LOGE("Unsupported feature_normalize_type: %{public}s",
meta_data.feature_normalize_type.c_str());
#else
SHERPA_ONNX_LOGE("Unsupported feature_normalize_type: %s",
meta_data.feature_normalize_type.c_str());
#endif
exit(-1);
}
}

View File

@@ -3,6 +3,15 @@
// Copyright (c) 2024 Xiaomi Corporation
#include "sherpa-onnx/csrc/speaker-embedding-extractor-impl.h"
#if __ANDROID_API__ >= 9
#include "android/asset_manager.h"
#include "android/asset_manager_jni.h"
#endif
#if __OHOS__
#include "rawfile/raw_file_manager.h"
#endif
#include "sherpa-onnx/csrc/macros.h"
#include "sherpa-onnx/csrc/onnx-utils.h"
#include "sherpa-onnx/csrc/speaker-embedding-extractor-general-impl.h"
@@ -35,7 +44,11 @@ static ModelType GetModelType(char *model_data, size_t model_data_length,
if (debug) {
std::ostringstream os;
PrintModelMetadata(os, meta_data);
#if __OHOS__
SHERPA_ONNX_LOGE("%{public}s", os.str().c_str());
#else
SHERPA_ONNX_LOGE("%s", os.str().c_str());
#endif
}
Ort::AllocatorWithDefaultOptions allocator;
@@ -59,7 +72,11 @@ static ModelType GetModelType(char *model_data, size_t model_data_length,
} else if (model_type == "nemo") {
return ModelType::kNeMo;
} else {
#if __OHOS__
SHERPA_ONNX_LOGE("Unsupported model_type: %{public}s", model_type.c_str());
#else
SHERPA_ONNX_LOGE("Unsupported model_type: %s", model_type.c_str());
#endif
return ModelType::kUnknown;
}
}
@@ -91,10 +108,10 @@ SpeakerEmbeddingExtractorImpl::Create(
return nullptr;
}
#if __ANDROID_API__ >= 9
template <typename Manager>
std::unique_ptr<SpeakerEmbeddingExtractorImpl>
SpeakerEmbeddingExtractorImpl::Create(
AAssetManager *mgr, const SpeakerEmbeddingExtractorConfig &config) {
Manager *mgr, const SpeakerEmbeddingExtractorConfig &config) {
ModelType model_type = ModelType::kUnknown;
{
@@ -120,6 +137,17 @@ SpeakerEmbeddingExtractorImpl::Create(
// unreachable code
return nullptr;
}
#if __ANDROID_API__ >= 9
template std::unique_ptr<SpeakerEmbeddingExtractorImpl>
SpeakerEmbeddingExtractorImpl::Create(
AAssetManager *mgr, const SpeakerEmbeddingExtractorConfig &config);
#endif
#if __OHOS__
template std::unique_ptr<SpeakerEmbeddingExtractorImpl>
SpeakerEmbeddingExtractorImpl::Create(
NativeResourceManager *mgr, const SpeakerEmbeddingExtractorConfig &config);
#endif
} // namespace sherpa_onnx

View File

@@ -9,11 +9,6 @@
#include <string>
#include <vector>
#if __ANDROID_API__ >= 9
#include "android/asset_manager.h"
#include "android/asset_manager_jni.h"
#endif
#include "sherpa-onnx/csrc/speaker-embedding-extractor.h"
namespace sherpa_onnx {
@@ -25,10 +20,9 @@ class SpeakerEmbeddingExtractorImpl {
static std::unique_ptr<SpeakerEmbeddingExtractorImpl> Create(
const SpeakerEmbeddingExtractorConfig &config);
#if __ANDROID_API__ >= 9
template <typename Manager>
static std::unique_ptr<SpeakerEmbeddingExtractorImpl> Create(
AAssetManager *mgr, const SpeakerEmbeddingExtractorConfig &config);
#endif
Manager *mgr, const SpeakerEmbeddingExtractorConfig &config);
virtual int32_t Dim() const = 0;

View File

@@ -8,6 +8,15 @@
#include <utility>
#include <vector>
#if __ANDROID_API__ >= 9
#include "android/asset_manager.h"
#include "android/asset_manager_jni.h"
#endif
#if __OHOS__
#include "rawfile/raw_file_manager.h"
#endif
#include "sherpa-onnx/csrc/macros.h"
#include "sherpa-onnx/csrc/onnx-utils.h"
#include "sherpa-onnx/csrc/session.h"
@@ -28,8 +37,8 @@ class SpeakerEmbeddingExtractorModel::Impl {
}
}
#if __ANDROID_API__ >= 9
Impl(AAssetManager *mgr, const SpeakerEmbeddingExtractorConfig &config)
template <typename Manager>
Impl(Manager *mgr, const SpeakerEmbeddingExtractorConfig &config)
: config_(config),
env_(ORT_LOGGING_LEVEL_ERROR),
sess_opts_(GetSessionOptions(config)),
@@ -39,7 +48,6 @@ class SpeakerEmbeddingExtractorModel::Impl {
Init(buf.data(), buf.size());
}
}
#endif
Ort::Value Compute(Ort::Value x) const {
std::array<Ort::Value, 1> inputs = {std::move(x)};
@@ -68,7 +76,11 @@ class SpeakerEmbeddingExtractorModel::Impl {
if (config_.debug) {
std::ostringstream os;
PrintModelMetadata(os, meta_data);
#if __OHOS__
SHERPA_ONNX_LOGE("%{public}s", os.str().c_str());
#else
SHERPA_ONNX_LOGE("%s", os.str().c_str());
#endif
}
Ort::AllocatorWithDefaultOptions allocator; // used in the macro below
@@ -84,8 +96,14 @@ class SpeakerEmbeddingExtractorModel::Impl {
std::string framework;
SHERPA_ONNX_READ_META_DATA_STR(framework, "framework");
if (framework != "wespeaker" && framework != "3d-speaker") {
#if __OHOS__
SHERPA_ONNX_LOGE(
"Expect a wespeaker or a 3d-speaker model, given: %{public}s",
framework.c_str());
#else
SHERPA_ONNX_LOGE("Expect a wespeaker or a 3d-speaker model, given: %s",
framework.c_str());
#endif
exit(-1);
}
}
@@ -111,11 +129,10 @@ SpeakerEmbeddingExtractorModel::SpeakerEmbeddingExtractorModel(
const SpeakerEmbeddingExtractorConfig &config)
: impl_(std::make_unique<Impl>(config)) {}
#if __ANDROID_API__ >= 9
template <typename Manager>
SpeakerEmbeddingExtractorModel::SpeakerEmbeddingExtractorModel(
AAssetManager *mgr, const SpeakerEmbeddingExtractorConfig &config)
Manager *mgr, const SpeakerEmbeddingExtractorConfig &config)
: impl_(std::make_unique<Impl>(mgr, config)) {}
#endif
SpeakerEmbeddingExtractorModel::~SpeakerEmbeddingExtractorModel() = default;
@@ -128,4 +145,14 @@ Ort::Value SpeakerEmbeddingExtractorModel::Compute(Ort::Value x) const {
return impl_->Compute(std::move(x));
}
#if __ANDROID_API__ >= 9
template SpeakerEmbeddingExtractorModel::SpeakerEmbeddingExtractorModel(
AAssetManager *mgr, const SpeakerEmbeddingExtractorConfig &config);
#endif
#if __OHOS__
template SpeakerEmbeddingExtractorModel::SpeakerEmbeddingExtractorModel(
NativeResourceManager *mgr, const SpeakerEmbeddingExtractorConfig &config);
#endif
} // namespace sherpa_onnx

View File

@@ -6,11 +6,6 @@
#include <memory>
#if __ANDROID_API__ >= 9
#include "android/asset_manager.h"
#include "android/asset_manager_jni.h"
#endif
#include "onnxruntime_cxx_api.h" // NOLINT
#include "sherpa-onnx/csrc/speaker-embedding-extractor-model-meta-data.h"
#include "sherpa-onnx/csrc/speaker-embedding-extractor.h"
@@ -22,10 +17,9 @@ class SpeakerEmbeddingExtractorModel {
explicit SpeakerEmbeddingExtractorModel(
const SpeakerEmbeddingExtractorConfig &config);
#if __ANDROID_API__ >= 9
SpeakerEmbeddingExtractorModel(AAssetManager *mgr,
template <typename Manager>
SpeakerEmbeddingExtractorModel(Manager *mgr,
const SpeakerEmbeddingExtractorConfig &config);
#endif
~SpeakerEmbeddingExtractorModel();

View File

@@ -22,11 +22,10 @@ class SpeakerEmbeddingExtractorNeMoImpl : public SpeakerEmbeddingExtractorImpl {
const SpeakerEmbeddingExtractorConfig &config)
: model_(config) {}
#if __ANDROID_API__ >= 9
template <typename Manager>
SpeakerEmbeddingExtractorNeMoImpl(
AAssetManager *mgr, const SpeakerEmbeddingExtractorConfig &config)
Manager *mgr, const SpeakerEmbeddingExtractorConfig &config)
: model_(mgr, config) {}
#endif
int32_t Dim() const override { return model_.GetMetaData().output_dim; }
@@ -54,9 +53,15 @@ class SpeakerEmbeddingExtractorNeMoImpl : public SpeakerEmbeddingExtractorImpl {
std::vector<float> Compute(OnlineStream *s) const override {
int32_t num_frames = s->NumFramesReady() - s->GetNumProcessedFrames();
if (num_frames <= 0) {
#if __OHOS__
SHERPA_ONNX_LOGE(
"Please make sure IsReady(s) returns true. num_frames: %{public}d",
num_frames);
#else
SHERPA_ONNX_LOGE(
"Please make sure IsReady(s) returns true. num_frames: %d",
num_frames);
#endif
return {};
}
@@ -72,8 +77,14 @@ class SpeakerEmbeddingExtractorNeMoImpl : public SpeakerEmbeddingExtractorImpl {
if (meta_data.feature_normalize_type == "per_feature") {
NormalizePerFeature(features.data(), num_frames, feat_dim);
} else {
#if __OHOS__
SHERPA_ONNX_LOGE("Unsupported feature_normalize_type: %{public}s",
meta_data.feature_normalize_type.c_str());
#else
SHERPA_ONNX_LOGE("Unsupported feature_normalize_type: %s",
meta_data.feature_normalize_type.c_str());
#endif
exit(-1);
}
}

View File

@@ -8,6 +8,15 @@
#include <utility>
#include <vector>
#if __ANDROID_API__ >= 9
#include "android/asset_manager.h"
#include "android/asset_manager_jni.h"
#endif
#if __OHOS__
#include "rawfile/raw_file_manager.h"
#endif
#include "sherpa-onnx/csrc/macros.h"
#include "sherpa-onnx/csrc/onnx-utils.h"
#include "sherpa-onnx/csrc/session.h"
@@ -28,8 +37,8 @@ class SpeakerEmbeddingExtractorNeMoModel::Impl {
}
}
#if __ANDROID_API__ >= 9
Impl(AAssetManager *mgr, const SpeakerEmbeddingExtractorConfig &config)
template <typename Manager>
Impl(Manager *mgr, const SpeakerEmbeddingExtractorConfig &config)
: config_(config),
env_(ORT_LOGGING_LEVEL_ERROR),
sess_opts_(GetSessionOptions(config)),
@@ -39,7 +48,6 @@ class SpeakerEmbeddingExtractorNeMoModel::Impl {
Init(buf.data(), buf.size());
}
}
#endif
Ort::Value Compute(Ort::Value x, Ort::Value x_lens) const {
std::array<Ort::Value, 2> inputs = {std::move(x), std::move(x_lens)};
@@ -73,7 +81,11 @@ class SpeakerEmbeddingExtractorNeMoModel::Impl {
if (config_.debug) {
std::ostringstream os;
PrintModelMetadata(os, meta_data);
#if __OHOS__
SHERPA_ONNX_LOGE("%{public}s", os.str().c_str());
#else
SHERPA_ONNX_LOGE("%s", os.str().c_str());
#endif
}
Ort::AllocatorWithDefaultOptions allocator; // used in the macro below
@@ -93,7 +105,12 @@ class SpeakerEmbeddingExtractorNeMoModel::Impl {
std::string framework;
SHERPA_ONNX_READ_META_DATA_STR(framework, "framework");
if (framework != "nemo") {
#if __OHOS__
SHERPA_ONNX_LOGE("Expect a NeMo model, given: %{public}s",
framework.c_str());
#else
SHERPA_ONNX_LOGE("Expect a NeMo model, given: %s", framework.c_str());
#endif
exit(-1);
}
}
@@ -119,11 +136,10 @@ SpeakerEmbeddingExtractorNeMoModel::SpeakerEmbeddingExtractorNeMoModel(
const SpeakerEmbeddingExtractorConfig &config)
: impl_(std::make_unique<Impl>(config)) {}
#if __ANDROID_API__ >= 9
template <typename Manager>
SpeakerEmbeddingExtractorNeMoModel::SpeakerEmbeddingExtractorNeMoModel(
AAssetManager *mgr, const SpeakerEmbeddingExtractorConfig &config)
Manager *mgr, const SpeakerEmbeddingExtractorConfig &config)
: impl_(std::make_unique<Impl>(mgr, config)) {}
#endif
SpeakerEmbeddingExtractorNeMoModel::~SpeakerEmbeddingExtractorNeMoModel() =
default;
@@ -142,4 +158,14 @@ OrtAllocator *SpeakerEmbeddingExtractorNeMoModel::Allocator() const {
return impl_->Allocator();
}
#if __ANDROID_API__ >= 9
template SpeakerEmbeddingExtractorNeMoModel::SpeakerEmbeddingExtractorNeMoModel(
AAssetManager *mgr, const SpeakerEmbeddingExtractorConfig &config);
#endif
#if __OHOS__
template SpeakerEmbeddingExtractorNeMoModel::SpeakerEmbeddingExtractorNeMoModel(
NativeResourceManager *mgr, const SpeakerEmbeddingExtractorConfig &config);
#endif
} // namespace sherpa_onnx

View File

@@ -6,11 +6,6 @@
#include <memory>
#if __ANDROID_API__ >= 9
#include "android/asset_manager.h"
#include "android/asset_manager_jni.h"
#endif
#include "onnxruntime_cxx_api.h" // NOLINT
#include "sherpa-onnx/csrc/speaker-embedding-extractor-nemo-model-meta-data.h"
#include "sherpa-onnx/csrc/speaker-embedding-extractor.h"
@@ -22,10 +17,9 @@ class SpeakerEmbeddingExtractorNeMoModel {
explicit SpeakerEmbeddingExtractorNeMoModel(
const SpeakerEmbeddingExtractorConfig &config);
#if __ANDROID_API__ >= 9
template <typename Manager>
SpeakerEmbeddingExtractorNeMoModel(
AAssetManager *mgr, const SpeakerEmbeddingExtractorConfig &config);
#endif
Manager *mgr, const SpeakerEmbeddingExtractorConfig &config);
~SpeakerEmbeddingExtractorNeMoModel();

View File

@@ -6,6 +6,15 @@
#include <vector>
#if __ANDROID_API__ >= 9
#include "android/asset_manager.h"
#include "android/asset_manager_jni.h"
#endif
#if __OHOS__
#include "rawfile/raw_file_manager.h"
#endif
#include "sherpa-onnx/csrc/file-utils.h"
#include "sherpa-onnx/csrc/macros.h"
#include "sherpa-onnx/csrc/speaker-embedding-extractor-impl.h"
@@ -55,11 +64,10 @@ SpeakerEmbeddingExtractor::SpeakerEmbeddingExtractor(
const SpeakerEmbeddingExtractorConfig &config)
: impl_(SpeakerEmbeddingExtractorImpl::Create(config)) {}
#if __ANDROID_API__ >= 9
template <typename Manager>
SpeakerEmbeddingExtractor::SpeakerEmbeddingExtractor(
AAssetManager *mgr, const SpeakerEmbeddingExtractorConfig &config)
Manager *mgr, const SpeakerEmbeddingExtractorConfig &config)
: impl_(SpeakerEmbeddingExtractorImpl::Create(mgr, config)) {}
#endif
SpeakerEmbeddingExtractor::~SpeakerEmbeddingExtractor() = default;
@@ -77,4 +85,14 @@ std::vector<float> SpeakerEmbeddingExtractor::Compute(OnlineStream *s) const {
return impl_->Compute(s);
}
#if __ANDROID_API__ >= 9
template SpeakerEmbeddingExtractor::SpeakerEmbeddingExtractor(
AAssetManager *mgr, const SpeakerEmbeddingExtractorConfig &config);
#endif
#if __OHOS__
template SpeakerEmbeddingExtractor::SpeakerEmbeddingExtractor(
NativeResourceManager *mgr, const SpeakerEmbeddingExtractorConfig &config);
#endif
} // namespace sherpa_onnx

View File

@@ -9,11 +9,6 @@
#include <string>
#include <vector>
#if __ANDROID_API__ >= 9
#include "android/asset_manager.h"
#include "android/asset_manager_jni.h"
#endif
#include "sherpa-onnx/csrc/online-stream.h"
#include "sherpa-onnx/csrc/parse-options.h"
@@ -45,10 +40,9 @@ class SpeakerEmbeddingExtractor {
explicit SpeakerEmbeddingExtractor(
const SpeakerEmbeddingExtractorConfig &config);
#if __ANDROID_API__ >= 9
SpeakerEmbeddingExtractor(AAssetManager *mgr,
template <typename Manager>
SpeakerEmbeddingExtractor(Manager *mgr,
const SpeakerEmbeddingExtractorConfig &config);
#endif
~SpeakerEmbeddingExtractor();