diff --git a/cmake/cmake_extension.py b/cmake/cmake_extension.py index 638bcd41..1e863193 100644 --- a/cmake/cmake_extension.py +++ b/cmake/cmake_extension.py @@ -167,6 +167,7 @@ class BuildExtension(build_ext): if enable_alsa(): binaries += ["sherpa-onnx-alsa"] binaries += ["sherpa-onnx-offline-tts-play-alsa"] + binaries += ["sherpa-onnx-alsa-offline-speaker-identification"] if is_windows(): binaries += ["kaldi-native-fbank-core.dll"] diff --git a/setup.py b/setup.py index cc8199d3..47decb0c 100644 --- a/setup.py +++ b/setup.py @@ -60,6 +60,7 @@ def get_binaries_to_install(): if enable_alsa(): binaries += ["sherpa-onnx-alsa"] binaries += ["sherpa-onnx-offline-tts-play-alsa"] + binaries += ["sherpa-onnx-alsa-offline-speaker-identification"] if is_windows(): binaries += ["kaldi-native-fbank-core.dll"] diff --git a/sherpa-onnx/csrc/CMakeLists.txt b/sherpa-onnx/csrc/CMakeLists.txt index cc576634..5799fe13 100644 --- a/sherpa-onnx/csrc/CMakeLists.txt +++ b/sherpa-onnx/csrc/CMakeLists.txt @@ -231,10 +231,12 @@ endif() if(SHERPA_ONNX_HAS_ALSA AND SHERPA_ONNX_ENABLE_BINARY) add_executable(sherpa-onnx-alsa sherpa-onnx-alsa.cc alsa.cc) add_executable(sherpa-onnx-offline-tts-play-alsa sherpa-onnx-offline-tts-play-alsa.cc alsa-play.cc) + add_executable(sherpa-onnx-alsa-offline-speaker-identification sherpa-onnx-alsa-offline-speaker-identification.cc alsa.cc) set(exes sherpa-onnx-alsa sherpa-onnx-offline-tts-play-alsa + sherpa-onnx-alsa-offline-speaker-identification ) foreach(exe IN LISTS exes) target_link_libraries(${exe} sherpa-onnx-core) diff --git a/sherpa-onnx/csrc/sherpa-onnx-alsa-offline-speaker-identification.cc b/sherpa-onnx/csrc/sherpa-onnx-alsa-offline-speaker-identification.cc new file mode 100644 index 00000000..5ecf99a5 --- /dev/null +++ b/sherpa-onnx/csrc/sherpa-onnx-alsa-offline-speaker-identification.cc @@ -0,0 +1,287 @@ +// sherpa-onnx/csrc/sherpa-onnx-alsa-offline-speaker-identification.cc +// +// Copyright (c) 2024 Xiaomi Corporation + +#include +#include +#include + +#include +#include +#include // NOLINT +#include +#include // NOLINT + +#include "sherpa-onnx/csrc/alsa.h" +#include "sherpa-onnx/csrc/macros.h" +#include "sherpa-onnx/csrc/microphone.h" +#include "sherpa-onnx/csrc/speaker-embedding-extractor.h" +#include "sherpa-onnx/csrc/speaker-embedding-manager.h" +#include "sherpa-onnx/csrc/wave-reader.h" + +enum class State { + kIdle, + kRecording, + kComputing, +}; + +State state = State::kIdle; + +// true to stop the program and exit +bool stop = false; + +std::vector samples; +std::mutex samples_mutex; + +static void DetectKeyPress() { + SHERPA_ONNX_LOGE("\nPress Enter to start"); + int32_t key; + while (!stop && (key = getchar())) { + if (key != 0x0a) { + continue; + } + + switch (state) { + case State::kIdle: + SHERPA_ONNX_LOGE("\nStart recording. Press Enter to stop recording"); + state = State::kRecording; + { + std::lock_guard lock(samples_mutex); + samples.clear(); + } + break; + case State::kRecording: + SHERPA_ONNX_LOGE("\nStop recording. Computing ..."); + state = State::kComputing; + break; + case State::kComputing: + break; + } + } +} + +static void Record(const char *device_name, int32_t expected_sample_rate) { + sherpa_onnx::Alsa alsa(device_name); + + if (alsa.GetExpectedSampleRate() != expected_sample_rate) { + fprintf(stderr, "sample rate: %d != %d\n", alsa.GetExpectedSampleRate(), + expected_sample_rate); + exit(-1); + } + + int32_t chunk = 0.1 * alsa.GetActualSampleRate(); + while (!stop) { + std::lock_guard lock(samples_mutex); + const std::vector &s = alsa.Read(chunk); + samples.insert(samples.end(), s.begin(), s.end()); + } +} + +static void Handler(int32_t sig) { + stop = true; + fprintf(stderr, "\nCaught Ctrl + C. Press Enter to exit\n"); +} + +static std::vector> ComputeEmbeddings( + const std::vector &filenames, + sherpa_onnx::SpeakerEmbeddingExtractor *extractor) { + std::vector> embedding_list; + embedding_list.reserve(filenames.size()); + + for (const auto &f : filenames) { + int32_t sampling_rate = -1; + + bool is_ok = false; + const std::vector samples = + sherpa_onnx::ReadWave(f, &sampling_rate, &is_ok); + + if (!is_ok) { + fprintf(stderr, "Failed to read %s\n", f.c_str()); + exit(-1); + } + + auto s = extractor->CreateStream(); + s->AcceptWaveform(sampling_rate, samples.data(), samples.size()); + s->InputFinished(); + auto embedding = extractor->Compute(s.get()); + embedding_list.push_back(embedding); + } + return embedding_list; +} + +static std::unordered_map> +ReadSpeakerFile(const std::string &filename) { + std::unordered_map> ans; + + std::ifstream is(filename); + if (!is) { + fprintf(stderr, "Failed to open %s", filename.c_str()); + exit(0); + } + + std::string line; + std::string name; + std::string path; + + while (std::getline(is, line)) { + std::istringstream iss(line); + name.clear(); + path.clear(); + + iss >> name >> path; + if (!iss || !iss.eof() || name.empty() || path.empty()) { + fprintf(stderr, "Invalid line: %s\n", line.c_str()); + exit(-1); + } + ans[name].push_back(path); + } + + return ans; +} + +int32_t main(int32_t argc, char *argv[]) { + signal(SIGINT, Handler); + + const char *kUsageMessage = R"usage( +This program shows how to use non-streaming speaker identification. +Usage: + +(1) Prepare a text file containing speaker related files. + +Each line in the text file contains two columns. The first column is the +speaker name, while the second column contains the wave file of the speaker. + +If the text file contains multiple wave files for the same speaker, then the +embeddings of these files are averaged. + +An example text file is given below: + + foo /path/to/a.wav + bar /path/to/b.wav + foo /path/to/c.wav + foobar /path/to/d.wav + +Each wave file should contain only a single channel; the sample format +should be int16_t; the sample rate can be arbitrary. + +(2) Download a model for computing speaker embeddings + +Please visit +https://github.com/k2-fsa/sherpa-onnx/releases/tag/speaker-recongition-models +to download a model. An example is given below: + + wget https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-recongition-models/wespeaker_zh_cnceleb_resnet34.onnx + +Note that `zh` means Chinese, while `en` means English. + +(3) Run it ! + + ./bin/sherpa-onnx-alsa-offline-speaker-identification \ + --model=/path/to/your-model.onnx \ + --speaker-file=/path/to/speaker.txt \ + device_name + +The device name specifies which microphone to use in case there are several +on your system. You can use + + arecord -l + +to find all available microphones on your computer. For instance, if it outputs + +**** List of CAPTURE Hardware Devices **** +card 3: UACDemoV10 [UACDemoV1.0], device 0: USB Audio [USB Audio] + Subdevices: 1/1 + Subdevice #0: subdevice #0 + +and if you want to select card 3 and the device 0 on that card, please use: + plughw:3,0 +as the device_name. + +)usage"; + + sherpa_onnx::ParseOptions po(kUsageMessage); + float threshold = 0.5; + std::string speaker_file; + + po.Register("threshold", &threshold, + "Threshold for comparing embedding scores."); + + po.Register("speaker-file", &speaker_file, "Path to speaker.txt"); + + sherpa_onnx::SpeakerEmbeddingExtractorConfig config; + config.Register(&po); + + po.Read(argc, argv); + if (po.NumArgs() != 1) { + fprintf(stderr, "Please provide only 1 argument: the device name\n"); + po.PrintUsage(); + exit(EXIT_FAILURE); + } + + fprintf(stderr, "%s\n", config.ToString().c_str()); + + if (!config.Validate()) { + fprintf(stderr, "Errors in config! Please use --help to view the usage.\n"); + return -1; + } + + SHERPA_ONNX_LOGE("\nCreating extractor ..."); + sherpa_onnx::SpeakerEmbeddingExtractor extractor(config); + SHERPA_ONNX_LOGE("\nextractor created!"); + + sherpa_onnx::SpeakerEmbeddingManager manager(extractor.Dim()); + + auto name2files = ReadSpeakerFile(speaker_file); + for (const auto &p : name2files) { + SHERPA_ONNX_LOGE("\nProcessing speaker %s", p.first.c_str()); + auto embedding_list = ComputeEmbeddings(p.second, &extractor); + manager.Add(p.first, embedding_list); + } + + std::string device_name = po.GetArg(1); + fprintf(stderr, "Use recording device: %s\n", device_name.c_str()); + int32_t sample_rate = 16000; + + std::thread t(DetectKeyPress); + std::thread t2(Record, device_name.c_str(), sample_rate); + + while (!stop) { + switch (state) { + case State::kIdle: + break; + case State::kRecording: + break; + case State::kComputing: { + std::vector buf; + { + std::lock_guard lock(samples_mutex); + buf = std::move(samples); + } + + auto s = extractor.CreateStream(); + s->AcceptWaveform(sample_rate, buf.data(), buf.size()); + s->InputFinished(); + auto embedding = extractor.Compute(s.get()); + auto name = manager.Search(embedding.data(), threshold); + + if (name.empty()) { + name = "--Unknown--"; + } + + SHERPA_ONNX_LOGE("\nDone!\nDetected speaker is: %s", name.c_str()); + + state = State::kIdle; + SHERPA_ONNX_LOGE("\nPress Enter to start"); + break; + } + } + + using namespace std::chrono_literals; + std::this_thread::sleep_for(20ms); // sleep for 20ms + } + + t.join(); + t2.join(); + + return 0; +} diff --git a/sherpa-onnx/csrc/sherpa-onnx-alsa.cc b/sherpa-onnx/csrc/sherpa-onnx-alsa.cc index 16d9bab4..ccd909bb 100644 --- a/sherpa-onnx/csrc/sherpa-onnx-alsa.cc +++ b/sherpa-onnx/csrc/sherpa-onnx-alsa.cc @@ -34,14 +34,14 @@ Usage: --provider=cpu \ --num-threads=2 \ --decoding-method=greedy_search \ - device_name \ + device_name Please refer to https://k2-fsa.github.io/sherpa/onnx/pretrained_models/index.html for a list of pre-trained models to download. The device name specifies which microphone to use in case there are several -on you system. You can use +on your system. You can use arecord -l