Add Go API for speaker identification (#718)

go-api-examples/README.md

@@ -26,4 +26,8 @@ for details.
 - [./vad-spoken-language-identification](./vad-spoken-language-identification) It shows how to use silero VAD + Whisper
   for spoken language identification.
 
+- [./speaker-identification](./speaker-identification) It shows how to use Go API for speaker identification.
+
+- [./vad-speaker-identification](./vad-speaker-identification) It shows how to use Go API for VAD + speaker identification.
+
 [sherpa-onnx]: https://github.com/k2-fsa/sherpa-onnx

go-api-examples/speaker-identification/go.mod

@@ -0,0 +1,3 @@
+module speaker-identification
+
+go 1.12

go-api-examples/speaker-identification/main.go

@@ -0,0 +1,146 @@
+package main
+
+import (
+	sherpa "github.com/k2-fsa/sherpa-onnx-go/sherpa_onnx"
+	"log"
+)
+
+func createSpeakerEmbeddingExtractor() *sherpa.SpeakerEmbeddingExtractor {
+	config := sherpa.SpeakerEmbeddingExtractorConfig{}
+
+	// Please download the model from
+	// https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-recongition-models/3dspeaker_speech_campplus_sv_zh-cn_16k-common.onnx
+	//
+	// You can find more models at
+	// https://github.com/k2-fsa/sherpa-onnx/releases/tag/speaker-recongition-models
+
+	config.Model = "./3dspeaker_speech_campplus_sv_zh-cn_16k-common.onnx"
+	config.NumThreads = 1
+	config.Debug = 1
+	config.Provider = "cpu"
+
+	ex := sherpa.NewSpeakerEmbeddingExtractor(&config)
+	return ex
+}
+
+func computeEmbeddings(ex *sherpa.SpeakerEmbeddingExtractor, files []string) [][]float32 {
+	embeddings := make([][]float32, len(files))
+
+	for i, f := range files {
+		wave := sherpa.ReadWave(f)
+
+		stream := ex.CreateStream()
+		defer sherpa.DeleteOnlineStream(stream)
+		stream.AcceptWaveform(wave.SampleRate, wave.Samples)
+		stream.InputFinished()
+		embeddings[i] = ex.Compute(stream)
+	}
+
+	return embeddings
+
+}
+
+func registerSpeakers(ex *sherpa.SpeakerEmbeddingExtractor, manager *sherpa.SpeakerEmbeddingManager) {
+	// Please download the test data from
+	// https://github.com/csukuangfj/sr-data
+	spk1_files := []string{
+		"./sr-data/enroll/fangjun-sr-1.wav",
+		"./sr-data/enroll/fangjun-sr-2.wav",
+		"./sr-data/enroll/fangjun-sr-3.wav",
+	}
+
+	spk2_files := []string{
+		"./sr-data/enroll/leijun-sr-1.wav",
+		"./sr-data/enroll/leijun-sr-2.wav",
+	}
+
+	spk1_embeddings := computeEmbeddings(ex, spk1_files)
+	spk2_embeddings := computeEmbeddings(ex, spk2_files)
+
+	ok := manager.RegisterV("fangjun", spk1_embeddings)
+	if !ok {
+		panic("Failed to register fangjun")
+	}
+
+	ok = manager.RegisterV("leijun", spk2_embeddings)
+	if !ok {
+		panic("Failed to register leijun")
+	}
+
+	if !manager.Contains("fangjun") {
+		panic("Failed to find fangjun")
+	}
+
+	if !manager.Contains("leijun") {
+		panic("Failed to find leijun")
+	}
+
+	if manager.NumSpeakers() != 2 {
+		panic("There should be only 2 speakers")
+	}
+
+	all_speakers := manager.AllSpeakers()
+	log.Printf("All speakers: %v\n", all_speakers)
+}
+
+func main() {
+	log.SetFlags(log.LstdFlags | log.Lmicroseconds)
+
+	ex := createSpeakerEmbeddingExtractor()
+	defer sherpa.DeleteSpeakerEmbeddingExtractor(ex)
+
+	manager := sherpa.NewSpeakerEmbeddingManager(ex.Dim())
+	defer sherpa.DeleteSpeakerEmbeddingManager(manager)
+	registerSpeakers(ex, manager)
+
+	// Please download the test data from
+	// https://github.com/csukuangfj/sr-data
+	test1 := "./sr-data/test/fangjun-test-sr-1.wav"
+	embeddings := computeEmbeddings(ex, []string{test1})[0]
+	threshold := float32(0.6)
+	name := manager.Search(embeddings, threshold)
+	if len(name) > 0 {
+		log.Printf("%v matches %v", test1, name)
+	} else {
+		log.Printf("No matches found for %v", test1)
+	}
+
+	test2 := "./sr-data/test/leijun-test-sr-1.wav"
+	embeddings = computeEmbeddings(ex, []string{test2})[0]
+	name = manager.Search(embeddings, threshold)
+	if len(name) > 0 {
+		log.Printf("%v matches %v", test2, name)
+	} else {
+		log.Printf("No matches found for %v", test2)
+	}
+
+	test3 := "./sr-data/test/liudehua-test-sr-1.wav"
+	embeddings = computeEmbeddings(ex, []string{test3})[0]
+	name = manager.Search(embeddings, threshold)
+	if len(name) > 0 {
+		log.Printf("%v matches %v", test3, name)
+	} else {
+		log.Printf("No matches found for %v", test3)
+	}
+
+	if !manager.Remove("fangjun") {
+		panic("Failed to deregister fangjun")
+	} else {
+		log.Print("fangjun deregistered\n")
+	}
+
+	test1 = "./sr-data/test/fangjun-test-sr-1.wav"
+	embeddings = computeEmbeddings(ex, []string{test1})[0]
+	name = manager.Search(embeddings, threshold)
+	if len(name) > 0 {
+		log.Printf("%v matches %v", test1, name)
+	} else {
+		log.Printf("No matches found for %v", test1)
+	}
+}
+
+func chk(err error) {
+	if err != nil {
+		panic(err)
+	}
+}

go-api-examples/speaker-identification/run.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+if [ ! -f ./3dspeaker_speech_campplus_sv_zh-cn_16k-common.onnx ]; then
+  curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-recongition-models/3dspeaker_speech_campplus_sv_zh-cn_16k-common.onnx
+fi
+
+if [ ! -f ./sr-data/enroll/fangjun-sr-1.wav ]; then
+  git clone https://github.com/csukuangfj/sr-data
+fi
+
+go mod tidy
+go build
+./speaker-identification

go-api-examples/vad-asr-paraformer/main.go

@@ -104,7 +104,7 @@ func main() {
 
 			duration := float32(len(speechSegment.Samples)) / float32(config.SampleRate)
 
-			audio := &sherpa.GeneratedAudio{}
+			audio := &sherpa.Wave{}
 			audio.Samples = speechSegment.Samples
 			audio.SampleRate = config.SampleRate
 
@@ -120,7 +120,7 @@ func main() {
 	chk(s.Stop())
 }
 
-func decode(recognizer *sherpa.OfflineRecognizer, audio *sherpa.GeneratedAudio, id int) {
+func decode(recognizer *sherpa.OfflineRecognizer, audio *sherpa.Wave, id int) {
 	stream := sherpa.NewOfflineStream(recognizer)
 	defer sherpa.DeleteOfflineStream(stream)
 	stream.AcceptWaveform(audio.SampleRate, audio.Samples)

go-api-examples/vad-asr-whisper/main.go

@@ -102,7 +102,7 @@ func main() {
 
 			duration := float32(len(speechSegment.Samples)) / float32(config.SampleRate)
 
-			audio := &sherpa.GeneratedAudio{}
+			audio := &sherpa.Wave{}
 			audio.Samples = speechSegment.Samples
 			audio.SampleRate = config.SampleRate
 
@@ -118,7 +118,7 @@ func main() {
 	chk(s.Stop())
 }
 
-func decode(recognizer *sherpa.OfflineRecognizer, audio *sherpa.GeneratedAudio, id int) {
+func decode(recognizer *sherpa.OfflineRecognizer, audio *sherpa.Wave, id int) {
 	stream := sherpa.NewOfflineStream(recognizer)
 	defer sherpa.DeleteOfflineStream(stream)
 	stream.AcceptWaveform(audio.SampleRate, audio.Samples)

go-api-examples/vad-speaker-identification/go.mod

@@ -0,0 +1,3 @@
+module vad-speaker-identification
+
+go 1.12

go-api-examples/vad-speaker-identification/main.go

@@ -0,0 +1,221 @@
+package main
+
+import (
+	"fmt"
+	"github.com/gordonklaus/portaudio"
+	sherpa "github.com/k2-fsa/sherpa-onnx-go/sherpa_onnx"
+	"log"
+)
+
+func createSpeakerEmbeddingExtractor() *sherpa.SpeakerEmbeddingExtractor {
+	config := sherpa.SpeakerEmbeddingExtractorConfig{}
+
+	// Please download the model from
+	// https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-recongition-models/3dspeaker_speech_campplus_sv_zh-cn_16k-common.onnx
+	//
+	// You can find more models at
+	// https://github.com/k2-fsa/sherpa-onnx/releases/tag/speaker-recongition-models
+
+	config.Model = "./3dspeaker_speech_campplus_sv_zh-cn_16k-common.onnx"
+	config.NumThreads = 2
+	config.Debug = 1
+	config.Provider = "cpu"
+
+	ex := sherpa.NewSpeakerEmbeddingExtractor(&config)
+	return ex
+}
+
+func computeEmbeddings(ex *sherpa.SpeakerEmbeddingExtractor, files []string) [][]float32 {
+	embeddings := make([][]float32, len(files))
+
+	for i, f := range files {
+		wave := sherpa.ReadWave(f)
+
+		stream := ex.CreateStream()
+		defer sherpa.DeleteOnlineStream(stream)
+		stream.AcceptWaveform(wave.SampleRate, wave.Samples)
+		stream.InputFinished()
+		embeddings[i] = ex.Compute(stream)
+	}
+
+	return embeddings
+
+}
+
+func registerSpeakers(ex *sherpa.SpeakerEmbeddingExtractor, manager *sherpa.SpeakerEmbeddingManager) {
+	// Please download the test data from
+	// https://github.com/csukuangfj/sr-data
+	spk1_files := []string{
+		"./sr-data/enroll/fangjun-sr-1.wav",
+		"./sr-data/enroll/fangjun-sr-2.wav",
+		"./sr-data/enroll/fangjun-sr-3.wav",
+	}
+
+	spk2_files := []string{
+		"./sr-data/enroll/leijun-sr-1.wav",
+		"./sr-data/enroll/leijun-sr-2.wav",
+	}
+
+	spk1_embeddings := computeEmbeddings(ex, spk1_files)
+	spk2_embeddings := computeEmbeddings(ex, spk2_files)
+
+	ok := manager.RegisterV("fangjun", spk1_embeddings)
+	if !ok {
+		panic("Failed to register fangjun")
+	}
+
+	ok = manager.RegisterV("leijun", spk2_embeddings)
+	if !ok {
+		panic("Failed to register leijun")
+	}
+
+	if !manager.Contains("fangjun") {
+		panic("Failed to find fangjun")
+	}
+
+	if !manager.Contains("leijun") {
+		panic("Failed to find leijun")
+	}
+
+	if manager.NumSpeakers() != 2 {
+		panic("There should be only 2 speakers")
+	}
+
+	all_speakers := manager.AllSpeakers()
+	log.Printf("All speakers: %v\n", all_speakers)
+}
+
+func createVad() *sherpa.VoiceActivityDetector {
+	config := sherpa.VadModelConfig{}
+
+	// Please download silero_vad.onnx from
+	// https://github.com/snakers4/silero-vad/blob/master/files/silero_vad.onnx
+
+	config.SileroVad.Model = "./silero_vad.onnx"
+	config.SileroVad.Threshold = 0.5
+	config.SileroVad.MinSilenceDuration = 0.5
+	config.SileroVad.MinSpeechDuration = 0.5
+	config.SileroVad.WindowSize = 512
+	config.SampleRate = 16000
+	config.NumThreads = 1
+	config.Provider = "cpu"
+	config.Debug = 1
+
+	var bufferSizeInSeconds float32 = 20
+
+	vad := sherpa.NewVoiceActivityDetector(&config, bufferSizeInSeconds)
+	return vad
+}
+
+func main() {
+	log.SetFlags(log.LstdFlags | log.Lmicroseconds)
+
+	vad := createVad()
+	defer sherpa.DeleteVoiceActivityDetector(vad)
+
+	ex := createSpeakerEmbeddingExtractor()
+	defer sherpa.DeleteSpeakerEmbeddingExtractor(ex)
+
+	manager := sherpa.NewSpeakerEmbeddingManager(ex.Dim())
+	defer sherpa.DeleteSpeakerEmbeddingManager(manager)
+	registerSpeakers(ex, manager)
+
+	err := portaudio.Initialize()
+	if err != nil {
+		log.Fatalf("Unable to initialize portaudio: %v\n", err)
+	}
+	defer portaudio.Terminate()
+
+	default_device, err := portaudio.DefaultInputDevice()
+	if err != nil {
+		log.Fatal("Failed to get default input device: %v\n", err)
+	}
+	log.Printf("Selected default input device: %s\n", default_device.Name)
+	param := portaudio.StreamParameters{}
+	param.Input.Device = default_device
+	param.Input.Channels = 1
+	param.Input.Latency = default_device.DefaultHighInputLatency
+
+	param.SampleRate = 16000
+	param.FramesPerBuffer = 0
+	param.Flags = portaudio.ClipOff
+
+	// you can choose another value for 0.1 if you want
+	samplesPerCall := int32(param.SampleRate * 0.1) // 0.1 second
+	samples := make([]float32, samplesPerCall)
+
+	s, err := portaudio.OpenStream(param, samples)
+	if err != nil {
+		log.Fatalf("Failed to open the stream")
+	}
+
+	defer s.Close()
+	chk(s.Start())
+
+	log.Print("Started! Please speak")
+	printed := false
+
+	k := 0
+	for {
+		chk(s.Read())
+		vad.AcceptWaveform(samples)
+
+		if vad.IsSpeech() && !printed {
+			printed = true
+			log.Print("Detected speech\n")
+		}
+
+		if !vad.IsSpeech() {
+			printed = false
+		}
+
+		for !vad.IsEmpty() {
+			speechSegment := vad.Front()
+			vad.Pop()
+
+			audio := &sherpa.Wave{}
+			audio.Samples = speechSegment.Samples
+			audio.SampleRate = 16000
+
+			// Now decode it
+			go decode(ex, manager, audio, k)
+
+			k += 1
+		}
+	}
+
+	chk(s.Stop())
+
+}
+
+func chk(err error) {
+	if err != nil {
+		panic(err)
+	}
+}
+
+func decode(ex *sherpa.SpeakerEmbeddingExtractor, manager *sherpa.SpeakerEmbeddingManager, audio *sherpa.GeneratedAudio, id int) {
+	stream := ex.CreateStream()
+	defer sherpa.DeleteOnlineStream(stream)
+
+	stream.AcceptWaveform(audio.SampleRate, audio.Samples)
+	stream.InputFinished()
+	embeddings := ex.Compute(stream)
+	threshold := float32(0.5)
+	name := manager.Search(embeddings, threshold)
+	if len(name) > 0 {
+		log.Printf("Found speaker: %v\n", name)
+	} else {
+		log.Print("Unknown speaker\n")
+		name = "Unknown"
+	}
+
+	duration := float32(len(audio.Samples)) / float32(audio.SampleRate)
+
+	filename := fmt.Sprintf("seg-%d-%.2f-seconds-%s.wav", id, duration, name)
+	ok := audio.Save(filename)
+	if ok {
+		log.Printf("Saved to %s", filename)
+	}
+	log.Print("----------\n")
+}

go-api-examples/vad-speaker-identification/run.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+if [ ! -f ./3dspeaker_speech_campplus_sv_zh-cn_16k-common.onnx ]; then
+  curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-recongition-models/3dspeaker_speech_campplus_sv_zh-cn_16k-common.onnx
+fi
+
+if [ ! -f ./sr-data/enroll/fangjun-sr-1.wav ]; then
+  git clone https://github.com/csukuangfj/sr-data
+fi
+
+go mod tidy
+go build
+./speaker-identification

go-api-examples/vad-spoken-language-identification/main.go

@@ -99,7 +99,7 @@ func main() {
 
 			duration := float32(len(speechSegment.Samples)) / float32(config.SampleRate)
 
-			audio := &sherpa.GeneratedAudio{}
+			audio := &sherpa.Wave{}
 			audio.Samples = speechSegment.Samples
 			audio.SampleRate = config.SampleRate
 
@@ -115,7 +115,7 @@ func main() {
 	chk(s.Stop())
 }
 
-func decode(slid *sherpa.SpokenLanguageIdentification, audio *sherpa.GeneratedAudio, id int) {
+func decode(slid *sherpa.SpokenLanguageIdentification, audio *sherpa.Wave, id int) {
 	stream := slid.CreateStream()
 	defer sherpa.DeleteOfflineStream(stream)