EngineX-Iluvatar/enginex-mr_series-sherpa-onnx
Archived
16
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

Fix keyword spotting. (#1689)

Browse Source 
Reset the stream right after detecting a keyword
This commit is contained in:
Fangjun Kuang
2025-01-20 16:41:10 +08:00
committed by GitHub
parent b943341fb1
commit 8b989a851c
43 changed files with 813 additions and 293 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
python-api-examples/keyword-spotter-from-microphone.py
View File

@@ -169,6 +169,8 @@ def main():
print("Started! Please speak")
idx = 0
sample_rate = 16000
samples_per_read = int(0.1 * sample_rate) # 0.1 second = 100 ms
stream = keyword_spotter.create_stream()
@@ -179,9 +181,12 @@ def main():
stream.accept_waveform(sample_rate, samples)
while keyword_spotter.is_ready(stream):
keyword_spotter.decode_stream(stream)
result = keyword_spotter.get_result(stream)
if result:
print("\r{}".format(result), end="", flush=True)
result = keyword_spotter.get_result(stream)
if result:
print(f"{idx}: {result }")
idx += 1
# Remember to reset stream right after detecting a keyword
keyword_spotter.reset_stream(stream)
if __name__ == "__main__":

235
python-api-examples/keyword-spotter.py
View File

@@ -18,122 +18,6 @@ import numpy as np
import sherpa_onnx
def get_args():
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter
)
parser.add_argument(
"--tokens",
type=str,
help="Path to tokens.txt",
)
parser.add_argument(
"--encoder",
type=str,
help="Path to the transducer encoder model",
)
parser.add_argument(
"--decoder",
type=str,
help="Path to the transducer decoder model",
)
parser.add_argument(
"--joiner",
type=str,
help="Path to the transducer joiner model",
)
parser.add_argument(
"--num-threads",
type=int,
default=1,
help="Number of threads for neural network computation",
)
parser.add_argument(
"--provider",
type=str,
default="cpu",
help="Valid values: cpu, cuda, coreml",
)
parser.add_argument(
"--max-active-paths",
type=int,
default=4,
help="""
It specifies number of active paths to keep during decoding.
""",
)
parser.add_argument(
"--num-trailing-blanks",
type=int,
default=1,
help="""The number of trailing blanks a keyword should be followed. Setting
to a larger value (e.g. 8) when your keywords has overlapping tokens
between each other.
""",
)
parser.add_argument(
"--keywords-file",
type=str,
help="""
The file containing keywords, one words/phrases per line, and for each
phrase the bpe/cjkchar/pinyin are separated by a space. For example:
▁HE LL O ▁WORLD
x iǎo ài t óng x ué
""",
)
parser.add_argument(
"--keywords-score",
type=float,
default=1.0,
help="""
The boosting score of each token for keywords. The larger the easier to
survive beam search.
""",
)
parser.add_argument(
"--keywords-threshold",
type=float,
default=0.25,
help="""
The trigger threshold (i.e. probability) of the keyword. The larger the
harder to trigger.
""",
)
parser.add_argument(
"sound_files",
type=str,
nargs="+",
help="The input sound file(s) to decode. Each file must be of WAVE"
"format with a single channel, and each sample has 16-bit, "
"i.e., int16_t. "
"The sample rate of the file can be arbitrary and does not need to "
"be 16 kHz",
)
return parser.parse_args()
def assert_file_exists(filename: str):
assert Path(filename).is_file(), (
f"{filename} does not exist!\n"
"Please refer to "
"https://k2-fsa.github.io/sherpa/onnx/kws/pretrained_models/index.html to download it"
)
def read_wave(wave_filename: str) -> Tuple[np.ndarray, int]:
"""
Args:
@@ -159,83 +43,74 @@ def read_wave(wave_filename: str) -> Tuple[np.ndarray, int]:
return samples_float32, f.getframerate()
def create_keyword_spotter():
kws = sherpa_onnx.KeywordSpotter(
tokens="./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01/tokens.txt",
encoder="./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01/encoder-epoch-12-avg-2-chunk-16-left-64.onnx",
decoder="./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01/decoder-epoch-12-avg-2-chunk-16-left-64.onnx",
joiner="./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01/joiner-epoch-12-avg-2-chunk-16-left-64.onnx",
num_threads=2,
keywords_file="./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01/test_wavs/test_keywords.txt",
provider="cpu",
)
return kws
def main():
args = get_args()
assert_file_exists(args.tokens)
assert_file_exists(args.encoder)
assert_file_exists(args.decoder)
assert_file_exists(args.joiner)
kws = create_keyword_spotter()
assert Path(
args.keywords_file
).is_file(), (
f"keywords_file : {args.keywords_file} not exist, please provide a valid path."
wave_filename = (
"./sherpa-onnx-kws-zipformer-wenetspeech-3.3M-2024-01-01/test_wavs/3.wav"
)
keyword_spotter = sherpa_onnx.KeywordSpotter(
tokens=args.tokens,
encoder=args.encoder,
decoder=args.decoder,
joiner=args.joiner,
num_threads=args.num_threads,
max_active_paths=args.max_active_paths,
keywords_file=args.keywords_file,
keywords_score=args.keywords_score,
keywords_threshold=args.keywords_threshold,
num_trailing_blanks=args.num_trailing_blanks,
provider=args.provider,
)
samples, sample_rate = read_wave(wave_filename)
print("Started!")
start_time = time.time()
tail_paddings = np.zeros(int(0.66 * sample_rate), dtype=np.float32)
streams = []
total_duration = 0
for wave_filename in args.sound_files:
assert_file_exists(wave_filename)
samples, sample_rate = read_wave(wave_filename)
duration = len(samples) / sample_rate
total_duration += duration
print("----------Use pre-defined keywords----------")
s = kws.create_stream()
s.accept_waveform(sample_rate, samples)
s.accept_waveform(sample_rate, tail_paddings)
s.input_finished()
while kws.is_ready(s):
kws.decode_stream(s)
r = kws.get_result(s)
if r != "":
# Remember to call reset right after detected a keyword
kws.reset_stream(s)
s = keyword_spotter.create_stream()
print(f"Detected {r}")
s.accept_waveform(sample_rate, samples)
print("----------Use pre-defined keywords + add a new keyword----------")
tail_paddings = np.zeros(int(0.66 * sample_rate), dtype=np.float32)
s.accept_waveform(sample_rate, tail_paddings)
s = kws.create_stream("y ǎn y uán @演员")
s.accept_waveform(sample_rate, samples)
s.accept_waveform(sample_rate, tail_paddings)
s.input_finished()
while kws.is_ready(s):
kws.decode_stream(s)
r = kws.get_result(s)
if r != "":
# Remember to call reset right after detected a keyword
kws.reset_stream(s)
s.input_finished()
print(f"Detected {r}")
streams.append(s)
print("----------Use pre-defined keywords + add 2 new keywords----------")
results = [""] * len(streams)
while True:
ready_list = []
for i, s in enumerate(streams):
if keyword_spotter.is_ready(s):
ready_list.append(s)
r = keyword_spotter.get_result(s)
if r:
results[i] += f"{r}/"
print(f"{r} is detected.")
if len(ready_list) == 0:
break
keyword_spotter.decode_streams(ready_list)
end_time = time.time()
print("Done!")
s = kws.create_stream("y ǎn y uán @演员/zh ī m íng @知名")
s.accept_waveform(sample_rate, samples)
s.accept_waveform(sample_rate, tail_paddings)
s.input_finished()
while kws.is_ready(s):
kws.decode_stream(s)
r = kws.get_result(s)
if r != "":
# Remember to call reset right after detected a keyword
kws.reset_stream(s)
for wave_filename, result in zip(args.sound_files, results):
print(f"{wave_filename}\n{result}")
print("-" * 10)
elapsed_seconds = end_time - start_time
rtf = elapsed_seconds / total_duration
print(f"num_threads: {args.num_threads}")
print(f"Wave duration: {total_duration:.3f} s")
print(f"Elapsed time: {elapsed_seconds:.3f} s")
print(
f"Real time factor (RTF): {elapsed_seconds:.3f}/{total_duration:.3f} = {rtf:.3f}"
)
print(f"Detected {r}")
if __name__ == "__main__":