Add Dart API for keyword spotter (#1162)

2024-07-22 10:53:34 +08:00
parent 22a262f5e4
commit ac8223bd8a
17 changed files with 518 additions and 5 deletions
--- a/flutter/sherpa_onnx/lib/sherpa_onnx.dart
+++ b/flutter/sherpa_onnx/lib/sherpa_onnx.dart
@@ -3,6 +3,7 @@ import 'dart:io';
 import 'dart:ffi';

 export 'src/feature_config.dart';
+export 'src/keyword_spotter.dart';
 export 'src/offline_recognizer.dart';
 export 'src/offline_stream.dart';
 export 'src/online_recognizer.dart';
--- a/flutter/sherpa_onnx/lib/src/keyword_spotter.dart
+++ b/flutter/sherpa_onnx/lib/src/keyword_spotter.dart
@@ -0,0 +1,164 @@
+// Copyright (c)  2024  Xiaomi Corporation
+import 'dart:convert';
+import 'dart:ffi';
+
+import 'package:ffi/ffi.dart';
+
+import './feature_config.dart';
+import './online_stream.dart';
+import './online_recognizer.dart';
+import './sherpa_onnx_bindings.dart';
+import './utils.dart';
+
+class KeywordSpotterConfig {
+  const KeywordSpotterConfig({
+    this.feat = const FeatureConfig(),
+    required this.model,
+    this.maxActivePaths = 4,
+    this.numTrailingBlanks = 1,
+    this.keywordsScore = 1.0,
+    this.keywordsThreshold = 0.25,
+    this.keywordsFile = '',
+  });
+
+  @override
+  String toString() {
+    return 'KeywordSpotterConfig(feat: $feat, model: $model, maxActivePaths: $maxActivePaths, numTrailingBlanks: $numTrailingBlanks, keywordsScore: $keywordsScore, keywordsThreshold: $keywordsThreshold, keywordsFile: $keywordsFile)';
+  }
+
+  final FeatureConfig feat;
+  final OnlineModelConfig model;
+
+  final int maxActivePaths;
+  final int numTrailingBlanks;
+
+  final double keywordsScore;
+  final double keywordsThreshold;
+  final String keywordsFile;
+}
+
+class KeywordResult {
+  KeywordResult({required this.keyword});
+
+  @override
+  String toString() {
+    return 'KeywordResult(keyword: $keyword)';
+  }
+
+  final String keyword;
+}
+
+class KeywordSpotter {
+  KeywordSpotter._({required this.ptr, required this.config});
+
+  /// The user is responsible to call the OnlineRecognizer.free()
+  /// method of the returned instance to avoid memory leak.
+  factory KeywordSpotter(KeywordSpotterConfig config) {
+    final c = calloc<SherpaOnnxKeywordSpotterConfig>();
+    c.ref.feat.sampleRate = config.feat.sampleRate;
+    c.ref.feat.featureDim = config.feat.featureDim;
+
+    // transducer
+    c.ref.model.transducer.encoder =
+        config.model.transducer.encoder.toNativeUtf8();
+    c.ref.model.transducer.decoder =
+        config.model.transducer.decoder.toNativeUtf8();
+    c.ref.model.transducer.joiner =
+        config.model.transducer.joiner.toNativeUtf8();
+
+    // paraformer
+    c.ref.model.paraformer.encoder =
+        config.model.paraformer.encoder.toNativeUtf8();
+    c.ref.model.paraformer.decoder =
+        config.model.paraformer.decoder.toNativeUtf8();
+
+    // zipformer2Ctc
+    c.ref.model.zipformer2Ctc.model =
+        config.model.zipformer2Ctc.model.toNativeUtf8();
+
+    c.ref.model.tokens = config.model.tokens.toNativeUtf8();
+    c.ref.model.numThreads = config.model.numThreads;
+    c.ref.model.provider = config.model.provider.toNativeUtf8();
+    c.ref.model.debug = config.model.debug ? 1 : 0;
+    c.ref.model.modelType = config.model.modelType.toNativeUtf8();
+    c.ref.model.modelingUnit = config.model.modelingUnit.toNativeUtf8();
+    c.ref.model.bpeVocab = config.model.bpeVocab.toNativeUtf8();
+
+    c.ref.maxActivePaths = config.maxActivePaths;
+    c.ref.numTrailingBlanks = config.numTrailingBlanks;
+    c.ref.keywordsScore = config.keywordsScore;
+    c.ref.keywordsThreshold = config.keywordsThreshold;
+    c.ref.keywordsFile = config.keywordsFile.toNativeUtf8();
+
+    final ptr = SherpaOnnxBindings.createKeywordSpotter?.call(c) ?? nullptr;
+
+    calloc.free(c.ref.keywordsFile);
+    calloc.free(c.ref.model.bpeVocab);
+    calloc.free(c.ref.model.modelingUnit);
+    calloc.free(c.ref.model.modelType);
+    calloc.free(c.ref.model.provider);
+    calloc.free(c.ref.model.tokens);
+    calloc.free(c.ref.model.zipformer2Ctc.model);
+    calloc.free(c.ref.model.paraformer.encoder);
+    calloc.free(c.ref.model.paraformer.decoder);
+
+    calloc.free(c.ref.model.transducer.encoder);
+    calloc.free(c.ref.model.transducer.decoder);
+    calloc.free(c.ref.model.transducer.joiner);
+    calloc.free(c);
+
+    return KeywordSpotter._(ptr: ptr, config: config);
+  }
+
+  void free() {
+    SherpaOnnxBindings.destroyKeywordSpotter?.call(ptr);
+    ptr = nullptr;
+  }
+
+  /// The user has to invoke stream.free() on the returned instance
+  /// to avoid memory leak
+  OnlineStream createStream({String keywords = ''}) {
+    if (keywords == '') {
+      final p = SherpaOnnxBindings.createKeywordStream?.call(ptr) ?? nullptr;
+      return OnlineStream(ptr: p);
+    }
+
+    final utf8 = keywords.toNativeUtf8();
+    final p =
+        SherpaOnnxBindings.createKeywordStreamWithKeywords?.call(ptr, utf8) ??
+            nullptr;
+    calloc.free(utf8);
+    return OnlineStream(ptr: p);
+  }
+
+  bool isReady(OnlineStream stream) {
+    int ready =
+        SherpaOnnxBindings.isKeywordStreamReady?.call(ptr, stream.ptr) ?? 0;
+
+    return ready == 1;
+  }
+
+  KeywordResult getResult(OnlineStream stream) {
+    final json =
+        SherpaOnnxBindings.getKeywordResultAsJson?.call(ptr, stream.ptr) ??
+            nullptr;
+    if (json == nullptr) {
+      return KeywordResult(keyword: '');
+    }
+
+    final parsedJson = jsonDecode(toDartString(json));
+
+    SherpaOnnxBindings.freeKeywordResultJson?.call(json);
+
+    return KeywordResult(
+      keyword: parsedJson['keyword'],
+    );
+  }
+
+  void decode(OnlineStream stream) {
+    SherpaOnnxBindings.decodeKeywordStream?.call(ptr, stream.ptr);
+  }
+
+  Pointer<SherpaOnnxKeywordSpotter> ptr;
+  KeywordSpotterConfig config;
+}
--- a/flutter/sherpa_onnx/lib/src/sherpa_onnx_bindings.dart
+++ b/flutter/sherpa_onnx/lib/src/sherpa_onnx_bindings.dart
@@ -283,6 +283,28 @@ final class SherpaOnnxSpeakerEmbeddingExtractorConfig extends Struct {
  external Pointer<Utf8> provider;
 }

+final class SherpaOnnxKeywordSpotterConfig extends Struct {
+  external SherpaOnnxFeatureConfig feat;
+
+  external SherpaOnnxOnlineModelConfig model;
+
+  @Int32()
+  external int maxActivePaths;
+
+  @Int32()
+  external int numTrailingBlanks;
+
+  @Float()
+  external double keywordsScore;
+
+  @Float()
+  external double keywordsThreshold;
+
+  external Pointer<Utf8> keywordsFile;
+}
+
+final class SherpaOnnxKeywordSpotter extends Opaque {}
+
 final class SherpaOnnxOfflineTts extends Opaque {}

 final class SherpaOnnxCircularBuffer extends Opaque {}
@@ -301,6 +323,48 @@ final class SherpaOnnxSpeakerEmbeddingExtractor extends Opaque {}

 final class SherpaOnnxSpeakerEmbeddingManager extends Opaque {}

+typedef CreateKeywordSpotterNative = Pointer<SherpaOnnxKeywordSpotter> Function(
+    Pointer<SherpaOnnxKeywordSpotterConfig>);
+
+typedef CreateKeywordSpotter = CreateKeywordSpotterNative;
+
+typedef DestroyKeywordSpotterNative = Void Function(
+    Pointer<SherpaOnnxKeywordSpotter>);
+
+typedef DestroyKeywordSpotter = void Function(
+    Pointer<SherpaOnnxKeywordSpotter>);
+
+typedef CreateKeywordStreamNative = Pointer<SherpaOnnxOnlineStream> Function(
+    Pointer<SherpaOnnxKeywordSpotter>);
+
+typedef CreateKeywordStream = CreateKeywordStreamNative;
+
+typedef CreateKeywordStreamWithKeywordsNative = Pointer<SherpaOnnxOnlineStream>
+    Function(Pointer<SherpaOnnxKeywordSpotter>, Pointer<Utf8>);
+
+typedef CreateKeywordStreamWithKeywords = CreateKeywordStreamWithKeywordsNative;
+
+typedef IsKeywordStreamReadyNative = Int32 Function(
+    Pointer<SherpaOnnxKeywordSpotter>, Pointer<SherpaOnnxOnlineStream>);
+
+typedef IsKeywordStreamReady = int Function(
+    Pointer<SherpaOnnxKeywordSpotter>, Pointer<SherpaOnnxOnlineStream>);
+
+typedef DecodeKeywordStreamNative = Void Function(
+    Pointer<SherpaOnnxKeywordSpotter>, Pointer<SherpaOnnxOnlineStream>);
+
+typedef DecodeKeywordStream = void Function(
+    Pointer<SherpaOnnxKeywordSpotter>, Pointer<SherpaOnnxOnlineStream>);
+
+typedef GetKeywordResultAsJsonNative = Pointer<Utf8> Function(
+    Pointer<SherpaOnnxKeywordSpotter>, Pointer<SherpaOnnxOnlineStream>);
+
+typedef GetKeywordResultAsJson = GetKeywordResultAsJsonNative;
+
+typedef FreeKeywordResultJsonNative = Void Function(Pointer<Utf8>);
+
+typedef FreeKeywordResultJson = void Function(Pointer<Utf8>);
+
 typedef SherpaOnnxCreateOfflineTtsNative = Pointer<SherpaOnnxOfflineTts>
    Function(Pointer<SherpaOnnxOfflineTtsConfig>);

@@ -735,6 +799,15 @@ typedef SherpaOnnxFreeWaveNative = Void Function(Pointer<SherpaOnnxWave>);
 typedef SherpaOnnxFreeWave = void Function(Pointer<SherpaOnnxWave>);

 class SherpaOnnxBindings {
+  static CreateKeywordSpotter? createKeywordSpotter;
+  static DestroyKeywordSpotter? destroyKeywordSpotter;
+  static CreateKeywordStream? createKeywordStream;
+  static CreateKeywordStreamWithKeywords? createKeywordStreamWithKeywords;
+  static IsKeywordStreamReady? isKeywordStreamReady;
+  static DecodeKeywordStream? decodeKeywordStream;
+  static GetKeywordResultAsJson? getKeywordResultAsJson;
+  static FreeKeywordResultJson? freeKeywordResultJson;
+
  static SherpaOnnxCreateOfflineTts? createOfflineTts;
  static SherpaOnnxDestroyOfflineTts? destroyOfflineTts;
  static SherpaOnnxOfflineTtsSampleRate? offlineTtsSampleRate;
@@ -879,6 +952,46 @@ class SherpaOnnxBindings {
  static SherpaOnnxFreeWave? freeWave;

  static void init(DynamicLibrary dynamicLibrary) {
+    createKeywordSpotter ??= dynamicLibrary
+        .lookup<NativeFunction<CreateKeywordSpotterNative>>(
+            'CreateKeywordSpotter')
+        .asFunction();
+
+    destroyKeywordSpotter ??= dynamicLibrary
+        .lookup<NativeFunction<DestroyKeywordSpotterNative>>(
+            'DestroyKeywordSpotter')
+        .asFunction();
+
+    createKeywordStream ??= dynamicLibrary
+        .lookup<NativeFunction<CreateKeywordStreamNative>>(
+            'CreateKeywordStream')
+        .asFunction();
+
+    createKeywordStreamWithKeywords ??= dynamicLibrary
+        .lookup<NativeFunction<CreateKeywordStreamWithKeywordsNative>>(
+            'CreateKeywordStreamWithKeywords')
+        .asFunction();
+
+    isKeywordStreamReady ??= dynamicLibrary
+        .lookup<NativeFunction<IsKeywordStreamReadyNative>>(
+            'IsKeywordStreamReady')
+        .asFunction();
+
+    decodeKeywordStream ??= dynamicLibrary
+        .lookup<NativeFunction<DecodeKeywordStreamNative>>(
+            'DecodeKeywordStream')
+        .asFunction();
+
+    getKeywordResultAsJson ??= dynamicLibrary
+        .lookup<NativeFunction<GetKeywordResultAsJsonNative>>(
+            'GetKeywordResultAsJson')
+        .asFunction();
+
+    freeKeywordResultJson ??= dynamicLibrary
+        .lookup<NativeFunction<FreeKeywordResultJsonNative>>(
+            'FreeKeywordResultJson')
+        .asFunction();
+
    createOfflineTts ??= dynamicLibrary
        .lookup<NativeFunction<SherpaOnnxCreateOfflineTtsNative>>(
            'SherpaOnnxCreateOfflineTts')
--- a/flutter/sherpa_onnx/pubspec.yaml
+++ b/flutter/sherpa_onnx/pubspec.yaml
@@ -31,20 +31,24 @@ dependencies:
    sdk: flutter

  sherpa_onnx_android: ^1.10.17
-    # path: ../sherpa_onnx_android
+  # sherpa_onnx_android:
+  #   path: ../sherpa_onnx_android

  sherpa_onnx_macos: ^1.10.17
-    # path: ../sherpa_onnx_macos
+  # sherpa_onnx_macos:
+  #   path: ../sherpa_onnx_macos

  sherpa_onnx_linux: ^1.10.17
-    # path: ../sherpa_onnx_linux
+  # sherpa_onnx_linux:
+  #   path: ../sherpa_onnx_linux
    #
  sherpa_onnx_windows: ^1.10.17
-    # path: ../sherpa_onnx_windows
+  # sherpa_onnx_windows:
+  #   path: ../sherpa_onnx_windows

  sherpa_onnx_ios: ^1.10.17
  # sherpa_onnx_ios:
-    # path: ../sherpa_onnx_ios
+  #   path: ../sherpa_onnx_ios

 dev_dependencies:
  flutter_lints: ^3.0.0