Add Dart API for speech enhancement GTCRN models (#1993)

.github/scripts/test-dart.sh

@@ -4,6 +4,12 @@ set -ex
 
 cd dart-api-examples
 
+pushd speech-enhancement-gtcrn
+echo "speech enhancement with gtcrn models"
+./run.sh
+ls -lh
+popd
+
 pushd tts
 
 echo '----------matcha tts----------'

.github/workflows/test-dart.yaml

@@ -115,6 +115,7 @@ jobs:
           cp scripts/dart/add-punctuations-pubspec.yaml dart-api-examples/add-punctuations/pubspec.yaml
           cp scripts/dart/speaker-id-pubspec.yaml dart-api-examples/speaker-identification/pubspec.yaml
           cp scripts/dart/speaker-diarization-pubspec.yaml dart-api-examples/speaker-diarization/pubspec.yaml
+          cp scripts/dart/speech-enhancement-gtcrn-pubspec.yaml dart-api-examples/speech-enhancement-gtcrn/pubspec.yaml
 
           cp scripts/dart/sherpa-onnx-pubspec.yaml flutter/sherpa_onnx/pubspec.yaml
 

dart-api-examples/README.md

@@ -19,6 +19,7 @@ https://pub.dev/packages/sherpa_onnx
 | [./tts](./tts)| Example for text to speech|
 | [./vad-with-non-streaming-asr](./vad-with-non-streaming-asr)| Example for voice activity detection with non-streaming speech recognition. You can use it to generate subtitles.|
 | [./vad](./vad)| Example for voice activity detection|
+| [./speech-enhancement-gtcrn](./speech-enhancement-gtcrn)| Example for speech enhancement/denoising|
 
 ## How to create an example in this folder
 

dart-api-examples/speech-enhancement-gtcrn/.gitignore

@@ -0,0 +1,3 @@
+# https://dart.dev/guides/libraries/private-files
+# Created by `dart pub`
+.dart_tool/

dart-api-examples/speech-enhancement-gtcrn/CHANGELOG.md

@@ -0,0 +1,3 @@
+## 1.0.0
+
+- Initial version.

dart-api-examples/speech-enhancement-gtcrn/README.md

@@ -0,0 +1,2 @@
+A sample command-line application with an entrypoint in `bin/`, library code
+in `lib/`, and example unit test in `test/`.

dart-api-examples/speech-enhancement-gtcrn/analysis_options.yaml

@@ -0,0 +1,30 @@
+# This file configures the static analysis results for your project (errors,
+# warnings, and lints).
+#
+# This enables the 'recommended' set of lints from `package:lints`.
+# This set helps identify many issues that may lead to problems when running
+# or consuming Dart code, and enforces writing Dart using a single, idiomatic
+# style and format.
+#
+# If you want a smaller set of lints you can change this to specify
+# 'package:lints/core.yaml'. These are just the most critical lints
+# (the recommended set includes the core lints).
+# The core lints are also what is used by pub.dev for scoring packages.
+
+include: package:lints/recommended.yaml
+
+# Uncomment the following section to specify additional rules.
+
+# linter:
+#   rules:
+#     - camel_case_types
+
+# analyzer:
+#   exclude:
+#     - path/to/excluded/files/**
+
+# For more information about the core and recommended set of lints, see
+# https://dart.dev/go/core-lints
+
+# For additional information about configuring this file, see
+# https://dart.dev/guides/language/analysis-options

dart-api-examples/speech-enhancement-gtcrn/bin/init.dart

@@ -0,0 +1 @@
+../../vad/bin/init.dart

dart-api-examples/speech-enhancement-gtcrn/bin/speech_enhancement_gtcrn.dart

@@ -0,0 +1,51 @@
+// Copyright (c)  2025  Xiaomi Corporation
+import 'dart:io';
+
+import 'package:args/args.dart';
+import 'package:sherpa_onnx/sherpa_onnx.dart' as sherpa_onnx;
+import './init.dart';
+
+void main(List<String> arguments) async {
+  await initSherpaOnnx();
+
+  final parser = ArgParser()
+    ..addOption('model', help: 'Path to gtcrn onnx model')
+    ..addOption('input-wav', help: 'Path to input.wav')
+    ..addOption('output-wav', help: 'Path to output.wav');
+
+  final res = parser.parse(arguments);
+  if (res['model'] == null ||
+      res['input-wav'] == null ||
+      res['output-wav'] == null) {
+    print(parser.usage);
+    exit(1);
+  }
+
+  final model = res['model'] as String;
+  final inputWav = res['input-wav'] as String;
+  final outputWav = res['output-wav'] as String;
+
+  final config = sherpa_onnx.OfflineSpeechDenoiserConfig(
+      model: sherpa_onnx.OfflineSpeechDenoiserModelConfig(
+    gtcrn: sherpa_onnx.OfflineSpeechDenoiserGtcrnModelConfig(model: model),
+    numThreads: 1,
+    debug: true,
+    provider: 'cpu',
+  ));
+
+  final sd = sherpa_onnx.OfflineSpeechDenoiser(config);
+
+  final waveData = sherpa_onnx.readWave(inputWav);
+
+  final denoised =
+      sd.run(samples: waveData.samples, sampleRate: waveData.sampleRate);
+
+  sd.free();
+
+  sherpa_onnx.writeWave(
+      filename: outputWav,
+      samples: denoised.samples,
+      sampleRate: denoised.sampleRate);
+
+  print('Saved to $outputWav');
+}

dart-api-examples/speech-enhancement-gtcrn/pubspec.yaml

@@ -0,0 +1,20 @@
+name: speech_enhancement_gtcrn
+
+description: >
+  This example demonstrates how to use the Dart API for speech enhancement/denoising.
+
+version: 1.0.0
+
+environment:
+  sdk: ">=3.0.0 <4.0.0"
+
+# Add regular dependencies here.
+dependencies:
+  sherpa_onnx: ^1.10.46
+  # sherpa_onnx:
+  #   path: ../../flutter/sherpa_onnx
+  path: ^1.9.0
+  args: ^2.5.0
+
+dev_dependencies:
+  lints: ^3.0.0

dart-api-examples/speech-enhancement-gtcrn/run.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+set -ex
+
+dart pub get
+
+if [ ! -f ./gtcrn_simple.onnx ]; then
+  curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speech-enhancement-models/gtcrn_simple.onnx
+fi
+
+if [ ! -f ./inp_16k.wav ]; then
+  curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speech-enhancement-models/inp_16k.wav
+fi
+
+
+dart run \
+  ./bin/speech_enhancement_gtcrn.dart \
+  --model ./gtcrn_simple.onnx \
+  --input-wav ./inp_16k.wav \
+  --output-wav ./enhanced-16k.wav
+
+ls -lh *.wav

flutter/sherpa_onnx/example/example.md

@@ -9,6 +9,8 @@
 
 ## Pure dart-examples
 
+Hint: All of the following functions can be used in Flutter, even if some of them are only provided in pure dart api examples.
+
 | Functions | URL | Supported Platforms|
 |---|---|---|
 |Speaker diarization| [Address](https://github.com/k2-fsa/sherpa-onnx/tree/master/dart-api-examples/speaker-diarization)| macOS, Windows, Linux|
@@ -20,3 +22,5 @@
 |Speaker identification and verification| [Address](https://github.com/k2-fsa/sherpa-onnx/tree/master/dart-api-examples/speaker-identification)| macOS, Windows, Linux|
 |Audio tagging| [Address](https://github.com/k2-fsa/sherpa-onnx/tree/master/dart-api-examples/audio-tagging)| macOS, Windows, Linux|
 |Keyword spotter| [Address](https://github.com/k2-fsa/sherpa-onnx/tree/master/dart-api-examples/keyword-spotter)| macOS, Windows, Linux|
+|Add punctuions| [Address](https://github.com/k2-fsa/sherpa-onnx/tree/master/dart-api-examples/add-punctuations)| macOS, Windows, Linux|
+|Speech enhancement/denoising| [Address](https://github.com/k2-fsa/sherpa-onnx/tree/master/dart-api-examples/speech-enhancement-gtcrn)| macOS, Windows, Linux|

flutter/sherpa_onnx/lib/sherpa_onnx.dart

@@ -8,6 +8,7 @@ export 'src/keyword_spotter.dart';
 export 'src/offline_punctuation.dart';
 export 'src/offline_recognizer.dart';
 export 'src/offline_speaker_diarization.dart';
+export 'src/offline_speech_denoiser.dart';
 export 'src/offline_stream.dart';
 export 'src/online_punctuation.dart';
 export 'src/online_recognizer.dart';

flutter/sherpa_onnx/lib/src/offline_speech_denoiser.dart

@@ -0,0 +1,169 @@
+// Copyright (c)  2025  Xiaomi Corporation
+import 'dart:ffi';
+import 'dart:typed_data';
+
+import 'package:ffi/ffi.dart';
+import './sherpa_onnx_bindings.dart';
+
+class OfflineSpeechDenoiserGtcrnModelConfig {
+  const OfflineSpeechDenoiserGtcrnModelConfig({
+    this.model = '',
+  });
+
+  factory OfflineSpeechDenoiserGtcrnModelConfig.fromJson(
+      Map<String, dynamic> json) {
+    return OfflineSpeechDenoiserGtcrnModelConfig(
+      model: json['model'] as String? ?? '',
+    );
+  }
+
+  @override
+  String toString() {
+    return 'OfflineSpeechDenoiserGtcrnModelConfig(model: $model)';
+  }
+
+  Map<String, dynamic> toJson() => {
+        'model': model,
+      };
+
+  final String model;
+}
+
+class OfflineSpeechDenoiserModelConfig {
+  const OfflineSpeechDenoiserModelConfig({
+    this.gtcrn = const OfflineSpeechDenoiserGtcrnModelConfig(),
+    this.numThreads = 1,
+    this.debug = true,
+    this.provider = 'cpu',
+  });
+
+  factory OfflineSpeechDenoiserModelConfig.fromJson(Map<String, dynamic> json) {
+    return OfflineSpeechDenoiserModelConfig(
+      gtcrn: json['gtcrn'] != null
+          ? OfflineSpeechDenoiserGtcrnModelConfig.fromJson(
+              json['gtcrn'] as Map<String, dynamic>)
+          : const OfflineSpeechDenoiserGtcrnModelConfig(),
+      numThreads: json['numThreads'] as int? ?? 1,
+      debug: json['debug'] as bool? ?? true,
+      provider: json['provider'] as String? ?? 'cpu',
+    );
+  }
+
+  @override
+  String toString() {
+    return 'OfflineSpeechDenoiserModelConfig(gtcrn: $gtcrn, numThreads: $numThreads, debug: $debug, provider: $provider)';
+  }
+
+  Map<String, dynamic> toJson() => {
+        'gtcrn': gtcrn.toJson(),
+        'numThreads': numThreads,
+        'debug': debug,
+        'provider': provider,
+      };
+
+  final OfflineSpeechDenoiserGtcrnModelConfig gtcrn;
+  final int numThreads;
+  final bool debug;
+  final String provider;
+}
+
+class OfflineSpeechDenoiserConfig {
+  const OfflineSpeechDenoiserConfig({
+    this.model = const OfflineSpeechDenoiserModelConfig(),
+  });
+
+  factory OfflineSpeechDenoiserConfig.fromJson(Map<String, dynamic> json) {
+    return OfflineSpeechDenoiserConfig(
+      model: json['model'] != null
+          ? OfflineSpeechDenoiserModelConfig.fromJson(
+              json['model'] as Map<String, dynamic>)
+          : const OfflineSpeechDenoiserModelConfig(),
+    );
+  }
+
+  @override
+  String toString() {
+    return 'OfflineSpeechDenoiserConfig(model: $model)';
+  }
+
+  Map<String, dynamic> toJson() => {
+        'model': model.toJson(),
+      };
+
+  final OfflineSpeechDenoiserModelConfig model;
+}
+
+class DenoisedAudio {
+  DenoisedAudio({
+    required this.samples,
+    required this.sampleRate,
+  });
+
+  final Float32List samples;
+  final int sampleRate;
+}
+
+class OfflineSpeechDenoiser {
+  OfflineSpeechDenoiser.fromPtr({required this.ptr, required this.config});
+
+  OfflineSpeechDenoiser._({required this.ptr, required this.config});
+
+  /// The user is responsible to call the OfflineSpeechDenoiser.free()
+  /// method of the returned instance to avoid memory leak.
+  factory OfflineSpeechDenoiser(OfflineSpeechDenoiserConfig config) {
+    final c = calloc<SherpaOnnxOfflineSpeechDenoiserConfig>();
+    c.ref.model.gtcrn.model = config.model.gtcrn.model.toNativeUtf8();
+
+    c.ref.model.numThreads = config.model.numThreads;
+    c.ref.model.debug = config.model.debug ? 1 : 0;
+    c.ref.model.provider = config.model.provider.toNativeUtf8();
+
+    final ptr =
+        SherpaOnnxBindings.sherpaOnnxCreateOfflineSpeechDenoiser?.call(c) ??
+            nullptr;
+
+    calloc.free(c.ref.model.provider);
+    calloc.free(c.ref.model.gtcrn.model);
+
+    return OfflineSpeechDenoiser._(ptr: ptr, config: config);
+  }
+
+  void free() {
+    SherpaOnnxBindings.sherpaOnnxDestroyOfflineSpeechDenoiser?.call(ptr);
+    ptr = nullptr;
+  }
+
+  DenoisedAudio run({required Float32List samples, required int sampleRate}) {
+    final n = samples.length;
+    final Pointer<Float> psamples = calloc<Float>(n);
+
+    final pList = psamples.asTypedList(n);
+    pList.setAll(0, samples);
+
+    final p = SherpaOnnxBindings.sherpaOnnxOfflineSpeechDenoiserRun
+            ?.call(ptr, psamples, n, sampleRate) ??
+        nullptr;
+
+    calloc.free(psamples);
+
+    if (p == nullptr) {
+      return DenoisedAudio(samples: Float32List(0), sampleRate: 0);
+    }
+
+    final denoisedSamples = p.ref.samples.asTypedList(p.ref.n);
+    final denoisedSampleRate = p.ref.sampleRate;
+    final newSamples = Float32List.fromList(denoisedSamples);
+
+    SherpaOnnxBindings.sherpaOnnxDestroyDenoisedAudio?.call(p);
+
+    return DenoisedAudio(samples: newSamples, sampleRate: denoisedSampleRate);
+  }
+
+  int get sampleRate =>
+      SherpaOnnxBindings.sherpaOnnxOfflineSpeechDenoiserGetSampleRate
+          ?.call(ptr) ??
+      0;
+
+  Pointer<SherpaOnnxOfflineSpeechDenoiser> ptr;
+  OfflineSpeechDenoiserConfig config;
+}

flutter/sherpa_onnx/lib/src/sherpa_onnx_bindings.dart

@@ -2,6 +2,36 @@
 import 'dart:ffi';
 import 'package:ffi/ffi.dart';
 
+final class SherpaOnnxOfflineSpeechDenoiserGtcrnModelConfig extends Struct {
+  external Pointer<Utf8> model;
+}
+
+final class SherpaOnnxOfflineSpeechDenoiserModelConfig extends Struct {
+  external SherpaOnnxOfflineSpeechDenoiserGtcrnModelConfig gtcrn;
+
+  @Int32()
+  external int numThreads;
+
+  @Int32()
+  external int debug;
+
+  external Pointer<Utf8> provider;
+}
+
+final class SherpaOnnxOfflineSpeechDenoiserConfig extends Struct {
+  external SherpaOnnxOfflineSpeechDenoiserModelConfig model;
+}
+
+final class SherpaOnnxDenoisedAudio extends Struct {
+  external Pointer<Float> samples;
+
+  @Int32()
+  external int n;
+
+  @Int32()
+  external int sampleRate;
+}
+
 final class SherpaOnnxSpeakerEmbeddingExtractorConfig extends Struct {
   external Pointer<Utf8> model;
 
@@ -517,6 +547,41 @@ final class SherpaOnnxOfflineSpeakerDiarization extends Opaque {}
 
 final class SherpaOnnxOfflineSpeakerDiarizationResult extends Opaque {}
 
+final class SherpaOnnxOfflineSpeechDenoiser extends Opaque {}
+
+typedef SherpaOnnxCreateOfflineSpeechDenoiserNative
+    = Pointer<SherpaOnnxOfflineSpeechDenoiser> Function(
+        Pointer<SherpaOnnxOfflineSpeechDenoiserConfig>);
+
+typedef SherpaOnnxCreateOfflineSpeechDenoiser
+    = SherpaOnnxCreateOfflineSpeechDenoiserNative;
+
+typedef SherpaOnnxDestroyOfflineSpeechDenoiserNative = Void Function(
+    Pointer<SherpaOnnxOfflineSpeechDenoiser>);
+
+typedef SherpaOnnxDestroyOfflineSpeechDenoiser = void Function(
+    Pointer<SherpaOnnxOfflineSpeechDenoiser>);
+
+typedef SherpaOnnxOfflineSpeechDenoiserGetSampleRateNative = Int32 Function(
+    Pointer<SherpaOnnxOfflineSpeechDenoiser>);
+
+typedef SherpaOnnxOfflineSpeechDenoiserGetSampleRate = int Function(
+    Pointer<SherpaOnnxOfflineSpeechDenoiser>);
+
+typedef SherpaOnnxOfflineSpeechDenoiserRunNative
+    = Pointer<SherpaOnnxDenoisedAudio> Function(
+        Pointer<SherpaOnnxOfflineSpeechDenoiser>, Pointer<Float>, Int32, Int32);
+
+typedef SherpaOnnxOfflineSpeechDenoiserRun
+    = Pointer<SherpaOnnxDenoisedAudio> Function(
+        Pointer<SherpaOnnxOfflineSpeechDenoiser>, Pointer<Float>, int, int);
+
+typedef SherpaOnnxDestroyDenoisedAudioNative = Void Function(
+    Pointer<SherpaOnnxDenoisedAudio>);
+
+typedef SherpaOnnxDestroyDenoisedAudio = void Function(
+    Pointer<SherpaOnnxDenoisedAudio>);
+
 typedef SherpaOnnxCreateOfflineSpeakerDiarizationNative
     = Pointer<SherpaOnnxOfflineSpeakerDiarization> Function(
         Pointer<SherpaOnnxOfflineSpeakerDiarizationConfig>);
@@ -1172,6 +1237,17 @@ typedef SherpaOnnxFreeWaveNative = Void Function(Pointer<SherpaOnnxWave>);
 typedef SherpaOnnxFreeWave = void Function(Pointer<SherpaOnnxWave>);
 
 class SherpaOnnxBindings {
+  static SherpaOnnxCreateOfflineSpeechDenoiser?
+      sherpaOnnxCreateOfflineSpeechDenoiser;
+
+  static SherpaOnnxDestroyOfflineSpeechDenoiser?
+      sherpaOnnxDestroyOfflineSpeechDenoiser;
+
+  static SherpaOnnxOfflineSpeechDenoiserGetSampleRate?
+      sherpaOnnxOfflineSpeechDenoiserGetSampleRate;
+  static SherpaOnnxOfflineSpeechDenoiserRun? sherpaOnnxOfflineSpeechDenoiserRun;
+  static SherpaOnnxDestroyDenoisedAudio? sherpaOnnxDestroyDenoisedAudio;
+
   static SherpaOnnxCreateOfflineSpeakerDiarization?
       sherpaOnnxCreateOfflineSpeakerDiarization;
   static SherpaOnnxDestroyOfflineSpeakerDiarization?
@@ -1370,6 +1446,33 @@ class SherpaOnnxBindings {
   static SherpaOnnxFreeWave? freeWave;
 
   static void init(DynamicLibrary dynamicLibrary) {
+    sherpaOnnxCreateOfflineSpeechDenoiser ??= dynamicLibrary
+        .lookup<NativeFunction<SherpaOnnxCreateOfflineSpeechDenoiserNative>>(
+            'SherpaOnnxCreateOfflineSpeechDenoiser')
+        .asFunction();
+
+    sherpaOnnxDestroyOfflineSpeechDenoiser ??= dynamicLibrary
+        .lookup<NativeFunction<SherpaOnnxDestroyOfflineSpeechDenoiserNative>>(
+            'SherpaOnnxDestroyOfflineSpeechDenoiser')
+        .asFunction();
+
+    sherpaOnnxOfflineSpeechDenoiserGetSampleRate ??= dynamicLibrary
+        .lookup<
+                NativeFunction<
+                    SherpaOnnxOfflineSpeechDenoiserGetSampleRateNative>>(
+            'SherpaOnnxOfflineSpeechDenoiserGetSampleRate')
+        .asFunction();
+
+    sherpaOnnxOfflineSpeechDenoiserRun ??= dynamicLibrary
+        .lookup<NativeFunction<SherpaOnnxOfflineSpeechDenoiserRunNative>>(
+            'SherpaOnnxOfflineSpeechDenoiserRun')
+        .asFunction();
+
+    sherpaOnnxDestroyDenoisedAudio ??= dynamicLibrary
+        .lookup<NativeFunction<SherpaOnnxDestroyDenoisedAudioNative>>(
+            'SherpaOnnxDestroyDenoisedAudio')
+        .asFunction();
+
     sherpaOnnxCreateOfflineSpeakerDiarization ??= dynamicLibrary
         .lookup<
                 NativeFunction<

scripts/dart/speech-enhancement-gtcrn-pubspec.yaml

@@ -0,0 +1,18 @@
+name: speech_enhancement_gtcrn
+
+description: >
+  This example demonstrates how to use the Dart API for speech enhancement/denoising.
+
+version: 1.0.0
+
+environment:
+  sdk: ">=3.0.0 <4.0.0"
+
+dependencies:
+  sherpa_onnx:
+    path: ../../flutter/sherpa_onnx
+  path: ^1.9.0
+  args: ^2.5.0
+
+dev_dependencies:
+  lints: ^3.0.0