Dart API for adding punctuations to text (#1182)

.github/scripts/test-dart.sh

@@ -4,6 +4,11 @@ set -ex
 
 cd dart-api-examples
 
+pushd add-punctuations
+echo '----------CT Transformer----------'
+./run-ct-transformer.sh
+popd
+
 pushd audio-tagging
 echo '----------zipformer----------'
 ./run-zipformer.sh

.github/workflows/test-dart.yaml

@@ -111,6 +111,7 @@ jobs:
           cp scripts/dart/kws-pubspec.yaml dart-api-examples/keyword-spotter/pubspec.yaml
           cp scripts/dart/vad-non-streaming-asr-pubspec.yaml dart-api-examples/vad-with-non-streaming-asr/pubspec.yaml
           cp scripts/dart/audio-tagging-pubspec.yaml dart-api-examples/audio-tagging/pubspec.yaml
+          cp scripts/dart/add-punctuations-pubspec.yaml dart-api-examples/add-punctuations/pubspec.yaml
 
           cp scripts/dart/sherpa-onnx-pubspec.yaml flutter/sherpa_onnx/pubspec.yaml
 

CHANGELOG.md

@@ -1,6 +1,7 @@
 ## 1.10.20
 
 * Add Dart API for audio tagging
+* Add Dart API for adding punctuations to text
 
 ## 1.10.19
 

CMakeLists.txt

@@ -11,7 +11,7 @@ project(sherpa-onnx)
 # ./nodejs-addon-examples
 # ./dart-api-examples/
 # ./CHANGELOG.md
-set(SHERPA_ONNX_VERSION "1.10.19")
+set(SHERPA_ONNX_VERSION "1.10.20")
 
 # Disable warning about
 #

dart-api-examples/README.md

@@ -16,6 +16,7 @@ https://pub.dev/packages/sherpa_onnx
 | [./vad](./vad)| Example for voice activity detection|
 | [./vad-with-non-streaming-asr](./vad-with-non-streaming-asr)| Example for voice activity detection with non-streaming speech recognition. You can use it to generate subtitles.|
 | [./audio-tagging](./audio-tagging)| Example for audio tagging.|
+| [./add-punctuations](./add-punctuations)| Example for adding punctuations to text.|
 
 ## How to create an example in this folder
 

dart-api-examples/add-punctuations/.gitignore

@@ -0,0 +1,3 @@
+# https://dart.dev/guides/libraries/private-files
+# Created by `dart pub`
+.dart_tool/

dart-api-examples/add-punctuations/README.md

@@ -0,0 +1,8 @@
+# Introduction
+
+This example shows how to use the Dart API from sherpa-onnx to add punctuations to text.
+
+| File | Description|
+|------|------------|
+|[./bin/punctuations.dart](./bin/punctuations.dart)| Use a [CT Transformer model](https://modelscope.cn/models/iic/punc_ct-transformer_zh-cn-common-vocab272727-pytorch/summary) to add punctuations to text. See [./run-ct-transformer.sh](./run-ct-transformer.sh)|
+

dart-api-examples/add-punctuations/analysis_options.yaml

@@ -0,0 +1,30 @@
+# This file configures the static analysis results for your project (errors,
+# warnings, and lints).
+#
+# This enables the 'recommended' set of lints from `package:lints`.
+# This set helps identify many issues that may lead to problems when running
+# or consuming Dart code, and enforces writing Dart using a single, idiomatic
+# style and format.
+#
+# If you want a smaller set of lints you can change this to specify
+# 'package:lints/core.yaml'. These are just the most critical lints
+# (the recommended set includes the core lints).
+# The core lints are also what is used by pub.dev for scoring packages.
+
+include: package:lints/recommended.yaml
+
+# Uncomment the following section to specify additional rules.
+
+# linter:
+#   rules:
+#     - camel_case_types
+
+# analyzer:
+#   exclude:
+#     - path/to/excluded/files/**
+
+# For more information about the core and recommended set of lints, see
+# https://dart.dev/go/core-lints
+
+# For additional information about configuring this file, see
+# https://dart.dev/guides/language/analysis-options

dart-api-examples/add-punctuations/bin/init.dart

@@ -0,0 +1 @@
+../../vad/bin/init.dart

dart-api-examples/add-punctuations/bin/punctuations.dart

@@ -0,0 +1,46 @@
+// Copyright (c)  2024  Xiaomi Corporation
+import 'dart:io';
+
+import 'package:args/args.dart';
+import 'package:sherpa_onnx/sherpa_onnx.dart' as sherpa_onnx;
+import './init.dart';
+
+void main(List<String> arguments) async {
+  await initSherpaOnnx();
+
+  final parser = ArgParser()..addOption('model', help: 'Path to model.onnx');
+
+  final res = parser.parse(arguments);
+  if (res['model'] == null) {
+    print(parser.usage);
+    exit(1);
+  }
+
+  final modelFile = res['model'] as String;
+  final modelConfig = sherpa_onnx.OfflinePunctuationModelConfig(
+    ctTransformer: modelFile,
+    numThreads: 1,
+    provider: 'cpu',
+    debug: false,
+  );
+
+  final config = sherpa_onnx.OfflinePunctuationConfig(model: modelConfig);
+
+  final punct = sherpa_onnx.OfflinePunctuation(config: config);
+
+  final texts = [
+    '这是一个测试你好吗How are you我很好thank you are you ok谢谢你',
+    '我们都是木头人不会说话不会动',
+    'The African blogosphere is rapidly expanding bringing more voices online in the form of commentaries opinions analyses rants and poetry',
+  ];
+
+  for (final t in texts) {
+    final textWithPunct = punct.addPunct(t);
+    print('----------');
+    print('Before: $t');
+    print('After: $textWithPunct');
+  }
+  print('----------');
+
+  punct.free();
+}

dart-api-examples/add-punctuations/pubspec.yaml

@@ -0,0 +1,17 @@
+name: add_punctuations
+
+description: >
+  This example demonstrates how to use the Dart API to add punctuations to text.
+
+version: 1.0.0
+
+environment:
+  sdk: ^3.4.0
+
+dependencies:
+  sherpa_onnx: ^1.10.20
+  path: ^1.9.0
+  args: ^2.5.0
+
+dev_dependencies:
+  lints: ^3.0.0

dart-api-examples/add-punctuations/run-ct-transformer.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+set -ex
+
+dart pub get
+
+if [[ ! -f ./sherpa-onnx-punct-ct-transformer-zh-en-vocab272727-2024-04-12/model.onnx ]]; then
+  curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/punctuation-models/sherpa-onnx-punct-ct-transformer-zh-en-vocab272727-2024-04-12.tar.bz2
+  tar xvf sherpa-onnx-punct-ct-transformer-zh-en-vocab272727-2024-04-12.tar.bz2
+  rm sherpa-onnx-punct-ct-transformer-zh-en-vocab272727-2024-04-12.tar.bz2
+fi
+
+dart run \
+  ./bin/punctuations.dart \
+  --model ./sherpa-onnx-punct-ct-transformer-zh-en-vocab272727-2024-04-12/model.onnx

dart-api-examples/audio-tagging/pubspec.yaml

@@ -9,7 +9,7 @@ environment:
   sdk: ^3.4.0
 
 dependencies:
-  sherpa_onnx: ^1.10.19
+  sherpa_onnx: ^1.10.20
   path: ^1.9.0
   args: ^2.5.0
 

dart-api-examples/keyword-spotter/pubspec.yaml

@@ -9,7 +9,7 @@ environment:
   sdk: ^3.4.0
 
 dependencies:
-  sherpa_onnx: ^1.10.19
+  sherpa_onnx: ^1.10.20
   # sherpa_onnx:
   #   path: ../../flutter/sherpa_onnx
   path: ^1.9.0

dart-api-examples/non-streaming-asr/pubspec.yaml

@@ -10,7 +10,7 @@ environment:
 
 # Add regular dependencies here.
 dependencies:
-  sherpa_onnx: ^1.10.19
+  sherpa_onnx: ^1.10.20
   path: ^1.9.0
   args: ^2.5.0
 

dart-api-examples/streaming-asr/pubspec.yaml

@@ -11,7 +11,7 @@ environment:
 
 # Add regular dependencies here.
 dependencies:
-  sherpa_onnx: ^1.10.19
+  sherpa_onnx: ^1.10.20
   path: ^1.9.0
   args: ^2.5.0
 

dart-api-examples/tts/pubspec.yaml

@@ -8,7 +8,7 @@ environment:
 
 # Add regular dependencies here.
 dependencies:
-  sherpa_onnx: ^1.10.19
+  sherpa_onnx: ^1.10.20
   path: ^1.9.0
   args: ^2.5.0
 

dart-api-examples/vad-with-non-streaming-asr/pubspec.yaml

@@ -10,7 +10,7 @@ environment:
   sdk: ^3.4.0
 
 dependencies:
-  sherpa_onnx: ^1.10.19
+  sherpa_onnx: ^1.10.20
   path: ^1.9.0
   args: ^2.5.0
 

dart-api-examples/vad/pubspec.yaml

@@ -9,7 +9,7 @@ environment:
   sdk: ^3.4.0
 
 dependencies:
-  sherpa_onnx: ^1.10.19
+  sherpa_onnx: ^1.10.20
   path: ^1.9.0
   args: ^2.5.0
 

flutter-examples/streaming_asr/pubspec.yaml

@@ -5,7 +5,7 @@ description: >
 
 publish_to: 'none'
 
-version: 1.10.19
+version: 1.10.20
 
 topics:
   - speech-recognition
@@ -30,7 +30,7 @@ dependencies:
   record: ^5.1.0
   url_launcher: ^6.2.6
 
-  sherpa_onnx: ^1.10.19
+  sherpa_onnx: ^1.10.20
   # sherpa_onnx:
     # path: ../../flutter/sherpa_onnx
 

flutter-examples/tts/pubspec.yaml

@@ -5,7 +5,7 @@ description: >
 
 publish_to: 'none' # Remove this line if you wish to publish to pub.dev
 
-version: 1.10.19
+version: 1.10.20
 
 environment:
   sdk: '>=3.4.0 <4.0.0'
@@ -17,7 +17,7 @@ dependencies:
   cupertino_icons: ^1.0.6
   path_provider: ^2.1.3
   path: ^1.9.0
-  sherpa_onnx: ^1.10.19
+  sherpa_onnx: ^1.10.20
   url_launcher: ^6.2.6
   audioplayers: ^5.0.0
 

flutter/sherpa_onnx/lib/sherpa_onnx.dart

@@ -9,6 +9,7 @@ export 'src/offline_recognizer.dart';
 export 'src/offline_stream.dart';
 export 'src/online_recognizer.dart';
 export 'src/online_stream.dart';
+export 'src/punctuation.dart';
 export 'src/speaker_identification.dart';
 export 'src/tts.dart';
 export 'src/vad.dart';

flutter/sherpa_onnx/lib/src/punctuation.dart

@@ -0,0 +1,91 @@
+// Copyright (c)  2024  Xiaomi Corporation
+import 'dart:ffi';
+import 'package:ffi/ffi.dart';
+
+import './sherpa_onnx_bindings.dart';
+
+class OfflinePunctuationModelConfig {
+  OfflinePunctuationModelConfig(
+      {required this.ctTransformer,
+      this.numThreads = 1,
+      this.provider = 'cpu',
+      this.debug = true});
+
+  @override
+  String toString() {
+    return 'OfflinePunctuationModelConfig(ctTransformer: $ctTransformer, numThreads: $numThreads, provider: $provider, debug: $debug)';
+  }
+
+  final String ctTransformer;
+  final int numThreads;
+  final String provider;
+  final bool debug;
+}
+
+class OfflinePunctuationConfig {
+  OfflinePunctuationConfig({
+    required this.model,
+  });
+
+  @override
+  String toString() {
+    return 'OfflinePunctuationConfig(model: $model)';
+  }
+
+  final OfflinePunctuationModelConfig model;
+}
+
+class OfflinePunctuation {
+  OfflinePunctuation._({required this.ptr, required this.config});
+
+  // The user has to invoke OfflinePunctuation.free() to avoid memory leak.
+  factory OfflinePunctuation({required OfflinePunctuationConfig config}) {
+    final c = calloc<SherpaOnnxOfflinePunctuationConfig>();
+
+    final ctTransformerPtr = config.model.ctTransformer.toNativeUtf8();
+    c.ref.model.ctTransformer = ctTransformerPtr;
+    c.ref.model.numThreads = config.model.numThreads;
+    c.ref.model.debug = config.model.debug ? 1 : 0;
+
+    final providerPtr = config.model.provider.toNativeUtf8();
+    c.ref.model.provider = providerPtr;
+
+    final ptr =
+        SherpaOnnxBindings.sherpaOnnxCreateOfflinePunctuation?.call(c) ??
+            nullptr;
+
+    calloc.free(providerPtr);
+    calloc.free(ctTransformerPtr);
+    calloc.free(c);
+
+    return OfflinePunctuation._(ptr: ptr, config: config);
+  }
+
+  void free() {
+    SherpaOnnxBindings.sherpaOnnxDestroyOfflinePunctuation?.call(ptr);
+    ptr = nullptr;
+  }
+
+  String addPunct(String text) {
+    final textPtr = text.toNativeUtf8();
+
+    final p = SherpaOnnxBindings.sherpaOfflinePunctuationAddPunct
+            ?.call(ptr, textPtr) ??
+        nullptr;
+
+    calloc.free(textPtr);
+
+    if (p == nullptr) {
+      return '';
+    }
+
+    final ans = p.toDartString();
+
+    SherpaOnnxBindings.sherpaOfflinePunctuationFreeText?.call(p);
+
+    return ans;
+  }
+
+  Pointer<SherpaOnnxOfflinePunctuation> ptr;
+  final OfflinePunctuationConfig config;
+}

flutter/sherpa_onnx/lib/src/sherpa_onnx_bindings.dart

@@ -2,6 +2,22 @@
 import 'dart:ffi';
 import 'package:ffi/ffi.dart';
 
+final class SherpaOnnxOfflinePunctuationModelConfig extends Struct {
+  external Pointer<Utf8> ctTransformer;
+
+  @Int32()
+  external int numThreads;
+
+  @Int32()
+  external int debug;
+
+  external Pointer<Utf8> provider;
+}
+
+final class SherpaOnnxOfflinePunctuationConfig extends Struct {
+  external SherpaOnnxOfflinePunctuationModelConfig model;
+}
+
 final class SherpaOnnxOfflineZipformerAudioTaggingModelConfig extends Struct {
   external Pointer<Utf8> model;
 }
@@ -338,6 +354,8 @@ final class SherpaOnnxKeywordSpotterConfig extends Struct {
   external Pointer<Utf8> keywordsFile;
 }
 
+final class SherpaOnnxOfflinePunctuation extends Opaque {}
+
 final class SherpaOnnxAudioTagging extends Opaque {}
 
 final class SherpaOnnxKeywordSpotter extends Opaque {}
@@ -360,6 +378,29 @@ final class SherpaOnnxSpeakerEmbeddingExtractor extends Opaque {}
 
 final class SherpaOnnxSpeakerEmbeddingManager extends Opaque {}
 
+typedef SherpaOnnxCreateOfflinePunctuationNative
+    = Pointer<SherpaOnnxOfflinePunctuation> Function(
+        Pointer<SherpaOnnxOfflinePunctuationConfig>);
+
+typedef SherpaOnnxCreateOfflinePunctuation
+    = SherpaOnnxCreateOfflinePunctuationNative;
+
+typedef SherpaOnnxDestroyOfflinePunctuationNative = Void Function(
+    Pointer<SherpaOnnxOfflinePunctuation>);
+
+typedef SherpaOnnxDestroyOfflinePunctuation = void Function(
+    Pointer<SherpaOnnxOfflinePunctuation>);
+
+typedef SherpaOfflinePunctuationAddPunctNative = Pointer<Utf8> Function(
+    Pointer<SherpaOnnxOfflinePunctuation>, Pointer<Utf8>);
+
+typedef SherpaOfflinePunctuationAddPunct
+    = SherpaOfflinePunctuationAddPunctNative;
+
+typedef SherpaOfflinePunctuationFreeTextNative = Void Function(Pointer<Utf8>);
+
+typedef SherpaOfflinePunctuationFreeText = void Function(Pointer<Utf8>);
+
 typedef SherpaOnnxCreateAudioTaggingNative = Pointer<SherpaOnnxAudioTagging>
     Function(Pointer<SherpaOnnxAudioTaggingConfig>);
 
@@ -875,6 +916,12 @@ typedef SherpaOnnxFreeWaveNative = Void Function(Pointer<SherpaOnnxWave>);
 typedef SherpaOnnxFreeWave = void Function(Pointer<SherpaOnnxWave>);
 
 class SherpaOnnxBindings {
+  static SherpaOnnxCreateOfflinePunctuation? sherpaOnnxCreateOfflinePunctuation;
+  static SherpaOnnxDestroyOfflinePunctuation?
+      sherpaOnnxDestroyOfflinePunctuation;
+  static SherpaOfflinePunctuationAddPunct? sherpaOfflinePunctuationAddPunct;
+  static SherpaOfflinePunctuationFreeText? sherpaOfflinePunctuationFreeText;
+
   static SherpaOnnxCreateAudioTagging? sherpaOnnxCreateAudioTagging;
   static SherpaOnnxDestroyAudioTagging? sherpaOnnxDestroyAudioTagging;
   static SherpaOnnxAudioTaggingCreateOfflineStream?
@@ -1036,6 +1083,26 @@ class SherpaOnnxBindings {
   static SherpaOnnxFreeWave? freeWave;
 
   static void init(DynamicLibrary dynamicLibrary) {
+    sherpaOnnxCreateOfflinePunctuation ??= dynamicLibrary
+        .lookup<NativeFunction<SherpaOnnxCreateOfflinePunctuationNative>>(
+            'SherpaOnnxCreateOfflinePunctuation')
+        .asFunction();
+
+    sherpaOnnxDestroyOfflinePunctuation ??= dynamicLibrary
+        .lookup<NativeFunction<SherpaOnnxDestroyOfflinePunctuationNative>>(
+            'SherpaOnnxDestroyOfflinePunctuation')
+        .asFunction();
+
+    sherpaOfflinePunctuationAddPunct ??= dynamicLibrary
+        .lookup<NativeFunction<SherpaOfflinePunctuationAddPunctNative>>(
+            'SherpaOfflinePunctuationAddPunct')
+        .asFunction();
+
+    sherpaOfflinePunctuationFreeText ??= dynamicLibrary
+        .lookup<NativeFunction<SherpaOfflinePunctuationFreeTextNative>>(
+            'SherpaOfflinePunctuationFreeText')
+        .asFunction();
+
     sherpaOnnxCreateAudioTagging ??= dynamicLibrary
         .lookup<NativeFunction<SherpaOnnxCreateAudioTaggingNative>>(
             'SherpaOnnxCreateAudioTagging')

flutter/sherpa_onnx/pubspec.yaml

@@ -17,7 +17,7 @@ topics:
   - voice-activity-detection
 
 # remember to change the version in ../sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec
-version: 1.10.19
+version: 1.10.20
 
 homepage: https://github.com/k2-fsa/sherpa-onnx
 
@@ -30,23 +30,23 @@ dependencies:
   flutter:
     sdk: flutter
 
-  sherpa_onnx_android: ^1.10.19
+  sherpa_onnx_android: ^1.10.20
   # sherpa_onnx_android:
   #   path: ../sherpa_onnx_android
 
-  sherpa_onnx_macos: ^1.10.19
+  sherpa_onnx_macos: ^1.10.20
   # sherpa_onnx_macos:
   #   path: ../sherpa_onnx_macos
 
-  sherpa_onnx_linux: ^1.10.19
+  sherpa_onnx_linux: ^1.10.20
   # sherpa_onnx_linux:
   #   path: ../sherpa_onnx_linux
     #
-  sherpa_onnx_windows: ^1.10.19
+  sherpa_onnx_windows: ^1.10.20
   # sherpa_onnx_windows:
   #   path: ../sherpa_onnx_windows
 
-  sherpa_onnx_ios: ^1.10.19
+  sherpa_onnx_ios: ^1.10.20
   # sherpa_onnx_ios:
   #   path: ../sherpa_onnx_ios
 

flutter/sherpa_onnx_ios/ios/sherpa_onnx_ios.podspec

@@ -7,7 +7,7 @@
 # https://groups.google.com/g/dart-ffi/c/nUATMBy7r0c
 Pod::Spec.new do |s|
   s.name             = 'sherpa_onnx_ios'
-  s.version          = '1.10.19'
+  s.version          = '1.10.20'
   s.summary          = 'A new Flutter FFI plugin project.'
   s.description      = <<-DESC
 A new Flutter FFI plugin project.

flutter/sherpa_onnx_macos/macos/sherpa_onnx_macos.podspec

@@ -4,7 +4,7 @@
 #
 Pod::Spec.new do |s|
   s.name             = 'sherpa_onnx_macos'
-  s.version          = '1.10.19'
+  s.version          = '1.10.20'
   s.summary          = 'sherpa-onnx Flutter FFI plugin project.'
   s.description      = <<-DESC
 sherpa-onnx Flutter FFI plugin project.

nodejs-addon-examples/package.json

@@ -1,5 +1,5 @@
 {
   "dependencies": {
-    "sherpa-onnx-node": "^1.10.19"
+    "sherpa-onnx-node": "^1.10.20"
   }
 }

scripts/dart/add-punctuations-pubspec.yaml

@@ -0,0 +1,18 @@
+name: add_punctuations
+
+description: >
+  This example demonstrates how to use the Dart API to add punctuations to text.
+
+version: 1.0.0
+
+environment:
+  sdk: ^3.4.0
+
+dependencies:
+  sherpa_onnx:
+    path: ../../flutter/sherpa_onnx
+  path: ^1.9.0
+  args: ^2.5.0
+
+dev_dependencies:
+  lints: ^3.0.0

scripts/dart/sherpa-onnx-pubspec.yaml

@@ -17,7 +17,7 @@ topics:
   - voice-activity-detection
 
 # remember to change the version in ../sherpa_onnx_macos/macos/sherpa_onnx.podspec
-version: 1.10.19
+version: 1.10.20
 
 homepage: https://github.com/k2-fsa/sherpa-onnx