Add Kotlin and Java API for online punctuation models (#1936)

.github/workflows/run-java-test.yaml

@@ -105,6 +105,22 @@ jobs:
             make -j4
             ls -lh lib
 
+      - name:  Run java test (Online add punctuations)
+        shell: bash
+        run: |
+          cd ./java-api-examples
+          ./run-online-add-punctuation-zh-en.sh
+          # Delete model files to save space
+          rm -rf sherpa-onnx-online-*
+
+      - name:  Run java test (Offline add punctuations)
+        shell: bash
+        run: |
+          cd ./java-api-examples
+          ./run-offline-add-punctuation-zh-en.sh
+          # Delete model files to save space
+          rm -rf sherpa-onnx-punct-*
+
       - name:  Run java test (Non-Streaming ASR)
         shell: bash
         run: |
@@ -196,13 +212,6 @@ jobs:
           ./run-audio-tagging-ced-from-file.sh
           rm -rf sherpa-onnx-ced-*
 
-      - name:  Run java test (add punctuations)
-        shell: bash
-        run: |
-          cd ./java-api-examples
-          ./run-add-punctuation-zh-en.sh
-          # Delete model files to save space
-          rm -rf sherpa-onnx-punct-*
 
       - name:  Run java test (Spoken language identification)
         shell: bash

java-api-examples/OfflineAddPunctuation.java

@@ -5,7 +5,7 @@
 // The model supports both English and Chinese.
 import com.k2fsa.sherpa.onnx.*;
 
-public class AddPunctuation {
+public class OfflineAddPunctuation {
   public static void main(String[] args) {
     // please download the model from
     // https://github.com/k2-fsa/sherpa-onnx/releases/tag/punctuation-models

java-api-examples/OnlineAddPunctuation.java

@@ -0,0 +1,41 @@
+// Copyright 2025 Xiaomi Corporation
+
+// This file shows how to use a punctuation model to add punctuations to text.
+//
+// The model supports ONLY English.
+import com.k2fsa.sherpa.onnx.*;
+
+public class OnlineAddPunctuation {
+  public static void main(String[] args) {
+    // please download the model from
+    // https://github.com/k2-fsa/sherpa-onnx/releases/download/punctuation-models/sherpa-onnx-online-punct-en-2024-08-06.tar.bz2
+    String model = "./sherpa-onnx-online-punct-en-2024-08-06/model.int8.onnx";
+    String bpeVocab = "./sherpa-onnx-online-punct-en-2024-08-06/bpe.vocab";
+    OnlinePunctuationModelConfig modelConfig =
+        OnlinePunctuationModelConfig.builder()
+            .setCnnBilstm(model)
+            .setBpeVocab(bpeVocab)
+            .setNumThreads(1)
+            .setDebug(true)
+            .build();
+    OnlinePunctuationConfig config =
+        OnlinePunctuationConfig.builder().setModel(modelConfig).build();
+
+    OnlinePunctuation punct = new OnlinePunctuation(config);
+
+    String[] sentences =
+        new String[] {
+          "how are you doing fantastic thank you how about you",
+          "The African blogosphere is rapidly expanding bringing more voices online in the form of"
+              + " commentaries opinions analyses rants and poetry",
+        };
+
+    System.out.println("---");
+    for (String text : sentences) {
+      String out = punct.addPunctuation(text);
+      System.out.printf("Input: %s\n", text);
+      System.out.printf("Output: %s\n", out);
+      System.out.println("---");
+    }
+  }
+}

java-api-examples/run-offline-add-punctuation-zh-en.sh

@@ -34,4 +34,4 @@ fi
 java \
   -Djava.library.path=$PWD/../build/lib \
   -cp ../sherpa-onnx/java-api/build/sherpa-onnx.jar \
-  ./AddPunctuation.java
+  ./OfflineAddPunctuation.java

java-api-examples/run-online-add-punctuation-zh-en.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+
+set -ex
+
+if [[ ! -f ../build/lib/libsherpa-onnx-jni.dylib  && ! -f ../build/lib/libsherpa-onnx-jni.so ]]; then
+  mkdir -p ../build
+  pushd ../build
+  cmake \
+    -DSHERPA_ONNX_ENABLE_PYTHON=OFF \
+    -DSHERPA_ONNX_ENABLE_TESTS=OFF \
+    -DSHERPA_ONNX_ENABLE_CHECK=OFF \
+    -DBUILD_SHARED_LIBS=ON \
+    -DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF \
+    -DSHERPA_ONNX_ENABLE_JNI=ON \
+    ..
+
+  make -j4
+  ls -lh lib
+  popd
+fi
+
+if [ ! -f ../sherpa-onnx/java-api/build/sherpa-onnx.jar ]; then
+  pushd ../sherpa-onnx/java-api
+  make
+  popd
+fi
+
+if [ ! -f ./sherpa-onnx-online-punct-en-2024-08-06/model.int8.onnx ]; then
+  curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/punctuation-models/sherpa-onnx-online-punct-en-2024-08-06.tar.bz2
+  tar xvf sherpa-onnx-online-punct-en-2024-08-06.tar.bz2
+  rm sherpa-onnx-online-punct-en-2024-08-06.tar.bz2
+fi
+
+java \
+  -Djava.library.path=$PWD/../build/lib \
+  -cp ../sherpa-onnx/java-api/build/sherpa-onnx.jar \
+  ./OnlineAddPunctuation.java

kotlin-api-examples/OnlinePunctuation.kt

@@ -0,0 +1 @@
+../sherpa-onnx/kotlin-api/OnlinePunctuation.kt

kotlin-api-examples/run.sh

@@ -302,16 +302,16 @@ function testInverseTextNormalizationOnlineAsr() {
   java -Djava.library.path=../build/lib -jar $out_filename
 }
 
-function testPunctuation() {
+function testOfflinePunctuation() {
   if [ ! -f ./sherpa-onnx-punct-ct-transformer-zh-en-vocab272727-2024-04-12/model.onnx ]; then
     curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/punctuation-models/sherpa-onnx-punct-ct-transformer-zh-en-vocab272727-2024-04-12.tar.bz2
     tar xvf sherpa-onnx-punct-ct-transformer-zh-en-vocab272727-2024-04-12.tar.bz2
     rm sherpa-onnx-punct-ct-transformer-zh-en-vocab272727-2024-04-12.tar.bz2
   fi
 
-  out_filename=test_punctuation.jar
+  out_filename=test_offline_punctuation.jar
   kotlinc-jvm -include-runtime -d $out_filename \
-    ./test_punctuation.kt \
+    ./test_offline_punctuation.kt \
     ./OfflinePunctuation.kt \
     faked-asset-manager.kt \
     faked-log.kt
@@ -321,6 +321,25 @@ function testPunctuation() {
   java -Djava.library.path=../build/lib -jar $out_filename
 }
 
+function testOnlinePunctuation() {
+  if [ ! -f ./sherpa-onnx-online-punct-en-2024-08-06/model.int8.onnx ]; then
+    curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/punctuation-models/sherpa-onnx-online-punct-en-2024-08-06.tar.bz2
+    tar xvf sherpa-onnx-online-punct-en-2024-08-06.tar.bz2
+    rm sherpa-onnx-online-punct-en-2024-08-06.tar.bz2
+  fi
+
+  out_filename=test_online_punctuation.jar
+  kotlinc-jvm -include-runtime -d $out_filename \
+    ./test_online_punctuation.kt \
+    ./OnlinePunctuation.kt \
+    faked-asset-manager.kt \
+    faked-log.kt
+
+  ls -lh $out_filename
+
+  java -Djava.library.path=../build/lib -jar $out_filename
+}
+
 function testOfflineSpeakerDiarization() {
   if [ ! -f ./sherpa-onnx-pyannote-segmentation-3-0/model.onnx ]; then
     curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/speaker-segmentation-models/sherpa-onnx-pyannote-segmentation-3-0.tar.bz2
@@ -359,6 +378,7 @@ testTts
 testAudioTagging
 testSpokenLanguageIdentification
 testOfflineAsr
-testPunctuation
+testOfflinePunctuation
+testOnlinePunctuation
 testInverseTextNormalizationOfflineAsr
 testInverseTextNormalizationOnlineAsr

kotlin-api-examples/test_online_punctuation.kt

@@ -0,0 +1,30 @@
+package com.k2fsa.sherpa.onnx
+
+fun main() {
+  testPunctuation()
+}
+
+// https://github.com/k2-fsa/sherpa-onnx/releases/download/punctuation-models/sherpa-onnx-online-punct-en-2024-08-06.tar.bz2
+fun testPunctuation() {
+  val config = OnlinePunctuationConfig(
+      model=OnlinePunctuationModelConfig(
+          cnnBilstm="./sherpa-onnx-online-punct-en-2024-08-06/model.int8.onnx",
+          bpeVocab="./sherpa-onnx-online-punct-en-2024-08-06/bpe.vocab",
+          numThreads=1,
+          debug=true,
+          provider="cpu",
+      )
+  )
+  val punct = OnlinePunctuation(config = config)
+  val sentences = arrayOf(
+        "how are you doing fantastic thank you what is about you",
+        "The African blogosphere is rapidly expanding bringing more voices online in the form of commentaries opinions analyses rants and poetry",
+  )
+  println("---")
+  for (text in sentences) {
+    val out = punct.addPunctuation(text)
+    println("Input: $text")
+    println("Output: $out")
+    println("---")
+  }
+}

sherpa-onnx/java-api/Makefile

@@ -53,6 +53,10 @@ java_files += OfflinePunctuationModelConfig.java
 java_files += OfflinePunctuationConfig.java
 java_files += OfflinePunctuation.java
 
+java_files += OnlinePunctuationModelConfig.java
+java_files += OnlinePunctuationConfig.java
+java_files += OnlinePunctuation.java
+
 java_files += OfflineZipformerAudioTaggingModelConfig.java
 java_files += AudioTaggingModelConfig.java
 java_files += AudioTaggingConfig.java

sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OnlinePunctuation.java

@@ -0,0 +1,39 @@
+// Copyright 2024 Xiaomi Corporation
+
+package com.k2fsa.sherpa.onnx;
+
+public class OnlinePunctuation {
+    static {
+        System.loadLibrary("sherpa-onnx-jni");
+    }
+
+    private long ptr = 0;
+
+    public OnlinePunctuation(OnlinePunctuationConfig config) {
+        ptr = newFromFile(config);
+    }
+
+    public String addPunctuation(String text) {
+        return addPunctuation(ptr, text);
+    }
+
+    @Override
+    protected void finalize() throws Throwable {
+        release();
+    }
+
+    // You'd better call it manually if it is not used anymore
+    public void release() {
+        if (this.ptr == 0) {
+            return;
+        }
+        delete(this.ptr);
+        this.ptr = 0;
+    }
+
+    private native void delete(long ptr);
+
+    private native long newFromFile(OnlinePunctuationConfig config);
+
+    private native String addPunctuation(long ptr, String text);
+}

sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OnlinePunctuationConfig.java

@@ -0,0 +1,33 @@
+// Copyright 2024 Xiaomi Corporation
+
+package com.k2fsa.sherpa.onnx;
+
+public class OnlinePunctuationConfig {
+    private final OnlinePunctuationModelConfig model;
+
+    private OnlinePunctuationConfig(Builder builder) {
+        this.model = builder.model;
+    }
+
+    public static Builder builder() {
+        return new Builder();
+    }
+
+    public OnlinePunctuationModelConfig getModel() {
+        return model;
+    }
+
+
+    public static class Builder {
+        private OnlinePunctuationModelConfig model = OnlinePunctuationModelConfig.builder().build();
+
+        public OnlinePunctuationConfig build() {
+            return new OnlinePunctuationConfig(this);
+        }
+
+        public Builder setModel(OnlinePunctuationModelConfig model) {
+            this.model = model;
+            return this;
+        }
+    }
+}

sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OnlinePunctuationModelConfig.java

@@ -0,0 +1,68 @@
+// Copyright 2024 Xiaomi Corporation
+
+package com.k2fsa.sherpa.onnx;
+
+public class OnlinePunctuationModelConfig {
+    private final String cnnBilstm;
+    private final String bpeVocab;
+    private final int numThreads;
+    private final boolean debug;
+    private final String provider;
+
+    private OnlinePunctuationModelConfig(Builder builder) {
+        this.cnnBilstm = builder.cnnBilstm;
+        this.bpeVocab = builder.bpeVocab;
+        this.numThreads = builder.numThreads;
+        this.debug = builder.debug;
+        this.provider = builder.provider;
+    }
+
+    public static Builder builder() {
+        return new Builder();
+    }
+
+    public String getCnnBilstm() {
+        return cnnBilstm;
+    }
+
+    public String getBpeVocab() {
+        return bpeVocab;
+    }
+
+    public static class Builder {
+        private String cnnBilstm = "";
+        private String bpeVocab = "";
+        private int numThreads = 1;
+        private boolean debug = true;
+        private String provider = "cpu";
+
+        public OnlinePunctuationModelConfig build() {
+            return new OnlinePunctuationModelConfig(this);
+        }
+
+        public Builder setCnnBilstm(String cnnBilstm) {
+            this.cnnBilstm = cnnBilstm;
+            return this;
+        }
+
+        public Builder setBpeVocab(String bpeVocab) {
+            this.bpeVocab = bpeVocab;
+            return this;
+        }
+
+        public Builder setNumThreads(int numThreads) {
+            this.numThreads = numThreads;
+            return this;
+        }
+
+        public Builder setDebug(boolean debug) {
+            this.debug = debug;
+            return this;
+        }
+
+        public Builder setProvider(String provider) {
+            this.provider = provider;
+            return this;
+        }
+    }
+}

sherpa-onnx/jni/CMakeLists.txt

@@ -17,6 +17,7 @@ set(sources
   offline-punctuation.cc
   offline-recognizer.cc
   offline-stream.cc
+  online-punctuation.cc
   online-recognizer.cc
   online-stream.cc
   speaker-embedding-extractor.cc

sherpa-onnx/jni/online-punctuation.cc

@@ -0,0 +1,117 @@
+// sherpa-onnx/jni/online-punctuation.cc
+//
+// Copyright (c)  2025  Xiaomi Corporation
+
+#include "sherpa-onnx/csrc/online-punctuation.h"
+
+#include "sherpa-onnx/csrc/macros.h"
+#include "sherpa-onnx/jni/common.h"
+
+namespace sherpa_onnx {
+
+static OnlinePunctuationConfig GetOnlinePunctuationConfig(JNIEnv *env,
+                                                          jobject config) {
+  OnlinePunctuationConfig ans;
+
+  jclass cls = env->GetObjectClass(config);
+  jfieldID fid;
+
+  fid = env->GetFieldID(cls, "model",
+                        "Lcom/k2fsa/sherpa/onnx/OnlinePunctuationModelConfig;");
+  jobject model_config = env->GetObjectField(config, fid);
+  jclass model_config_cls = env->GetObjectClass(model_config);
+
+  fid = env->GetFieldID(model_config_cls, "cnnBilstm", "Ljava/lang/String;");
+  jstring s = (jstring)env->GetObjectField(model_config, fid);
+  const char *p = env->GetStringUTFChars(s, nullptr);
+  ans.model.cnn_bilstm = p;
+  env->ReleaseStringUTFChars(s, p);
+
+  fid = env->GetFieldID(model_config_cls, "bpeVocab", "Ljava/lang/String;");
+  s = (jstring)env->GetObjectField(model_config, fid);
+  p = env->GetStringUTFChars(s, nullptr);
+  ans.model.bpe_vocab = p;
+  env->ReleaseStringUTFChars(s, p);
+
+  fid = env->GetFieldID(model_config_cls, "numThreads", "I");
+  ans.model.num_threads = env->GetIntField(model_config, fid);
+
+  fid = env->GetFieldID(model_config_cls, "debug", "Z");
+  ans.model.debug = env->GetBooleanField(model_config, fid);
+
+  fid = env->GetFieldID(model_config_cls, "provider", "Ljava/lang/String;");
+  s = (jstring)env->GetObjectField(model_config, fid);
+  p = env->GetStringUTFChars(s, nullptr);
+  ans.model.provider = p;
+  env->ReleaseStringUTFChars(s, p);
+
+  return ans;
+}
+
+}  // namespace sherpa_onnx
+
+SHERPA_ONNX_EXTERN_C
+JNIEXPORT jlong JNICALL
+Java_com_k2fsa_sherpa_onnx_OnlinePunctuation_newFromAsset(JNIEnv *env,
+                                                          jobject /*obj*/,
+                                                          jobject asset_manager,
+                                                          jobject _config) {
+#if __ANDROID_API__ >= 9
+  AAssetManager *mgr = AAssetManager_fromJava(env, asset_manager);
+  if (!mgr) {
+    SHERPA_ONNX_LOGE("Failed to get asset manager: %p", mgr);
+    return 0;
+  }
+#endif
+  auto config = sherpa_onnx::GetOnlinePunctuationConfig(env, _config);
+  SHERPA_ONNX_LOGE("config:\n%s", config.ToString().c_str());
+
+  auto model = new sherpa_onnx::OnlinePunctuation(
+#if __ANDROID_API__ >= 9
+      mgr,
+#endif
+      config);
+
+  return (jlong)model;
+}
+
+SHERPA_ONNX_EXTERN_C
+JNIEXPORT jlong JNICALL
+Java_com_k2fsa_sherpa_onnx_OnlinePunctuation_newFromFile(JNIEnv *env,
+                                                         jobject /*obj*/,
+                                                         jobject _config) {
+  auto config = sherpa_onnx::GetOnlinePunctuationConfig(env, _config);
+  SHERPA_ONNX_LOGE("config:\n%s", config.ToString().c_str());
+
+  if (!config.Validate()) {
+    SHERPA_ONNX_LOGE("Errors found in config!");
+    return 0;
+  }
+
+  auto model = new sherpa_onnx::OnlinePunctuation(config);
+
+  return (jlong)model;
+}
+
+SHERPA_ONNX_EXTERN_C
+JNIEXPORT void JNICALL Java_com_k2fsa_sherpa_onnx_OnlinePunctuation_delete(
+    JNIEnv * /*env*/, jobject /*obj*/, jlong ptr) {
+  delete reinterpret_cast<sherpa_onnx::OnlinePunctuation *>(ptr);
+}
+
+SHERPA_ONNX_EXTERN_C
+JNIEXPORT jstring JNICALL
+Java_com_k2fsa_sherpa_onnx_OnlinePunctuation_addPunctuation(JNIEnv *env,
+                                                            jobject /*obj*/,
+                                                            jlong ptr,
+                                                            jstring text) {
+  auto punct = reinterpret_cast<const sherpa_onnx::OnlinePunctuation *>(ptr);
+
+  const char *ptext = env->GetStringUTFChars(text, nullptr);
+
+  std::string result = punct->AddPunctuationWithCase(ptext);
+
+  env->ReleaseStringUTFChars(text, ptext);
+
+  return env->NewStringUTF(result.c_str());
+}

sherpa-onnx/kotlin-api/OnlinePunctuation.kt

@@ -0,0 +1,61 @@
+package com.k2fsa.sherpa.onnx
+
+import android.content.res.AssetManager
+
+data class OnlinePunctuationModelConfig(
+    var cnnBilstm: String = "",
+    var bpeVocab: String = "",
+    var numThreads: Int = 1,
+    var debug: Boolean = false,
+    var provider: String = "cpu",
+)
+
+
+data class OnlinePunctuationConfig(
+    var model: OnlinePunctuationModelConfig,
+)
+
+class OnlinePunctuation(
+    assetManager: AssetManager? = null,
+    config: OnlinePunctuationConfig,
+) {
+    private var ptr: Long
+
+    init {
+        ptr = if (assetManager != null) {
+            newFromAsset(assetManager, config)
+        } else {
+            newFromFile(config)
+        }
+    }
+
+    protected fun finalize() {
+        if (ptr != 0L) {
+            delete(ptr)
+            ptr = 0
+        }
+    }
+
+    fun release() = finalize()
+
+    fun addPunctuation(text: String) = addPunctuation(ptr, text)
+
+    private external fun delete(ptr: Long)
+
+    private external fun addPunctuation(ptr: Long, text: String): String
+
+    private external fun newFromAsset(
+        assetManager: AssetManager,
+        config: OnlinePunctuationConfig,
+    ): Long
+
+    private external fun newFromFile(
+        config: OnlinePunctuationConfig,
+    ): Long
+
+    companion object {
+        init {
+            System.loadLibrary("sherpa-onnx-jni")
+        }
+    }
+}