diff --git a/java-api-examples/Makefile b/java-api-examples/Makefile index 2ce75439..cdc17503 100755 --- a/java-api-examples/Makefile +++ b/java-api-examples/Makefile @@ -17,6 +17,8 @@ LIB_BUILD_DIR = ./lib EXAMPLE_FILE = DecodeFile.java +EXAMPLE_Mic = DecodeMic.java + JAVAC = javac BUILD_DIR = build @@ -29,8 +31,11 @@ vpath %.class $(BUILD_DIR) vpath %.java src -buildexample: +buildfile: $(JAVAC) -cp lib/sherpaonnx.jar -d $(BUILD_DIR) -encoding UTF-8 src/$(EXAMPLE_FILE) + +buildmic: + $(JAVAC) -cp lib/sherpaonnx.jar -d $(BUILD_DIR) -encoding UTF-8 src/$(EXAMPLE_Mic) rebuild: clean all @@ -43,10 +48,13 @@ clean: mkdir -p ./lib -run: +runfile: - java -cp ./lib/sherpaonnx.jar:build $(RUNJFLAGS) DecodeFile + java -cp ./lib/sherpaonnx.jar:build $(RUNJFLAGS) DecodeFile +runmic: + + java -cp ./lib/sherpaonnx.jar:build $(RUNJFLAGS) DecodeMic buildlib: $(LIB_FILES:.java=.class) @@ -58,4 +66,8 @@ buildlib: $(LIB_FILES:.java=.class) packjar: jar cvfe lib/sherpaonnx.jar . -C $(BUILD_DIR) . -all: clean buildlib packjar buildexample run +all: clean buildlib packjar buildfile buildmic + + + + diff --git a/java-api-examples/README.md b/java-api-examples/README.md index ebb498f2..8f1bc5af 100755 --- a/java-api-examples/README.md +++ b/java-api-examples/README.md @@ -1,17 +1,22 @@ - 0.Introduction ---- -Java wrapper `com.k2fsa.sherpaonnx.OnlineRecognizer` for `sherpa-onnx`. Java is a cross-platform language; you can build jni .so lib according to your system, and then use the same java api for all your platform. -``` xml +-------------- + +Java wrapper `com.k2fsa.sherpa.onnx.OnlineRecognizer` for `sherpa-onnx`. Java is a cross-platform language; you can build jni .so lib according to your system, and then use the same java api for all your platform. + +```xml Depend on: Openjdk 1.8 ``` + --- + 1.Compile libsherpa-onnx-jni.so ---- +------------------------------- + Compile sherpa-onnx/jni/jni.cc according to your system. Example for Ubuntu 18.04 LTS, Openjdk 1.8.0_362: -``` xml + +```xml git clone https://github.com/k2-fsa/sherpa-onnx cd sherpa-onnx mkdir build @@ -19,14 +24,19 @@ Example for Ubuntu 18.04 LTS, Openjdk 1.8.0_362: cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=ON -DSHERPA_ONNX_ENABLE_JNI=ON .. make -j6 ``` + --- + 2.Download asr model files ---- +-------------------------- + [click here for more detail](https://k2-fsa.github.io/sherpa/onnx/pretrained_models/index.html) ---- +-------------------------- + 3.Config model config.cfg ---- -``` xml +------------------------- + +```xml #model config sample_rate=16000 feature_dim=80 @@ -42,11 +52,15 @@ Example for Ubuntu 18.04 LTS, Openjdk 1.8.0_362: decoding_method=greedy_search max_active_paths=4 ``` + --- + 4.A simple java example ---- +----------------------- + refer to [java_api_example](https://github.com/k2-fsa/sherpa-onnx/blob/master/java-api-examples/src/DecodeFile.java) for more detail. -``` java + +```java import com.k2fsa.sherpa.onnx.OnlineRecognizer; import com.k2fsa.sherpa.onnx.OnlineStream; String cfgpath=appdir+"/modelconfig.cfg"; @@ -71,18 +85,34 @@ refer to [java_api_example](https://github.com/k2-fsa/sherpa-onnx/blob/master/ja rcgOjb.reSet(streamObj); rcgOjb.releaseStream(streamObj); // release stream rcgOjb.release(); // release recognizer - - ``` + --- -5.Makefile ---- -package jar and run app example -package path: /sherpa-onnx/java-api-examples/lib/sherpaonnx.jar -``` bash + +5.Makefile +---------- + +OS Ubuntu 18.04 LTS +Build package path: /sherpa-onnx/java-api-examples/lib/sherpaonnx.jar + +5.1 Build + +```bash cd sherpa-onnx/java-api-examples - make all - - ``` + make all +``` + +5.2 Run DecodeFile example + +```bash + make runfile +``` + +5.3 Run DecodeMic example + +```bash + make runmic +``` + diff --git a/java-api-examples/src/DecodeMic.java b/java-api-examples/src/DecodeMic.java new file mode 100755 index 00000000..c1a7f432 --- /dev/null +++ b/java-api-examples/src/DecodeMic.java @@ -0,0 +1,223 @@ +/* + * // Copyright 2022-2023 by zhaoming + */ +/* +Real-time speech recognition from a microphone with com.k2fsa.sherpa.onnx Java API + +example for cfgFile modelconfig.cfg + sample_rate=16000 + feature_dim=80 + rule1_min_trailing_silence=2.4 + rule2_min_trailing_silence=1.2 + rule3_min_utterance_length=20 + encoder=/sherpa-onnx/build/bin/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/encoder-epoch-99-avg-1.onnx + decoder=/sherpa-onnx/build/bin/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/decoder-epoch-99-avg-1.onnx + joiner=/sherpa-onnx/build/bin/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/joiner-epoch-99-avg-1.onnx + tokens=/sherpa-onnx/build/bin/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20/tokens.txt + num_threads=4 + enable_endpoint_detection=true + decoding_method=greedy_search + max_active_paths=4 + +*/ +import com.k2fsa.sherpa.onnx.OnlineRecognizer; +import com.k2fsa.sherpa.onnx.OnlineStream; +import java.io.*; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.ShortBuffer; +import java.nio.charset.StandardCharsets; +import javax.sound.sampled.AudioFormat; +import javax.sound.sampled.AudioSystem; +import javax.sound.sampled.DataLine; +import javax.sound.sampled.TargetDataLine; + +/** Microphone Example */ +public class DecodeMic { + MicRcgThread micRcgThread = null; // thread handle + + OnlineRecognizer rcgOjb; // the recognizer + + OnlineStream streamObj; // the stream + + public DecodeMic() { + + micRcgThread = new MicRcgThread(); // create a new instance for MicRcgThread + } + + public void open() { + micRcgThread.start(); // start to capture microphone data + } + + public void close() { + micRcgThread.stop(); // close capture + } + + /** init asr engine with config file */ + public void initModelWithCfg(String cfgFile) { + try { + + // set setSoPath() before running this + rcgOjb = new OnlineRecognizer(cfgFile); + + streamObj = rcgOjb.createStream(); // create a stream for asr engine to feed data + } catch (Exception e) { + System.err.println(e); + e.printStackTrace(); + } + } + + /** read data from mic and feed to asr engine */ + class MicRcgThread implements Runnable { + + TargetDataLine capline; // line for capture mic data + + Thread thread; // this thread + int segmentId = 0; // record the segment id when detect endpoint + String preText = ""; // decoded text + + public MicRcgThread() {} + + public void start() { + + thread = new Thread(this); + + thread.start(); // start thread + } + + public void stop() { + capline.stop(); + capline.close(); + capline = null; + thread = null; + } + + /** feed captured microphone data to asr */ + public void decodeSample(byte[] samplebytes) { + try { + ByteBuffer byteBuf = ByteBuffer.wrap(samplebytes); // create a bytebuf for samples + byteBuf.order(ByteOrder.LITTLE_ENDIAN); // set bytebuf to little endian + ShortBuffer shortBuf = byteBuf.asShortBuffer(); // covert to short type + short[] arrShort = new short[shortBuf.capacity()]; // array for copy short data + float[] arrFloat = new float[shortBuf.capacity()]; // array for copy float data + shortBuf.get(arrShort); // put date to arrShort + + for (int i = 0; i < arrShort.length; i++) { + arrFloat[i] = arrShort[i] / 32768f; // loop to covert short data to float -1 to 1 + } + streamObj.acceptWaveform(arrFloat); // feed asr engine with float data + while (rcgOjb.isReady(streamObj)) { // if engine is ready for unprocessed data + + rcgOjb.decodeStream(streamObj); // decode for this stream + } + boolean isEndpoint = + rcgOjb.isEndpoint( + streamObj); // endpoint check, make sure enable_endpoint_detection=true in config + // file + String nowText = rcgOjb.getResult(streamObj); // get asr result + String recText = ""; + byte[] utf8Data; // for covert text to utf8 + if (isEndpoint && nowText.length() > 0) { + rcgOjb.reSet(streamObj); // reSet stream when detect endpoint + segmentId++; + preText = nowText; + recText = "text(seg_" + String.valueOf(segmentId) + "):" + nowText + "\n"; + utf8Data = recText.getBytes(StandardCharsets.UTF_8); + System.out.println(new String(utf8Data)); + } + + if (!nowText.equals(preText)) { // if preText not equal nowtext + preText = nowText; + recText = nowText + "\n"; + utf8Data = recText.getBytes(StandardCharsets.UTF_8); + System.out.println(new String(utf8Data)); + } + } catch (Exception e) { + System.err.println(e); + e.printStackTrace(); + } + } + + /** run mic capture thread */ + public void run() { + System.out.println("Started! Please speak..."); + + AudioFormat.Encoding encoding = AudioFormat.Encoding.PCM_SIGNED; // the pcm format + float rate = 16000.0f; // using 16 kHz + int channels = 1; // single channel + int sampleSize = 16; // sampleSize 16bit + boolean isBigEndian = false; // using little endian + + AudioFormat format = + new AudioFormat( + encoding, rate, sampleSize, channels, (sampleSize / 8) * channels, rate, isBigEndian); + + DataLine.Info info = new DataLine.Info(TargetDataLine.class, format); + + // check system support such data format + if (!AudioSystem.isLineSupported(info)) { + System.out.println(info + " not supported."); + return; + } + + // open a line for capture. + + try { + capline = (TargetDataLine) AudioSystem.getLine(info); + capline.open(format, capline.getBufferSize()); + } catch (Exception ex) { + System.out.println(ex); + return; + } + + // the buf size for mic captured each time + int bufferLengthInBytes = capline.getBufferSize() / 8 * format.getFrameSize(); + byte[] micData = new byte[bufferLengthInBytes]; + int numBytesRead; + + capline.start(); // start to capture mic data + + while (thread != null) { + // read data from line + if ((numBytesRead = capline.read(micData, 0, bufferLengthInBytes)) == -1) { + break; + } + + decodeSample(micData); // decode mic data + } + + // stop and close + + try { + if (capline != null) { + capline.stop(); + capline.close(); + capline = null; + } + + } catch (Exception ex) { + System.err.println(ex); + } + } + } // End class DecodeMic + + public static void main(String s[]) { + try { + String appDir = System.getProperty("user.dir"); + System.out.println("appdir=" + appDir); + String cfgPath = appDir + "/modelconfig.cfg"; + String soPath = appDir + "/../build/lib/libsherpa-onnx-jni.so"; + OnlineRecognizer.setSoPath(soPath); // set so. lib for OnlineRecognizer + + DecodeMic decodeEx = new DecodeMic(); + decodeEx.initModelWithCfg(cfgPath); // init asr engine + decodeEx.open(); // open thread for mic + System.out.print("Press Enter to EXIT!\n"); + char i = (char) System.in.read(); + decodeEx.close(); + } catch (Exception e) { + System.err.println(e); + e.printStackTrace(); + } + } +}