Add Flutter GUI example for VAD with a microphone. (#905)
This commit is contained in:
Fangjun Kuang
255
sherpa-onnx/flutter/example/lib/home.dart
Normal file
255
sherpa-onnx/flutter/example/lib/home.dart
Normal file
@@ -0,0 +1,255 @@
|
||||
// Copyright (c) 2024 Xiaomi Corporation
|
||||
import 'dart:async';
|
||||
|
||||
import 'package:flutter/foundation.dart';
|
||||
import 'package:flutter/material.dart';
|
||||
import 'package:path/path.dart' as p;
|
||||
import 'package:path_provider/path_provider.dart';
|
||||
import 'package:record/record.dart';
|
||||
|
||||
import 'package:sherpa_onnx/sherpa_onnx.dart' as sherpa_onnx;
|
||||
|
||||
import './utils.dart';
|
||||
|
||||
class HomeScreen extends StatefulWidget {
|
||||
const HomeScreen({super.key});
|
||||
|
||||
@override
|
||||
State<HomeScreen> createState() => _HomeScreenState();
|
||||
}
|
||||
|
||||
class _HomeScreenState extends State<HomeScreen> {
|
||||
late final AudioRecorder _audioRecorder;
|
||||
|
||||
bool _printed = false;
|
||||
var _color = Colors.black;
|
||||
bool _isInitialized = false;
|
||||
|
||||
sherpa_onnx.VoiceActivityDetector? _vad;
|
||||
sherpa_onnx.CircularBuffer? _buffer;
|
||||
|
||||
StreamSubscription<RecordState>? _recordSub;
|
||||
RecordState _recordState = RecordState.stop;
|
||||
|
||||
@override
|
||||
void initState() {
|
||||
_audioRecorder = AudioRecorder();
|
||||
|
||||
_recordSub = _audioRecorder.onStateChanged().listen((recordState) {
|
||||
_updateRecordState(recordState);
|
||||
});
|
||||
|
||||
super.initState();
|
||||
}
|
||||
|
||||
Future<void> _start() async {
|
||||
if (!_isInitialized) {
|
||||
sherpa_onnx.initBindings();
|
||||
final src = 'assets/silero_vad.onnx';
|
||||
final modelPath = await copyAssetFile(src: src, dst: 'silero_vad.onnx');
|
||||
|
||||
final sileroVadConfig = sherpa_onnx.SileroVadModelConfig(
|
||||
model: modelPath,
|
||||
minSpeechDuration: 0.25,
|
||||
minSilenceDuration: 0.5,
|
||||
);
|
||||
|
||||
final config = sherpa_onnx.VadModelConfig(
|
||||
sileroVad: sileroVadConfig,
|
||||
numThreads: 1,
|
||||
debug: true,
|
||||
);
|
||||
|
||||
_vad = sherpa_onnx.VoiceActivityDetector(
|
||||
config: config, bufferSizeInSeconds: 30);
|
||||
|
||||
_buffer = sherpa_onnx.CircularBuffer(capacity: 16000 * 30);
|
||||
print(_buffer!.ptr);
|
||||
|
||||
_isInitialized = true;
|
||||
}
|
||||
|
||||
try {
|
||||
if (await _audioRecorder.hasPermission()) {
|
||||
const encoder = AudioEncoder.pcm16bits;
|
||||
|
||||
if (!await _isEncoderSupported(encoder)) {
|
||||
return;
|
||||
}
|
||||
|
||||
final devs = await _audioRecorder.listInputDevices();
|
||||
debugPrint(devs.toString());
|
||||
|
||||
const config = RecordConfig(
|
||||
encoder: encoder,
|
||||
sampleRate: 16000,
|
||||
numChannels: 1,
|
||||
);
|
||||
|
||||
final stream = await _audioRecorder.startStream(config);
|
||||
|
||||
final dir = await getApplicationDocumentsDirectory();
|
||||
|
||||
stream.listen(
|
||||
(data) {
|
||||
final samplesFloat32 =
|
||||
convertBytesToFloat32(Uint8List.fromList(data));
|
||||
|
||||
_buffer!.push(samplesFloat32);
|
||||
|
||||
final windowSize = _vad!.config.sileroVad.windowSize;
|
||||
while (_buffer!.size > windowSize) {
|
||||
final samples =
|
||||
_buffer!.get(startIndex: _buffer!.head, n: windowSize);
|
||||
_buffer!.pop(windowSize);
|
||||
_vad!.acceptWaveform(samples);
|
||||
if (_vad!.isDetected() && !_printed) {
|
||||
print('detected');
|
||||
_printed = true;
|
||||
|
||||
setState(() => _color = Colors.red);
|
||||
}
|
||||
|
||||
if (!_vad!.isDetected()) {
|
||||
_printed = false;
|
||||
setState(() => _color = Colors.black);
|
||||
}
|
||||
|
||||
while (!_vad!.isEmpty()) {
|
||||
final segment = _vad!.front();
|
||||
final duration = segment.samples.length / 16000;
|
||||
final d = DateTime.now();
|
||||
final filename = p.join(dir.path,
|
||||
'${d.year}-${d.month}-${d.day}-${d.hour}-${d.minute}-${d.second}-duration-${duration.toStringAsPrecision(3)}s.wav');
|
||||
|
||||
bool ok = sherpa_onnx.writeWave(
|
||||
filename: filename,
|
||||
samples: segment.samples,
|
||||
sampleRate: 16000);
|
||||
if (!ok) {
|
||||
print('Failed to write $filename');
|
||||
} else {
|
||||
print('Saved to write $filename');
|
||||
}
|
||||
|
||||
_vad!.pop();
|
||||
}
|
||||
}
|
||||
},
|
||||
onDone: () {
|
||||
print('stream stopped.');
|
||||
},
|
||||
);
|
||||
}
|
||||
} catch (e) {
|
||||
print(e);
|
||||
}
|
||||
}
|
||||
|
||||
Future<void> _stop() async {
|
||||
_buffer!.reset();
|
||||
_vad!.clear();
|
||||
|
||||
await _audioRecorder.stop();
|
||||
}
|
||||
|
||||
Future<void> _pause() => _audioRecorder.pause();
|
||||
|
||||
Future<void> _resume() => _audioRecorder.resume();
|
||||
|
||||
void _updateRecordState(RecordState recordState) {
|
||||
setState(() => _recordState = recordState);
|
||||
}
|
||||
|
||||
Future<bool> _isEncoderSupported(AudioEncoder encoder) async {
|
||||
final isSupported = await _audioRecorder.isEncoderSupported(
|
||||
encoder,
|
||||
);
|
||||
|
||||
if (!isSupported) {
|
||||
debugPrint('${encoder.name} is not supported on this platform.');
|
||||
debugPrint('Supported encoders are:');
|
||||
|
||||
for (final e in AudioEncoder.values) {
|
||||
if (await _audioRecorder.isEncoderSupported(e)) {
|
||||
debugPrint('- ${encoder.name}');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return isSupported;
|
||||
}
|
||||
|
||||
@override
|
||||
Widget build(BuildContext context) {
|
||||
return MaterialApp(
|
||||
home: Scaffold(
|
||||
body: Column(
|
||||
mainAxisAlignment: MainAxisAlignment.center,
|
||||
children: [
|
||||
Container(
|
||||
width: 100.0,
|
||||
height: 100.0,
|
||||
decoration: BoxDecoration(
|
||||
shape: BoxShape.circle,
|
||||
color: _color,
|
||||
),
|
||||
),
|
||||
const SizedBox(height: 50),
|
||||
Row(
|
||||
mainAxisAlignment: MainAxisAlignment.center,
|
||||
children: <Widget>[
|
||||
_buildRecordStopControl(),
|
||||
const SizedBox(width: 20),
|
||||
_buildText(),
|
||||
],
|
||||
),
|
||||
],
|
||||
),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
@override
|
||||
void dispose() {
|
||||
_recordSub?.cancel();
|
||||
_audioRecorder.dispose();
|
||||
_vad?.free();
|
||||
_buffer?.free();
|
||||
super.dispose();
|
||||
}
|
||||
|
||||
Widget _buildRecordStopControl() {
|
||||
late Icon icon;
|
||||
late Color color;
|
||||
|
||||
if (_recordState != RecordState.stop) {
|
||||
icon = const Icon(Icons.stop, color: Colors.red, size: 30);
|
||||
color = Colors.red.withOpacity(0.1);
|
||||
} else {
|
||||
final theme = Theme.of(context);
|
||||
icon = Icon(Icons.mic, color: theme.primaryColor, size: 30);
|
||||
color = theme.primaryColor.withOpacity(0.1);
|
||||
}
|
||||
|
||||
return ClipOval(
|
||||
child: Material(
|
||||
color: color,
|
||||
child: InkWell(
|
||||
child: SizedBox(width: 56, height: 56, child: icon),
|
||||
onTap: () {
|
||||
(_recordState != RecordState.stop) ? _stop() : _start();
|
||||
},
|
||||
),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
Widget _buildText() {
|
||||
if (_recordState == RecordState.stop) {
|
||||
return const Text("Start");
|
||||
} else {
|
||||
return const Text("Stop");
|
||||
}
|
||||
}
|
||||
}
|
||||
40
sherpa-onnx/flutter/example/lib/info.dart
Normal file
40
sherpa-onnx/flutter/example/lib/info.dart
Normal file
@@ -0,0 +1,40 @@
|
||||
// Copyright (c) 2024 Xiaomi Corporation
|
||||
import 'package:flutter/material.dart';
|
||||
import 'package:url_launcher/url_launcher.dart';
|
||||
|
||||
class InfoScreen extends StatelessWidget {
|
||||
@override
|
||||
Widget build(BuildContext context) {
|
||||
const double height = 20;
|
||||
return Container(
|
||||
child: Padding(
|
||||
padding: const EdgeInsets.all(8.0),
|
||||
child: Column(
|
||||
crossAxisAlignment: CrossAxisAlignment.start,
|
||||
children: <Widget>[
|
||||
Text('Everything is open-sourced.'),
|
||||
SizedBox(height: height),
|
||||
InkWell(
|
||||
child: Text('Code: https://github.com/k2-fsa/sherpa-onnx'),
|
||||
onTap: () => launch('https://k2-fsa.github.io/sherpa/onnx/'),
|
||||
),
|
||||
SizedBox(height: height),
|
||||
InkWell(
|
||||
child: Text('Doc: https://k2-fsa.github.io/sherpa/onnx/'),
|
||||
onTap: () => launch('https://k2-fsa.github.io/sherpa/onnx/'),
|
||||
),
|
||||
SizedBox(height: height),
|
||||
Text('QQ 群: 744602236'),
|
||||
SizedBox(height: height),
|
||||
InkWell(
|
||||
child: Text(
|
||||
'微信群: https://k2-fsa.github.io/sherpa/social-groups.html'),
|
||||
onTap: () =>
|
||||
launch('https://k2-fsa.github.io/sherpa/social-groups.html'),
|
||||
),
|
||||
],
|
||||
),
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,8 @@ import 'package:flutter/material.dart';
|
||||
|
||||
import "./speaker_identification_test.dart";
|
||||
import "./vad_test.dart";
|
||||
import './home.dart';
|
||||
import './info.dart';
|
||||
|
||||
void main() {
|
||||
runApp(const MyApp());
|
||||
@@ -11,110 +13,56 @@ void main() {
|
||||
|
||||
class MyApp extends StatelessWidget {
|
||||
const MyApp({Key? key}) : super(key: key);
|
||||
// This widget is the root of your application.
|
||||
@override
|
||||
Widget build(BuildContext context) {
|
||||
return MaterialApp(
|
||||
title: 'Flutter Demo',
|
||||
title: 'Next-gen Kaldi',
|
||||
theme: ThemeData(
|
||||
// This is the theme of your application.
|
||||
//
|
||||
// Try running your application with "flutter run". You'll see the
|
||||
// application has a blue toolbar. Then, without quitting the app, try
|
||||
// changing the primarySwatch below to Colors.green and then invoke
|
||||
// "hot reload" (press "r" in the console where you ran "flutter run",
|
||||
// or simply save your changes to "hot reload" in a Flutter IDE).
|
||||
// Notice that the counter didn't reset back to zero; the application
|
||||
// is not restarted.
|
||||
primarySwatch: Colors.blue,
|
||||
),
|
||||
home: const MyHomePage(title: 'Flutter Demo Home Page'),
|
||||
home: const MyHomePage(title: 'Next-gen Kaldi: VAD demo'),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
class MyHomePage extends StatefulWidget {
|
||||
const MyHomePage({Key? key, required this.title}) : super(key: key);
|
||||
// This widget is the home page of your application. It is stateful, meaning
|
||||
// that it has a State object (defined below) that contains fields that affect
|
||||
// how it looks.
|
||||
// This class is the configuration for the state. It holds the values (in this
|
||||
// case the title) provided by the parent (in this case the App widget) and
|
||||
// used by the build method of the State. Fields in a Widget subclass are
|
||||
// always marked "final".
|
||||
final String title;
|
||||
@override
|
||||
State<MyHomePage> createState() => _MyHomePageState();
|
||||
}
|
||||
|
||||
class _MyHomePageState extends State<MyHomePage> {
|
||||
int _counter = 0;
|
||||
Future<void> _incrementCounter() async {
|
||||
if (_counter <= 10) {
|
||||
sherpa_onnx.initBindings();
|
||||
await testSpeakerID();
|
||||
// await testVad();
|
||||
}
|
||||
|
||||
setState(() {
|
||||
// This call to setState tells the Flutter framework that something has
|
||||
// changed in this State, which causes it to rerun the build method below
|
||||
// so that the display can reflect the updated values. If we changed
|
||||
// _counter without calling setState(), then the build method would not be
|
||||
// called again, and so nothing would appear to happen.
|
||||
_counter++;
|
||||
});
|
||||
}
|
||||
|
||||
int _currentIndex = 0;
|
||||
final List<Widget> _tabs = [
|
||||
HomeScreen(),
|
||||
InfoScreen(),
|
||||
];
|
||||
@override
|
||||
Widget build(BuildContext context) {
|
||||
// This method is rerun every time setState is called, for instance as done
|
||||
// by the _incrementCounter method above.
|
||||
//
|
||||
// The Flutter framework has been optimized to make rerunning build methods
|
||||
// fast, so that you can just rebuild anything that needs updating rather
|
||||
// than having to individually change instances of widgets.
|
||||
return Scaffold(
|
||||
appBar: AppBar(
|
||||
// Here we take the value from the MyHomePage object that was created by
|
||||
// the App.build method, and use it to set our appbar title.
|
||||
title: Text(widget.title),
|
||||
),
|
||||
body: Center(
|
||||
// Center is a layout widget. It takes a single child and positions it
|
||||
// in the middle of the parent.
|
||||
child: Column(
|
||||
// Column is also a layout widget. It takes a list of children and
|
||||
// arranges them vertically. By default, it sizes itself to fit its
|
||||
// children horizontally, and tries to be as tall as its parent.
|
||||
//
|
||||
// Invoke "debug painting" (press "p" in the console, choose the
|
||||
// "Toggle Debug Paint" action from the Flutter Inspector in Android
|
||||
// Studio, or the "Toggle Debug Paint" command in Visual Studio Code)
|
||||
// to see the wireframe for each widget.
|
||||
//
|
||||
// Column has various properties to control how it sizes itself and
|
||||
// how it positions its children. Here we use mainAxisAlignment to
|
||||
// center the children vertically; the main axis here is the vertical
|
||||
// axis because Columns are vertical (the cross axis would be
|
||||
// horizontal).
|
||||
mainAxisAlignment: MainAxisAlignment.center,
|
||||
children: <Widget>[
|
||||
const Text(
|
||||
'You have pushed the button this many times:',
|
||||
),
|
||||
Text(
|
||||
'$_counter',
|
||||
style: Theme.of(context).textTheme.headlineSmall,
|
||||
),
|
||||
],
|
||||
),
|
||||
body: _tabs[_currentIndex],
|
||||
bottomNavigationBar: BottomNavigationBar(
|
||||
currentIndex: _currentIndex,
|
||||
onTap: (int index) {
|
||||
setState(() {
|
||||
_currentIndex = index;
|
||||
});
|
||||
},
|
||||
items: [
|
||||
BottomNavigationBarItem(
|
||||
icon: Icon(Icons.home),
|
||||
label: 'Home',
|
||||
),
|
||||
BottomNavigationBarItem(
|
||||
icon: Icon(Icons.info),
|
||||
label: 'Info',
|
||||
),
|
||||
],
|
||||
),
|
||||
floatingActionButton: FloatingActionButton(
|
||||
onPressed: _incrementCounter,
|
||||
tooltip: 'Increment',
|
||||
child: const Icon(Icons.add),
|
||||
), // This trailing comma makes auto-formatting nicer for build methods.
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
import 'package:path/path.dart';
|
||||
import 'package:path_provider/path_provider.dart';
|
||||
import 'package:flutter/services.dart' show rootBundle;
|
||||
import 'dart:typed_data';
|
||||
import "dart:io";
|
||||
|
||||
// Copy the asset file from src to dst
|
||||
@@ -16,3 +17,16 @@ Future<String> copyAssetFile({required String src, required String dst}) async {
|
||||
|
||||
return target;
|
||||
}
|
||||
|
||||
Float32List convertBytesToFloat32(Uint8List bytes, [endian = Endian.little]) {
|
||||
final values = Float32List(bytes.length ~/ 2);
|
||||
|
||||
final data = ByteData.view(bytes.buffer);
|
||||
|
||||
for (var i = 0; i < bytes.length; i += 2) {
|
||||
int short = data.getInt16(i, endian);
|
||||
values[i ~/ 2] = short / 32678.0;
|
||||
}
|
||||
|
||||
return values;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user