初始化项目，由ModelHub XC社区提供模型
Model: wbbbbb/wav2vec2-large-chinese-zh-cn Source: Original Platform
This commit is contained in:
ModelHub XC
28
.gitattributes
vendored
Normal file
28
.gitattributes
vendored
Normal file
@@ -0,0 +1,28 @@
|
||||
*.7z filter=lfs diff=lfs merge=lfs -text
|
||||
*.arrow filter=lfs diff=lfs merge=lfs -text
|
||||
*.bin filter=lfs diff=lfs merge=lfs -text
|
||||
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
||||
*.ftz filter=lfs diff=lfs merge=lfs -text
|
||||
*.gz filter=lfs diff=lfs merge=lfs -text
|
||||
*.h5 filter=lfs diff=lfs merge=lfs -text
|
||||
*.joblib filter=lfs diff=lfs merge=lfs -text
|
||||
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.model filter=lfs diff=lfs merge=lfs -text
|
||||
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
||||
*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||
*.ot filter=lfs diff=lfs merge=lfs -text
|
||||
*.parquet filter=lfs diff=lfs merge=lfs -text
|
||||
*.pb filter=lfs diff=lfs merge=lfs -text
|
||||
*.pt filter=lfs diff=lfs merge=lfs -text
|
||||
*.pth filter=lfs diff=lfs merge=lfs -text
|
||||
*.rar filter=lfs diff=lfs merge=lfs -text
|
||||
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tflite filter=lfs diff=lfs merge=lfs -text
|
||||
*.tgz filter=lfs diff=lfs merge=lfs -text
|
||||
*.wasm filter=lfs diff=lfs merge=lfs -text
|
||||
*.xz filter=lfs diff=lfs merge=lfs -text
|
||||
*.zip filter=lfs diff=lfs merge=lfs -text
|
||||
*.zstandard filter=lfs diff=lfs merge=lfs -text
|
||||
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||
model.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||
157
README.md
Normal file
157
README.md
Normal file
@@ -0,0 +1,157 @@
|
||||
---
|
||||
language: zh
|
||||
datasets:
|
||||
- common_voice
|
||||
metrics:
|
||||
- wer
|
||||
- cer
|
||||
tags:
|
||||
- audio
|
||||
- automatic-speech-recognition
|
||||
- speech
|
||||
- xlsr-fine-tuning-week
|
||||
license: apache-2.0
|
||||
model-index:
|
||||
- name: XLSR Wav2Vec2 Chinese (zh-CN) by wbbbbb
|
||||
results:
|
||||
- task:
|
||||
name: Speech Recognition
|
||||
type: automatic-speech-recognition
|
||||
dataset:
|
||||
name: Common Voice zh-CN
|
||||
type: common_voice
|
||||
args: zh-CN
|
||||
metrics:
|
||||
- name: Test WER
|
||||
type: wer
|
||||
value: 70.47
|
||||
- name: Test CER
|
||||
type: cer
|
||||
value: 12.30
|
||||
---
|
||||
# Fine-tuned XLSR-53 large model for speech recognition in Chinese
|
||||
|
||||
Fine-tuned [facebook/wav2vec2-large-xlsr-53](https://huggingface.co/facebook/wav2vec2-large-xlsr-53) on Chinese using the train and validation splits of [Common Voice 6.1](https://huggingface.co/datasets/common_voice), [CSS10](https://github.com/Kyubyong/css10) and [ST-CMDS](http://www.openslr.org/38/).
|
||||
When using this model, make sure that your speech input is sampled at 16kHz.
|
||||
|
||||
This model has been fine-tuned on RTX3090 for 50h
|
||||
|
||||
The script used for training can be found here: https://github.com/jonatasgrosman/wav2vec2-sprint
|
||||
|
||||
## Usage
|
||||
|
||||
The model can be used directly (without a language model) as follows...
|
||||
|
||||
Using the [HuggingSound](https://github.com/jonatasgrosman/huggingsound) library:
|
||||
|
||||
```python
|
||||
from huggingsound import SpeechRecognitionModel
|
||||
model = SpeechRecognitionModel("wbbbbb/wav2vec2-large-chinese-zh-cn")
|
||||
audio_paths = ["/path/to/file.mp3", "/path/to/another_file.wav"]
|
||||
transcriptions = model.transcribe(audio_paths)
|
||||
```
|
||||
|
||||
|
||||
|
||||
## Evaluation
|
||||
|
||||
The model can be evaluated as follows on the Chinese (zh-CN) test data of Common Voice.
|
||||
|
||||
```python
|
||||
import torch
|
||||
import re
|
||||
import librosa
|
||||
from datasets import load_dataset, load_metric
|
||||
from transformers import Wav2Vec2ForCTC, Wav2Vec2Processor
|
||||
import warnings
|
||||
import os
|
||||
|
||||
os.environ["KMP_AFFINITY"] = ""
|
||||
|
||||
|
||||
LANG_ID = "zh-CN"
|
||||
MODEL_ID = "zh-CN-output-aishell"
|
||||
DEVICE = "cuda"
|
||||
|
||||
test_dataset = load_dataset("common_voice", LANG_ID, split="test")
|
||||
|
||||
wer = load_metric("wer")
|
||||
cer = load_metric("cer")
|
||||
|
||||
|
||||
|
||||
processor = Wav2Vec2Processor.from_pretrained(MODEL_ID)
|
||||
model = Wav2Vec2ForCTC.from_pretrained(MODEL_ID)
|
||||
model.to(DEVICE)
|
||||
|
||||
# Preprocessing the datasets.
|
||||
# We need to read the audio files as arrays
|
||||
def speech_file_to_array_fn(batch):
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore")
|
||||
speech_array, sampling_rate = librosa.load(batch["path"], sr=16_000)
|
||||
batch["speech"] = speech_array
|
||||
batch["sentence"] = (
|
||||
re.sub("([^\u4e00-\u9fa5\u0030-\u0039])", "", batch["sentence"]).lower() + " "
|
||||
)
|
||||
return batch
|
||||
|
||||
|
||||
test_dataset = test_dataset.map(
|
||||
speech_file_to_array_fn,
|
||||
num_proc=15,
|
||||
remove_columns=['client_id', 'up_votes', 'down_votes', 'age', 'gender', 'accent', 'locale', 'segment'],
|
||||
)
|
||||
|
||||
# Preprocessing the datasets.
|
||||
# We need to read the audio files as arrays
|
||||
def evaluate(batch):
|
||||
inputs = processor(
|
||||
batch["speech"], sampling_rate=16_000, return_tensors="pt", padding=True
|
||||
)
|
||||
|
||||
with torch.no_grad():
|
||||
logits = model(
|
||||
inputs.input_values.to(DEVICE),
|
||||
attention_mask=inputs.attention_mask.to(DEVICE),
|
||||
).logits
|
||||
|
||||
pred_ids = torch.argmax(logits, dim=-1)
|
||||
batch["pred_strings"] = processor.batch_decode(pred_ids)
|
||||
return batch
|
||||
|
||||
|
||||
result = test_dataset.map(evaluate, batched=True, batch_size=8)
|
||||
|
||||
predictions = [x.lower() for x in result["pred_strings"]]
|
||||
references = [x.lower() for x in result["sentence"]]
|
||||
|
||||
print(
|
||||
f"WER: {wer.compute(predictions=predictions, references=references, chunk_size=1000) * 100}"
|
||||
)
|
||||
print(f"CER: {cer.compute(predictions=predictions, references=references) * 100}")
|
||||
|
||||
```
|
||||
|
||||
**Test Result**:
|
||||
|
||||
In the table below I report the Word Error Rate (WER) and the Character Error Rate (CER) of the model. I ran the evaluation script described above on other models as well (on 2022-07-18). Note that the table below may show different results from those already reported, this may have been caused due to some specificity of the other evaluation scripts used.
|
||||
|
||||
| Model | WER | CER |
|
||||
| ------------- | ------------- | ------------- |
|
||||
| wbbbbb/wav2vec2-large-chinese-zh-cn | **70.47%** | **12.30%** |
|
||||
| jonatasgrosman/wav2vec2-large-xlsr-53-chinese-zh-cn | **82.37%** | **19.03%** |
|
||||
| ydshieh/wav2vec2-large-xlsr-53-chinese-zh-cn-gpt | 84.01% | 20.95% |
|
||||
|
||||
|
||||
## Citation
|
||||
If you want to cite this model you can use this:
|
||||
|
||||
```bibtex
|
||||
@misc{grosman2021xlsr53-large-chinese,
|
||||
title={Fine-tuned {XLSR}-53 large model for speech recognition in {C}hinese},
|
||||
author={Grosman, Jonatas},
|
||||
howpublished={\url{https://huggingface.co/wbbbbb/wav2vec2-large-chinese-zh-cn}},
|
||||
year={2021}
|
||||
}
|
||||
```
|
||||
115
config.json
Normal file
115
config.json
Normal file
@@ -0,0 +1,115 @@
|
||||
{
|
||||
"_name_or_path": "TencentGameMate/chinese-wav2vec2-large",
|
||||
"activation_dropout": 0.0,
|
||||
"adapter_kernel_size": 3,
|
||||
"adapter_stride": 2,
|
||||
"add_adapter": false,
|
||||
"apply_spec_augment": true,
|
||||
"architectures": [
|
||||
"Wav2Vec2ForPreTraining"
|
||||
],
|
||||
"attention_dropout": 0.0,
|
||||
"bos_token_id": 1,
|
||||
"classifier_proj_size": 256,
|
||||
"codevector_dim": 768,
|
||||
"contrastive_logits_temperature": 0.1,
|
||||
"conv_bias": true,
|
||||
"conv_dim": [
|
||||
512,
|
||||
512,
|
||||
512,
|
||||
512,
|
||||
512,
|
||||
512,
|
||||
512
|
||||
],
|
||||
"conv_kernel": [
|
||||
10,
|
||||
3,
|
||||
3,
|
||||
3,
|
||||
3,
|
||||
2,
|
||||
2
|
||||
],
|
||||
"conv_stride": [
|
||||
5,
|
||||
2,
|
||||
2,
|
||||
2,
|
||||
2,
|
||||
2,
|
||||
2
|
||||
],
|
||||
"ctc_loss_reduction": "mean",
|
||||
"ctc_zero_infinity": false,
|
||||
"diversity_loss_weight": 0.1,
|
||||
"do_stable_layer_norm": true,
|
||||
"eos_token_id": 2,
|
||||
"feat_extract_activation": "gelu",
|
||||
"feat_extract_dropout": 0.0,
|
||||
"feat_extract_norm": "layer",
|
||||
"feat_proj_dropout": 0.0,
|
||||
"feat_quantizer_dropout": 0.0,
|
||||
"final_dropout": 0.0,
|
||||
"hidden_act": "gelu",
|
||||
"hidden_dropout": 0.0,
|
||||
"hidden_size": 1024,
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 4096,
|
||||
"layer_norm_eps": 1e-05,
|
||||
"layerdrop": 0.1,
|
||||
"mask_channel_length": 10,
|
||||
"mask_channel_min_space": 1,
|
||||
"mask_channel_other": 0.0,
|
||||
"mask_channel_prob": 0.0,
|
||||
"mask_channel_selection": "static",
|
||||
"mask_feature_length": 10,
|
||||
"mask_feature_min_masks": 0,
|
||||
"mask_feature_prob": 0.0,
|
||||
"mask_time_length": 10,
|
||||
"mask_time_min_masks": 2,
|
||||
"mask_time_min_space": 1,
|
||||
"mask_time_other": 0.0,
|
||||
"mask_time_prob": 0.05,
|
||||
"mask_time_selection": "static",
|
||||
"model_type": "wav2vec2",
|
||||
"num_adapter_layers": 3,
|
||||
"num_attention_heads": 16,
|
||||
"num_codevector_groups": 2,
|
||||
"num_codevectors_per_group": 320,
|
||||
"num_conv_pos_embedding_groups": 16,
|
||||
"num_conv_pos_embeddings": 128,
|
||||
"num_feat_extract_layers": 7,
|
||||
"num_hidden_layers": 24,
|
||||
"num_negatives": 100,
|
||||
"output_hidden_size": 1024,
|
||||
"pad_token_id": 5168,
|
||||
"proj_codevector_dim": 768,
|
||||
"tdnn_dilation": [
|
||||
1,
|
||||
2,
|
||||
3,
|
||||
1,
|
||||
1
|
||||
],
|
||||
"tdnn_dim": [
|
||||
512,
|
||||
512,
|
||||
512,
|
||||
512,
|
||||
1500
|
||||
],
|
||||
"tdnn_kernel": [
|
||||
5,
|
||||
3,
|
||||
3,
|
||||
1,
|
||||
1
|
||||
],
|
||||
"torch_dtype": "float32",
|
||||
"transformers_version": "4.21.0.dev0",
|
||||
"use_weighted_layer_sum": false,
|
||||
"vocab_size": 5171,
|
||||
"xvector_output_dim": 512
|
||||
}
|
||||
3
events.out.tfevents.1657844204.4b5a3e58fbcb.699120.0
Normal file
3
events.out.tfevents.1657844204.4b5a3e58fbcb.699120.0
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:c374dba0c398629c6af644acd8b0fe9abe49bb0a9ea716cdc626fb91bf20d549
|
||||
size 109381
|
||||
3
model.safetensors
Normal file
3
model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:40dc8eaaef20377a3ed6bd97b0a61daf10f6499d1f27994f8a0f1ce9cf70cfec
|
||||
size 1283008572
|
||||
9
preprocessor_config.json
Normal file
9
preprocessor_config.json
Normal file
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"do_normalize": true,
|
||||
"feature_extractor_type": "Wav2Vec2FeatureExtractor",
|
||||
"feature_size": 1,
|
||||
"padding_side": "right",
|
||||
"padding_value": 0,
|
||||
"return_attention_mask": true,
|
||||
"sampling_rate": 16000
|
||||
}
|
||||
3
pytorch_model.bin
Normal file
3
pytorch_model.bin
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:0972b384a86152ecac856ad7a88c43b427c8e5cb7eb96198ac36ee7d55922f7a
|
||||
size 1283099825
|
||||
92
special_tokens_map.json
Normal file
92
special_tokens_map.json
Normal file
@@ -0,0 +1,92 @@
|
||||
{
|
||||
"additional_special_tokens": [
|
||||
{
|
||||
"content": "<s>",
|
||||
"lstrip": false,
|
||||
"normalized": true,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
{
|
||||
"content": "</s>",
|
||||
"lstrip": false,
|
||||
"normalized": true,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
{
|
||||
"content": "<s>",
|
||||
"lstrip": false,
|
||||
"normalized": true,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
{
|
||||
"content": "</s>",
|
||||
"lstrip": false,
|
||||
"normalized": true,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
{
|
||||
"content": "<s>",
|
||||
"lstrip": false,
|
||||
"normalized": true,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
{
|
||||
"content": "</s>",
|
||||
"lstrip": false,
|
||||
"normalized": true,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
{
|
||||
"content": "<s>",
|
||||
"lstrip": false,
|
||||
"normalized": true,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
{
|
||||
"content": "</s>",
|
||||
"lstrip": false,
|
||||
"normalized": true,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
{
|
||||
"content": "<s>",
|
||||
"lstrip": false,
|
||||
"normalized": true,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
{
|
||||
"content": "</s>",
|
||||
"lstrip": false,
|
||||
"normalized": true,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
{
|
||||
"content": "<s>",
|
||||
"lstrip": false,
|
||||
"normalized": true,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
{
|
||||
"content": "</s>",
|
||||
"lstrip": false,
|
||||
"normalized": true,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
}
|
||||
],
|
||||
"bos_token": "<s>",
|
||||
"eos_token": "</s>",
|
||||
"pad_token": "[PAD]",
|
||||
"unk_token": "[UNK]"
|
||||
}
|
||||
5171
vocab.json
Normal file
5171
vocab.json
Normal file
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user