From 6041d8b4ead587fd9de6020d417d164f26aa8351 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Sun, 18 Oct 2026 03:11:16 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: comodoro/wav2vec2-xls-r-300m-sk-cv8 Source: Original Platform --- .gitattributes | 27 +++++++ README.md | 131 +++++++++++++++++++++++++++++++ added_tokens.json | 1 + config.json | 108 ++++++++++++++++++++++++++ eval.py | 161 +++++++++++++++++++++++++++++++++++++++ preprocessor_config.json | 10 +++ pytorch_model.bin | 3 + special_tokens_map.json | 1 + tokenizer_config.json | 1 + vocab.json | 1 + 10 files changed, 444 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 added_tokens.json create mode 100644 config.json create mode 100755 eval.py create mode 100644 preprocessor_config.json create mode 100644 pytorch_model.bin create mode 100644 special_tokens_map.json create mode 100644 tokenizer_config.json create mode 100644 vocab.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..6d34772 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,27 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bin.* filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zstandard filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..2178d17 --- /dev/null +++ b/README.md @@ -0,0 +1,131 @@ +--- +language: +- sk +license: apache-2.0 +tags: +- automatic-speech-recognition +- mozilla-foundation/common_voice_8_0 +- robust-speech-event +- xlsr-fine-tuning-week +- hf-asr-leaderboard +datasets: +- common_voice +model-index: +- name: Slovak comodoro Wav2Vec2 XLSR 300M CV8 + results: + - task: + name: Automatic Speech Recognition + type: automatic-speech-recognition + dataset: + name: Common Voice 8 + type: mozilla-foundation/common_voice_8_0 + args: sk + metrics: + - name: Test WER + type: wer + value: 49.6 + - name: Test CER + type: cer + value: 13.3 + - task: + name: Automatic Speech Recognition + type: automatic-speech-recognition + dataset: + name: Robust Speech Event - Dev Data + type: speech-recognition-community-v2/dev_data + args: sk + metrics: + - name: Test WER + type: wer + value: 81.7 + - task: + name: Automatic Speech Recognition + type: automatic-speech-recognition + dataset: + name: Robust Speech Event - Test Data + type: speech-recognition-community-v2/eval_data + args: sk + metrics: + - name: Test WER + type: wer + value: 80.26 +--- + + +# wav2vec2-xls-r-300m-cs-cv8 + +This model is a fine-tuned version of [facebook/wav2vec2-xls-r-300m](https://huggingface.co/facebook/wav2vec2-xls-r-300m) on the common_voice 8.0 dataset. +It achieves the following results on the evaluation set: + +- WER: 0.49575384615384616 +- CER: 0.13333333333333333 + +## Usage + +The model can be used directly (without a language model) as follows: + +```python +import torch +import torchaudio +from datasets import load_dataset +from transformers import Wav2Vec2ForCTC, Wav2Vec2Processor + +test_dataset = load_dataset("mozilla-foundation/common_voice_8_0", "sk", split="test[:2%]") + +processor = Wav2Vec2Processor.from_pretrained("comodoro/wav2vec2-xls-r-300m-sk-cv8") +model = Wav2Vec2ForCTC.from_pretrained("comodoro/wav2vec2-xls-r-300m-sk-cv8") + +resampler = torchaudio.transforms.Resample(48_000, 16_000) + +# Preprocessing the datasets. +# We need to read the aduio files as arrays +def speech_file_to_array_fn(batch): + speech_array, sampling_rate = torchaudio.load(batch["path"]) + batch["speech"] = resampler(speech_array).squeeze().numpy() + return batch + +test_dataset = test_dataset.map(speech_file_to_array_fn) +inputs = processor(test_dataset[:2]["speech"], sampling_rate=16_000, return_tensors="pt", padding=True) + +with torch.no_grad(): + logits = model(inputs.input_values, attention_mask=inputs.attention_mask).logits + +predicted_ids = torch.argmax(logits, dim=-1) + +print("Prediction:", processor.batch_decode(predicted_ids)) +print("Reference:", test_dataset[:2]["sentence"]) +``` + +## Evaluation + +The model can be evaluated using the attached `eval.py` script: +``` +python eval.py --model_id comodoro/wav2vec2-xls-r-300m-sk-cv8 --dataset mozilla-foundation/common_voice_8_0 --split test --config sk +``` + +## Training and evaluation data + +The Common Voice 8.0 `train` and `validation` datasets were used for training + +### Training hyperparameters + +The following hyperparameters were used during training: + +- learning_rate: 7e-4 +- train_batch_size: 32 +- eval_batch_size: 8 +- seed: 42 +- gradient_accumulation_steps: 20 +- total_train_batch_size: 640 +- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08 +- lr_scheduler_type: linear +- lr_scheduler_warmup_steps: 500 +- num_epochs: 50 +- mixed_precision_training: Native AMP + +### Framework versions + +- Transformers 4.16.0.dev0 +- Pytorch 1.10.1+cu102 +- Datasets 1.17.1.dev0 +- Tokenizers 0.11.0 diff --git a/added_tokens.json b/added_tokens.json new file mode 100644 index 0000000..6273cd4 --- /dev/null +++ b/added_tokens.json @@ -0,0 +1 @@ +{"": 47, "": 48} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..7145283 --- /dev/null +++ b/config.json @@ -0,0 +1,108 @@ +{ + "_name_or_path": "facebook/wav2vec2-xls-r-300m", + "activation_dropout": 0.0, + "adapter_kernel_size": 3, + "adapter_stride": 2, + "add_adapter": false, + "apply_spec_augment": true, + "architectures": [ + "Wav2Vec2ForCTC" + ], + "attention_dropout": 0.1, + "bos_token_id": 1, + "classifier_proj_size": 256, + "codevector_dim": 768, + "contrastive_logits_temperature": 0.1, + "conv_bias": true, + "conv_dim": [ + 512, + 512, + 512, + 512, + 512, + 512, + 512 + ], + "conv_kernel": [ + 10, + 3, + 3, + 3, + 3, + 2, + 2 + ], + "conv_stride": [ + 5, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "ctc_loss_reduction": "mean", + "ctc_zero_infinity": false, + "diversity_loss_weight": 0.1, + "do_stable_layer_norm": true, + "eos_token_id": 2, + "feat_extract_activation": "gelu", + "feat_extract_dropout": 0.0, + "feat_extract_norm": "layer", + "feat_proj_dropout": 0.0, + "feat_quantizer_dropout": 0.0, + "final_dropout": 0.0, + "gradient_checkpointing": false, + "hidden_act": "gelu", + "hidden_dropout": 0.1, + "hidden_size": 1024, + "initializer_range": 0.02, + "intermediate_size": 4096, + "layer_norm_eps": 1e-05, + "layerdrop": 0.1, + "mask_feature_length": 10, + "mask_feature_min_masks": 0, + "mask_feature_prob": 0.0, + "mask_time_length": 10, + "mask_time_min_masks": 2, + "mask_time_prob": 0.05, + "model_type": "wav2vec2", + "num_adapter_layers": 3, + "num_attention_heads": 16, + "num_codevector_groups": 2, + "num_codevectors_per_group": 320, + "num_conv_pos_embedding_groups": 16, + "num_conv_pos_embeddings": 128, + "num_feat_extract_layers": 7, + "num_hidden_layers": 24, + "num_negatives": 100, + "output_hidden_size": 1024, + "pad_token_id": 46, + "proj_codevector_dim": 768, + "tdnn_dilation": [ + 1, + 2, + 3, + 1, + 1 + ], + "tdnn_dim": [ + 512, + 512, + 512, + 512, + 1500 + ], + "tdnn_kernel": [ + 5, + 3, + 3, + 1, + 1 + ], + "torch_dtype": "float32", + "transformers_version": "4.16.0.dev0", + "use_weighted_layer_sum": false, + "vocab_size": 49, + "xvector_output_dim": 512 +} diff --git a/eval.py b/eval.py new file mode 100755 index 0000000..246108d --- /dev/null +++ b/eval.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 +from datasets import load_dataset, load_metric, Audio, Dataset +from transformers import pipeline, AutoFeatureExtractor +import re +import argparse +import unicodedata +from typing import Dict + + +def log_results(result: Dataset, args: Dict[str, str]): + """ DO NOT CHANGE. This function computes and logs the result metrics. """ + + log_outputs = args.log_outputs + dataset_id = "_".join(args.dataset.split("/") + [args.config, args.split]) + + # load metric + wer = load_metric("wer") + cer = load_metric("cer") + + # compute metrics + wer_result = wer.compute(references=result["target"], predictions=result["prediction"]) + cer_result = cer.compute(references=result["target"], predictions=result["prediction"]) + + # print & log results + result_str = ( + f"WER: {wer_result}\n" + f"CER: {cer_result}" + ) + print(result_str) + + with open(f"{dataset_id}_eval_results.txt", "w") as f: + f.write(result_str) + + # log all results in text file. Possibly interesting for analysis + if log_outputs is not None: + pred_file = f"log_{dataset_id}_predictions.txt" + target_file = f"log_{dataset_id}_targets.txt" + + with open(pred_file, "w") as p, open(target_file, "w") as t: + + # mapping function to write output + def write_to_file(batch, i): + p.write(f"{i}" + "\n") + p.write(batch["prediction"] + "\n") + t.write(f"{i}" + "\n") + t.write(batch["target"] + "\n") + + result.map(write_to_file, with_indices=True) + + +def normalize_text(text: str) -> str: + """ DO ADAPT FOR YOUR USE CASE. this function normalizes the target text. """ + + + CHARS = { + 'ü': 'ue', + 'ö': 'oe', + 'ï': 'i', + 'ë': 'e', + 'ã': 'a', + 'à': 'á', + 'ø': 'o', + 'è': 'é', + 'ê': 'é', + 'å': 'ó', + 'î': 'i', + 'ñ': 'ň', + 'ç': 's', + 'ż': 'ž', + 'ł': 'w', + 'ć': 'č', + 'þ': 't', + 'ß': 'ss', + 'ę': 'en', + 'ą': 'an', + 'æ': 'ae', + } + + def replace_chars(sentence): + result = '' + for ch in sentence: + new = CHARS[ch] if ch in CHARS else ch + result += new + + return result + + chars_to_ignore_regex = '[\,\?\.\!\-\;\:\/\"\“\„\%\”\�\–\'\`\«\»\—\’\…\³]' + + text = text.lower() + # normalize non-standard (stylized) unicode characters + text = unicodedata.normalize('NFKC', text) + # remove punctuation + text = re.sub(chars_to_ignore_regex, "", text) + text = replace_chars(text) + + # Let's also make sure we split on all kinds of newlines, spaces, etc... + text = " ".join(text.split()) + + return text + + +def main(args): + # load dataset + dataset = load_dataset(args.dataset, args.config, split=args.split, use_auth_token=True) + + # for testing: only process the first two examples as a test + # dataset = dataset.select(range(10)) + + # load processor + feature_extractor = AutoFeatureExtractor.from_pretrained(args.model_id) + sampling_rate = feature_extractor.sampling_rate + + # resample audio + dataset = dataset.cast_column("audio", Audio(sampling_rate=sampling_rate)) + + # load eval pipeline + asr = pipeline("automatic-speech-recognition", model=args.model_id) + + # map function to decode audio + def map_to_pred(batch): + prediction = asr(batch["audio"]["array"], chunk_length_s=args.chunk_length_s, stride_length_s=args.stride_length_s) + + batch["prediction"] = prediction["text"] + batch["target"] = normalize_text(batch["sentence"]) + return batch + + # run inference on all examples + result = dataset.map(map_to_pred, remove_columns=dataset.column_names) + + # compute and log_results + # do not change function below + log_results(result, args) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument( + "--model_id", type=str, required=True, help="Model identifier. Should be loadable with 🤗 Transformers" + ) + parser.add_argument( + "--dataset", type=str, required=True, help="Dataset name to evaluate the `model_id`. Should be loadable with 🤗 Datasets" + ) + parser.add_argument( + "--config", type=str, required=True, help="Config of the dataset. *E.g.* `'en'` for Common Voice" + ) + parser.add_argument( + "--split", type=str, required=True, help="Split of the dataset. *E.g.* `'test'`" + ) + parser.add_argument( + "--chunk_length_s", type=float, default=None, help="Chunk length in seconds. Defaults to None. For long audio files a good value would be 5.0 seconds." + ) + parser.add_argument( + "--stride_length_s", type=float, default=None, help="Stride of the audio chunks. Defaults to None. For long audio files a good value would be 1.0 seconds." + ) + parser.add_argument( + "--log_outputs", action='store_true', help="If defined, write outputs to log file for analysis." + ) + args = parser.parse_args() + + main(args) diff --git a/preprocessor_config.json b/preprocessor_config.json new file mode 100644 index 0000000..67dee51 --- /dev/null +++ b/preprocessor_config.json @@ -0,0 +1,10 @@ +{ + "do_normalize": true, + "feature_extractor_type": "Wav2Vec2FeatureExtractor", + "feature_size": 1, + "padding_side": "right", + "padding_value": 0.0, + "processor_class": "Wav2Vec2Processor", + "return_attention_mask": true, + "sampling_rate": 16000 +} diff --git a/pytorch_model.bin b/pytorch_model.bin new file mode 100644 index 0000000..a50386f --- /dev/null +++ b/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:721ace680bba9be71df42335fd08f2b631aa72bdd168b1804d9eb98fd9e25c90 +size 1262124593 diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..623bcb0 --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1 @@ +{"bos_token": "", "eos_token": "", "unk_token": "[UNK]", "pad_token": "[PAD]", "additional_special_tokens": [{"content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}]} \ No newline at end of file diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..e89b92f --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1 @@ +{"unk_token": "[UNK]", "bos_token": "", "eos_token": "", "pad_token": "[PAD]", "do_lower_case": false, "word_delimiter_token": "|", "special_tokens_map_file": null, "tokenizer_file": null, "name_or_path": "./", "tokenizer_class": "Wav2Vec2CTCTokenizer", "processor_class": "Wav2Vec2Processor"} \ No newline at end of file diff --git a/vocab.json b/vocab.json new file mode 100644 index 0000000..aeb2ec3 --- /dev/null +++ b/vocab.json @@ -0,0 +1 @@ +{"a": 1, "b": 2, "c": 3, "d": 4, "e": 5, "f": 6, "g": 7, "h": 8, "i": 9, "j": 10, "k": 11, "l": 12, "m": 13, "n": 14, "o": 15, "p": 16, "q": 17, "r": 18, "s": 19, "t": 20, "u": 21, "v": 22, "w": 23, "x": 24, "y": 25, "z": 26, "á": 27, "ä": 28, "é": 29, "í": 30, "ó": 31, "ô": 32, "ú": 33, "ý": 34, "č": 35, "ď": 36, "ĺ": 37, "ľ": 38, "ň": 39, "ŕ": 40, "š": 41, "ť": 42, "ž": 43, "̌": 44, "|": 0, "[UNK]": 45, "[PAD]": 46} \ No newline at end of file