From 14a51929790ebc6b59e8437034b1d76cc907dc04 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Tue, 20 Oct 2026 22:51:22 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: NbAiLab/nb-wav2vec2-1b-bokmaal-v2 Source: Original Platform --- .gitattributes | 39 ++ README.md | 3 + README.org.md | 1078 +++++++++++++++++++++++++++++++++ add_kenlm.py | 37 ++ added_tokens.json | 1 + alphabet.json | 1 + cardinal_numbers.py | 690 +++++++++++++++++++++ config.json | 107 ++++ eval.py | 254 ++++++++ language_model/5gram.bin | 3 + language_model/attrs.json | 1 + language_model/unigrams.txt | 3 + model.safetensors | 3 + preprocessor_config.json | 10 + pytorch_model.bin | 3 + run.sh | 38 ++ run_speech_recognition_ctc.py | 807 ++++++++++++++++++++++++ special_tokens_map.json | 1 + tokenizer_config.json | 1 + training_args.bin | 3 + vocab.json | 1 + 21 files changed, 3084 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 README.org.md create mode 100644 add_kenlm.py create mode 100644 added_tokens.json create mode 100644 alphabet.json create mode 100644 cardinal_numbers.py create mode 100644 config.json create mode 100644 eval.py create mode 100644 language_model/5gram.bin create mode 100644 language_model/attrs.json create mode 100644 language_model/unigrams.txt create mode 100644 model.safetensors create mode 100644 preprocessor_config.json create mode 100644 pytorch_model.bin create mode 100755 run.sh create mode 100644 run_speech_recognition_ctc.py create mode 100644 special_tokens_map.json create mode 100644 tokenizer_config.json create mode 100644 training_args.bin create mode 100644 vocab.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..3297853 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,39 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zstandard filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +wandb/run-20220829_122920-1y92iq2k/run-1y92iq2k.wandb filter=lfs diff=lfs merge=lfs -text +wandb/run-20220829_122920-1y92iq2k/logs/debug-internal.log filter=lfs diff=lfs merge=lfs -text +wandb/run-20220829_122920-1y92iq2k/files/output.log filter=lfs diff=lfs merge=lfs -text +*.wandb filter=lfs diff=lfs merge=lfs -text +wandb/run-20220919_091325-3hu9r4oi/logs/debug-internal.log filter=lfs diff=lfs merge=lfs -text +wandb/run-20220919_091325-3hu9r4oi/files/output.log filter=lfs diff=lfs merge=lfs -text +language_model/unigrams.txt filter=lfs diff=lfs merge=lfs -text +model.safetensors filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..154df82 --- /dev/null +++ b/README.md @@ -0,0 +1,3 @@ +--- +license: apache-2.0 +--- diff --git a/README.org.md b/README.org.md new file mode 100644 index 0000000..4269ffe --- /dev/null +++ b/README.org.md @@ -0,0 +1,1078 @@ +--- +license: apache-2.0 +tags: +- generated_from_trainer +base_model: facebook/wav2vec2-xls-r-1b +model-index: +- name: wav2vec2-1b-npsc-nst + results: [] +--- + + + +# wav2vec2-1b-npsc-nst + +FROM: https://huggingface.co/NbAiLab/wav2vec2-1b-npsc-nst + +This model is a fine-tuned version of [facebook/wav2vec2-xls-r-1b](https://huggingface.co/facebook/wav2vec2-xls-r-1b) on the None dataset. +It achieves the following results on the evaluation set: +- Loss: 0.0633 +- Wer: 0.0358 + +## Model description + +More information needed + +## Intended uses & limitations + +More information needed + +## Training and evaluation data + +More information needed + +## Training procedure + +### Training hyperparameters + +The following hyperparameters were used during training: +- learning_rate: 2e-05 +- train_batch_size: 12 +- eval_batch_size: 12 +- seed: 42 +- gradient_accumulation_steps: 2 +- total_train_batch_size: 24 +- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08 +- lr_scheduler_type: linear +- lr_scheduler_warmup_steps: 2000 +- num_epochs: 40.0 +- mixed_precision_training: Native AMP + +### Training results + +| Training Loss | Epoch | Step | Validation Loss | Wer | +|:-------------:|:-----:|:------:|:---------------:|:------:| +| 3.2301 | 0.04 | 500 | 3.1633 | 0.9980 | +| 3.0299 | 0.08 | 1000 | 3.0472 | 0.9958 | +| 1.7041 | 0.12 | 1500 | 1.5009 | 0.7927 | +| 0.9346 | 0.16 | 2000 | 0.5483 | 0.4676 | +| 0.7126 | 0.2 | 2500 | 0.4635 | 0.3379 | +| 0.4732 | 0.24 | 3000 | 0.2517 | 0.2402 | +| 0.4312 | 0.28 | 3500 | 0.1964 | 0.1957 | +| 0.373 | 0.32 | 4000 | 0.1775 | 0.1739 | +| 0.3233 | 0.35 | 4500 | 0.1721 | 0.1597 | +| 0.2865 | 0.39 | 5000 | 0.1507 | 0.1551 | +| 0.2901 | 0.43 | 5500 | 0.1427 | 0.1413 | +| 0.2656 | 0.47 | 6000 | 0.1399 | 0.1377 | +| 0.2641 | 0.51 | 6500 | 0.1332 | 0.1310 | +| 0.2489 | 0.55 | 7000 | 0.1309 | 0.1317 | +| 0.2531 | 0.59 | 7500 | 0.1247 | 0.1291 | +| 0.2524 | 0.63 | 8000 | 0.1247 | 0.1232 | +| 0.2142 | 0.67 | 8500 | 0.1248 | 0.1270 | +| 0.2233 | 0.71 | 9000 | 0.1121 | 0.1201 | +| 0.2313 | 0.75 | 9500 | 0.1102 | 0.1161 | +| 0.2207 | 0.79 | 10000 | 0.1117 | 0.1200 | +| 0.2252 | 0.83 | 10500 | 0.1087 | 0.1162 | +| 0.208 | 0.87 | 11000 | 0.1036 | 0.1129 | +| 0.2054 | 0.91 | 11500 | 0.1009 | 0.1075 | +| 0.2008 | 0.95 | 12000 | 0.0986 | 0.1072 | +| 0.2019 | 0.98 | 12500 | 0.0983 | 0.1057 | +| 0.1957 | 1.02 | 13000 | 0.0959 | 0.1032 | +| 0.181 | 1.06 | 13500 | 0.0958 | 0.1031 | +| 0.1883 | 1.1 | 14000 | 0.0912 | 0.1013 | +| 0.1747 | 1.14 | 14500 | 0.0898 | 0.0995 | +| 0.1821 | 1.18 | 15000 | 0.0920 | 0.0990 | +| 0.1793 | 1.22 | 15500 | 0.0879 | 0.0960 | +| 0.1853 | 1.26 | 16000 | 0.0908 | 0.0953 | +| 0.1903 | 1.3 | 16500 | 0.0864 | 0.0959 | +| 0.1709 | 1.34 | 17000 | 0.0873 | 0.0970 | +| 0.1676 | 1.38 | 17500 | 0.0886 | 0.0950 | +| 0.1837 | 1.42 | 18000 | 0.0842 | 0.0932 | +| 0.1679 | 1.46 | 18500 | 0.0870 | 0.0922 | +| 0.1782 | 1.5 | 19000 | 0.0847 | 0.0908 | +| 0.1684 | 1.54 | 19500 | 0.0838 | 0.0897 | +| 0.1748 | 1.58 | 20000 | 0.0834 | 0.0874 | +| 0.1682 | 1.62 | 20500 | 0.0832 | 0.0876 | +| 0.1646 | 1.65 | 21000 | 0.0814 | 0.0884 | +| 0.1716 | 1.69 | 21500 | 0.0818 | 0.0874 | +| 0.1661 | 1.73 | 22000 | 0.0808 | 0.0865 | +| 0.1797 | 1.77 | 22500 | 0.0810 | 0.0866 | +| 0.1754 | 1.81 | 23000 | 0.0775 | 0.0854 | +| 0.1648 | 1.85 | 23500 | 0.0785 | 0.0855 | +| 0.1393 | 1.89 | 24000 | 0.0805 | 0.0873 | +| 0.1598 | 1.93 | 24500 | 0.0796 | 0.0861 | +| 0.1544 | 1.97 | 25000 | 0.0775 | 0.0844 | +| 0.1357 | 2.01 | 25500 | 0.0768 | 0.0815 | +| 0.1436 | 2.05 | 26000 | 0.0778 | 0.0820 | +| 0.1394 | 2.09 | 26500 | 0.0750 | 0.0815 | +| 0.1541 | 2.13 | 27000 | 0.0755 | 0.0806 | +| 0.1422 | 2.17 | 27500 | 0.0704 | 0.0771 | +| 0.1395 | 2.21 | 28000 | 0.0695 | 0.0744 | +| 0.1385 | 2.25 | 28500 | 0.0688 | 0.0720 | +| 0.1297 | 2.29 | 29000 | 0.0715 | 0.0735 | +| 0.1377 | 2.32 | 29500 | 0.0709 | 0.0722 | +| 0.1385 | 2.36 | 30000 | 0.0670 | 0.0718 | +| 0.154 | 2.4 | 30500 | 0.0678 | 0.0711 | +| 0.1316 | 2.44 | 31000 | 0.0664 | 0.0725 | +| 0.1413 | 2.48 | 31500 | 0.0673 | 0.0701 | +| 0.144 | 2.52 | 32000 | 0.0660 | 0.0694 | +| 0.1396 | 2.56 | 32500 | 0.0653 | 0.0687 | +| 0.1364 | 2.6 | 33000 | 0.0637 | 0.0691 | +| 0.1295 | 2.64 | 33500 | 0.0631 | 0.0688 | +| 0.1461 | 2.68 | 34000 | 0.0643 | 0.0674 | +| 0.1435 | 2.72 | 34500 | 0.0648 | 0.0681 | +| 0.1349 | 2.76 | 35000 | 0.0634 | 0.0669 | +| 0.124 | 2.8 | 35500 | 0.0643 | 0.0672 | +| 0.121 | 2.84 | 36000 | 0.0653 | 0.0677 | +| 0.1264 | 2.88 | 36500 | 0.0635 | 0.0680 | +| 0.1239 | 2.92 | 37000 | 0.0629 | 0.0655 | +| 0.1293 | 2.95 | 37500 | 0.0627 | 0.0663 | +| 0.1319 | 2.99 | 38000 | 0.0622 | 0.0669 | +| 0.121 | 3.03 | 38500 | 0.0628 | 0.0659 | +| 0.1258 | 3.07 | 39000 | 0.0615 | 0.0647 | +| 0.1312 | 3.11 | 39500 | 0.0611 | 0.0655 | +| 0.1246 | 3.15 | 40000 | 0.0635 | 0.0658 | +| 0.124 | 3.19 | 40500 | 0.0653 | 0.0673 | +| 0.1294 | 3.23 | 41000 | 0.0622 | 0.0645 | +| 0.1179 | 3.27 | 41500 | 0.0614 | 0.0639 | +| 0.1232 | 3.31 | 42000 | 0.0611 | 0.0640 | +| 0.1315 | 3.35 | 42500 | 0.0626 | 0.0638 | +| 0.2402 | 3.39 | 43000 | 0.0616 | 0.0633 | +| 0.1245 | 3.43 | 43500 | 0.0610 | 0.0639 | +| 0.1192 | 3.47 | 44000 | 0.0597 | 0.0642 | +| 0.1227 | 3.51 | 44500 | 0.0614 | 0.0644 | +| 0.1176 | 3.55 | 45000 | 0.0607 | 0.0625 | +| 0.1176 | 3.59 | 45500 | 0.0585 | 0.0620 | +| 0.1123 | 3.62 | 46000 | 0.0594 | 0.0624 | +| 0.1204 | 3.66 | 46500 | 0.0590 | 0.0635 | +| 0.1142 | 3.7 | 47000 | 0.0584 | 0.0633 | +| 0.1171 | 3.74 | 47500 | 0.0603 | 0.0624 | +| 0.1131 | 3.78 | 48000 | 0.0592 | 0.0619 | +| 0.1088 | 3.82 | 48500 | 0.0603 | 0.0615 | +| 0.1135 | 3.86 | 49000 | 0.0587 | 0.0611 | +| 0.1067 | 3.9 | 49500 | 0.0581 | 0.0620 | +| 0.1088 | 3.94 | 50000 | 0.0585 | 0.0601 | +| 0.1153 | 3.98 | 50500 | 0.0574 | 0.0618 | +| 0.1131 | 4.02 | 51000 | 0.0586 | 0.0611 | +| 0.1045 | 4.06 | 51500 | 0.0579 | 0.0605 | +| 0.1096 | 4.1 | 52000 | 0.0590 | 0.0601 | +| 0.102 | 4.14 | 52500 | 0.0590 | 0.0602 | +| 0.1019 | 4.18 | 53000 | 0.0579 | 0.0607 | +| 0.1098 | 4.22 | 53500 | 0.0591 | 0.0593 | +| 0.1105 | 4.25 | 54000 | 0.0587 | 0.0600 | +| 0.1079 | 4.29 | 54500 | 0.0585 | 0.0598 | +| 0.1017 | 4.33 | 55000 | 0.0593 | 0.0592 | +| 0.1095 | 4.37 | 55500 | 0.0569 | 0.0598 | +| 0.106 | 4.41 | 56000 | 0.0549 | 0.0597 | +| 0.1108 | 4.45 | 56500 | 0.0551 | 0.0601 | +| 0.2399 | 4.49 | 57000 | 0.0559 | 0.0585 | +| 0.1094 | 4.53 | 57500 | 0.0561 | 0.0594 | +| 0.1077 | 4.57 | 58000 | 0.0541 | 0.0598 | +| 0.1119 | 4.61 | 58500 | 0.0558 | 0.0595 | +| 0.1054 | 4.65 | 59000 | 0.0551 | 0.0601 | +| 0.1025 | 4.69 | 59500 | 0.0560 | 0.0584 | +| 0.1104 | 4.73 | 60000 | 0.0559 | 0.0591 | +| 0.1105 | 4.77 | 60500 | 0.0549 | 0.0574 | +| 0.0965 | 4.81 | 61000 | 0.0562 | 0.0576 | +| 0.1011 | 4.85 | 61500 | 0.0552 | 0.0579 | +| 0.1053 | 4.89 | 62000 | 0.0546 | 0.0576 | +| 0.1046 | 4.92 | 62500 | 0.0559 | 0.0570 | +| 0.0969 | 4.96 | 63000 | 0.0559 | 0.0579 | +| 0.1112 | 5.0 | 63500 | 0.0570 | 0.0580 | +| 0.1043 | 5.04 | 64000 | 0.0561 | 0.0570 | +| 0.0966 | 5.08 | 64500 | 0.0561 | 0.0580 | +| 0.0928 | 5.12 | 65000 | 0.0548 | 0.0573 | +| 0.0994 | 5.16 | 65500 | 0.0547 | 0.0570 | +| 0.0987 | 5.2 | 66000 | 0.0525 | 0.0590 | +| 0.1043 | 5.24 | 66500 | 0.0546 | 0.0572 | +| 0.0941 | 5.28 | 67000 | 0.0551 | 0.0573 | +| 0.1073 | 5.32 | 67500 | 0.0528 | 0.0577 | +| 0.0976 | 5.36 | 68000 | 0.0541 | 0.0579 | +| 0.0993 | 5.4 | 68500 | 0.0561 | 0.0574 | +| 0.1001 | 5.44 | 69000 | 0.0537 | 0.0568 | +| 0.0961 | 5.48 | 69500 | 0.0546 | 0.0578 | +| 0.0944 | 5.52 | 70000 | 0.0532 | 0.0564 | +| 0.0991 | 5.56 | 70500 | 0.0524 | 0.0567 | +| 0.1047 | 5.59 | 71000 | 0.0533 | 0.0575 | +| 0.0927 | 5.63 | 71500 | 0.0528 | 0.0572 | +| 0.1031 | 5.67 | 72000 | 0.0534 | 0.0556 | +| 0.102 | 5.71 | 72500 | 0.0528 | 0.0562 | +| 0.0936 | 5.75 | 73000 | 0.0528 | 0.0559 | +| 0.0954 | 5.79 | 73500 | 0.0512 | 0.0544 | +| 0.1025 | 5.83 | 74000 | 0.0534 | 0.0558 | +| 0.0934 | 5.87 | 74500 | 0.0519 | 0.0556 | +| 0.1 | 5.91 | 75000 | 0.0516 | 0.0554 | +| 0.0941 | 5.95 | 75500 | 0.0528 | 0.0553 | +| 0.0902 | 5.99 | 76000 | 0.0514 | 0.0551 | +| 0.0988 | 6.03 | 76500 | 0.0536 | 0.0550 | +| 0.0904 | 6.07 | 77000 | 0.0536 | 0.0554 | +| 0.0925 | 6.11 | 77500 | 0.0546 | 0.0553 | +| 0.0938 | 6.15 | 78000 | 0.0528 | 0.0548 | +| 0.0909 | 6.19 | 78500 | 0.0551 | 0.0549 | +| 0.0879 | 6.22 | 79000 | 0.0536 | 0.0549 | +| 0.0878 | 6.26 | 79500 | 0.0538 | 0.0556 | +| 0.0888 | 6.3 | 80000 | 0.0536 | 0.0549 | +| 0.0811 | 6.34 | 80500 | 0.0540 | 0.0565 | +| 0.0934 | 6.38 | 81000 | 0.0530 | 0.0553 | +| 0.0884 | 6.42 | 81500 | 0.0540 | 0.0547 | +| 0.0908 | 6.46 | 82000 | 0.0513 | 0.0539 | +| 0.0932 | 6.5 | 82500 | 0.0541 | 0.0547 | +| 0.097 | 6.54 | 83000 | 0.0519 | 0.0547 | +| 0.1023 | 6.58 | 83500 | 0.0527 | 0.0538 | +| 0.0862 | 6.62 | 84000 | 0.0516 | 0.0542 | +| 0.0882 | 6.66 | 84500 | 0.0532 | 0.0544 | +| 0.0947 | 6.7 | 85000 | 0.0520 | 0.0536 | +| 0.0891 | 6.74 | 85500 | 0.0524 | 0.0539 | +| 0.097 | 6.78 | 86000 | 0.0513 | 0.0535 | +| 0.0935 | 6.82 | 86500 | 0.0527 | 0.0541 | +| 0.0882 | 6.86 | 87000 | 0.0522 | 0.0539 | +| 0.0873 | 6.89 | 87500 | 0.0529 | 0.0532 | +| 0.0944 | 6.93 | 88000 | 0.0484 | 0.0530 | +| 0.0908 | 6.97 | 88500 | 0.0501 | 0.0542 | +| 0.0843 | 7.01 | 89000 | 0.0510 | 0.0537 | +| 0.0885 | 7.05 | 89500 | 0.0544 | 0.0536 | +| 0.0878 | 7.09 | 90000 | 0.0508 | 0.0529 | +| 0.083 | 7.13 | 90500 | 0.0500 | 0.0523 | +| 0.086 | 7.17 | 91000 | 0.0520 | 0.0531 | +| 0.0843 | 7.21 | 91500 | 0.0524 | 0.0544 | +| 0.079 | 7.25 | 92000 | 0.0517 | 0.0537 | +| 0.0806 | 7.29 | 92500 | 0.0513 | 0.0531 | +| 0.0758 | 7.33 | 93000 | 0.0518 | 0.0525 | +| 0.0865 | 7.37 | 93500 | 0.0518 | 0.0533 | +| 0.0872 | 7.41 | 94000 | 0.0515 | 0.0536 | +| 0.0795 | 7.45 | 94500 | 0.0518 | 0.0529 | +| 0.0939 | 7.49 | 95000 | 0.0504 | 0.0524 | +| 0.0856 | 7.52 | 95500 | 0.0519 | 0.0523 | +| 0.0894 | 7.56 | 96000 | 0.0506 | 0.0525 | +| 0.0794 | 7.6 | 96500 | 0.0520 | 0.0516 | +| 0.0839 | 7.64 | 97000 | 0.0527 | 0.0518 | +| 0.0873 | 7.68 | 97500 | 0.0518 | 0.0524 | +| 0.0793 | 7.72 | 98000 | 0.0509 | 0.0522 | +| 0.0762 | 7.76 | 98500 | 0.0517 | 0.0525 | +| 0.0881 | 7.8 | 99000 | 0.0502 | 0.0517 | +| 0.0832 | 7.84 | 99500 | 0.0504 | 0.0522 | +| 0.0777 | 7.88 | 100000 | 0.0502 | 0.0518 | +| 0.0844 | 7.92 | 100500 | 0.0512 | 0.0511 | +| 0.0848 | 7.96 | 101000 | 0.0493 | 0.0520 | +| 0.0926 | 8.0 | 101500 | 0.0478 | 0.0520 | +| 0.0767 | 8.04 | 102000 | 0.0492 | 0.0522 | +| 0.082 | 8.08 | 102500 | 0.0502 | 0.0514 | +| 0.0767 | 8.12 | 103000 | 0.0508 | 0.0509 | +| 0.072 | 8.16 | 103500 | 0.0510 | 0.0511 | +| 0.0813 | 8.19 | 104000 | 0.0503 | 0.0510 | +| 0.0737 | 8.23 | 104500 | 0.0488 | 0.0524 | +| 0.0782 | 8.27 | 105000 | 0.0494 | 0.0513 | +| 0.0763 | 8.31 | 105500 | 0.0498 | 0.0507 | +| 0.0789 | 8.35 | 106000 | 0.0505 | 0.0509 | +| 0.0755 | 8.39 | 106500 | 0.0508 | 0.0511 | +| 0.0718 | 8.43 | 107000 | 0.0513 | 0.0504 | +| 0.0784 | 8.47 | 107500 | 0.0492 | 0.0505 | +| 0.071 | 8.51 | 108000 | 0.0480 | 0.0504 | +| 0.0827 | 8.55 | 108500 | 0.0488 | 0.0506 | +| 0.0746 | 8.59 | 109000 | 0.0501 | 0.0510 | +| 0.0772 | 8.63 | 109500 | 0.0496 | 0.0501 | +| 0.0712 | 8.67 | 110000 | 0.0509 | 0.0509 | +| 0.0799 | 8.71 | 110500 | 0.0496 | 0.0511 | +| 0.0895 | 8.75 | 111000 | 0.0494 | 0.0508 | +| 0.0799 | 8.79 | 111500 | 0.0481 | 0.0507 | +| 0.084 | 8.83 | 112000 | 0.0488 | 0.0510 | +| 0.0811 | 8.86 | 112500 | 0.0489 | 0.0511 | +| 0.0842 | 8.9 | 113000 | 0.0490 | 0.0513 | +| 0.0701 | 8.94 | 113500 | 0.0489 | 0.0507 | +| 0.0893 | 8.98 | 114000 | 0.0473 | 0.0500 | +| 0.0654 | 9.02 | 114500 | 0.0500 | 0.0504 | +| 0.0774 | 9.06 | 115000 | 0.0501 | 0.0511 | +| 0.0691 | 9.1 | 115500 | 0.0513 | 0.0505 | +| 0.0724 | 9.14 | 116000 | 0.0506 | 0.0500 | +| 0.0721 | 9.18 | 116500 | 0.0502 | 0.0507 | +| 0.0715 | 9.22 | 117000 | 0.0501 | 0.0503 | +| 0.0768 | 9.26 | 117500 | 0.0496 | 0.0495 | +| 0.0758 | 9.3 | 118000 | 0.0491 | 0.0499 | +| 0.0681 | 9.34 | 118500 | 0.0484 | 0.0493 | +| 0.0835 | 9.38 | 119000 | 0.0504 | 0.0496 | +| 0.0775 | 9.42 | 119500 | 0.0500 | 0.0496 | +| 0.0797 | 9.46 | 120000 | 0.0492 | 0.0499 | +| 0.0736 | 9.49 | 120500 | 0.0502 | 0.0502 | +| 0.0704 | 9.53 | 121000 | 0.0497 | 0.0499 | +| 0.0735 | 9.57 | 121500 | 0.0505 | 0.0504 | +| 0.0712 | 9.61 | 122000 | 0.0489 | 0.0495 | +| 0.068 | 9.65 | 122500 | 0.0491 | 0.0496 | +| 0.0779 | 9.69 | 123000 | 0.0497 | 0.0493 | +| 0.0768 | 9.73 | 123500 | 0.0494 | 0.0496 | +| 0.0761 | 9.77 | 124000 | 0.0505 | 0.0498 | +| 0.0707 | 9.81 | 124500 | 0.0504 | 0.0493 | +| 0.0776 | 9.85 | 125000 | 0.0482 | 0.0494 | +| 0.0713 | 9.89 | 125500 | 0.0477 | 0.0490 | +| 0.0789 | 9.93 | 126000 | 0.0489 | 0.0499 | +| 0.0713 | 9.97 | 126500 | 0.0484 | 0.0497 | +| 0.0777 | 10.01 | 127000 | 0.0487 | 0.0493 | +| 0.0599 | 10.05 | 127500 | 0.0509 | 0.0496 | +| 0.0631 | 10.09 | 128000 | 0.0498 | 0.0501 | +| 0.0648 | 10.13 | 128500 | 0.0497 | 0.0493 | +| 0.0611 | 10.16 | 129000 | 0.0498 | 0.0490 | +| 0.0756 | 10.2 | 129500 | 0.0484 | 0.0492 | +| 0.0721 | 10.24 | 130000 | 0.0506 | 0.0489 | +| 0.0748 | 10.28 | 130500 | 0.0502 | 0.0487 | +| 0.0678 | 10.32 | 131000 | 0.0497 | 0.0490 | +| 0.0667 | 10.36 | 131500 | 0.0506 | 0.0489 | +| 0.0703 | 10.4 | 132000 | 0.0504 | 0.0485 | +| 0.0737 | 10.44 | 132500 | 0.0495 | 0.0491 | +| 0.0706 | 10.48 | 133000 | 0.0489 | 0.0492 | +| 0.0761 | 10.52 | 133500 | 0.0487 | 0.0492 | +| 0.0694 | 10.56 | 134000 | 0.0489 | 0.0485 | +| 0.0632 | 10.6 | 134500 | 0.0488 | 0.0487 | +| 0.0774 | 10.64 | 135000 | 0.0488 | 0.0480 | +| 0.0681 | 10.68 | 135500 | 0.0479 | 0.0482 | +| 0.0653 | 10.72 | 136000 | 0.0499 | 0.0486 | +| 0.0666 | 10.76 | 136500 | 0.0478 | 0.0488 | +| 0.0796 | 10.8 | 137000 | 0.0491 | 0.0481 | +| 0.0717 | 10.83 | 137500 | 0.0486 | 0.0481 | +| 0.0666 | 10.87 | 138000 | 0.0484 | 0.0493 | +| 0.0661 | 10.91 | 138500 | 0.0475 | 0.0480 | +| 0.0646 | 10.95 | 139000 | 0.0491 | 0.0486 | +| 0.0705 | 10.99 | 139500 | 0.0482 | 0.0487 | +| 0.0647 | 11.03 | 140000 | 0.0497 | 0.0477 | +| 0.0577 | 11.07 | 140500 | 0.0496 | 0.0478 | +| 0.0769 | 11.11 | 141000 | 0.0502 | 0.0482 | +| 0.0653 | 11.15 | 141500 | 0.0502 | 0.0474 | +| 0.0613 | 11.19 | 142000 | 0.0485 | 0.0473 | +| 0.0666 | 11.23 | 142500 | 0.0489 | 0.0479 | +| 0.0633 | 11.27 | 143000 | 0.0508 | 0.0478 | +| 0.0632 | 11.31 | 143500 | 0.0482 | 0.0480 | +| 0.0672 | 11.35 | 144000 | 0.0480 | 0.0487 | +| 0.0662 | 11.39 | 144500 | 0.0478 | 0.0476 | +| 0.0689 | 11.43 | 145000 | 0.0486 | 0.0473 | +| 0.0615 | 11.46 | 145500 | 0.0485 | 0.0470 | +| 0.0651 | 11.5 | 146000 | 0.0497 | 0.0485 | +| 0.0708 | 11.54 | 146500 | 0.0486 | 0.0474 | +| 0.0595 | 11.58 | 147000 | 0.0476 | 0.0473 | +| 0.0643 | 11.62 | 147500 | 0.0499 | 0.0482 | +| 0.0668 | 11.66 | 148000 | 0.0499 | 0.0473 | +| 0.0633 | 11.7 | 148500 | 0.0481 | 0.0471 | +| 0.0636 | 11.74 | 149000 | 0.0501 | 0.0477 | +| 0.0615 | 11.78 | 149500 | 0.0488 | 0.0476 | +| 0.0599 | 11.82 | 150000 | 0.0474 | 0.0469 | +| 0.0714 | 11.86 | 150500 | 0.0475 | 0.0477 | +| 0.0753 | 11.9 | 151000 | 0.0471 | 0.0463 | +| 0.061 | 11.94 | 151500 | 0.0472 | 0.0478 | +| 0.0692 | 11.98 | 152000 | 0.0472 | 0.0471 | +| 0.0682 | 12.02 | 152500 | 0.0484 | 0.0470 | +| 0.0643 | 12.06 | 153000 | 0.0480 | 0.0476 | +| 0.0651 | 12.1 | 153500 | 0.0470 | 0.0470 | +| 0.0616 | 12.13 | 154000 | 0.0502 | 0.0467 | +| 0.0545 | 12.17 | 154500 | 0.0491 | 0.0463 | +| 0.0547 | 12.21 | 155000 | 0.0492 | 0.0472 | +| 0.0574 | 12.25 | 155500 | 0.0498 | 0.0464 | +| 0.0585 | 12.29 | 156000 | 0.0482 | 0.0462 | +| 0.058 | 12.33 | 156500 | 0.0479 | 0.0460 | +| 0.067 | 12.37 | 157000 | 0.0468 | 0.0465 | +| 0.0656 | 12.41 | 157500 | 0.0487 | 0.0468 | +| 0.0591 | 12.45 | 158000 | 0.0489 | 0.0465 | +| 0.067 | 12.49 | 158500 | 0.0489 | 0.0467 | +| 0.0553 | 12.53 | 159000 | 0.0510 | 0.0463 | +| 0.0649 | 12.57 | 159500 | 0.0481 | 0.0465 | +| 0.0609 | 12.61 | 160000 | 0.0473 | 0.0460 | +| 0.063 | 12.65 | 160500 | 0.0471 | 0.0463 | +| 0.0602 | 12.69 | 161000 | 0.0468 | 0.0465 | +| 0.0631 | 12.73 | 161500 | 0.0470 | 0.0462 | +| 0.0603 | 12.76 | 162000 | 0.0481 | 0.0459 | +| 0.0607 | 12.8 | 162500 | 0.0485 | 0.0456 | +| 0.0571 | 12.84 | 163000 | 0.0492 | 0.0467 | +| 0.0636 | 12.88 | 163500 | 0.0484 | 0.0454 | +| 0.0614 | 12.92 | 164000 | 0.0493 | 0.0462 | +| 0.0601 | 12.96 | 164500 | 0.0481 | 0.0463 | +| 0.0647 | 13.0 | 165000 | 0.0499 | 0.0463 | +| 0.0551 | 13.04 | 165500 | 0.0487 | 0.0456 | +| 0.0531 | 13.08 | 166000 | 0.0503 | 0.0456 | +| 0.0526 | 13.12 | 166500 | 0.0475 | 0.0455 | +| 0.054 | 13.16 | 167000 | 0.0483 | 0.0463 | +| 0.0521 | 13.2 | 167500 | 0.0489 | 0.0459 | +| 0.0643 | 13.24 | 168000 | 0.0493 | 0.0463 | +| 0.0517 | 13.28 | 168500 | 0.0474 | 0.0459 | +| 0.0678 | 13.32 | 169000 | 0.0477 | 0.0462 | +| 0.0692 | 13.36 | 169500 | 0.0488 | 0.0456 | +| 0.0604 | 13.4 | 170000 | 0.0488 | 0.0459 | +| 0.0593 | 13.43 | 170500 | 0.0475 | 0.0463 | +| 0.0546 | 13.47 | 171000 | 0.0486 | 0.0461 | +| 0.0596 | 13.51 | 171500 | 0.0473 | 0.0463 | +| 0.058 | 13.55 | 172000 | 0.0490 | 0.0457 | +| 0.0601 | 13.59 | 172500 | 0.0498 | 0.0463 | +| 0.0523 | 13.63 | 173000 | 0.0492 | 0.0469 | +| 0.1803 | 13.67 | 173500 | 0.0475 | 0.0464 | +| 0.0542 | 13.71 | 174000 | 0.0486 | 0.0457 | +| 0.0579 | 13.75 | 174500 | 0.0489 | 0.0462 | +| 0.062 | 13.79 | 175000 | 0.0477 | 0.0462 | +| 0.0608 | 13.83 | 175500 | 0.0497 | 0.0458 | +| 0.059 | 13.87 | 176000 | 0.0484 | 0.0456 | +| 0.0588 | 13.91 | 176500 | 0.0489 | 0.0459 | +| 0.0633 | 13.95 | 177000 | 0.0490 | 0.0462 | +| 0.0522 | 13.99 | 177500 | 0.0474 | 0.0458 | +| 0.0464 | 14.03 | 178000 | 0.0483 | 0.0454 | +| 0.0546 | 14.07 | 178500 | 0.0485 | 0.0446 | +| 0.056 | 14.1 | 179000 | 0.0488 | 0.0454 | +| 0.0633 | 14.14 | 179500 | 0.0472 | 0.0458 | +| 0.0601 | 14.18 | 180000 | 0.0480 | 0.0450 | +| 0.0598 | 14.22 | 180500 | 0.0487 | 0.0447 | +| 0.0539 | 14.26 | 181000 | 0.0484 | 0.0455 | +| 0.058 | 14.3 | 181500 | 0.0483 | 0.0454 | +| 0.0557 | 14.34 | 182000 | 0.0490 | 0.0459 | +| 0.0569 | 14.38 | 182500 | 0.0491 | 0.0458 | +| 0.0614 | 14.42 | 183000 | 0.0503 | 0.0462 | +| 0.0596 | 14.46 | 183500 | 0.0493 | 0.0458 | +| 0.0547 | 14.5 | 184000 | 0.0491 | 0.0454 | +| 0.0536 | 14.54 | 184500 | 0.0497 | 0.0450 | +| 0.0598 | 14.58 | 185000 | 0.0483 | 0.0457 | +| 0.0555 | 14.62 | 185500 | 0.0486 | 0.0452 | +| 0.0634 | 14.66 | 186000 | 0.0468 | 0.0450 | +| 0.0649 | 14.7 | 186500 | 0.0493 | 0.0448 | +| 0.0646 | 14.73 | 187000 | 0.0481 | 0.0451 | +| 0.0551 | 14.77 | 187500 | 0.0505 | 0.0453 | +| 0.0649 | 14.81 | 188000 | 0.0482 | 0.0446 | +| 0.0664 | 14.85 | 188500 | 0.0484 | 0.0446 | +| 0.0544 | 14.89 | 189000 | 0.0483 | 0.0451 | +| 0.0579 | 14.93 | 189500 | 0.0476 | 0.0444 | +| 0.0554 | 14.97 | 190000 | 0.0475 | 0.0449 | +| 0.0498 | 15.01 | 190500 | 0.0504 | 0.0449 | +| 0.0508 | 15.05 | 191000 | 0.0491 | 0.0447 | +| 0.0568 | 15.09 | 191500 | 0.0482 | 0.0446 | +| 0.0572 | 15.13 | 192000 | 0.0479 | 0.0450 | +| 0.0527 | 15.17 | 192500 | 0.0492 | 0.0449 | +| 0.0515 | 15.21 | 193000 | 0.0499 | 0.0457 | +| 0.053 | 15.25 | 193500 | 0.0490 | 0.0446 | +| 0.0479 | 15.29 | 194000 | 0.0510 | 0.0449 | +| 0.0545 | 15.33 | 194500 | 0.0499 | 0.0448 | +| 0.0518 | 15.37 | 195000 | 0.0497 | 0.0444 | +| 0.0476 | 15.4 | 195500 | 0.0489 | 0.0446 | +| 0.054 | 15.44 | 196000 | 0.0494 | 0.0438 | +| 0.0562 | 15.48 | 196500 | 0.0491 | 0.0443 | +| 0.058 | 15.52 | 197000 | 0.0481 | 0.0439 | +| 0.0546 | 15.56 | 197500 | 0.0491 | 0.0447 | +| 0.0448 | 15.6 | 198000 | 0.0504 | 0.0445 | +| 0.0588 | 15.64 | 198500 | 0.0487 | 0.0438 | +| 0.0555 | 15.68 | 199000 | 0.0486 | 0.0438 | +| 0.052 | 15.72 | 199500 | 0.0491 | 0.0439 | +| 0.052 | 15.76 | 200000 | 0.0486 | 0.0442 | +| 0.0551 | 15.8 | 200500 | 0.0481 | 0.0441 | +| 0.0548 | 15.84 | 201000 | 0.0482 | 0.0442 | +| 0.0535 | 15.88 | 201500 | 0.0464 | 0.0445 | +| 0.0539 | 15.92 | 202000 | 0.0478 | 0.0443 | +| 0.0482 | 15.96 | 202500 | 0.0498 | 0.0440 | +| 0.0513 | 16.0 | 203000 | 0.0475 | 0.0441 | +| 0.051 | 16.03 | 203500 | 0.0485 | 0.0437 | +| 0.0524 | 16.07 | 204000 | 0.0500 | 0.0439 | +| 0.0578 | 16.11 | 204500 | 0.0478 | 0.0441 | +| 0.0528 | 16.15 | 205000 | 0.0480 | 0.0433 | +| 0.0515 | 16.19 | 205500 | 0.0490 | 0.0439 | +| 0.0516 | 16.23 | 206000 | 0.0490 | 0.0440 | +| 0.0476 | 16.27 | 206500 | 0.0488 | 0.0440 | +| 0.0504 | 16.31 | 207000 | 0.0485 | 0.0438 | +| 0.052 | 16.35 | 207500 | 0.0488 | 0.0447 | +| 0.1711 | 16.39 | 208000 | 0.0501 | 0.0444 | +| 0.0512 | 16.43 | 208500 | 0.0498 | 0.0443 | +| 0.0507 | 16.47 | 209000 | 0.0472 | 0.0443 | +| 0.0536 | 16.51 | 209500 | 0.0490 | 0.0438 | +| 0.0546 | 16.55 | 210000 | 0.0508 | 0.0436 | +| 0.046 | 16.59 | 210500 | 0.0482 | 0.0442 | +| 0.0504 | 16.63 | 211000 | 0.0474 | 0.0437 | +| 0.0584 | 16.67 | 211500 | 0.0480 | 0.0437 | +| 0.0525 | 16.7 | 212000 | 0.0487 | 0.0429 | +| 0.0482 | 16.74 | 212500 | 0.0479 | 0.0438 | +| 0.054 | 16.78 | 213000 | 0.0494 | 0.0435 | +| 0.0472 | 16.82 | 213500 | 0.0502 | 0.0436 | +| 0.0445 | 16.86 | 214000 | 0.0498 | 0.0446 | +| 0.0518 | 16.9 | 214500 | 0.0489 | 0.0436 | +| 0.0511 | 16.94 | 215000 | 0.0480 | 0.0435 | +| 0.049 | 16.98 | 215500 | 0.0497 | 0.0441 | +| 0.0448 | 17.02 | 216000 | 0.0496 | 0.0434 | +| 0.0489 | 17.06 | 216500 | 0.0488 | 0.0441 | +| 0.0516 | 17.1 | 217000 | 0.0489 | 0.0440 | +| 0.0486 | 17.14 | 217500 | 0.0511 | 0.0436 | +| 0.0525 | 17.18 | 218000 | 0.0495 | 0.0442 | +| 0.0491 | 17.22 | 218500 | 0.0512 | 0.0439 | +| 0.0469 | 17.26 | 219000 | 0.0512 | 0.0441 | +| 0.0469 | 17.3 | 219500 | 0.0485 | 0.0440 | +| 0.0575 | 17.34 | 220000 | 0.0490 | 0.0435 | +| 0.0439 | 17.37 | 220500 | 0.0515 | 0.0438 | +| 0.051 | 17.41 | 221000 | 0.0498 | 0.0442 | +| 0.0607 | 17.45 | 221500 | 0.0490 | 0.0440 | +| 0.0512 | 17.49 | 222000 | 0.0505 | 0.0441 | +| 0.0496 | 17.53 | 222500 | 0.0494 | 0.0442 | +| 0.0466 | 17.57 | 223000 | 0.0508 | 0.0438 | +| 0.0484 | 17.61 | 223500 | 0.0498 | 0.0434 | +| 0.0458 | 17.65 | 224000 | 0.0506 | 0.0432 | +| 0.0539 | 17.69 | 224500 | 0.0492 | 0.0432 | +| 0.0455 | 17.73 | 225000 | 0.0508 | 0.0432 | +| 0.0484 | 17.77 | 225500 | 0.0511 | 0.0430 | +| 0.0493 | 17.81 | 226000 | 0.0496 | 0.0428 | +| 0.0453 | 17.85 | 226500 | 0.0498 | 0.0431 | +| 0.0448 | 17.89 | 227000 | 0.0509 | 0.0429 | +| 0.0494 | 17.93 | 227500 | 0.0489 | 0.0433 | +| 0.0489 | 17.97 | 228000 | 0.0494 | 0.0431 | +| 0.0455 | 18.0 | 228500 | 0.0505 | 0.0427 | +| 0.0479 | 18.04 | 229000 | 0.0523 | 0.0423 | +| 0.0448 | 18.08 | 229500 | 0.0513 | 0.0427 | +| 0.0466 | 18.12 | 230000 | 0.0500 | 0.0424 | +| 0.0506 | 18.16 | 230500 | 0.0501 | 0.0429 | +| 0.051 | 18.2 | 231000 | 0.0488 | 0.0431 | +| 0.0455 | 18.24 | 231500 | 0.0525 | 0.0436 | +| 0.0446 | 18.28 | 232000 | 0.0513 | 0.0426 | +| 0.0427 | 18.32 | 232500 | 0.0493 | 0.0426 | +| 0.0499 | 18.36 | 233000 | 0.0511 | 0.0426 | +| 0.0558 | 18.4 | 233500 | 0.0521 | 0.0424 | +| 0.0412 | 18.44 | 234000 | 0.0501 | 0.0424 | +| 0.0472 | 18.48 | 234500 | 0.0495 | 0.0429 | +| 0.049 | 18.52 | 235000 | 0.0506 | 0.0425 | +| 0.0455 | 18.56 | 235500 | 0.0492 | 0.0426 | +| 0.0475 | 18.6 | 236000 | 0.0502 | 0.0430 | +| 0.0464 | 18.64 | 236500 | 0.0489 | 0.0424 | +| 0.047 | 18.67 | 237000 | 0.0477 | 0.0425 | +| 0.0523 | 18.71 | 237500 | 0.0499 | 0.0423 | +| 0.0458 | 18.75 | 238000 | 0.0492 | 0.0427 | +| 0.0443 | 18.79 | 238500 | 0.0486 | 0.0426 | +| 0.0446 | 18.83 | 239000 | 0.0503 | 0.0425 | +| 0.049 | 18.87 | 239500 | 0.0504 | 0.0425 | +| 0.049 | 18.91 | 240000 | 0.0496 | 0.0419 | +| 0.049 | 18.95 | 240500 | 0.0510 | 0.0423 | +| 0.0527 | 18.99 | 241000 | 0.0488 | 0.0423 | +| 0.041 | 19.03 | 241500 | 0.0518 | 0.0424 | +| 0.0449 | 19.07 | 242000 | 0.0516 | 0.0422 | +| 0.044 | 19.11 | 242500 | 0.0528 | 0.0423 | +| 0.0402 | 19.15 | 243000 | 0.0509 | 0.0419 | +| 0.0384 | 19.19 | 243500 | 0.0511 | 0.0424 | +| 0.0483 | 19.23 | 244000 | 0.0513 | 0.0423 | +| 0.0409 | 19.27 | 244500 | 0.0515 | 0.0429 | +| 0.0445 | 19.31 | 245000 | 0.0515 | 0.0421 | +| 0.0434 | 19.34 | 245500 | 0.0508 | 0.0428 | +| 0.0462 | 19.38 | 246000 | 0.0497 | 0.0424 | +| 0.0474 | 19.42 | 246500 | 0.0490 | 0.0421 | +| 0.0462 | 19.46 | 247000 | 0.0495 | 0.0425 | +| 0.0495 | 19.5 | 247500 | 0.0509 | 0.0429 | +| 0.0403 | 19.54 | 248000 | 0.0522 | 0.0428 | +| 0.0418 | 19.58 | 248500 | 0.0505 | 0.0421 | +| 0.0429 | 19.62 | 249000 | 0.0509 | 0.0424 | +| 0.0396 | 19.66 | 249500 | 0.0508 | 0.0419 | +| 0.0412 | 19.7 | 250000 | 0.0522 | 0.0420 | +| 0.041 | 19.74 | 250500 | 0.0513 | 0.0421 | +| 0.0471 | 19.78 | 251000 | 0.0495 | 0.0419 | +| 0.0426 | 19.82 | 251500 | 0.0490 | 0.0420 | +| 0.0484 | 19.86 | 252000 | 0.0483 | 0.0421 | +| 0.0456 | 19.9 | 252500 | 0.0511 | 0.0421 | +| 0.04 | 19.94 | 253000 | 0.0528 | 0.0417 | +| 0.0434 | 19.97 | 253500 | 0.0509 | 0.0415 | +| 0.0413 | 20.01 | 254000 | 0.0510 | 0.0419 | +| 0.0397 | 20.05 | 254500 | 0.0514 | 0.0420 | +| 0.0395 | 20.09 | 255000 | 0.0529 | 0.0419 | +| 0.0427 | 20.13 | 255500 | 0.0514 | 0.0420 | +| 0.0393 | 20.17 | 256000 | 0.0524 | 0.0423 | +| 0.0409 | 20.21 | 256500 | 0.0524 | 0.0421 | +| 0.0438 | 20.25 | 257000 | 0.0533 | 0.0422 | +| 0.0393 | 20.29 | 257500 | 0.0518 | 0.0421 | +| 0.0434 | 20.33 | 258000 | 0.0526 | 0.0420 | +| 0.043 | 20.37 | 258500 | 0.0517 | 0.0423 | +| 0.0464 | 20.41 | 259000 | 0.0518 | 0.0425 | +| 0.0392 | 20.45 | 259500 | 0.0520 | 0.0417 | +| 0.0438 | 20.49 | 260000 | 0.0523 | 0.0417 | +| 0.0437 | 20.53 | 260500 | 0.0514 | 0.0416 | +| 0.0461 | 20.57 | 261000 | 0.0524 | 0.0418 | +| 0.0456 | 20.61 | 261500 | 0.0515 | 0.0423 | +| 0.0426 | 20.64 | 262000 | 0.0517 | 0.0422 | +| 0.0403 | 20.68 | 262500 | 0.0515 | 0.0418 | +| 0.0385 | 20.72 | 263000 | 0.0522 | 0.0421 | +| 0.0397 | 20.76 | 263500 | 0.0512 | 0.0420 | +| 0.1716 | 20.8 | 264000 | 0.0508 | 0.0422 | +| 0.0378 | 20.84 | 264500 | 0.0521 | 0.0419 | +| 0.0398 | 20.88 | 265000 | 0.0517 | 0.0418 | +| 0.0444 | 20.92 | 265500 | 0.0492 | 0.0416 | +| 0.0385 | 20.96 | 266000 | 0.0518 | 0.0413 | +| 0.0462 | 21.0 | 266500 | 0.0506 | 0.0416 | +| 0.0386 | 21.04 | 267000 | 0.0520 | 0.0411 | +| 0.0345 | 21.08 | 267500 | 0.0538 | 0.0415 | +| 0.0435 | 21.12 | 268000 | 0.0567 | 0.0416 | +| 0.038 | 21.16 | 268500 | 0.0547 | 0.0416 | +| 0.0399 | 21.2 | 269000 | 0.0531 | 0.0415 | +| 0.0453 | 21.24 | 269500 | 0.0525 | 0.0415 | +| 0.037 | 21.27 | 270000 | 0.0541 | 0.0415 | +| 0.0404 | 21.31 | 270500 | 0.0543 | 0.0410 | +| 0.0418 | 21.35 | 271000 | 0.0548 | 0.0412 | +| 0.0384 | 21.39 | 271500 | 0.0548 | 0.0414 | +| 0.0439 | 21.43 | 272000 | 0.0526 | 0.0412 | +| 0.0398 | 21.47 | 272500 | 0.0520 | 0.0416 | +| 0.0396 | 21.51 | 273000 | 0.0525 | 0.0409 | +| 0.0427 | 21.55 | 273500 | 0.0529 | 0.0416 | +| 0.0448 | 21.59 | 274000 | 0.0514 | 0.0412 | +| 0.0384 | 21.63 | 274500 | 0.0531 | 0.0416 | +| 0.0432 | 21.67 | 275000 | 0.0526 | 0.0412 | +| 0.0361 | 21.71 | 275500 | 0.0530 | 0.0413 | +| 0.0416 | 21.75 | 276000 | 0.0500 | 0.0412 | +| 0.0374 | 21.79 | 276500 | 0.0511 | 0.0414 | +| 0.0373 | 21.83 | 277000 | 0.0522 | 0.0413 | +| 0.0399 | 21.87 | 277500 | 0.0533 | 0.0416 | +| 0.0409 | 21.91 | 278000 | 0.0517 | 0.0416 | +| 0.0391 | 21.94 | 278500 | 0.0497 | 0.0411 | +| 0.0424 | 21.98 | 279000 | 0.0520 | 0.0414 | +| 0.0426 | 22.02 | 279500 | 0.0547 | 0.0414 | +| 0.0434 | 22.06 | 280000 | 0.0509 | 0.0429 | +| 0.0368 | 22.1 | 280500 | 0.0527 | 0.0415 | +| 0.0411 | 22.14 | 281000 | 0.0546 | 0.0428 | +| 0.0398 | 22.18 | 281500 | 0.0553 | 0.0427 | +| 0.0455 | 22.22 | 282000 | 0.0541 | 0.0432 | +| 0.0402 | 22.26 | 282500 | 0.0554 | 0.0423 | +| 0.0404 | 22.3 | 283000 | 0.0541 | 0.0419 | +| 0.0369 | 22.34 | 283500 | 0.0529 | 0.0419 | +| 0.0386 | 22.38 | 284000 | 0.0532 | 0.0413 | +| 0.0343 | 22.42 | 284500 | 0.0540 | 0.0408 | +| 0.0357 | 22.46 | 285000 | 0.0523 | 0.0416 | +| 0.0447 | 22.5 | 285500 | 0.0531 | 0.0415 | +| 0.0363 | 22.54 | 286000 | 0.0555 | 0.0416 | +| 0.0332 | 22.58 | 286500 | 0.0538 | 0.0412 | +| 0.0347 | 22.61 | 287000 | 0.0537 | 0.0413 | +| 0.0405 | 22.65 | 287500 | 0.0518 | 0.0418 | +| 0.0342 | 22.69 | 288000 | 0.0545 | 0.0411 | +| 0.0382 | 22.73 | 288500 | 0.0529 | 0.0416 | +| 0.0341 | 22.77 | 289000 | 0.0534 | 0.0412 | +| 0.038 | 22.81 | 289500 | 0.0535 | 0.0408 | +| 0.0386 | 22.85 | 290000 | 0.0545 | 0.0412 | +| 0.0388 | 22.89 | 290500 | 0.0518 | 0.0416 | +| 0.0394 | 22.93 | 291000 | 0.0537 | 0.0412 | +| 0.035 | 22.97 | 291500 | 0.0545 | 0.0421 | +| 0.0337 | 23.01 | 292000 | 0.0541 | 0.0410 | +| 0.0381 | 23.05 | 292500 | 0.0538 | 0.0408 | +| 0.0377 | 23.09 | 293000 | 0.0549 | 0.0415 | +| 0.0359 | 23.13 | 293500 | 0.0545 | 0.0412 | +| 0.0389 | 23.17 | 294000 | 0.0538 | 0.0413 | +| 0.0327 | 23.21 | 294500 | 0.0539 | 0.0410 | +| 0.0444 | 23.24 | 295000 | 0.0526 | 0.0409 | +| 0.0386 | 23.28 | 295500 | 0.0538 | 0.0416 | +| 0.0375 | 23.32 | 296000 | 0.0524 | 0.0411 | +| 0.0388 | 23.36 | 296500 | 0.0542 | 0.0416 | +| 0.0314 | 23.4 | 297000 | 0.0539 | 0.0407 | +| 0.0371 | 23.44 | 297500 | 0.0533 | 0.0411 | +| 0.0431 | 23.48 | 298000 | 0.0533 | 0.0411 | +| 0.0297 | 23.52 | 298500 | 0.0551 | 0.0406 | +| 0.0385 | 23.56 | 299000 | 0.0536 | 0.0406 | +| 0.0379 | 23.6 | 299500 | 0.0531 | 0.0410 | +| 0.0359 | 23.64 | 300000 | 0.0552 | 0.0415 | +| 0.0353 | 23.68 | 300500 | 0.0552 | 0.0411 | +| 0.0432 | 23.72 | 301000 | 0.0517 | 0.0408 | +| 0.0346 | 23.76 | 301500 | 0.0536 | 0.0404 | +| 0.0332 | 23.8 | 302000 | 0.0552 | 0.0406 | +| 0.0417 | 23.84 | 302500 | 0.0510 | 0.0401 | +| 0.0326 | 23.88 | 303000 | 0.0540 | 0.0403 | +| 0.0367 | 23.91 | 303500 | 0.0527 | 0.0395 | +| 0.0346 | 23.95 | 304000 | 0.0522 | 0.0400 | +| 0.0373 | 23.99 | 304500 | 0.0526 | 0.0403 | +| 0.0308 | 24.03 | 305000 | 0.0544 | 0.0404 | +| 0.0361 | 24.07 | 305500 | 0.0550 | 0.0404 | +| 0.0332 | 24.11 | 306000 | 0.0546 | 0.0404 | +| 0.034 | 24.15 | 306500 | 0.0546 | 0.0405 | +| 0.0382 | 24.19 | 307000 | 0.0534 | 0.0402 | +| 0.0436 | 24.23 | 307500 | 0.0530 | 0.0404 | +| 0.0349 | 24.27 | 308000 | 0.0530 | 0.0405 | +| 0.0282 | 24.31 | 308500 | 0.0538 | 0.0403 | +| 0.0355 | 24.35 | 309000 | 0.0521 | 0.0400 | +| 0.0372 | 24.39 | 309500 | 0.0542 | 0.0401 | +| 0.036 | 24.43 | 310000 | 0.0553 | 0.0405 | +| 0.0354 | 24.47 | 310500 | 0.0546 | 0.0407 | +| 0.0368 | 24.51 | 311000 | 0.0537 | 0.0407 | +| 0.0358 | 24.54 | 311500 | 0.0525 | 0.0399 | +| 0.0376 | 24.58 | 312000 | 0.0561 | 0.0403 | +| 0.0365 | 24.62 | 312500 | 0.0551 | 0.0406 | +| 0.041 | 24.66 | 313000 | 0.0537 | 0.0399 | +| 0.0359 | 24.7 | 313500 | 0.0545 | 0.0404 | +| 0.0414 | 24.74 | 314000 | 0.0546 | 0.0405 | +| 0.0307 | 24.78 | 314500 | 0.0562 | 0.0405 | +| 0.0384 | 24.82 | 315000 | 0.0547 | 0.0409 | +| 0.0304 | 24.86 | 315500 | 0.0550 | 0.0407 | +| 0.0364 | 24.9 | 316000 | 0.0547 | 0.0405 | +| 0.0345 | 24.94 | 316500 | 0.0539 | 0.0411 | +| 0.037 | 24.98 | 317000 | 0.0518 | 0.0409 | +| 0.0328 | 25.02 | 317500 | 0.0554 | 0.0402 | +| 0.0336 | 25.06 | 318000 | 0.0551 | 0.0401 | +| 0.033 | 25.1 | 318500 | 0.0569 | 0.0402 | +| 0.0323 | 25.14 | 319000 | 0.0545 | 0.0401 | +| 0.0312 | 25.18 | 319500 | 0.0543 | 0.0398 | +| 0.0284 | 25.21 | 320000 | 0.0536 | 0.0397 | +| 0.0319 | 25.25 | 320500 | 0.0532 | 0.0399 | +| 0.0336 | 25.29 | 321000 | 0.0559 | 0.0395 | +| 0.0347 | 25.33 | 321500 | 0.0551 | 0.0400 | +| 0.0344 | 25.37 | 322000 | 0.0565 | 0.0401 | +| 0.0403 | 25.41 | 322500 | 0.0536 | 0.0397 | +| 0.0318 | 25.45 | 323000 | 0.0549 | 0.0402 | +| 0.0321 | 25.49 | 323500 | 0.0540 | 0.0400 | +| 0.0318 | 25.53 | 324000 | 0.0549 | 0.0397 | +| 0.0327 | 25.57 | 324500 | 0.0556 | 0.0398 | +| 0.0349 | 25.61 | 325000 | 0.0537 | 0.0404 | +| 0.0319 | 25.65 | 325500 | 0.0554 | 0.0402 | +| 0.0341 | 25.69 | 326000 | 0.0557 | 0.0399 | +| 0.035 | 25.73 | 326500 | 0.0539 | 0.0399 | +| 0.031 | 25.77 | 327000 | 0.0533 | 0.0398 | +| 0.0298 | 25.81 | 327500 | 0.0538 | 0.0401 | +| 0.032 | 25.85 | 328000 | 0.0549 | 0.0398 | +| 0.0337 | 25.88 | 328500 | 0.0545 | 0.0395 | +| 0.0349 | 25.92 | 329000 | 0.0543 | 0.0394 | +| 0.0348 | 25.96 | 329500 | 0.0558 | 0.0400 | +| 0.0299 | 26.0 | 330000 | 0.0560 | 0.0393 | +| 0.0302 | 26.04 | 330500 | 0.0569 | 0.0398 | +| 0.0298 | 26.08 | 331000 | 0.0580 | 0.0396 | +| 0.0319 | 26.12 | 331500 | 0.0554 | 0.0394 | +| 0.0325 | 26.16 | 332000 | 0.0554 | 0.0393 | +| 0.035 | 26.2 | 332500 | 0.0562 | 0.0394 | +| 0.0297 | 26.24 | 333000 | 0.0567 | 0.0394 | +| 0.0312 | 26.28 | 333500 | 0.0570 | 0.0397 | +| 0.03 | 26.32 | 334000 | 0.0566 | 0.0399 | +| 0.0309 | 26.36 | 334500 | 0.0553 | 0.0394 | +| 0.0317 | 26.4 | 335000 | 0.0568 | 0.0392 | +| 0.0314 | 26.44 | 335500 | 0.0564 | 0.0398 | +| 0.0327 | 26.48 | 336000 | 0.0567 | 0.0394 | +| 0.0297 | 26.51 | 336500 | 0.0567 | 0.0389 | +| 0.0355 | 26.55 | 337000 | 0.0553 | 0.0395 | +| 0.0368 | 26.59 | 337500 | 0.0551 | 0.0394 | +| 0.0326 | 26.63 | 338000 | 0.0563 | 0.0393 | +| 0.0329 | 26.67 | 338500 | 0.0558 | 0.0400 | +| 0.031 | 26.71 | 339000 | 0.0565 | 0.0398 | +| 0.0302 | 26.75 | 339500 | 0.0568 | 0.0392 | +| 0.0262 | 26.79 | 340000 | 0.0577 | 0.0395 | +| 0.0259 | 26.83 | 340500 | 0.0560 | 0.0394 | +| 0.0304 | 26.87 | 341000 | 0.0558 | 0.0397 | +| 0.0308 | 26.91 | 341500 | 0.0557 | 0.0397 | +| 0.0308 | 26.95 | 342000 | 0.0563 | 0.0393 | +| 0.0392 | 26.99 | 342500 | 0.0540 | 0.0390 | +| 0.031 | 27.03 | 343000 | 0.0556 | 0.0395 | +| 0.0284 | 27.07 | 343500 | 0.0586 | 0.0395 | +| 0.0275 | 27.11 | 344000 | 0.0574 | 0.0393 | +| 0.0302 | 27.15 | 344500 | 0.0571 | 0.0393 | +| 0.0323 | 27.18 | 345000 | 0.0570 | 0.0391 | +| 0.0312 | 27.22 | 345500 | 0.0556 | 0.0394 | +| 0.0291 | 27.26 | 346000 | 0.0567 | 0.0396 | +| 0.0332 | 27.3 | 346500 | 0.0580 | 0.0390 | +| 0.0321 | 27.34 | 347000 | 0.0566 | 0.0392 | +| 0.0323 | 27.38 | 347500 | 0.0572 | 0.0389 | +| 0.0308 | 27.42 | 348000 | 0.0576 | 0.0389 | +| 0.0318 | 27.46 | 348500 | 0.0561 | 0.0393 | +| 0.0297 | 27.5 | 349000 | 0.0569 | 0.0391 | +| 0.0303 | 27.54 | 349500 | 0.0564 | 0.0393 | +| 0.0306 | 27.58 | 350000 | 0.0580 | 0.0388 | +| 0.029 | 27.62 | 350500 | 0.0582 | 0.0391 | +| 0.031 | 27.66 | 351000 | 0.0577 | 0.0390 | +| 0.0314 | 27.7 | 351500 | 0.0574 | 0.0390 | +| 0.0364 | 27.74 | 352000 | 0.0589 | 0.0391 | +| 0.0337 | 27.78 | 352500 | 0.0552 | 0.0390 | +| 0.0301 | 27.81 | 353000 | 0.0567 | 0.0385 | +| 0.0285 | 27.85 | 353500 | 0.0564 | 0.0385 | +| 0.0323 | 27.89 | 354000 | 0.0553 | 0.0385 | +| 0.027 | 27.93 | 354500 | 0.0573 | 0.0385 | +| 0.0301 | 27.97 | 355000 | 0.0550 | 0.0386 | +| 0.0249 | 28.01 | 355500 | 0.0578 | 0.0387 | +| 0.026 | 28.05 | 356000 | 0.0585 | 0.0387 | +| 0.0292 | 28.09 | 356500 | 0.0562 | 0.0390 | +| 0.0343 | 28.13 | 357000 | 0.0580 | 0.0391 | +| 0.0314 | 28.17 | 357500 | 0.0579 | 0.0390 | +| 0.0283 | 28.21 | 358000 | 0.0598 | 0.0390 | +| 0.0283 | 28.25 | 358500 | 0.0573 | 0.0386 | +| 0.033 | 28.29 | 359000 | 0.0561 | 0.0387 | +| 0.0241 | 28.33 | 359500 | 0.0572 | 0.0386 | +| 0.0291 | 28.37 | 360000 | 0.0564 | 0.0386 | +| 0.0294 | 28.41 | 360500 | 0.0589 | 0.0386 | +| 0.0301 | 28.45 | 361000 | 0.0579 | 0.0388 | +| 0.0303 | 28.48 | 361500 | 0.0569 | 0.0385 | +| 0.0284 | 28.52 | 362000 | 0.0573 | 0.0382 | +| 0.0298 | 28.56 | 362500 | 0.0569 | 0.0385 | +| 0.0271 | 28.6 | 363000 | 0.0575 | 0.0382 | +| 0.0285 | 28.64 | 363500 | 0.0572 | 0.0384 | +| 0.0275 | 28.68 | 364000 | 0.0570 | 0.0380 | +| 0.0312 | 28.72 | 364500 | 0.0578 | 0.0381 | +| 0.0248 | 28.76 | 365000 | 0.0573 | 0.0384 | +| 0.0297 | 28.8 | 365500 | 0.0594 | 0.0382 | +| 0.0326 | 28.84 | 366000 | 0.0567 | 0.0384 | +| 0.0237 | 28.88 | 366500 | 0.0577 | 0.0384 | +| 0.027 | 28.92 | 367000 | 0.0569 | 0.0384 | +| 0.0271 | 28.96 | 367500 | 0.0561 | 0.0385 | +| 0.0294 | 29.0 | 368000 | 0.0569 | 0.0383 | +| 0.0272 | 29.04 | 368500 | 0.0563 | 0.0379 | +| 0.0269 | 29.08 | 369000 | 0.0577 | 0.0382 | +| 0.0267 | 29.12 | 369500 | 0.0583 | 0.0385 | +| 0.0313 | 29.15 | 370000 | 0.0586 | 0.0381 | +| 0.0239 | 29.19 | 370500 | 0.0599 | 0.0378 | +| 0.0258 | 29.23 | 371000 | 0.0569 | 0.0384 | +| 0.0318 | 29.27 | 371500 | 0.0550 | 0.0380 | +| 0.024 | 29.31 | 372000 | 0.0582 | 0.0382 | +| 0.0304 | 29.35 | 372500 | 0.0585 | 0.0382 | +| 0.0285 | 29.39 | 373000 | 0.0567 | 0.0382 | +| 0.0262 | 29.43 | 373500 | 0.0572 | 0.0383 | +| 0.0254 | 29.47 | 374000 | 0.0581 | 0.0379 | +| 0.0325 | 29.51 | 374500 | 0.0576 | 0.0384 | +| 0.0282 | 29.55 | 375000 | 0.0578 | 0.0381 | +| 0.0282 | 29.59 | 375500 | 0.0592 | 0.0382 | +| 0.0274 | 29.63 | 376000 | 0.0593 | 0.0379 | +| 0.031 | 29.67 | 376500 | 0.0593 | 0.0382 | +| 0.0242 | 29.71 | 377000 | 0.0600 | 0.0380 | +| 0.0242 | 29.75 | 377500 | 0.0571 | 0.0377 | +| 0.0278 | 29.78 | 378000 | 0.0573 | 0.0379 | +| 0.0275 | 29.82 | 378500 | 0.0584 | 0.0377 | +| 0.0238 | 29.86 | 379000 | 0.0602 | 0.0377 | +| 0.0291 | 29.9 | 379500 | 0.0585 | 0.0377 | +| 0.0275 | 29.94 | 380000 | 0.0571 | 0.0374 | +| 0.0249 | 29.98 | 380500 | 0.0575 | 0.0377 | +| 0.026 | 30.02 | 381000 | 0.0579 | 0.0374 | +| 0.0253 | 30.06 | 381500 | 0.0584 | 0.0378 | +| 0.0258 | 30.1 | 382000 | 0.0593 | 0.0377 | +| 0.0276 | 30.14 | 382500 | 0.0601 | 0.0378 | +| 0.0279 | 30.18 | 383000 | 0.0589 | 0.0379 | +| 0.0335 | 30.22 | 383500 | 0.0583 | 0.0376 | +| 0.0243 | 30.26 | 384000 | 0.0608 | 0.0377 | +| 0.0257 | 30.3 | 384500 | 0.0583 | 0.0376 | +| 0.0289 | 30.34 | 385000 | 0.0583 | 0.0379 | +| 0.027 | 30.38 | 385500 | 0.0597 | 0.0382 | +| 0.0305 | 30.42 | 386000 | 0.0595 | 0.0381 | +| 0.0303 | 30.45 | 386500 | 0.0592 | 0.0374 | +| 0.0258 | 30.49 | 387000 | 0.0580 | 0.0375 | +| 0.0311 | 30.53 | 387500 | 0.0595 | 0.0372 | +| 0.0252 | 30.57 | 388000 | 0.0598 | 0.0376 | +| 0.0329 | 30.61 | 388500 | 0.0591 | 0.0374 | +| 0.034 | 30.65 | 389000 | 0.0580 | 0.0378 | +| 0.0296 | 30.69 | 389500 | 0.0584 | 0.0379 | +| 0.0248 | 30.73 | 390000 | 0.0580 | 0.0377 | +| 0.0312 | 30.77 | 390500 | 0.0579 | 0.0378 | +| 0.0255 | 30.81 | 391000 | 0.0594 | 0.0379 | +| 0.0233 | 30.85 | 391500 | 0.0592 | 0.0375 | +| 0.0256 | 30.89 | 392000 | 0.0587 | 0.0377 | +| 0.0269 | 30.93 | 392500 | 0.0570 | 0.0375 | +| 0.0278 | 30.97 | 393000 | 0.0581 | 0.0376 | +| 0.0272 | 31.01 | 393500 | 0.0573 | 0.0375 | +| 0.0268 | 31.05 | 394000 | 0.0589 | 0.0371 | +| 0.0281 | 31.09 | 394500 | 0.0604 | 0.0373 | +| 0.0289 | 31.12 | 395000 | 0.0580 | 0.0377 | +| 0.0219 | 31.16 | 395500 | 0.0586 | 0.0375 | +| 0.0255 | 31.2 | 396000 | 0.0585 | 0.0376 | +| 0.0278 | 31.24 | 396500 | 0.0592 | 0.0374 | +| 0.0252 | 31.28 | 397000 | 0.0603 | 0.0377 | +| 0.0274 | 31.32 | 397500 | 0.0602 | 0.0376 | +| 0.0204 | 31.36 | 398000 | 0.0619 | 0.0375 | +| 0.0255 | 31.4 | 398500 | 0.0608 | 0.0373 | +| 0.0217 | 31.44 | 399000 | 0.0608 | 0.0376 | +| 0.0222 | 31.48 | 399500 | 0.0597 | 0.0377 | +| 0.023 | 31.52 | 400000 | 0.0599 | 0.0374 | +| 0.0248 | 31.56 | 400500 | 0.0613 | 0.0378 | +| 0.0238 | 31.6 | 401000 | 0.0604 | 0.0375 | +| 0.0238 | 31.64 | 401500 | 0.0597 | 0.0375 | +| 0.0263 | 31.68 | 402000 | 0.0596 | 0.0374 | +| 0.0216 | 31.72 | 402500 | 0.0594 | 0.0374 | +| 0.0278 | 31.75 | 403000 | 0.0584 | 0.0377 | +| 0.0239 | 31.79 | 403500 | 0.0605 | 0.0375 | +| 0.0282 | 31.83 | 404000 | 0.0586 | 0.0374 | +| 0.0264 | 31.87 | 404500 | 0.0584 | 0.0378 | +| 0.0264 | 31.91 | 405000 | 0.0599 | 0.0374 | +| 0.0264 | 31.95 | 405500 | 0.0609 | 0.0373 | +| 0.0249 | 31.99 | 406000 | 0.0600 | 0.0371 | +| 0.0252 | 32.03 | 406500 | 0.0610 | 0.0373 | +| 0.0261 | 32.07 | 407000 | 0.0610 | 0.0375 | +| 0.0251 | 32.11 | 407500 | 0.0598 | 0.0379 | +| 0.0245 | 32.15 | 408000 | 0.0616 | 0.0375 | +| 0.0243 | 32.19 | 408500 | 0.0611 | 0.0375 | +| 0.0236 | 32.23 | 409000 | 0.0615 | 0.0374 | +| 0.0279 | 32.27 | 409500 | 0.0591 | 0.0381 | +| 0.032 | 32.31 | 410000 | 0.0603 | 0.0379 | +| 0.0293 | 32.35 | 410500 | 0.0615 | 0.0380 | +| 0.0242 | 32.39 | 411000 | 0.0624 | 0.0387 | +| 0.0211 | 32.42 | 411500 | 0.0605 | 0.0386 | +| 0.0248 | 32.46 | 412000 | 0.0613 | 0.0383 | +| 0.0225 | 32.5 | 412500 | 0.0604 | 0.0383 | +| 0.0238 | 32.54 | 413000 | 0.0603 | 0.0385 | +| 0.0222 | 32.58 | 413500 | 0.0608 | 0.0386 | +| 0.0248 | 32.62 | 414000 | 0.0600 | 0.0382 | +| 0.0227 | 32.66 | 414500 | 0.0612 | 0.0377 | +| 0.0254 | 32.7 | 415000 | 0.0595 | 0.0375 | +| 0.0233 | 32.74 | 415500 | 0.0611 | 0.0375 | +| 0.0272 | 32.78 | 416000 | 0.0601 | 0.0380 | +| 0.0244 | 32.82 | 416500 | 0.0604 | 0.0377 | +| 0.0253 | 32.86 | 417000 | 0.0604 | 0.0378 | +| 0.0267 | 32.9 | 417500 | 0.0589 | 0.0373 | +| 0.0268 | 32.94 | 418000 | 0.0600 | 0.0371 | +| 0.0239 | 32.98 | 418500 | 0.0598 | 0.0372 | +| 0.0258 | 33.02 | 419000 | 0.0589 | 0.0371 | +| 0.0216 | 33.05 | 419500 | 0.0612 | 0.0372 | +| 0.024 | 33.09 | 420000 | 0.0620 | 0.0370 | +| 0.0251 | 33.13 | 420500 | 0.0614 | 0.0371 | +| 0.0224 | 33.17 | 421000 | 0.0617 | 0.0374 | +| 0.024 | 33.21 | 421500 | 0.0629 | 0.0371 | +| 0.0256 | 33.25 | 422000 | 0.0613 | 0.0372 | +| 0.0252 | 33.29 | 422500 | 0.0612 | 0.0370 | +| 0.0265 | 33.33 | 423000 | 0.0600 | 0.0370 | +| 0.0227 | 33.37 | 423500 | 0.0611 | 0.0371 | +| 0.0403 | 33.41 | 424000 | 0.0613 | 0.0373 | +| 0.0278 | 33.45 | 424500 | 0.0612 | 0.0372 | +| 0.0275 | 33.49 | 425000 | 0.0611 | 0.0377 | +| 0.0231 | 33.53 | 425500 | 0.0604 | 0.0378 | +| 0.0245 | 33.57 | 426000 | 0.0620 | 0.0377 | +| 0.023 | 33.61 | 426500 | 0.0606 | 0.0376 | +| 0.0263 | 33.65 | 427000 | 0.0599 | 0.0378 | +| 0.0198 | 33.69 | 427500 | 0.0631 | 0.0377 | +| 0.0222 | 33.72 | 428000 | 0.0615 | 0.0379 | +| 0.0216 | 33.76 | 428500 | 0.0624 | 0.0376 | +| 0.0235 | 33.8 | 429000 | 0.0596 | 0.0374 | +| 0.0232 | 33.84 | 429500 | 0.0611 | 0.0368 | +| 0.0211 | 33.88 | 430000 | 0.0605 | 0.0372 | +| 0.0236 | 33.92 | 430500 | 0.0601 | 0.0366 | +| 0.029 | 33.96 | 431000 | 0.0596 | 0.0368 | +| 0.0287 | 34.0 | 431500 | 0.0594 | 0.0368 | +| 0.0264 | 34.04 | 432000 | 0.0614 | 0.0368 | +| 0.0218 | 34.08 | 432500 | 0.0620 | 0.0369 | +| 0.0261 | 34.12 | 433000 | 0.0603 | 0.0366 | +| 0.0228 | 34.16 | 433500 | 0.0594 | 0.0368 | +| 0.0217 | 34.2 | 434000 | 0.0609 | 0.0367 | +| 0.0234 | 34.24 | 434500 | 0.0613 | 0.0366 | +| 0.0196 | 34.28 | 435000 | 0.0621 | 0.0365 | +| 0.0265 | 34.32 | 435500 | 0.0606 | 0.0367 | +| 0.0276 | 34.36 | 436000 | 0.0606 | 0.0364 | +| 0.0254 | 34.39 | 436500 | 0.0604 | 0.0364 | +| 0.0234 | 34.43 | 437000 | 0.0611 | 0.0365 | +| 0.0216 | 34.47 | 437500 | 0.0617 | 0.0364 | +| 0.0246 | 34.51 | 438000 | 0.0608 | 0.0366 | +| 0.0318 | 34.55 | 438500 | 0.0619 | 0.0364 | +| 0.0247 | 34.59 | 439000 | 0.0604 | 0.0364 | +| 0.0223 | 34.63 | 439500 | 0.0617 | 0.0365 | +| 0.0216 | 34.67 | 440000 | 0.0612 | 0.0366 | +| 0.025 | 34.71 | 440500 | 0.0619 | 0.0364 | +| 0.0178 | 34.75 | 441000 | 0.0625 | 0.0365 | +| 0.0213 | 34.79 | 441500 | 0.0614 | 0.0362 | +| 0.0223 | 34.83 | 442000 | 0.0605 | 0.0364 | +| 0.0285 | 34.87 | 442500 | 0.0611 | 0.0362 | +| 0.0212 | 34.91 | 443000 | 0.0620 | 0.0364 | +| 0.0233 | 34.95 | 443500 | 0.0615 | 0.0363 | +| 0.0245 | 34.99 | 444000 | 0.0620 | 0.0364 | +| 0.0238 | 35.02 | 444500 | 0.0632 | 0.0361 | +| 0.0217 | 35.06 | 445000 | 0.0622 | 0.0364 | +| 0.0259 | 35.1 | 445500 | 0.0611 | 0.0362 | +| 0.0196 | 35.14 | 446000 | 0.0620 | 0.0363 | +| 0.023 | 35.18 | 446500 | 0.0616 | 0.0364 | +| 0.0244 | 35.22 | 447000 | 0.0618 | 0.0366 | +| 0.0225 | 35.26 | 447500 | 0.0610 | 0.0366 | +| 0.0222 | 35.3 | 448000 | 0.0618 | 0.0366 | +| 0.0268 | 35.34 | 448500 | 0.0603 | 0.0367 | +| 0.0232 | 35.38 | 449000 | 0.0610 | 0.0362 | +| 0.025 | 35.42 | 449500 | 0.0610 | 0.0360 | +| 0.0205 | 35.46 | 450000 | 0.0617 | 0.0362 | +| 0.0259 | 35.5 | 450500 | 0.0604 | 0.0365 | +| 0.0211 | 35.54 | 451000 | 0.0603 | 0.0365 | +| 0.025 | 35.58 | 451500 | 0.0613 | 0.0366 | +| 0.0209 | 35.62 | 452000 | 0.0617 | 0.0367 | +| 0.0232 | 35.66 | 452500 | 0.0627 | 0.0362 | +| 0.0256 | 35.69 | 453000 | 0.0615 | 0.0364 | +| 0.0258 | 35.73 | 453500 | 0.0603 | 0.0364 | +| 0.0242 | 35.77 | 454000 | 0.0615 | 0.0363 | +| 0.0256 | 35.81 | 454500 | 0.0618 | 0.0365 | +| 0.0232 | 35.85 | 455000 | 0.0619 | 0.0366 | +| 0.0217 | 35.89 | 455500 | 0.0610 | 0.0366 | +| 0.0214 | 35.93 | 456000 | 0.0624 | 0.0363 | +| 0.0183 | 35.97 | 456500 | 0.0624 | 0.0361 | +| 0.0209 | 36.01 | 457000 | 0.0623 | 0.0361 | +| 0.0237 | 36.05 | 457500 | 0.0613 | 0.0364 | +| 0.0179 | 36.09 | 458000 | 0.0622 | 0.0364 | +| 0.0239 | 36.13 | 458500 | 0.0615 | 0.0367 | +| 0.0203 | 36.17 | 459000 | 0.0610 | 0.0367 | +| 0.0213 | 36.21 | 459500 | 0.0625 | 0.0365 | +| 0.0162 | 36.25 | 460000 | 0.0631 | 0.0365 | +| 0.0225 | 36.29 | 460500 | 0.0622 | 0.0364 | +| 0.0198 | 36.32 | 461000 | 0.0616 | 0.0362 | +| 0.0184 | 36.36 | 461500 | 0.0622 | 0.0362 | +| 0.0232 | 36.4 | 462000 | 0.0616 | 0.0363 | +| 0.0247 | 36.44 | 462500 | 0.0612 | 0.0367 | +| 0.0208 | 36.48 | 463000 | 0.0622 | 0.0370 | +| 0.0299 | 36.52 | 463500 | 0.0624 | 0.0374 | +| 0.0231 | 36.56 | 464000 | 0.0617 | 0.0368 | +| 0.0202 | 36.6 | 464500 | 0.0627 | 0.0367 | +| 0.0224 | 36.64 | 465000 | 0.0608 | 0.0369 | +| 0.0227 | 36.68 | 465500 | 0.0630 | 0.0367 | +| 0.0177 | 36.72 | 466000 | 0.0628 | 0.0365 | +| 0.0179 | 36.76 | 466500 | 0.0629 | 0.0363 | +| 0.0216 | 36.8 | 467000 | 0.0627 | 0.0363 | +| 0.0193 | 36.84 | 467500 | 0.0623 | 0.0360 | +| 0.0195 | 36.88 | 468000 | 0.0621 | 0.0363 | +| 0.0199 | 36.92 | 468500 | 0.0621 | 0.0362 | +| 0.0182 | 36.96 | 469000 | 0.0619 | 0.0361 | +| 0.0228 | 36.99 | 469500 | 0.0620 | 0.0361 | +| 0.022 | 37.03 | 470000 | 0.0624 | 0.0361 | +| 0.0167 | 37.07 | 470500 | 0.0621 | 0.0360 | +| 0.0191 | 37.11 | 471000 | 0.0633 | 0.0361 | +| 0.0214 | 37.15 | 471500 | 0.0623 | 0.0361 | +| 0.0217 | 37.19 | 472000 | 0.0623 | 0.0361 | +| 0.0184 | 37.23 | 472500 | 0.0629 | 0.0358 | +| 0.0198 | 37.27 | 473000 | 0.0632 | 0.0358 | +| 0.0183 | 37.31 | 473500 | 0.0628 | 0.0359 | +| 0.0225 | 37.35 | 474000 | 0.0626 | 0.0358 | +| 0.0229 | 37.39 | 474500 | 0.0625 | 0.0359 | +| 0.0203 | 37.43 | 475000 | 0.0624 | 0.0361 | +| 0.0189 | 37.47 | 475500 | 0.0626 | 0.0360 | +| 0.0222 | 37.51 | 476000 | 0.0628 | 0.0362 | +| 0.0168 | 37.55 | 476500 | 0.0625 | 0.0363 | +| 0.019 | 37.59 | 477000 | 0.0624 | 0.0361 | +| 0.0198 | 37.63 | 477500 | 0.0621 | 0.0361 | +| 0.0196 | 37.66 | 478000 | 0.0618 | 0.0360 | +| 0.0212 | 37.7 | 478500 | 0.0624 | 0.0362 | +| 0.0177 | 37.74 | 479000 | 0.0635 | 0.0360 | +| 0.0178 | 37.78 | 479500 | 0.0634 | 0.0360 | +| 0.0195 | 37.82 | 480000 | 0.0633 | 0.0360 | +| 0.0199 | 37.86 | 480500 | 0.0633 | 0.0357 | +| 0.0178 | 37.9 | 481000 | 0.0640 | 0.0359 | +| 0.0197 | 37.94 | 481500 | 0.0632 | 0.0359 | +| 0.0203 | 37.98 | 482000 | 0.0629 | 0.0358 | +| 0.019 | 38.02 | 482500 | 0.0633 | 0.0358 | +| 0.02 | 38.06 | 483000 | 0.0632 | 0.0359 | +| 0.022 | 38.1 | 483500 | 0.0635 | 0.0359 | +| 0.0185 | 38.14 | 484000 | 0.0642 | 0.0358 | +| 0.0198 | 38.18 | 484500 | 0.0642 | 0.0358 | +| 0.0209 | 38.22 | 485000 | 0.0631 | 0.0358 | +| 0.0178 | 38.26 | 485500 | 0.0637 | 0.0359 | +| 0.0209 | 38.29 | 486000 | 0.0637 | 0.0360 | +| 0.0185 | 38.33 | 486500 | 0.0631 | 0.0360 | +| 0.0212 | 38.37 | 487000 | 0.0634 | 0.0359 | +| 0.0204 | 38.41 | 487500 | 0.0634 | 0.0359 | +| 0.0223 | 38.45 | 488000 | 0.0634 | 0.0360 | +| 0.0209 | 38.49 | 488500 | 0.0632 | 0.0361 | +| 0.0191 | 38.53 | 489000 | 0.0629 | 0.0362 | +| 0.02 | 38.57 | 489500 | 0.0631 | 0.0360 | +| 0.0184 | 38.61 | 490000 | 0.0636 | 0.0361 | +| 0.0253 | 38.65 | 490500 | 0.0637 | 0.0362 | +| 0.0171 | 38.69 | 491000 | 0.0630 | 0.0363 | +| 0.0226 | 38.73 | 491500 | 0.0636 | 0.0362 | +| 0.0209 | 38.77 | 492000 | 0.0635 | 0.0362 | +| 0.0218 | 38.81 | 492500 | 0.0632 | 0.0363 | +| 0.0193 | 38.85 | 493000 | 0.0633 | 0.0362 | +| 0.0178 | 38.89 | 493500 | 0.0640 | 0.0362 | +| 0.0198 | 38.93 | 494000 | 0.0638 | 0.0364 | +| 0.0178 | 38.96 | 494500 | 0.0638 | 0.0363 | +| 0.0205 | 39.0 | 495000 | 0.0635 | 0.0363 | +| 0.0196 | 39.04 | 495500 | 0.0633 | 0.0362 | +| 0.0264 | 39.08 | 496000 | 0.0634 | 0.0362 | +| 0.0214 | 39.12 | 496500 | 0.0635 | 0.0362 | +| 0.0171 | 39.16 | 497000 | 0.0638 | 0.0361 | +| 0.0214 | 39.2 | 497500 | 0.0635 | 0.0361 | +| 0.0207 | 39.24 | 498000 | 0.0635 | 0.0361 | +| 0.0207 | 39.28 | 498500 | 0.0636 | 0.0361 | +| 0.0196 | 39.32 | 499000 | 0.0636 | 0.0360 | +| 0.0194 | 39.36 | 499500 | 0.0634 | 0.0360 | +| 0.0178 | 39.4 | 500000 | 0.0634 | 0.0359 | +| 0.0226 | 39.44 | 500500 | 0.0635 | 0.0359 | +| 0.0203 | 39.48 | 501000 | 0.0632 | 0.0359 | +| 0.0228 | 39.52 | 501500 | 0.0631 | 0.0359 | +| 0.0184 | 39.56 | 502000 | 0.0636 | 0.0359 | +| 0.0207 | 39.59 | 502500 | 0.0635 | 0.0359 | +| 0.019 | 39.63 | 503000 | 0.0634 | 0.0359 | +| 0.016 | 39.67 | 503500 | 0.0634 | 0.0359 | +| 0.0215 | 39.71 | 504000 | 0.0633 | 0.0359 | +| 0.0188 | 39.75 | 504500 | 0.0633 | 0.0359 | +| 0.0231 | 39.79 | 505000 | 0.0632 | 0.0359 | +| 0.0191 | 39.83 | 505500 | 0.0633 | 0.0359 | +| 0.0172 | 39.87 | 506000 | 0.0633 | 0.0358 | +| 0.017 | 39.91 | 506500 | 0.0633 | 0.0358 | +| 0.0212 | 39.95 | 507000 | 0.0633 | 0.0358 | +| 0.0188 | 39.99 | 507500 | 0.0633 | 0.0358 | + + +### Framework versions + +- Transformers 4.18.0 +- Pytorch 1.11.0+cu113 +- Datasets 2.1.0 +- Tokenizers 0.12.1 diff --git a/add_kenlm.py b/add_kenlm.py new file mode 100644 index 0000000..aa5e1cb --- /dev/null +++ b/add_kenlm.py @@ -0,0 +1,37 @@ +import argparse +from transformers import AutoProcessor +from transformers import Wav2Vec2ProcessorWithLM +from pyctcdecode import build_ctcdecoder + + +def main(args): + processor = AutoProcessor.from_pretrained(args.model_name_or_path) + vocab_dict = processor.tokenizer.get_vocab() + sorted_vocab_dict = { + k.lower(): v for k, v in sorted(vocab_dict.items(), key=lambda item: item[1]) + } + decoder = build_ctcdecoder( + labels=list(sorted_vocab_dict.keys()), + kenlm_model_path=args.kenlm_model_path, + ) + processor_with_lm = Wav2Vec2ProcessorWithLM( + feature_extractor=processor.feature_extractor, + tokenizer=processor.tokenizer, + decoder=decoder, + ) + processor_with_lm.save_pretrained(args.model_name_or_path) + print( + f"Run: ~/bin/build_binary language_model/*.arpa language_model/5gram.bin -T $(pwd) && rm language_model/*.arpa") + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument('--model_name_or_path', default="./", help='Model name or path. Defaults to ./') + parser.add_argument('--kenlm_model_path', required=True, help='Path to KenLM arpa file.') + args = parser.parse_args() + return args + + +if __name__ == "__main__": + main(parse_args()) + diff --git a/added_tokens.json b/added_tokens.json new file mode 100644 index 0000000..9854ab7 --- /dev/null +++ b/added_tokens.json @@ -0,0 +1 @@ +{"": 39, "": 40} \ No newline at end of file diff --git a/alphabet.json b/alphabet.json new file mode 100644 index 0000000..c64a5de --- /dev/null +++ b/alphabet.json @@ -0,0 +1 @@ +{"labels": [" ", "(", ")", "0", "3", "7", "8", "9", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z", "\u00e5", "\u00e6", "\u00f8", "\u2047", "", "", ""], "is_bpe": false} \ No newline at end of file diff --git a/cardinal_numbers.py b/cardinal_numbers.py new file mode 100644 index 0000000..acb8b48 --- /dev/null +++ b/cardinal_numbers.py @@ -0,0 +1,690 @@ +#!/usr/bin/env python +# coding: utf-8 + +"""From https://github.com/peresolb/number-conversion/""" +import sys +import os +import nltk + + +# Dict for basic primitive numbers: 1-10 +b = { + 1: "én", + 2: "to", + 3: "tre", + 4: "fire", + 5: "fem", + 6: "seks", + 7: "sju", + 8: "åtte", + 9: "ni", + 10: "ti", +} +b_nn = { + 1: "ein", + 2: "to", + 3: "tre", + 4: "fire", + 5: "fem", + 6: "seks", + 7: "sju", + 8: "åtte", + 9: "ni", + 10: "ti", +} + +# Dict for teen primitive numbers: 11-19 +t = { + 11: "elleve", + 12: "tolv", + 13: "tretten", + 14: "fjorten", + 15: "femten", + 16: "seksten", + 17: "sytten", + 18: "atten", + 19: "nitten", +} + + +# Dict for two digit primitive numbers: 20-90 +do = { + 20: "tjue", + 30: "tretti", + 40: "førti", + 50: "femti", + 60: "seksti", + 70: "sytti", + 80: "åtti", + 90: "nitti", +} + + +# Dict for the 3 digit primitive number: 100 +doo = {100: "hundre"} + + +# Dict for the 4 digit primitive number: 1000 +dooo = {1000: "tusen"} + + +# Reverser function for primitive number dicts +def _revdict(numberdict): + newdict = {} + for k, v in numberdict.items(): + newdict[v] = k + return newdict + + +# The reverse of the primitive number dicts, where strings are keys and integers are values, +# are name of original dict underscore 'r' +b_r, b_nn_r, t_r, do_r, doo_r, dooo_r = ( + _revdict(b), + _revdict(b_nn), + _revdict(t), + _revdict(do), + _revdict(doo), + _revdict(dooo), +) + + +def _oneten(nr, reverse=False, nn=False): + """Function taking an int from 1-10 and returning the corresponding word, + or, if reverse=True, taking a numberword from 1-10 and returning the digit""" + if reverse == False: + if not type(nr) is int: + return None + if nr <= 10: + if nn == False: + return b[nr] + else: + return b_nn[nr] + else: + if not type(nr) is str: + return None + if nn == False: + if nr in b_r.keys(): + return b_r[nr] + else: + if nr in b_nn_r.keys(): + return b_nn_r[nr] + + +def _onedig(nr, reverse=False, nn=False): + if reverse == False: + if not _oneten(nr) == "ti": + if nn == False: + return _oneten(nr) + else: + return _oneten(nr, nn=True) + if reverse == True: + if not _oneten(nr, reverse=True) == 10: + if nn == False: + return _oneten(nr, reverse=True) + else: + return _oneten(nr, reverse=True, nn=True) + + +def _teen(nr, reverse=False): + """Function taking a primitive two-digit int in the teen range and returning the + corresponding word, or, if reverse=True, the corresponding number word""" + if reverse == False: + if not type(nr) is int: + return None + if nr in t.keys(): + return t[nr] + else: + if not type(nr) is str: + return None + if nr in t_r.keys(): + return t_r[nr] + + +def _twodig(nr, reverse=False): + """Function taking a primitive two-digit int in the range 20-90 and returning + the corresponding word, or, if reverse=True, the corresponding number word""" + if reverse == False: + if not type(nr) is int: + return None + if nr in do.keys(): + return do[nr] + else: + if not type(nr) is str: + return None + if nr in do_r.keys(): + return do_r[nr] + + +def _numparser(numword, nn=False): + """Parse word to see if they start with a wd in firstnumwords. + If yes, return firstnumword and second part""" + if not type(numword) is str: + return None + firstpart = "" + scndpart = "" + firstnumwords = list(do_r.keys()) + for s in firstnumwords: + if numword.startswith(s): + slength = len(s) + firstpart = s + scndpart = numword[slength:] + if nn == False: + if ( + scndpart in b_r.keys() and b_r[scndpart] < 10 + ): # Only return if second part is dig below 10 + return (firstpart, scndpart) + else: + if ( + scndpart in b_nn_r.keys() and b_nn_r[scndpart] < 10 + ): # Only return if second part is dig below 10 + return (firstpart, scndpart) + + +def _one_to_nineteen(nr, reverse=False, nn=False): + """Function taking a primitive two-digit int in the range 1-19 + and returning the corresponding word, or, if reverse=True, the corresponding number word""" + if reverse == False: + if not type(nr) is int: + return None + if nr < 11: + if nn == False: + return _oneten(nr) + else: + return _oneten(nr, nn=True) + elif nr < 20: + return _teen(nr) + else: + if not type(nr) is str: + return None + if nn == False: + if type(_oneten(nr, reverse=True)) is int: + return _oneten(nr, reverse=True) + elif type(_teen(nr, reverse=True)): + return _teen(nr, reverse=True) + else: + if type(_oneten(nr, reverse=True, nn=True)) is int: + return _oneten(nr, reverse=True, nn=True) + elif type(_teen(nr, reverse=True)): + return _teen(nr, reverse=True) + + +def _one_to_nn(nr, reverse=False, nn=False): + """Function taking an int in the range 1-99 and returning the corresponding word. Reverse as before""" + if reverse == False: + if not type(nr) is int: + return None + if nr > 0: + if nr < 20: + if nn == False: + return _one_to_nineteen(nr) + else: + return _one_to_nineteen(nr, nn=True) + elif nr < 100: + if nr in do.keys(): + return _twodig(nr) + else: + nrstring = str(nr) + frstdig = int(nrstring[0]) * 10 + scndig = int(nrstring[1]) + frstwd = _twodig(frstdig) + if nn == False: + scnwd = _onedig(scndig) + else: + scnwd = _onedig(scndig, nn=True) + nrwd = frstwd + scnwd + return nrwd + else: + if not type(nr) is str: + return None + if nn == False: + if type(_one_to_nineteen(nr, reverse=True)) is int: + return _one_to_nineteen(nr, reverse=True) + elif type(_twodig(nr, reverse=True)) is int: + return _twodig(nr, reverse=True) + else: + if _numparser(nr) == None: + return None + parsed = _numparser(nr) + first = _twodig(parsed[0], reverse=True) + second = _one_to_nineteen(parsed[1], reverse=True) + return first + second + else: + if type(_one_to_nineteen(nr, reverse=True, nn=True)) is int: + return _one_to_nineteen(nr, reverse=True, nn=True) + elif type(_twodig(nr, reverse=True)) is int: + return _twodig(nr, reverse=True) + else: + if _numparser(nr, nn=True) == None: + return None + parsed = _numparser(nr, nn=True) + first = _twodig(parsed[0], reverse=True) + second = _one_to_nineteen(parsed[1], reverse=True, nn=True) + return first + second + + +def _one_to_nnn(nr, reverse=False, nn=False): + """Function taking an int in the range 1-999 and returning the corresponding word. Reverse as before""" + if reverse == False: + if not type(nr) is int: + return None + if nr == 0: + return None + if nr < 100: # 1-99 + if nn == False: + return _one_to_nn(nr) + else: + return _one_to_nn(nr, nn=True) + elif nr < 1000: + if nr == 100: # 100 + return doo[100] + else: + nrstring = str(nr) # 435 181 + frstdig = int(nrstring[0]) # 4 1 + scndig = int(nrstring[1]) # 3 8 + thrdig = int(nrstring[2]) # 5 1 + scthdig = int(nrstring[1:]) # 35 81 + if nn == False: + frstwd = _onedig(frstdig) # fire + else: + frstwd = _onedig(frstdig, nn=True) + nrwd = "" + if scndig == 0: # 405 or 400 + if thrdig == 0: # 400 + nrwd = "%s %s" % (frstwd, doo[100]) # fire hundre + else: # 405 + if nn == False: + thrdwd = _one_to_nn(thrdig) # fem + else: + thrdwd = _one_to_nn(thrdig, nn=True) + if frstdig != 1: + nrwd = "%s %s og %s" % ( + frstwd, + doo[100], + thrdwd, + ) # fire hundre og fem + else: + nrwd = "%s og %s" % (doo[100], thrdwd) # hundre og fem + else: # 435 + scthwd = "" + if nn == False: + scthwd = _one_to_nn(scthdig) # trettifem + else: + scthwd = _one_to_nn(scthdig, nn=True) + if frstdig != 1: + nrwd = "%s %s og %s" % (frstwd, doo[100], scthwd) + else: + nrwd = "%s og %s" % (doo[100], scthwd) # hundre og trettifem + return nrwd + else: + if not type(nr) is str: + return None + if type(doo_r.get(nr, None)) is int: # hundre - 100 + return doo_r[nr] + elif ( + len(nr.split(" ")) == 1 + and type(_one_to_nn(nr, reverse=True)) is int + and nn == False + ): + return _one_to_nn(nr, reverse=True) # 44 + elif ( + len(nr.split(" ")) == 1 + and type(_one_to_nn(nr, reverse=True, nn=True)) is int + and nn == True + ): + return _one_to_nn(nr, reverse=True, nn=True) # 44 + elif len(nr.split(" ")) == 2: # to hundre + splitwords = nr.split(" ") + if nn == False and nr == "ett hundre": + return 100 + elif nn == True and nr == "eitt hundre": + return 100 + elif ( + type(_one_to_nn(splitwords[0], reverse=True)) is int + and splitwords[1] == "hundre" + ): + return _one_to_nn(splitwords[0], reverse=True) * 100 + elif len(nr.split(" ")) == 3: # hundre og tre + splitwords = nr.split(" ") + if splitwords[0] == "hundre" and splitwords[1] == "og": + if nn == False: + if type(_one_to_nn(splitwords[2], reverse=True)) is int: + return 100 + _one_to_nn(splitwords[2], reverse=True) + else: + if type(_one_to_nn(splitwords[2], reverse=True, nn=True)) is int: + return 100 + _one_to_nn(splitwords[2], reverse=True, nn=True) + else: + return None + elif len(nr.split(" ")) == 4: # ett hundre og trettifire, fire hundre og åtte + splitwords = nr.split(" ") + if nn == False: + if ( + splitwords[0] == "ett" + and splitwords[1] == "hundre" + and splitwords[2] == "og" + and type(_one_to_nn(splitwords[3], reverse=True)) is int + ): # ett hundre og trettifire + return 100 + _one_to_nn(splitwords[3], reverse=True) + elif ( + type(_one_to_nn(splitwords[0], reverse=True)) is int + and _one_to_nn(splitwords[0], reverse=True) < 10 + and splitwords[1] == "hundre" + and splitwords[2] == "og" + and type(_one_to_nn(splitwords[3], reverse=True)) is int + ): # fire hundre og trettifire + hundreds = _one_to_nn(splitwords[0], reverse=True) * 100 + tens = _one_to_nn(splitwords[3], reverse=True) + return hundreds + tens + else: + return None + else: + if ( + splitwords[0] == "eitt" + and splitwords[1] == "hundre" + and splitwords[2] == "og" + and type(_one_to_nn(splitwords[3], reverse=True, nn=True)) is int + ): # eit hundre og trettifire + return 100 + _one_to_nn(splitwords[3], reverse=True, nn=True) + elif ( + type(_one_to_nn(splitwords[0], reverse=True, nn=True)) is int + and _one_to_nn(splitwords[0], reverse=True, nn=True) < 10 + and splitwords[1] == "hundre" + and splitwords[2] == "og" + and type(_one_to_nn(splitwords[3], reverse=True, nn=True)) is int + ): # fire hundre og trettifire + hundreds = _one_to_nn(splitwords[0], reverse=True, nn=True) * 100 + tens = _one_to_nn(splitwords[3], reverse=True, nn=True) + return hundreds + tens + else: + return None + + +def _high_hundred(nr, nn=False): + """In Norwegian, as in English, it is possible to express the numbers 1100-1999 with hundreds, + e.g. "tolv hundre og nittiåtte", /twelve hundred and ninety-eight/. We want to be able to convert + these to integers. However, we don't need to produce them, so this algoritm only goes from strings to integers""" + if not type(nr) is str: + return None + if len(nr.split(" ")) > 1: + frstwd = nr.split(" ")[0] + if not type(_teen(frstwd, reverse=True)) is int: + return None + frstdig = _teen(frstwd, reverse=True) + if len(nr.split(" ")) == 2 and nr.split(" ")[1] == "hundre": # femten hundre + return frstdig * 100 + elif ( + len(nr.split(" ")) == 4 + and nr.split(" ")[1] == "hundre" + and nr.split(" ")[2] == "og" + ): + if ( + nn == False and type(_one_to_nn(nr.split(" ")[3], reverse=True)) is int + ): # femten hundre og førtito + lastdigs = _one_to_nn(nr.split(" ")[3], reverse=True) + return (frstdig * 100) + lastdigs + elif ( + nn == True + and type(_one_to_nn(nr.split(" ")[3], reverse=True, nn=True)) is int + ): # femten hundre og førtito + lastdigs = _one_to_nn(nr.split(" ")[3], reverse=True, nn=True) + return (frstdig * 100) + lastdigs + + +def _one_to_nnnnnn(nr, reverse=False, nn=False): + """Function taking an int in the range 1-999999 and returning the corresponding word. Reverse as before""" + if reverse == False: + if not type(nr) is int: + return None + if nr == 0: + return None + if nr < 1000: # 1-999 + if nn == False: + return _one_to_nnn(nr) + else: + return _one_to_nnn(nr, nn=True) + elif nr < 1000000: # 1000-999999 + if nr == 1000: # 1000 + if nn == False: + return "ett tusen" + else: + return "eitt tusen" + else: + nrstring = str(nr) # Starting with last three digits. e.g. 23[456] + ultdig = int(nrstring[-1]) # 6 + penultdig = int(nrstring[-2]) # 5 + antepenultdig = int(nrstring[-3]) # 4 + ult_and_penultdig = int(nrstring[-2:]) # 56 + ult_penult_antepenultdig = int(nrstring[-3:]) # 456 + tailstring = "" + if antepenultdig == 0: # 012, 002, 000 + if penultdig == 0: # 000, 002 + if ultdig == 0: # 000 + tailstring = "tusen" + else: # 002 + if nn == False: + ultstring = _one_to_nnn(ultdig) + tailstring = "tusen og %s" % ultstring # tusen og to + else: + ultstring = _one_to_nnn(ultdig, nn=True) + tailstring = "tusen og %s" % ultstring # tusen og to + else: # 012 + if nn == False: + ult_and_penultstring = _one_to_nnn(ult_and_penultdig) + tailstring = ( + "tusen og %s" % ult_and_penultstring + ) # tusen og tolv + else: + ult_and_penultstring = _one_to_nnn( + ult_and_penultdig, nn=True + ) + tailstring = ( + "tusen og %s" % ult_and_penultstring + ) # tusen og tolv + else: # 456 + if nn == False: + ult_penult_antepenultstring = _one_to_nnn( + ult_penult_antepenultdig + ) + if str(ult_penult_antepenultdig)[0] == "1": + tailstring = ( + "tusen ett %s" % ult_penult_antepenultstring + ) # tusen ett hundre + else: + tailstring = ( + "tusen %s" % ult_penult_antepenultstring + ) # tusen fire hundre og femtiseks + else: + ult_penult_antepenultstring = _one_to_nnn( + ult_penult_antepenultdig, nn=True + ) + if str(ult_penult_antepenultdig)[0] == "1": + tailstring = ( + "tusen eitt %s" % ult_penult_antepenultstring + ) # tusen ett hundre + else: + tailstring = ( + "tusen %s" % ult_penult_antepenultstring + ) # tusen fire hundre og femtiseks + startdigs = int( + nrstring[:-3] + ) # startstring can consist of the 1, 2 or 3 first digits + startstring = "" + if startdigs == 1: # 1001 starts with "ett" + if nn == False: + startstring = "ett" + else: + startstring = "eitt" + elif startdigs > 99 and startdigs < 200: # 155555 starts with ett + if nn == False: + startnumstring = _one_to_nnn(startdigs) + startstring = "ett %s" % startnumstring + else: + startnumstring = _one_to_nnn(startdigs, nn=True) + startstring = "eitt %s" % startnumstring + else: # the remaining numbers are purely compositional + if nn == False: + startstring = _one_to_nnn(startdigs) + else: + startstring = _one_to_nnn(startdigs, nn=True) + numstring = "%s %s" % (startstring, tailstring) + return numstring + else: + if not type(nr) is str: + return None + if type(_one_to_nnn(nr, reverse=True)) is int and nn == False: + return _one_to_nnn(nr, reverse=True) # 444 + elif type(_one_to_nnn(nr, reverse=True, nn=True)) is int and nn == True: + return _one_to_nnn(nr, reverse=True, nn=True) # 444 + elif nr == "tusen": # tusen - 1000 + return 1000 + elif ( + len(nr.split(" ")) > 1 and nr.split(" ")[-1] == "tusen" + ): # ett tusen, ett hundre tusen etc. + wdlist = nr.split(" ") + firstphrase = " ".join(wdlist[:-1]) + if type(_one_to_nnn(firstphrase, reverse=True)) is int and nn == False: + firstdig = _one_to_nnn(firstphrase, reverse=True) + return firstdig * 1000 + elif ( + type(_one_to_nnn(firstphrase, reverse=True, nn=True)) is int + and nn == True + ): + firstdig = _one_to_nnn(firstphrase, reverse=True, nn=True) + return firstdig * 1000 + elif ( + len(nr.split(" ")) == 2 and nr.split(" ")[0] == "ett" and nn == False + ): # ett tusen + return 1000 + elif len(nr.split(" ")) == 2 and nr.split(" ")[0] == "eitt" and nn == True: + return 1000 + else: # misspellings should not result in return value + return None + else: + if len(nr.split(" ")) > 1: # all other numbers should contain spaces + numwordlist = nr.split(" ") + if ( + "tusen" in numwordlist + ): # Find last part of numphrase, which starts with "tusen" + tusenind = numwordlist.index("tusen") # find index of "tusen" + lastwords = numwordlist[tusenind:] # words from 'tusen' + firstwords = numwordlist[:tusenind] # words until 'tusen' + lastdigs = 0 + if len(lastwords) == 3: + if lastwords[1] == "og": # 'tusen og fire' 'tusen og førtifire' + lastword = lastwords[-1] + if nn == False: + lastdigs = _one_to_nnn(lastword, reverse=True) + elif nn == True: + lastdigs = _one_to_nnn(lastword, reverse=True, nn=True) + elif ( + nn == False + and type(_one_to_nnn(" ".join(lastwords[1:]), reverse=True)) + is int + and lastwords[2] == "hundre" + ): # tusen to hundre + hundredphrase = " ".join(lastwords[1:]) + lastdigs = _one_to_nnn(hundredphrase, reverse=True) + elif ( + nn == True + and type( + _one_to_nnn( + " ".join(lastwords[1:]), reverse=True, nn=True + ) + ) + is int + and lastwords[2] == "hundre" + ): # tusen to hundre + hundredphrase = " ".join(lastwords[1:]) + lastdigs = _one_to_nnn(hundredphrase, reverse=True, nn=True) + else: # misspellings should not result in return value + return None + elif ( + len(lastwords) == 5 and lastwords[2] == "hundre" + ): # 'tusen fire hundre og fem' + hundredphrase = " ".join(lastwords[1:]) + if nn == False: + lastdigs = _one_to_nnn(hundredphrase, reverse=True) + else: + lastdigs = _one_to_nnn(hundredphrase, reverse=True, nn=True) + else: # misspellings should not result in return value + return None + firstdigs = 0 + firstphrase = " ".join(firstwords) + if len(firstwords) == 0: # as in 'tusen og tretti' + firstdigs = 1000 + elif ( + len(firstwords) == 1 and firstwords[0] == "ett" and nn == False + ): + firstdigs = 1000 + elif ( + len(firstwords) == 1 and firstwords[0] == "eitt" and nn == True + ): + firstdigs = 1000 + elif ( + type(_one_to_nnn(firstphrase, reverse=True)) is int + and nn == False + ): + firstdigs = _one_to_nnn(firstphrase, reverse=True) * 1000 + elif ( + type(_one_to_nnn(firstphrase, reverse=True, nn=True)) is int + and nn == True + ): + firstdigs = ( + _one_to_nnn(firstphrase, reverse=True, nn=True) * 1000 + ) + else: # misspellings should not result in return value + return None + if type(firstdigs) is int and type(lastdigs) is int: + return firstdigs + lastdigs + + +def convert_nums(nr, reverse=False, nn=False): + """Functions for converting numbers. Only works for numbers in range 0-999999 for now""" + if reverse == False: + if type(nr) is int: + returnstring = "" + if nr == 0: + returnstring = "null" + elif nr < 1000000: + if nn == False: + returnstring = _one_to_nnnnnn(nr) + else: + returnstring = _one_to_nnnnnn(nr, nn=True) + else: + return None + return returnstring + else: + if type(nr) is str: + returnint = 0 + if nr == "null": + returnint = 0 + elif nn == False and type(_one_to_nnnnnn(nr, reverse=True)) is int: + returnint = _one_to_nnnnnn(nr, reverse=True) + elif nn == True and type(_one_to_nnnnnn(nr, reverse=True, nn=True)) is int: + returnint = _one_to_nnnnnn(nr, reverse=True, nn=True) + elif nn == False and type(_high_hundred(nr)) is int: + returnint = _high_hundred(nr) + elif nn == True and type(_high_hundred(nr, nn=True)) is int: + returnint = _high_hundred(nr, nn=True) + else: + return None + return returnint + + +if __name__ == "__main__": + # testing + mydigit = 243564 + mynumstring = "hundre og atten tusen fire hundre og trettién" + mydigit_nn = 34381 + mynumstring_nn = "tre hundre og førtiein" + + print( + "Digit conversion bm: %s\nString conversion bm: %s" + % (convert_nums(mydigit), convert_nums(mynumstring, reverse=True)) + ) + print( + "Digit conversion nn: %s\nString conversion nn: %s" + % ( + convert_nums(mydigit_nn, nn=True), + convert_nums(mynumstring_nn, nn=True, reverse=True), + ) + ) diff --git a/config.json b/config.json new file mode 100644 index 0000000..ccbb394 --- /dev/null +++ b/config.json @@ -0,0 +1,107 @@ +{ + "_name_or_path": "facebook/wav2vec2-xls-r-1b", + "activation_dropout": 0.055, + "adapter_kernel_size": 3, + "adapter_stride": 2, + "add_adapter": false, + "apply_spec_augment": true, + "architectures": [ + "Wav2Vec2ForCTC" + ], + "attention_dropout": 0.094, + "bos_token_id": 1, + "classifier_proj_size": 256, + "codevector_dim": 1024, + "contrastive_logits_temperature": 0.1, + "conv_bias": true, + "conv_dim": [ + 512, + 512, + 512, + 512, + 512, + 512, + 512 + ], + "conv_kernel": [ + 10, + 3, + 3, + 3, + 3, + 2, + 2 + ], + "conv_stride": [ + 5, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "ctc_loss_reduction": "mean", + "ctc_zero_infinity": true, + "diversity_loss_weight": 0.1, + "do_stable_layer_norm": true, + "eos_token_id": 2, + "feat_extract_activation": "gelu", + "feat_extract_dropout": 0.0, + "feat_extract_norm": "layer", + "feat_proj_dropout": 0.04, + "feat_quantizer_dropout": 0.0, + "final_dropout": 0.0, + "hidden_act": "gelu", + "hidden_dropout": 0.047, + "hidden_size": 1280, + "initializer_range": 0.02, + "intermediate_size": 5120, + "layer_norm_eps": 1e-05, + "layerdrop": 0.041, + "mask_feature_length": 64, + "mask_feature_min_masks": 0, + "mask_feature_prob": 0.25, + "mask_time_length": 10, + "mask_time_min_masks": 2, + "mask_time_prob": 0.082, + "model_type": "wav2vec2", + "num_adapter_layers": 3, + "num_attention_heads": 16, + "num_codevector_groups": 2, + "num_codevectors_per_group": 320, + "num_conv_pos_embedding_groups": 16, + "num_conv_pos_embeddings": 128, + "num_feat_extract_layers": 7, + "num_hidden_layers": 48, + "num_negatives": 100, + "output_hidden_size": 1280, + "pad_token_id": 38, + "proj_codevector_dim": 1024, + "tdnn_dilation": [ + 1, + 2, + 3, + 1, + 1 + ], + "tdnn_dim": [ + 512, + 512, + 512, + 512, + 1500 + ], + "tdnn_kernel": [ + 5, + 3, + 3, + 1, + 1 + ], + "torch_dtype": "float32", + "transformers_version": "4.18.0", + "use_weighted_layer_sum": false, + "vocab_size": 41, + "xvector_output_dim": 512 +} diff --git a/eval.py b/eval.py new file mode 100644 index 0000000..194ff21 --- /dev/null +++ b/eval.py @@ -0,0 +1,254 @@ +#!/usr/bin/env python3 +import argparse +import re +from typing import Dict + +import torch +from datasets import Audio, Dataset, load_dataset, load_metric +from num2words import num2words as n2w +from slugify import slugify + +from transformers import AutoFeatureExtractor, AutoModelForCTC, pipeline, Wav2Vec2Processor, Wav2Vec2ProcessorWithLM, Wav2Vec2FeatureExtractor +# from pyctcdecode import BeamSearchDecoderCTC + +from cardinal_numbers import convert_nums + + +def log_results(result: Dataset, args: Dict[str, str]): + """DO NOT CHANGE. This function computes and logs the result metrics.""" + + log_outputs = args.log_outputs + lm = "withLM" if args.use_lm else "noLM" + model_id = args.model_id.replace("/", "_").replace(".", "") + if args.filter: + extra_args = [args.config, slugify(args.filter), args.split, lm] + else: + extra_args = [args.config, args.split, lm] + dataset_id = "_".join([model_id] + args.dataset.split("/") + extra_args) + + # load metric + wer = load_metric("wer") + cer = load_metric("cer") + + # compute metrics + wer_result = wer.compute(references=result["target"], predictions=result["prediction"]) + cer_result = cer.compute(references=result["target"], predictions=result["prediction"]) + + # print & log results + result_str = f"{dataset_id}\nWER: {wer_result}\nCER: {cer_result}" + print(result_str) + + with open(f"{dataset_id}_eval_results.txt", "w") as f: + f.write(result_str) + with open(f"{dataset_id}_eval_results.tsv", "w") as f: + f.write("\t".join([args.model_id, args.dataset, args.config, args.filter, args.split, str(lm), str(wer_result), str(cer_result)])) + + # log all results in text file. Possibly interesting for analysis + if log_outputs is not None: + pred_file = f"log_{dataset_id}_predictions.txt" + target_file = f"log_{dataset_id}_targets.txt" + + with open(pred_file, "w") as p, open(target_file, "w") as t: + # mapping function to write output + def write_to_file(batch, i): + p.write(f"{i}" + "\n") + p.write(batch["prediction"] + "\n") + t.write(f"{i}" + "\n") + t.write(batch["target"] + "\n") + + result.map(write_to_file, with_indices=True) + + +def normalize_text(original_text: str, dataset: str) -> str: + """DO ADAPT FOR YOUR USE CASE. this function normalizes the target text.""" + + text = original_text.lower() + if dataset.lower().endswith("fleurs"): + replacements = ( + (r"\be\.kr", "etter kristus fødsel"), + (r"\bf\.kr", "før kristi fødsel"), + (r"\bca[.]?\b", "circa"), + (r"(\d)\s*km/t", r"\1 kilometer i timen"), + (r"(\d)\s*km", r"\1 kilometer"), + (r"(\d)\s*cm", r"\1 centimeter"), + (r"(\d)\s*mm", r"\1 millimeter"), + (r"kl\.", "klokka"), + (r"f\.eks", "for eksempel"), + ) + for abrev, expasion in replacements: + text = re.sub(abrev, expasion, text) + text = re.sub(r'(\d+)[-–](\d+)', r'\1 til \2', text) # 1-89, 70-90 + text = re.sub(r'(\d{2}):00', r'\1', text) # 21:00 + text = re.sub(r"(\d{2}):0(\d{1})", r"\1 null \2", text) # 17:03 + text = re.sub(r"(\d{1,2}):(\d{1,2})", r"\1 \2", text) # 17:23 (time), 4:3 (aspect ratios) + text = re.sub(r"(1[1-9])00", r"\1 hundre", text) # 1800, 1900 + text = re.sub(r"(1[1-9])0([1-9])", r"\1 null \2 ", text) # 1901, 1909 + text = re.sub(r"(1[1-9])([1-9]\d)", r"\1 \2 ", text) # 1911, 1987 + text = re.sub(r"(20)0([1-9])", r"\1 null \2 ", text) # 2009 + text = re.sub(r"(20)(\d{2})", r"\1 \2 ", text) # 2009 + text = re.sub(r"(\d{1,3})[.](\d{1,2})", r"\1 dot \2 ", text) # 802.11n, 2.5ghz (in English) + text = re.sub(r"(\d{1,2})[ .](\d{3})", r"\1\2", text) # 10 000, 32.000 + text = re.sub(r'(\w+)-(\w+)', r'\1 \2', text) # n-standard + # text = re.compile(r"-?0?[1-9][\d.]*").sub(lambda x: n2w(x.group(0), lang="no"), text.replace(".", "")) + text = re.compile(r"-?0?[1-9][\d.]*").sub(lambda x: convert_nums(int(x.group(0)), nn=True), text.replace(".", "")) + + + chars_to_ignore_regex = '[\,\?\.\!\-\;\:\"\“\%\‘\”\�\'\–\_\\\+\#\/]' # noqa: W605 IMPORTANT: this should correspond to the chars that were ignored during training + text = re.sub(chars_to_ignore_regex, "", text) + " " + + if dataset.lower().endswith("nst"): + text = text.lower() + text = text.replace("(...vær stille under dette opptaket...)", "") + text = re.sub('[áàâ]', 'a', text) + text = re.sub('[ä]', 'æ', text) + text = re.sub('[éèëê]', 'e', text) + text = re.sub('[íìïî]', 'i', text) + text = re.sub('[óòöô]', 'o', text) + text = re.sub('[ö]', 'ø', text) + text = re.sub('[ç]', 'c', text) + text = re.sub('[úùüû]', 'u', text) + # text = re.sub('\\(?=(Punktum|Komma|Utropstegn|Spørsmålstegn))', ' ', text) + text = re.sub('\s+', ' ', text) + elif dataset.lower().endswith("npsc"): + text = re.sub('[áàâ]', 'a', text) + text = re.sub('[ä]', 'æ', text) + text = re.sub('[éèëê]', 'e', text) + text = re.sub('[íìïî]', 'i', text) + text = re.sub('[óòöô]', 'o', text) + text = re.sub('[ö]', 'ø', text) + text = re.sub('[ç]', 'c', text) + text = re.sub('[úùüû]', 'u', text) + text = re.sub('\s+', ' ', text) + elif dataset.lower().endswith("fleurs"): + text = re.sub('[áàâ]', 'a', text) + text = re.sub('[ä]', 'æ', text) + text = re.sub('[éèëê]', 'e', text) + text = re.sub('[íìïî]', 'i', text) + text = re.sub('[óòöô]', 'o', text) + text = re.sub('[ö]', 'ø', text) + text = re.sub('[ç]', 'c', text) + text = re.sub('[úùüû]', 'u', text) + text = re.sub('[«»]', '', text) + text = re.sub('\s+', ' ', text) + text = re.sub('', 'eee', text) + text = re.sub('', 'qqq', text) + text = re.sub('', 'mmm', text) + text = re.sub('', 'xxx', text) + + # # In addition, we can normalize the target text, e.g. removing new lines characters etc... + # # note that order is important here! + # token_sequences_to_ignore = ["\n\n", "\n", " ", " "] + + # for t in token_sequences_to_ignore: + # text = " ".join(text.split(t)) + + return text + + +def main(args): + # load dataset + dataset = load_dataset(args.dataset, args.config, split=args.split, use_auth_token=True) + if args.filter: + attribute, value = list(map(str.strip, args.filter.split(":"))) + dataset = dataset.filter( + lambda x: x[attribute] == value, + desc=f"Filtering on {args.filter}", + ) + # for testing: only process the first two examples as a test + # dataset = dataset.select(range(10)) + + # load processor + feature_extractor = AutoFeatureExtractor.from_pretrained(args.model_id) + sampling_rate = feature_extractor.sampling_rate + + # resample audio + dataset = dataset.cast_column("audio", Audio(sampling_rate=sampling_rate)) + + # load eval pipeline + if args.device is None: + args.device = 0 if torch.cuda.is_available() else -1 + # asr = pipeline("automatic-speech-recognition", model=args.model_id, device=args.device) + + model_instance = AutoModelForCTC.from_pretrained(args.model_id) + if args.use_lm: + processor = Wav2Vec2ProcessorWithLM.from_pretrained(args.model_id) + decoder = processor.decoder + else: + processor = Wav2Vec2Processor.from_pretrained(args.model_id) + decoder = None + asr = pipeline( + "automatic-speech-recognition", + model=model_instance, + tokenizer=processor.tokenizer, + feature_extractor=processor.feature_extractor, + decoder=decoder, + device=args.device + ) + + # feature_extractor_dict, _ = Wav2Vec2FeatureExtractor.get_feature_extractor_dict(args.model_id) + # feature_extractor_dict["processor_class"] = "Wav2Vec2Processor" if not args.use_lm else "Wav2Vec2ProcessorWithLM" + # feature_extractor = Wav2Vec2FeatureExtractor.from_dict(feature_extractor_dict) + + # asr = pipeline("automatic-speech-recognition", model=args.model_id, feature_extractor=feature_extractor, device=args.device, decoder=BeamSearchDecoderCTC.load_from_dir("./")) + + # map function to decode audio + def map_to_pred(batch): + prediction = asr( + batch["audio"]["array"], chunk_length_s=args.chunk_length_s, stride_length_s=args.stride_length_s + ) + + batch["prediction"] = prediction["text"] + batch["target"] = normalize_text(batch[args.text_column], args.dataset) + return batch + + # run inference on all examples + result = dataset.map(map_to_pred, remove_columns=dataset.column_names) + + # compute and log_results + # do not change function below + log_results(result, args) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument( + "--model_id", type=str, required=True, help="Model identifier. Should be loadable with 🤗 Transformers" + ) + parser.add_argument( + "--dataset", + type=str, + required=True, + help="Dataset name to evaluate the `model_id`. Should be loadable with 🤗 Datasets", + ) + parser.add_argument( + "--config", type=str, required=True, help="Config of the dataset. *E.g.* `'en'` for Common Voice" + ) + parser.add_argument( + "--filter", type=str, default="", help="Simple filter on attributes. *E.g.* `region_of_youth:Troms` would pnly keep those samplesfor which the condition is met" + ) + parser.add_argument("--split", type=str, required=True, help="Split of the dataset. *E.g.* `'test'`") + parser.add_argument( + "--text_column", type=str, default="text", help="Column name containing the transcription." + ) + parser.add_argument( + "--chunk_length_s", type=float, default=None, help="Chunk length in seconds. Defaults to 5 seconds." + ) + parser.add_argument( + "--stride_length_s", type=float, default=None, help="Stride of the audio chunks. Defaults to 1 second." + ) + parser.add_argument( + "--log_outputs", action="store_true", help="If defined, write outputs to log file for analysis." + ) + parser.add_argument( + "--device", + type=int, + default=None, + help="The device to run the pipeline on. -1 for CPU (default), 0 for the first GPU and so on.", + ) + parser.add_argument( + "--use_lm", action="store_true", help="If defined, use included language model as the decoder." + ) + args = parser.parse_args() + + main(args) diff --git a/language_model/5gram.bin b/language_model/5gram.bin new file mode 100644 index 0000000..1f72c6b --- /dev/null +++ b/language_model/5gram.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b41c24c63f2f0585bea83666369593f3b3e6d047f327a90f36ebca2c35ef0ff +size 4243671427 diff --git a/language_model/attrs.json b/language_model/attrs.json new file mode 100644 index 0000000..3c07595 --- /dev/null +++ b/language_model/attrs.json @@ -0,0 +1 @@ +{"alpha": 0.5, "beta": 1.5, "unk_score_offset": -10.0, "score_boundary": true} \ No newline at end of file diff --git a/language_model/unigrams.txt b/language_model/unigrams.txt new file mode 100644 index 0000000..161067a --- /dev/null +++ b/language_model/unigrams.txt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac3e71ca49838ca355df6fdcb8d89344a5a9bf9e1a76587cdf5df1367c19b9a9 +size 16759269 diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..217b6f6 --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ab493ecc739df905d0decdc09ce1d9ee4453987bb1bdbcf2e8b06642b3c47b6 +size 3850301028 diff --git a/preprocessor_config.json b/preprocessor_config.json new file mode 100644 index 0000000..ebf4fcf --- /dev/null +++ b/preprocessor_config.json @@ -0,0 +1,10 @@ +{ + "do_normalize": true, + "feature_extractor_type": "Wav2Vec2FeatureExtractor", + "feature_size": 1, + "padding_side": "right", + "padding_value": 0, + "processor_class": "Wav2Vec2ProcessorWithLM", + "return_attention_mask": true, + "sampling_rate": 16000 +} diff --git a/pytorch_model.bin b/pytorch_model.bin new file mode 100644 index 0000000..8d03494 --- /dev/null +++ b/pytorch_model.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0de5c9ba91acd5844605a71e621f73b3dd9f9acbfb283d3d835b44a491b189b6 +size 3850475057 diff --git a/run.sh b/run.sh new file mode 100755 index 0000000..4754fad --- /dev/null +++ b/run.sh @@ -0,0 +1,38 @@ +WANDB_ENTITY=NbAiLab WANDB_PROJECT=wav2vec2 python run_speech_recognition_ctc.py \ + --model_name_or_path="facebook/wav2vec2-xls-r-1b" \ + --hub_model_id="NbAiLab/wav2vec2-1b-npsc-nst" \ + --output_dir="./" \ + --num_train_epochs="40" \ + --per_device_train_batch_size="12" \ + --per_device_eval_batch_size="12" \ + --gradient_accumulation_steps="2" \ + --learning_rate="2e-5" \ + --warmup_steps="2000" \ + --length_column_name="input_length" \ + --evaluation_strategy="steps" \ + --text_column_name="text" \ + --save_steps="500" \ + --eval_steps="500" \ + --logging_steps="100" \ + --layerdrop="0.041" \ + --attention_dropout="0.094" \ + --activation_dropout="0.055" \ + --hidden_dropout="0.047" \ + --save_total_limit="3" \ + --freeze_feature_encoder \ + --feat_proj_dropout="0.04" \ + --mask_time_prob="0.082" \ + --mask_time_length="10" \ + --mask_feature_prob="0.25" \ + --mask_feature_length="64" \ + --gradient_checkpointing \ + --min_duration_in_seconds="0.5" \ + --max_duration_in_seconds="30.0" \ + --use_auth_token \ + --seed="42" \ + --fp16 \ + --group_by_length \ + --do_train --do_eval \ + --push_to_hub \ + --preprocessing_num_workers="32" \ + --ctc_zero_infinity diff --git a/run_speech_recognition_ctc.py b/run_speech_recognition_ctc.py new file mode 100644 index 0000000..e178b69 --- /dev/null +++ b/run_speech_recognition_ctc.py @@ -0,0 +1,807 @@ +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2021 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and + +""" Fine-tuning a 🤗 Transformers CTC model for automatic speech recognition""" + +import functools +import json +import logging +import os +import re +import sys +import warnings +from dataclasses import dataclass, field +from typing import Dict, List, Optional, Union + +import datasets +import numpy as np +import torch +from datasets import DatasetDict, load_dataset, load_metric + +import transformers +from transformers import ( + AutoConfig, + AutoFeatureExtractor, + AutoModelForCTC, + AutoProcessor, + AutoTokenizer, + HfArgumentParser, + Trainer, + TrainingArguments, + Wav2Vec2Processor, + set_seed, +) +from transformers.trainer_utils import get_last_checkpoint, is_main_process +from transformers.utils import check_min_version +from transformers.utils.versions import require_version + +# Will error if the minimal version of Transformers is not installed. Remove at your own risks. +check_min_version("4.16.0.dev0") + +require_version("datasets>=1.13.3", "To fix: pip install -r examples/pytorch/text-classification/requirements.txt") + +logger = logging.getLogger(__name__) + + +def list_field(default=None, metadata=None): + return field(default_factory=lambda: default, metadata=metadata) + + +@dataclass +class ModelArguments: + """ + Arguments pertaining to which model/config/tokenizer we are going to fine-tune from. + """ + + model_name_or_path: str = field( + metadata={"help": "Path to pretrained model or model identifier from huggingface.co/models"} + ) + tokenizer_name_or_path: Optional[str] = field( + default=None, + metadata={"help": "Path to pretrained tokenizer or tokenizer identifier from huggingface.co/models"}, + ) + cache_dir: Optional[str] = field( + default=None, + metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"}, + ) + freeze_feature_encoder: bool = field( + default=True, metadata={"help": "Whether to freeze the feature encoder layers of the model."} + ) + attention_dropout: float = field( + default=0.0, metadata={"help": "The dropout ratio for the attention probabilities."} + ) + activation_dropout: float = field( + default=0.0, metadata={"help": "The dropout ratio for activations inside the fully connected layer."} + ) + feat_proj_dropout: float = field(default=0.0, metadata={"help": "The dropout ratio for the projected features."}) + hidden_dropout: float = field( + default=0.0, + metadata={ + "help": "The dropout probability for all fully connected layers in the embeddings, encoder, and pooler." + }, + ) + final_dropout: float = field( + default=0.0, + metadata={"help": "The dropout probability for the final projection layer."}, + ) + mask_time_prob: float = field( + default=0.05, + metadata={ + "help": "Probability of each feature vector along the time axis to be chosen as the start of the vector" + "span to be masked. Approximately ``mask_time_prob * sequence_length // mask_time_length`` feature" + "vectors will be masked along the time axis." + }, + ) + mask_time_length: int = field( + default=10, + metadata={"help": "Length of vector span to mask along the time axis."}, + ) + mask_feature_prob: float = field( + default=0.0, + metadata={ + "help": "Probability of each feature vector along the feature axis to be chosen as the start of the vector" + "span to be masked. Approximately ``mask_feature_prob * sequence_length // mask_feature_length`` feature bins will be masked along the time axis." + }, + ) + mask_feature_length: int = field( + default=10, + metadata={"help": "Length of vector span to mask along the feature axis."}, + ) + layerdrop: float = field(default=0.0, metadata={"help": "The LayerDrop probability."}) + ctc_loss_reduction: Optional[str] = field( + default="mean", metadata={"help": "The way the ctc loss should be reduced. Should be one of 'mean' or 'sum'."} + ) + ctc_zero_infinity: Optional[bool] = field( + default=False, metadata={"help": "If True, will try yo aboud the CTC loss goinf to infinity."} + ) + + +@dataclass +class DataTrainingArguments: + """ + Arguments pertaining to what data we are going to input our model for training and eval. + + Using `HfArgumentParser` we can turn this class + into argparse arguments to be able to specify them on + the command line. + """ + + # dataset_name: str = field( + # metadata={"help": "The configuration name of the dataset to use (via the datasets library)."} + # ) + # dataset_config_name: str = field( + # default=None, metadata={"help": "The configuration name of the dataset to use (via the datasets library)."} + # ) + train_split_name: str = field( + default="train", + metadata={ + "help": "The name of the training data set split to use (via the datasets library). Defaults to 'train'" + }, + ) + eval_split_name: str = field( + default="test", + metadata={ + "help": "The name of the training data set split to use (via the datasets library). Defaults to 'train'" + }, + ) + audio_column_name: str = field( + default="audio", + metadata={"help": "The name of the dataset column containing the audio data. Defaults to 'audio'"}, + ) + text_column_name: str = field( + default="text", + metadata={"help": "The name of the dataset column containing the text data. Defaults to 'text'"}, + ) + overwrite_cache: bool = field( + default=False, metadata={"help": "Overwrite the cached preprocessed datasets or not."} + ) + preprocessing_num_workers: Optional[int] = field( + default=None, + metadata={"help": "The number of processes to use for the preprocessing."}, + ) + max_train_samples: Optional[int] = field( + default=None, + metadata={ + "help": "For debugging purposes or quicker training, truncate the number of training examples to this " + "value if set." + }, + ) + max_eval_samples: Optional[int] = field( + default=None, + metadata={ + "help": "For debugging purposes or quicker training, truncate the number of validation examples to this " + "value if set." + }, + ) + chars_to_ignore: Optional[List[str]] = list_field( + default=None, + metadata={"help": "A list of characters to remove from the transcripts."}, + ) + eval_metrics: List[str] = list_field( + default=["wer"], + metadata={"help": "A list of metrics the model should be evaluated on. E.g. `'wer cer'`"}, + ) + max_duration_in_seconds: float = field( + default=20.0, + metadata={ + "help": "Filter audio files that are longer than `max_duration_in_seconds` seconds to 'max_duration_in_seconds`" + }, + ) + min_duration_in_seconds: float = field( + default=0.0, metadata={"help": "Filter audio files that are shorter than `min_duration_in_seconds` seconds"} + ) + preprocessing_only: bool = field( + default=False, + metadata={ + "help": "Whether to only do data preprocessing and skip training. " + "This is especially useful when data preprocessing errors out in distributed training due to timeout. " + "In this case, one should run the preprocessing in a non-distributed setup with `preprocessing_only=True` " + "so that the cached datasets can consequently be loaded in distributed training" + }, + ) + use_auth_token: bool = field( + default=False, + metadata={ + "help": "If :obj:`True`, will use the token generated when running" + ":obj:`transformers-cli login` as HTTP bearer authorization for remote files." + }, + ) + unk_token: str = field( + default="[UNK]", + metadata={"help": "The unk token for the tokenizer"}, + ) + pad_token: str = field( + default="[PAD]", + metadata={"help": "The padding token for the tokenizer"}, + ) + word_delimiter_token: str = field( + default="|", + metadata={"help": "The word delimiter token for the tokenizer"}, + ) + phoneme_language: Optional[str] = field( + default=None, + metadata={ + "help": "The target language that should be used be" + " passed to the tokenizer for tokenization. Note that" + " this is only relevant if the model classifies the" + " input audio to a sequence of phoneme sequences." + }, + ) + + +@dataclass +class DataCollatorCTCWithPadding: + """ + Data collator that will dynamically pad the inputs received. + Args: + processor (:class:`~transformers.AutoProcessor`) + The processor used for proccessing the data. + padding (:obj:`bool`, :obj:`str` or :class:`~transformers.tokenization_utils_base.PaddingStrategy`, `optional`, defaults to :obj:`True`): + Select a strategy to pad the returned sequences (according to the model's padding side and padding index) + among: + * :obj:`True` or :obj:`'longest'`: Pad to the longest sequence in the batch (or no padding if only a single + sequence if provided). + * :obj:`'max_length'`: Pad to a maximum length specified with the argument :obj:`max_length` or to the + maximum acceptable input length for the model if that argument is not provided. + * :obj:`False` or :obj:`'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of + different lengths). + max_length (:obj:`int`, `optional`): + Maximum length of the ``input_values`` of the returned list and optionally padding length (see above). + max_length_labels (:obj:`int`, `optional`): + Maximum length of the ``labels`` returned list and optionally padding length (see above). + pad_to_multiple_of (:obj:`int`, `optional`): + If set will pad the sequence to a multiple of the provided value. + This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability >= + 7.5 (Volta). + """ + + processor: AutoProcessor + padding: Union[bool, str] = "longest" + pad_to_multiple_of: Optional[int] = None + pad_to_multiple_of_labels: Optional[int] = None + + def __call__(self, features: List[Dict[str, Union[List[int], torch.Tensor]]]) -> Dict[str, torch.Tensor]: + # split inputs and labels since they have to be of different lenghts and need + # different padding methods + input_features = [{"input_values": feature["input_values"]} for feature in features] + label_features = [{"input_ids": feature["labels"]} for feature in features] + + batch = self.processor.pad( + input_features, + padding=self.padding, + pad_to_multiple_of=self.pad_to_multiple_of, + return_tensors="pt", + ) + + with self.processor.as_target_processor(): + labels_batch = self.processor.pad( + label_features, + padding=self.padding, + pad_to_multiple_of=self.pad_to_multiple_of_labels, + return_tensors="pt", + ) + + # replace padding with -100 to ignore loss correctly + labels = labels_batch["input_ids"].masked_fill(labels_batch.attention_mask.ne(1), -100) + + batch["labels"] = labels + + return batch + + +def create_vocabulary_from_data( + datasets: DatasetDict, + word_delimiter_token: Optional[str] = None, + unk_token: Optional[str] = None, + pad_token: Optional[str] = None, +): + # Given training and test labels create vocabulary + alphabet = set() + + def extract_all_chars(batch): + all_text = " ".join(batch["target_text"]) + alphabet.update(all_text) + + datasets.map( + extract_all_chars, + batched=True, + batch_size=-1, + keep_in_memory=True, + remove_columns=datasets["train"].column_names, + ) + + # # take union of all unique characters in each dataset + # vocab_set = functools.reduce( + # lambda vocab_1, vocab_2: {"vocab": list(set(vocab_1["vocab"][0]) | set(vocab_2["vocab"][0]))}, vocabs.values() + # )["vocab"][0] + + vocab_dict = {v: k for k, v in enumerate(sorted(list(alphabet)))} + + # replace white space with delimiter token + if word_delimiter_token is not None: + vocab_dict[word_delimiter_token] = vocab_dict[" "] + del vocab_dict[" "] + + # add unk and pad token + if unk_token is not None: + vocab_dict[unk_token] = len(vocab_dict) + + if pad_token is not None: + vocab_dict[pad_token] = len(vocab_dict) + + return vocab_dict + + +def make_dataset(seed=42): + # Pre-processing dataset + import re + + def map_nst(entry): + text = entry["text"].lower() + text = text.replace("(...Vær stille under dette opptaket...)", "") + text = re.sub('[áàâ]', 'a', text) + text = re.sub('[ä]', 'æ', text) + text = re.sub('[éèëê]', 'e', text) + text = re.sub('[íìïî]', 'i', text) + text = re.sub('[óòöô]', 'o', text) + text = re.sub('[ö]', 'ø', text) + text = re.sub('[ç]', 'c', text) + text = re.sub('[úùüû]', 'u', text) + # text = re.sub('\\(?=(Punktum|Komma|Utropstegn|Spørsmålstegn))', ' ', text) + text = re.sub('\s+', ' ', text) + return {"text": text} + + def filter_nst(entry): + if not ((len(entry["text"]) <= len(entry["audio"]["array"]) // 320) and (len(entry["text"].strip()) >= 3)): + return False # Too short + if re.match(entry["type"], "pIW|CA"): + return False # Spelling out words + return True + + def filter_npsc(entry): + # False if there are digits in the text + if not ((len(entry["text"]) <= len(entry["audio"]["array"]) // 320) and (len(entry["text"].strip()) >= 3)): + return False # Too short + if re.search("\d", entry["text"]): + return False + return True + + def map_npsc(entry): + batch = {"text": entry["text"].lower()} + batch["text"] = re.sub('[áàâ]', 'a', batch["text"]) + batch["text"] = re.sub('[ä]', 'æ', batch["text"]) + batch["text"] = re.sub('[éèëê]', 'e', batch["text"]) + batch["text"] = re.sub('[íìïî]', 'i', batch["text"]) + batch["text"] = re.sub('[óòöô]', 'o', batch["text"]) + batch["text"] = re.sub('[ö]', 'ø', batch["text"]) + batch["text"] = re.sub('[ç]', 'c', batch["text"]) + batch["text"] = re.sub('[úùüû]', 'u', batch["text"]) + batch["text"] = re.sub('\s', ' ', batch["text"]) + batch["text"] = re.sub('', 'eee', batch["text"]) + batch["text"] = re.sub('', 'qqq', batch["text"]) + batch["text"] = re.sub('', 'mmm', batch["text"]) + batch["text"] = re.sub('', 'xxx', batch["text"]) + # batch["text"] = re.sub('', '?', batch["text"]) + if "<" in batch["text"]: + raise ValueError(batch["text"]) + return batch + + nst = datasets.load_dataset("NbAiLab/NST", "no-close") + npsc = datasets.load_dataset("NbAiLab/NPSC", "16K_mp3") + # TODO NST_hesitate + + split = len(npsc["train"]) / (len(npsc["train"]) + len(npsc["validation"])) # Use same train/val ratio as NPSC + nst_train = nst["train"].train_test_split(train_size=split, seed=seed) + nst["train"] = nst_train["train"] + nst["validation"] = nst_train["test"] + + nst = nst.filter(filter_nst).map(map_nst).shuffle(seed=seed) + npsc = npsc.filter(filter_npsc).map(map_npsc).shuffle(seed=seed) + + npsc_base = npsc.remove_columns([col for col in npsc["train"].column_names if col not in ["text", "audio"]]) + nst_base = nst.remove_columns([col for col in nst["train"].column_names if col not in ["text", "audio"]]) + + combined = {} + for split in "train", "validation", "test": + probs = np.array([len(nst_base[split]), len(npsc_base[split])]) # Weight by number of examples + probs = (probs / probs.sum()).tolist() + comb = datasets.interleave_datasets([nst_base[split], npsc_base[split]], probabilities=probs, seed=seed) + combined[split] = comb + + return datasets.DatasetDict(**combined) + + +def main(): + # See all possible arguments in src/transformers/training_args.py + # or by passing the --help flag to this script. + # We now keep distinct sets of args, for a cleaner separation of concerns. + + parser = HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)) + if len(sys.argv) == 2 and sys.argv[1].endswith(".json"): + # If we pass only one argument to the script and it's the path to a json file, + # let's parse it to get our arguments. + model_args, data_args, training_args = parser.parse_json_file(json_file=os.path.abspath(sys.argv[1])) + else: + model_args, data_args, training_args = parser.parse_args_into_dataclasses() + + # Detecting last checkpoint. + last_checkpoint = None + if os.path.isdir(training_args.output_dir) and training_args.do_train and not training_args.overwrite_output_dir: + last_checkpoint = get_last_checkpoint(training_args.output_dir) + if last_checkpoint is None and len(os.listdir(training_args.output_dir)) > 0: + raise ValueError( + f"Output directory ({training_args.output_dir}) already exists and is not empty. " + "Use --overwrite_output_dir to overcome." + ) + elif last_checkpoint is not None: + logger.info( + f"Checkpoint detected, resuming training at {last_checkpoint}. To avoid this behavior, change " + "the `--output_dir` or add `--overwrite_output_dir` to train from scratch." + ) + + # Setup logging + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + handlers=[logging.StreamHandler(sys.stdout)], + ) + logger.setLevel(logging.INFO if is_main_process(training_args.local_rank) else logging.WARN) + + # Log on each process the small summary: + logger.warning( + f"Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}" + f"distributed training: {bool(training_args.local_rank != -1)}, 16-bits training: {training_args.fp16}" + ) + # Set the verbosity to info of the Transformers logger (on main process only): + if is_main_process(training_args.local_rank): + transformers.utils.logging.set_verbosity_info() + logger.info("Training/evaluation parameters %s", training_args) + + # Set seed before initializing model. + set_seed(training_args.seed) + + # 1. First, let's load the dataset + raw_datasets = make_dataset(seed=training_args.seed) + + if training_args.do_train: + if data_args.audio_column_name not in raw_datasets["train"].column_names: + raise ValueError( + f"--audio_column_name '{data_args.audio_column_name}' not found in dataset '{data_args.dataset_name}'. " + "Make sure to set `--audio_column_name` to the correct audio column - one of " + f"{', '.join(raw_datasets['train'].column_names)}." + ) + + if data_args.text_column_name not in raw_datasets["train"].column_names: + raise ValueError( + f"--text_column_name {data_args.text_column_name} not found in dataset '{data_args.dataset_name}'. " + "Make sure to set `--text_column_name` to the correct text column - one of " + f"{', '.join(raw_datasets['train'].column_names)}." + ) + + if data_args.max_train_samples is not None: + raw_datasets["train"] = raw_datasets["train"].select(range(data_args.max_train_samples)) + + if training_args.do_eval: + if data_args.max_eval_samples is not None: + raw_datasets["eval"] = raw_datasets["eval"].select(range(data_args.max_eval_samples)) + + # 2. We remove some special characters from the datasets + # that make training complicated and do not help in transcribing the speech + # E.g. characters, such as `,` and `.` do not really have an acoustic characteristic + # that could be easily picked up by the model + # chars_to_ignore_regex = ( + # f'[{"".join(data_args.chars_to_ignore)}]' if data_args.chars_to_ignore is not None else None + # ) + chars_to_ignore_regex = '[\,\?\.\!\-\;\:\"\“\%\‘\”\�\'\–\_\\\+\#\/]' + + text_column_name = data_args.text_column_name + + def remove_special_characters(batch): + if chars_to_ignore_regex is not None: + batch["target_text"] = re.sub(chars_to_ignore_regex, "", batch[text_column_name]).lower() + " " + else: + batch["target_text"] = batch[text_column_name].lower() + " " + return batch + + with training_args.main_process_first(desc="dataset map special characters removal"): + raw_datasets = raw_datasets.map( + remove_special_characters, + remove_columns=[text_column_name], + desc="remove special characters from datasets", + ) + + # save special tokens for tokenizer + word_delimiter_token = data_args.word_delimiter_token + unk_token = data_args.unk_token + pad_token = data_args.pad_token + + # 3. Next, let's load the config as we might need it to create + # the tokenizer + # load config + config = AutoConfig.from_pretrained( + model_args.model_name_or_path, cache_dir=model_args.cache_dir, use_auth_token=data_args.use_auth_token + ) + + # 4. Next, if no tokenizer file is defined, + # we create the vocabulary of the model by extracting all unique characters from + # the training and evaluation datasets + # We need to make sure that only first rank saves vocabulary + # make sure all processes wait until vocab is created + tokenizer_name_or_path = model_args.tokenizer_name_or_path + tokenizer_kwargs = {} + if tokenizer_name_or_path is None: + # save vocab in training output dir + tokenizer_name_or_path = training_args.output_dir + + vocab_file = os.path.join(tokenizer_name_or_path, "vocab.json") + + with training_args.main_process_first(): + if training_args.overwrite_output_dir and os.path.isfile(vocab_file): + os.remove(vocab_file) + + with training_args.main_process_first(desc="dataset map vocabulary creation"): + if not os.path.isfile(vocab_file): + os.makedirs(tokenizer_name_or_path, exist_ok=True) + vocab_dict = create_vocabulary_from_data( + raw_datasets, + word_delimiter_token=word_delimiter_token, + unk_token=unk_token, + pad_token=pad_token, + ) + + # save vocab dict to be loaded into tokenizer + with open(vocab_file, "w") as file: + json.dump(vocab_dict, file) + + # if tokenizer has just been created + # it is defined by `tokenizer_class` if present in config else by `model_type` + tokenizer_kwargs = { + "config": config if config.tokenizer_class is not None else None, + "tokenizer_type": config.model_type if config.tokenizer_class is None else None, + "unk_token": unk_token, + "pad_token": pad_token, + "word_delimiter_token": word_delimiter_token, + } + + # 5. Now we can instantiate the feature extractor, tokenizer and model + # Note for distributed training, the .from_pretrained methods guarantee that only + # one local process can concurrently download model & vocab. + + # load feature_extractor and tokenizer + tokenizer = AutoTokenizer.from_pretrained( + tokenizer_name_or_path, + use_auth_token=data_args.use_auth_token, + **tokenizer_kwargs, + ) + feature_extractor = AutoFeatureExtractor.from_pretrained( + model_args.model_name_or_path, cache_dir=model_args.cache_dir, use_auth_token=data_args.use_auth_token + ) + + # adapt config + config.update( + { + "feat_proj_dropout": model_args.feat_proj_dropout, + "attention_dropout": model_args.attention_dropout, + "hidden_dropout": model_args.hidden_dropout, + "final_dropout": model_args.final_dropout, + "mask_time_prob": model_args.mask_time_prob, + "mask_time_length": model_args.mask_time_length, + "mask_feature_prob": model_args.mask_feature_prob, + "mask_feature_length": model_args.mask_feature_length, + "gradient_checkpointing": training_args.gradient_checkpointing, + "layerdrop": model_args.layerdrop, + "ctc_loss_reduction": model_args.ctc_loss_reduction, + "ctc_zero_infinity": model_args.ctc_zero_infinity, + "pad_token_id": tokenizer.pad_token_id, + "vocab_size": len(tokenizer), + "activation_dropout": model_args.activation_dropout, + } + ) + + # create model + model = AutoModelForCTC.from_pretrained( + model_args.model_name_or_path, + cache_dir=model_args.cache_dir, + config=config, + use_auth_token=data_args.use_auth_token, + ) + + # freeze encoder + if model_args.freeze_feature_encoder: + model.freeze_feature_encoder() + + # 6. Now we preprocess the datasets including loading the audio, resampling and normalization + # Thankfully, `datasets` takes care of automatically loading and resampling the audio, + # so that we just need to set the correct target sampling rate and normalize the input + # via the `feature_extractor` + + # make sure that dataset decodes audio with correct sampling rate + dataset_sampling_rate = next(iter(raw_datasets.values())).features[data_args.audio_column_name].sampling_rate + if dataset_sampling_rate != feature_extractor.sampling_rate: + raw_datasets = raw_datasets.cast_column( + data_args.audio_column_name, datasets.features.Audio(sampling_rate=feature_extractor.sampling_rate) + ) + + # derive max & min input length for sample rate & max duration + max_input_length = data_args.max_duration_in_seconds * feature_extractor.sampling_rate + min_input_length = data_args.min_duration_in_seconds * feature_extractor.sampling_rate + audio_column_name = data_args.audio_column_name + num_workers = data_args.preprocessing_num_workers + + # `phoneme_language` is only relevant if the model is fine-tuned on phoneme classification + phoneme_language = data_args.phoneme_language + + # Preprocessing the datasets. + # We need to read the audio files as arrays and tokenize the targets. + def prepare_dataset(batch): + # load audio + sample = batch[audio_column_name] + + inputs = feature_extractor(sample["array"], sampling_rate=sample["sampling_rate"]) + batch["input_values"] = inputs.input_values[0] + batch["input_length"] = len(batch["input_values"]) + + # encode targets + additional_kwargs = {} + if phoneme_language is not None: + additional_kwargs["phonemizer_lang"] = phoneme_language + + batch["labels"] = tokenizer(batch["target_text"], **additional_kwargs).input_ids + return batch + + with training_args.main_process_first(desc="dataset map preprocessing"): + vectorized_datasets = raw_datasets.map( + prepare_dataset, + remove_columns=next(iter(raw_datasets.values())).column_names, + num_proc=num_workers, + desc="preprocess datasets", + ) + + def is_audio_in_length_range(length): + return length > min_input_length and length < max_input_length + + # filter data that is shorter than min_input_length + vectorized_datasets = vectorized_datasets.filter( + is_audio_in_length_range, + num_proc=num_workers, + input_columns=["input_length"], + ) + + # 7. Next, we can prepare the training. + # Let's use word error rate (WER) as our evaluation metric, + # instantiate a data collator and the trainer + + # Define evaluation metrics during training, *i.e.* word error rate, character error rate + eval_metrics = {metric: load_metric(metric) for metric in data_args.eval_metrics} + + # for large datasets it is advised to run the preprocessing on a + # single machine first with ``args.preprocessing_only`` since there will mostly likely + # be a timeout when running the script in distributed mode. + # In a second step ``args.preprocessing_only`` can then be set to `False` to load the + # cached dataset + if data_args.preprocessing_only: + logger.info(f"Data preprocessing finished. Files cached at {vectorized_datasets.cache_files}") + return + + def compute_metrics(pred): + pred_logits = pred.predictions + pred_ids = np.argmax(pred_logits, axis=-1) + + pred.label_ids[pred.label_ids == -100] = tokenizer.pad_token_id + + pred_str = tokenizer.batch_decode(pred_ids) + # we do not want to group tokens when computing the metrics + label_str = tokenizer.batch_decode(pred.label_ids, group_tokens=False) + + metrics = {k: v.compute(predictions=pred_str, references=label_str) for k, v in eval_metrics.items()} + + return metrics + + # Now save everything to be able to create a single processor later + if is_main_process(training_args.local_rank): + # save feature extractor, tokenizer and config + feature_extractor.save_pretrained(training_args.output_dir) + tokenizer.save_pretrained(training_args.output_dir) + config.save_pretrained(training_args.output_dir) + + try: + processor = AutoProcessor.from_pretrained(training_args.output_dir) + except (OSError, KeyError): + warnings.warn( + "Loading a processor from a feature extractor config that does not" + " include a `processor_class` attribute is deprecated and will be removed in v5. Please add the following " + " attribute to your `preprocessor_config.json` file to suppress this warning: " + " `'processor_class': 'Wav2Vec2Processor'`", + FutureWarning, + ) + processor = Wav2Vec2Processor.from_pretrained(training_args.output_dir) + + # Instantiate custom data collator + data_collator = DataCollatorCTCWithPadding(processor=processor) + + # Initialize Trainer + trainer = Trainer( + model=model, + data_collator=data_collator, + args=training_args, + compute_metrics=compute_metrics, + train_dataset=vectorized_datasets["train"] if training_args.do_train else None, + eval_dataset=vectorized_datasets["validation"] if training_args.do_eval else None, + tokenizer=feature_extractor, + ) + + # 8. Finally, we can start training + + # Training + if training_args.do_train: + + # use last checkpoint if exist + if last_checkpoint is not None: + checkpoint = last_checkpoint + elif os.path.isdir(model_args.model_name_or_path): + checkpoint = model_args.model_name_or_path + else: + checkpoint = None + + train_result = trainer.train(resume_from_checkpoint=checkpoint) + trainer.save_model() + + metrics = train_result.metrics + max_train_samples = ( + data_args.max_train_samples + if data_args.max_train_samples is not None + else len(vectorized_datasets["train"]) + ) + metrics["train_samples"] = min(max_train_samples, len(vectorized_datasets["train"])) + + trainer.log_metrics("train", metrics) + trainer.save_metrics("train", metrics) + trainer.save_state() + + # Evaluation + results = {} + if training_args.do_eval: + logger.info("*** Evaluate ***") + metrics = trainer.evaluate() + max_eval_samples = ( + data_args.max_eval_samples if data_args.max_eval_samples is not None else len(vectorized_datasets["eval"]) + ) + metrics["eval_samples"] = min(max_eval_samples, len(vectorized_datasets["eval"])) + + trainer.log_metrics("eval", metrics) + trainer.save_metrics("eval", metrics) + + # Write model card and (optionally) push to hub + config_name = data_args.dataset_config_name if data_args.dataset_config_name is not None else "na" + kwargs = { + "finetuned_from": model_args.model_name_or_path, + "tasks": "speech-recognition", + "tags": ["automatic-speech-recognition", data_args.dataset_name], + "dataset_args": f"Config: {config_name}, Training split: {data_args.train_split_name}, Eval split: {data_args.eval_split_name}", + "dataset": f"{data_args.dataset_name.upper()} - {config_name.upper()}", + } + if "common_voice" in data_args.dataset_name: + kwargs["language"] = config_name + + if training_args.push_to_hub: + trainer.push_to_hub(**kwargs) + else: + trainer.create_model_card(**kwargs) + + return results + + +if __name__ == "__main__": + main() diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..ac9d0f5 --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1 @@ +{"bos_token": "", "eos_token": "", "unk_token": "[UNK]", "pad_token": "[PAD]", "additional_special_tokens": [{"content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}]} \ No newline at end of file diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..5cc8e4c --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1 @@ +{"unk_token": "[UNK]", "bos_token": "", "eos_token": "", "pad_token": "[PAD]", "do_lower_case": false, "word_delimiter_token": "|", "replace_word_delimiter_char": " ", "special_tokens_map_file": null, "name_or_path": "./", "tokenizer_class": "Wav2Vec2CTCTokenizer", "processor_class": "Wav2Vec2ProcessorWithLM"} \ No newline at end of file diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..0ac5f01 --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d63c31e6cdc4ad990144c981422252a8ee34040e3eb92deeb7c6f36e75ccdee2 +size 3055 diff --git a/vocab.json b/vocab.json new file mode 100644 index 0000000..9f20672 --- /dev/null +++ b/vocab.json @@ -0,0 +1 @@ +{"(": 1, ")": 2, "0": 3, "3": 4, "7": 5, "8": 6, "9": 7, "a": 8, "b": 9, "c": 10, "d": 11, "e": 12, "f": 13, "g": 14, "h": 15, "i": 16, "j": 17, "k": 18, "l": 19, "m": 20, "n": 21, "o": 22, "p": 23, "q": 24, "r": 25, "s": 26, "t": 27, "u": 28, "v": 29, "w": 30, "x": 31, "y": 32, "z": 33, "å": 34, "æ": 35, "ø": 36, "|": 0, "[UNK]": 37, "[PAD]": 38} \ No newline at end of file