From 545953eeabc9bda98d0d778fd75c6f240ccdcd86 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Fri, 21 Aug 2026 19:39:17 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: nikitastheo/phonebabylm-phone-deu-eng-sequential_interleaved Source: Original Platform --- .gitattributes | 35 + README.md | 24 + config.json | 33 + generation_config.json | 7 + model.safetensors | 3 + special_tokens_map.json | 30 + tokenizer.json | 174 +++ tokenizer_config.json | 37 + train_rank0.log | 2372 +++++++++++++++++++++++++++++++++++++++ training_metadata.json | 74 ++ vocab.json | 1 + 11 files changed, 2790 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 config.json create mode 100644 generation_config.json create mode 100644 model.safetensors create mode 100644 special_tokens_map.json create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json create mode 100644 train_rank0.log create mode 100644 training_metadata.json create mode 100644 vocab.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..a6344aa --- /dev/null +++ b/.gitattributes @@ -0,0 +1,35 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..2a16e43 --- /dev/null +++ b/README.md @@ -0,0 +1,24 @@ +--- +tags: +- causal-lm +- text-generation +library_name: transformers +--- + +# nikitastheo/phonebabylm-phone-deu-eng-sequential_interleaved + +Trained with train_clm.py, a Hugging Face Accelerate causal-LM training script (no `Trainer`). + +## Training details + +- **Base config**: `gpt_base_config.json` +- **Tokenizer**: `nikitastheo/phonelm-phone-deu-eng-tokenizer` +- **Max steps**: 30000 +- **Learning rate**: 0.0001 +- **LR scheduler**: linear +- **Warmup steps**: 3000 +- **Batch size (per device)**: 32 +- **Gradient accumulation steps**: 1 +- **Total train batch size**: 32 +- **Language switch steps**: 10000 + diff --git a/config.json b/config.json new file mode 100644 index 0000000..63268b3 --- /dev/null +++ b/config.json @@ -0,0 +1,33 @@ +{ + "activation_function": "gelu", + "architectures": [ + "GPT2LMHeadModel" + ], + "attn_pdrop": 0.1, + "bos_token_id": 2, + "dtype": "float32", + "embd_pdrop": 0.1, + "eos_token_id": 2, + "initializer_range": 0.02, + "layer_norm_epsilon": 1e-05, + "model_type": "gpt2", + "n_ctx": 512, + "n_embd": 768, + "n_head": 12, + "n_inner": 3072, + "n_layer": 12, + "n_positions": 512, + "pad_token_id": 1, + "reorder_and_upcast_attn": false, + "resid_pdrop": 0.1, + "scale_attn_by_inverse_layer_idx": false, + "scale_attn_weights": true, + "summary_activation": null, + "summary_first_dropout": 0.1, + "summary_proj_to_labels": true, + "summary_type": "cls_index", + "summary_use_proj": true, + "transformers_version": "4.57.6", + "use_cache": true, + "vocab_size": 71 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..32a3d06 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,7 @@ +{ + "_from_model_config": true, + "bos_token_id": 2, + "eos_token_id": 2, + "pad_token_id": 1, + "transformers_version": "4.57.6" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..e4895c3 --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a4ac61b4935cc87bb04dffbd037c203334c01c8c265e93a736a0ef4990e3f61f +size 342029952 diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..cdd0a22 --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,30 @@ +{ + "bos_token": { + "content": "UTT_BOUNDARY", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "UTT_BOUNDARY", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "PAD", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "unk_token": { + "content": "UNK", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..6266992 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,174 @@ +{ + "version": "1.0", + "truncation": null, + "padding": null, + "added_tokens": [ + { + "id": 0, + "content": "UNK", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1, + "content": "PAD", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2, + "content": "UTT_BOUNDARY", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + } + ], + "normalizer": { + "type": "Sequence", + "normalizers": [ + { + "type": "Replace", + "pattern": { + "String": "ː" + }, + "content": " ː " + }, + { + "type": "Strip", + "strip_left": true, + "strip_right": true + } + ] + }, + "pre_tokenizer": { + "type": "Whitespace" + }, + "post_processor": { + "type": "TemplateProcessing", + "single": [ + { + "SpecialToken": { + "id": "UTT_BOUNDARY", + "type_id": 0 + } + }, + { + "Sequence": { + "id": "A", + "type_id": 0 + } + } + ], + "pair": [ + { + "Sequence": { + "id": "A", + "type_id": 0 + } + }, + { + "Sequence": { + "id": "B", + "type_id": 1 + } + } + ], + "special_tokens": { + "UTT_BOUNDARY": { + "id": "UTT_BOUNDARY", + "ids": [ + 2 + ], + "tokens": [ + "UTT_BOUNDARY" + ] + } + } + }, + "decoder": null, + "model": { + "type": "WordLevel", + "vocab": { + "UNK": 0, + "PAD": 1, + "UTT_BOUNDARY": 2, + "d": 3, + "a": 4, + "s": 5, + "WORD_BOUNDARY": 6, + "f": 7, + "ɛ": 8, + "ɐ": 9, + "k": 10, + "ə": 11, + "l": 12, + "v": 13, + "o": 14, + "ː": 15, + "n": 16, + "t": 17, + "ɪ": 18, + "aɪ": 19, + "m": 20, + "z": 21, + "e": 22, + "ɡ": 23, + "h": 24, + "aʊ": 25, + "b": 26, + "u": 27, + "x": 28, + "ʊ": 29, + "i": 30, + "ʃ": 31, + "ʏ": 32, + "ts": 33, + "ɔ": 34, + "p": 35, + "ʀ": 36, + "ŋ": 37, + "œ": 38, + "ç": 39, + "y": 40, + "ø": 41, + "j": 42, + "pf": 43, + "ʊɐ": 44, + "eɪ": 45, + "d̠ʒ": 46, + "ɛɪ": 47, + "w": 48, + "ɒ": 49, + "ã": 50, + "t̠ʃ": 51, + "tɕ": 52, + "ð": 53, + "θ": 54, + "ɹ": 55, + "əʊ": 56, + "ʌ": 57, + "eə": 58, + "ɔɪ": 59, + "l̩": 60, + "ʒ": 61, + "ʊə": 62, + "ɔ̃": 63, + "iə": 64, + "œ̃": 65, + "ʔ": 66, + "æ": 67, + "ɑ": 68, + "ɜ": 69, + "oʊ": 70 + }, + "unk_token": "UNK" + } +} \ No newline at end of file diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..fe3ab68 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,37 @@ +{ + "add_prefix_space": false, + "added_tokens_decoder": { + "0": { + "content": "UNK", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "1": { + "content": "PAD", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "2": { + "content": "UTT_BOUNDARY", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "UTT_BOUNDARY", + "clean_up_tokenization_spaces": false, + "eos_token": "UTT_BOUNDARY", + "extra_special_tokens": {}, + "model_max_length": 1000000000000000019884624838656, + "pad_token": "PAD", + "tokenizer_class": "GPT2Tokenizer", + "unk_token": "UNK" +} diff --git a/train_rank0.log b/train_rank0.log new file mode 100644 index 0000000..c7c0c48 --- /dev/null +++ b/train_rank0.log @@ -0,0 +1,2372 @@ +07/16/2026 10:07:38 - INFO - __main__ - [RANK 0] Distributed environment: DistributedType.NO +Num processes: 1 +Process index: 0 +Local process index: 0 +Device: cuda + +Mixed precision type: no + +07/16/2026 10:07:38 - INFO - __main__ - [RANK 0] Training args: +{'data_dir': 'data/phone/deu-eng/tokenized_sequential_interleaved', 'model_name_or_path': None, 'config_name': 'gpt_base_config.json', 'tokenizer_name': 'nikitastheo/phonelm-phone-deu-eng-tokenizer', 'per_device_train_batch_size': 32, 'per_device_eval_batch_size': 32, 'learning_rate': 0.0001, 'weight_decay': 0.01, 'max_steps': 47675, 'gradient_accumulation_steps': 1, 'lr_scheduler_type': , 'warmup_steps': 4767, 'output_dir': 'models/phonebabylm-phone-deu-eng-sequential_interleaved', 'seed': 42, 'model_type': None, 'block_size': None, 'push_to_hub': True, 'hub_model_id': 'nikitastheo/phonebabylm-phone-deu-eng-sequential_interleaved', 'hub_token': None, 'private': False, 'trust_remote_code': False, 'with_tracking': True, 'tracking_dir': None, 'report_to': 'wandb', 'restart_lr_at_switch': False, 'language_switch_epoch': 5, 'model_revision': 'main', 'config_overrides': None, 'use_slow_tokenizer': False, 'override_vocab_size': None, 'dtype': None, 'logging_steps': 10, 'cache_dir': None, 'eval_steps': 4767, 'max_grad_norm': 1.0, 'adam_epsilon': 1e-06, 'run_name': 'phonebabylm-phone-deu-eng-sequential_interleaved', 'max_train_samples': None, 'max_eval_samples': None} +07/16/2026 10:07:38 - INFO - transformers.configuration_utils - loading configuration file gpt_base_config.json +07/16/2026 10:07:38 - INFO - transformers.configuration_utils - Model config GPT2Config { + "activation_function": "gelu", + "architectures": [ + "GPT2LMHeadModel" + ], + "attn_pdrop": 0.1, + "bos_token_id": 50256, + "embd_pdrop": 0.1, + "eos_token_id": 50256, + "initializer_range": 0.02, + "layer_norm_epsilon": 1e-05, + "model_type": "gpt2", + "n_ctx": 512, + "n_embd": 768, + "n_head": 12, + "n_inner": 3072, + "n_layer": 12, + "n_positions": 512, + "reorder_and_upcast_attn": false, + "resid_pdrop": 0.1, + "scale_attn_by_inverse_layer_idx": false, + "scale_attn_weights": true, + "summary_activation": null, + "summary_first_dropout": 0.1, + "summary_proj_to_labels": true, + "summary_type": "cls_index", + "summary_use_proj": true, + "transformers_version": "4.57.6", + "use_cache": true, + "vocab_size": 1 +} + +07/16/2026 10:07:38 - INFO - transformers.tokenization_utils_base - loading file vocab.json from cache at /home/geofila/.cache/huggingface/hub/models--nikitastheo--phonelm-phone-deu-eng-tokenizer/snapshots/9bea9a4137e9f8a784365dd064e257186231952d/vocab.json +07/16/2026 10:07:38 - INFO - transformers.tokenization_utils_base - loading file merges.txt from cache at None +07/16/2026 10:07:38 - INFO - transformers.tokenization_utils_base - loading file tokenizer.json from cache at /home/geofila/.cache/huggingface/hub/models--nikitastheo--phonelm-phone-deu-eng-tokenizer/snapshots/9bea9a4137e9f8a784365dd064e257186231952d/tokenizer.json +07/16/2026 10:07:38 - INFO - transformers.tokenization_utils_base - loading file added_tokens.json from cache at None +07/16/2026 10:07:38 - INFO - transformers.tokenization_utils_base - loading file special_tokens_map.json from cache at /home/geofila/.cache/huggingface/hub/models--nikitastheo--phonelm-phone-deu-eng-tokenizer/snapshots/9bea9a4137e9f8a784365dd064e257186231952d/special_tokens_map.json +07/16/2026 10:07:38 - INFO - transformers.tokenization_utils_base - loading file tokenizer_config.json from cache at /home/geofila/.cache/huggingface/hub/models--nikitastheo--phonelm-phone-deu-eng-tokenizer/snapshots/9bea9a4137e9f8a784365dd064e257186231952d/tokenizer_config.json +07/16/2026 10:07:38 - INFO - transformers.tokenization_utils_base - loading file chat_template.jinja from cache at None +07/16/2026 10:07:40 - INFO - transformers.generation.configuration_utils - Generate config GenerationConfig { + "bos_token_id": 2, + "eos_token_id": 2, + "pad_token_id": 1 +} + +07/16/2026 10:07:41 - INFO - __main__ - [RANK 0] Model parameters: 85.50M total (85.50M trainable) +07/16/2026 10:07:52 - INFO - __main__ - [RANK 0] Sample 83810 of the training set: {'input_ids': tensor([ 3, 16, 18, 9, 16, 31, 11, 14, 16, 10, 27, 20, 27, 47, 26, 16, 2, 18, + 9, 24, 11, 29, 3, 18, 10, 27, 3, 28, 5, 14, 16, 24, 4, 5, 24, 4, + 5, 24, 4, 5, 16, 2, 17, 8, 3, 14, 16, 24, 4, 5, 24, 4, 5, 24, + 4, 5, 16, 2, 3, 27, 14, 10, 26, 3, 16, 12, 18, 6, 3, 16, 37, 25, + 5, 14, 16, 18, 29, 3, 16, 31, 27, 29, 16, 23, 6, 21, 26, 16, 21, 3, + 28, 5, 16, 2, 17, 8, 3, 14, 16, 24, 4, 5, 24, 4, 5, 24, 4, 5, + 16, 2, 3, 27, 14, 10, 26, 3, 16, 12, 18, 6, 3, 16, 19, 9, 16, 10, + 14, 22, 5, 9, 26, 9, 16, 21, 3, 28, 5, 16, 2, 3, 27, 14, 10, 26, + 3, 16, 12, 18, 6, 3, 16, 19, 9, 16, 19, 9, 16, 10, 14, 22, 5, 9, + 26, 29, 16, 21, 3, 28, 5, 16, 2, 3, 27, 14, 10, 26, 3, 16, 12, 18, + 6, 3, 16, 19, 9, 16, 10, 11, 6, 23, 26, 29, 16, 21, 3, 28, 5, 16, + 2, 3, 27, 14, 10, 26, 3, 16, 12, 18, 6, 3, 16, 20, 28, 5, 3, 26, + 29, 16, 21, 3, 28, 5, 16, 2, 3, 27, 14, 10, 26, 3, 16, 12, 18, 6, + 3, 16, 9, 4, 5, 16, 2, 17, 8, 3, 14, 16, 24, 4, 5, 24, 4, 5, + 24, 4, 5, 16, 2, 18, 9, 24, 11, 29, 3, 18, 10, 27, 3, 28, 5, 14, + 16, 24, 4, 5, 24, 4, 5, 24, 4, 5, 16, 2, 17, 8, 3, 14, 16, 24, + 4, 5, 24, 4, 5, 24, 4, 5, 16, 2, 3, 27, 14, 10, 26, 3, 16, 12, + 18, 6, 3, 16, 19, 9, 16, 10, 14, 22, 5, 9, 26, 29, 16, 21, 3, 28, + 5, 16, 2, 3, 27, 14, 10, 26, 3, 16, 12, 18, 6, 3, 16, 19, 9, 16, + 10, 11, 6, 23, 26, 29, 16, 21, 3, 28, 5, 16, 2, 3, 27, 14, 10, 26, + 3, 16, 12, 18, 6, 3, 16, 19, 9, 16, 20, 28, 5, 3, 26, 29, 16, 21, + 3, 28, 5, 16, 2, 3, 27, 14, 10, 26, 3, 16, 12, 18, 6, 3, 16, 19, + 9, 16, 23, 6, 21, 26, 29, 16, 21, 3, 28, 5, 16, 2, 18, 9, 24, 11, + 29, 3, 18, 10, 27, 3, 28, 5, 14, 16, 24, 4, 5, 24, 4, 5, 24, 4, + 5, 16, 2, 3, 27, 14, 10, 26, 3, 16, 12, 18, 6, 3, 16, 10, 30, 36, + 16, 17, 27, 5, 6, 16, 2, 3, 27, 14, 10, 26, 3, 16, 12, 18, 6, 3, + 16, 31, 39, 5, 16, 36, 14, 25, 5, 36, 29, 3, 16, 40, 11, 35, 3, 16, + 19, 9, 16, 10, 11, 6, 23, 26, 29, 16, 2, 3, 27, 14, 10, 26, 3, 16, + 12, 18, 6, 3, 16, 18, 15, 16])} t WORD_BOUNDARY ɪ n WORD_BOUNDARY d ɛ ɐ WORD_BOUNDARY ɡ a ʀ a ʒ ə WORD_BOUNDARY UTT_BOUNDARY ɪ n v ɛ s t ɪ ɡ a t o ː ɐ WORD_BOUNDARY v e ː v e ː v e ː WORD_BOUNDARY UTT_BOUNDARY m ɔ t ɐ WORD_BOUNDARY v e ː v e ː v e ː WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY h i ː ɐ WORD_BOUNDARY ɪ s t WORD_BOUNDARY d a s WORD_BOUNDARY b l aʊ ə WORD_BOUNDARY aʊ t o ː WORD_BOUNDARY UTT_BOUNDARY m ɔ t ɐ WORD_BOUNDARY v e ː v e ː v e ː WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY aɪ n WORD_BOUNDARY ɡ ɐ y ː n ə n WORD_BOUNDARY aʊ t o ː WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY aɪ n WORD_BOUNDARY aɪ n WORD_BOUNDARY ɡ ɐ y ː n ə s WORD_BOUNDARY aʊ t o ː WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY aɪ n WORD_BOUNDARY ɡ ɛ l b ə s WORD_BOUNDARY aʊ t o ː WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY ʀ o ː t ə s WORD_BOUNDARY aʊ t o ː WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY n e ː WORD_BOUNDARY UTT_BOUNDARY m ɔ t ɐ WORD_BOUNDARY v e ː v e ː v e ː WORD_BOUNDARY UTT_BOUNDARY ɪ n v ɛ s t ɪ ɡ a t o ː ɐ WORD_BOUNDARY v e ː v e ː v e ː WORD_BOUNDARY UTT_BOUNDARY m ɔ t ɐ WORD_BOUNDARY v e ː v e ː v e ː WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY aɪ n WORD_BOUNDARY ɡ ɐ y ː n ə s WORD_BOUNDARY aʊ t o ː WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY aɪ n WORD_BOUNDARY ɡ ɛ l b ə s WORD_BOUNDARY aʊ t o ː WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY aɪ n WORD_BOUNDARY ʀ o ː t ə s WORD_BOUNDARY aʊ t o ː WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY aɪ n WORD_BOUNDARY b l aʊ ə s WORD_BOUNDARY aʊ t o ː WORD_BOUNDARY UTT_BOUNDARY ɪ n v ɛ s t ɪ ɡ a t o ː ɐ WORD_BOUNDARY v e ː v e ː v e ː WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY ɡ ʊ k WORD_BOUNDARY m a ː l WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY d u ː WORD_BOUNDARY k ɐ i ː k s t WORD_BOUNDARY j ɛ ts t WORD_BOUNDARY aɪ n WORD_BOUNDARY ɡ ɛ l b ə s WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY ɪ ç WORD_BOUNDARY +07/16/2026 10:07:52 - INFO - __main__ - [RANK 0] Sample 14592 of the training set: {'input_ids': tensor([27, 29, 16, 18, 15, 16, 31, 27, 5, 16, 24, 27, 29, 16, 23, 21, 26, 9, + 16, 36, 27, 9, 16, 2, 18, 9, 24, 11, 29, 3, 18, 10, 27, 3, 28, 5, + 14, 16, 33, 28, 5, 16, 40, 11, 35, 3, 16, 31, 25, 5, 16, 23, 25, 5, + 9, 26, 16, 37, 28, 5, 6, 26, 9, 16, 2, 18, 9, 24, 11, 29, 3, 18, + 10, 27, 3, 28, 5, 14, 16, 31, 27, 9, 16, 36, 27, 9, 16, 17, 27, 9, + 16, 11, 29, 16, 27, 5, 23, 14, 16, 9, 18, 15, 3, 16, 31, 14, 30, 36, + 26, 9, 16, 24, 11, 9, 16, 31, 39, 5, 16, 31, 25, 5, 16, 23, 25, 5, + 9, 26, 16, 37, 28, 5, 6, 29, 3, 16, 24, 19, 29, 3, 16, 31, 39, 5, + 16, 31, 27, 29, 16, 2, 17, 4, 5, 31, 25, 5, 27, 5, 16, 17, 27, 32, + 26, 9, 16, 24, 25, 5, 14, 16, 19, 9, 16, 10, 14, 27, 17, 28, 5, 7, + 28, 5, 9, 16, 2, 3, 27, 14, 10, 26, 3, 16, 12, 18, 6, 3, 16, 7, + 11, 14, 10, 11, 29, 26, 9, 16, 17, 30, 29, 16, 18, 15, 16, 31, 27, 29, + 16, 4, 5, 23, 26, 9, 16, 33, 28, 5, 16, 31, 42, 15, 12, 13, 25, 5, + 6, 26, 9, 16, 2, 18, 9, 24, 11, 29, 3, 18, 10, 27, 3, 28, 5, 14, + 16, 40, 27, 5, 16, 31, 27, 9, 16, 36, 27, 9, 29, 3, 16, 31, 4, 5, + 16, 31, 25, 5, 14, 16, 31, 27, 9, 16, 24, 27, 29, 16, 21, 29, 33, 39, + 5, 32, 26, 9, 16, 35, 30, 17, 16, 31, 14, 30, 36, 26, 9, 16, 10, 11, + 6, 16, 2, 3, 27, 14, 10, 26, 3, 16, 12, 18, 6, 3, 16, 37, 26, 17, + 16, 2, 18, 9, 24, 11, 29, 3, 18, 10, 27, 3, 28, 5, 14, 16, 37, 26, + 17, 16, 2, 17, 4, 5, 31, 25, 5, 27, 5, 16, 31, 27, 29, 16, 3, 19, + 6, 16, 10, 26, 37, 43, 14, 3, 16, 31, 27, 5, 16, 9, 18, 15, 3, 16, + 37, 18, 9, 16, 2, 18, 9, 24, 11, 29, 3, 18, 10, 27, 3, 28, 5, 14, + 16, 37, 39, 5, 16, 18, 17, 14, 16, 17, 27, 5, 6, 16, 13, 27, 29, 25, + 5, 14, 3, 16, 31, 27, 29, 16, 10, 11, 6, 16, 2, 17, 4, 5, 31, 25, + 5, 27, 5, 16, 37, 11, 9, 12, 26, 9, 16, 13, 6, 30, 29, 16, 36, 6, + 19, 9, 16, 10, 18, 34, 16, 27, 6, 19, 9, 16, 18, 9, 16, 31, 25, 5, + 16, 2, 18, 9, 24, 11, 29, 3, 18, 10, 27, 3, 28, 5, 14, 16, 2, 3, + 27, 14, 10, 26, 3, 16, 12, 18, 6, 3, 16, 18, 9, 16, 31, 25, 5, 16, + 2, 17, 4, 5, 31, 25, 5, 27])} a s WORD_BOUNDARY ɪ ç WORD_BOUNDARY d a ː WORD_BOUNDARY v a s WORD_BOUNDARY b aʊ ə n WORD_BOUNDARY k a n WORD_BOUNDARY UTT_BOUNDARY ɪ n v ɛ s t ɪ ɡ a t o ː ɐ WORD_BOUNDARY z o ː WORD_BOUNDARY j ɛ ts t WORD_BOUNDARY d i ː WORD_BOUNDARY b i ː n ə WORD_BOUNDARY h o ː l ə n WORD_BOUNDARY UTT_BOUNDARY ɪ n v ɛ s t ɪ ɡ a t o ː ɐ WORD_BOUNDARY d a n WORD_BOUNDARY k a n WORD_BOUNDARY m a n WORD_BOUNDARY ɛ s WORD_BOUNDARY a ː b ɐ WORD_BOUNDARY n ɪ ç t WORD_BOUNDARY d ɐ ʊ k ə n WORD_BOUNDARY v ɛ n WORD_BOUNDARY d u ː WORD_BOUNDARY d i ː WORD_BOUNDARY b i ː n ə WORD_BOUNDARY h o ː l s t WORD_BOUNDARY v aɪ s t WORD_BOUNDARY d u ː WORD_BOUNDARY d a s WORD_BOUNDARY UTT_BOUNDARY m e ː d i ː a ː WORD_BOUNDARY m a x ə n WORD_BOUNDARY v i ː ɐ WORD_BOUNDARY aɪ n WORD_BOUNDARY ɡ ɐ a m o ː f o ː n WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY f ɛ ɐ ɡ ɛ s ə n WORD_BOUNDARY m ʊ s WORD_BOUNDARY ɪ ç WORD_BOUNDARY d a s WORD_BOUNDARY e ː b ə n WORD_BOUNDARY z o ː WORD_BOUNDARY d ʊɐ ç ʃ p i ː l ə n WORD_BOUNDARY UTT_BOUNDARY ɪ n v ɛ s t ɪ ɡ a t o ː ɐ WORD_BOUNDARY j a ː WORD_BOUNDARY d a n WORD_BOUNDARY k a n s t WORD_BOUNDARY d e ː WORD_BOUNDARY d i ː ɐ WORD_BOUNDARY d a n WORD_BOUNDARY v a s WORD_BOUNDARY aʊ s z u ː x ə n WORD_BOUNDARY ts ʊ m WORD_BOUNDARY d ɐ ʊ k ə n WORD_BOUNDARY ɡ ɛ l WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY h ə m WORD_BOUNDARY UTT_BOUNDARY ɪ n v ɛ s t ɪ ɡ a t o ː ɐ WORD_BOUNDARY h ə m WORD_BOUNDARY UTT_BOUNDARY m e ː d i ː a ː WORD_BOUNDARY d a s WORD_BOUNDARY t aɪ l WORD_BOUNDARY ɡ ə h œ ɐ t WORD_BOUNDARY d a ː WORD_BOUNDARY n ɪ ç t WORD_BOUNDARY h ɪ n WORD_BOUNDARY UTT_BOUNDARY ɪ n v ɛ s t ɪ ɡ a t o ː ɐ WORD_BOUNDARY h u ː WORD_BOUNDARY ɪ m ɐ WORD_BOUNDARY m a ː l WORD_BOUNDARY p a s i ː ɐ t WORD_BOUNDARY d a s WORD_BOUNDARY ɡ ɛ l WORD_BOUNDARY UTT_BOUNDARY m e ː d i ː a ː WORD_BOUNDARY h ɛ n ʃ ə n WORD_BOUNDARY p l ʊ s WORD_BOUNDARY k l aɪ n WORD_BOUNDARY ɡ ɪ ŋ WORD_BOUNDARY a l aɪ n WORD_BOUNDARY ɪ n WORD_BOUNDARY d i ː WORD_BOUNDARY UTT_BOUNDARY ɪ n v ɛ s t ɪ ɡ a t o ː ɐ WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY ɪ n WORD_BOUNDARY d i ː WORD_BOUNDARY UTT_BOUNDARY m e ː d i ː a +07/16/2026 10:07:52 - INFO - __main__ - [RANK 0] Sample 3278 of the training set: {'input_ids': tensor([31, 26, 9, 26, 9, 16, 10, 11, 6, 31, 26, 16, 19, 9, 26, 16, 27, 9, + 35, 27, 5, 6, 16, 31, 11, 14, 33, 11, 6, 23, 26, 9, 16, 12, 19, 9, + 26, 16, 24, 25, 5, 31, 14, 16, 31, 25, 5, 16, 11, 14, 16, 31, 4, 5, + 17, 16, 11, 14, 17, 8, 14, 31, 26, 3, 26, 9, 16, 18, 9, 16, 11, 3, + 26, 6, 23, 14, 41, 36, 16, 21, 29, 10, 26, 37, 11, 9, 31, 18, 15, 3, + 16, 37, 27, 3, 26, 16, 33, 28, 5, 16, 7, 11, 14, 31, 18, 15, 3, 26, + 3, 26, 16, 33, 18, 15, 16, 31, 11, 14, 16, 7, 11, 14, 31, 27, 32, 3, + 16, 10, 4, 5, 10, 26, 9, 16, 31, 25, 5, 16, 23, 19, 31, 26, 9, 16, + 18, 17, 14, 16, 17, 4, 5, 14, 16, 18, 17, 14, 16, 17, 4, 5, 14, 16, + 7, 11, 14, 24, 18, 36, 26, 6, 3, 26, 9, 16, 33, 25, 5, 16, 33, 18, + 15, 16, 18, 9, 16, 24, 25, 5, 31, 14, 12, 13, 14, 41, 15, 26, 16, 9, + 27, 5, 32, 16, 17, 4, 5, 20, 26, 20, 26, 9, 16, 24, 8, 32, 26, 9, + 16, 6, 22, 5, 10, 26, 9, 37, 27, 7, 3, 14, 16, 27, 13, 14, 4, 5, + 31, 26, 16, 10, 27, 5, 23, 26, 9, 16, 33, 25, 5, 16, 11, 9, 3, 6, + 18, 15, 16, 31, 27, 29, 16, 7, 11, 14, 23, 14, 11, 15, 26, 9, 16, 35, + 39, 5, 16, 30, 9, 3, 16, 6, 4, 5, 36, 3, 26, 9, 16, 19, 9, 16, + 7, 8, 6, 12, 3, 11, 9, 31, 18, 10, 26, 29, 16, 10, 26, 12, 3, 11, + 9, 31, 9, 18, 29, 16, 27, 13, 16, 21, 32, 16, 23, 26, 36, 27, 9, 3, + 26, 9, 16, 33, 25, 5, 16, 31, 25, 5, 16, 3, 27, 5, 3, 16, 17, 18, + 3, 16, 7, 28, 5, 14, 23, 26, 31, 27, 32, 3, 16, 23, 26, 10, 27, 34, + 26, 9, 16, 35, 39, 5, 16, 37, 27, 5, 23, 26, 9, 16, 35, 24, 27, 5, + 14, 16, 24, 11, 5, 20, 26, 16, 11, 29, 16, 25, 5, 9, 26, 9, 16, 6, + 25, 5, 23, 14, 16, 10, 26, 24, 4, 5, 33, 26, 9, 16, 33, 27, 5, 10, + 3, 26, 9, 16, 33, 25, 5, 16, 24, 11, 9, 16, 33, 25, 5, 16, 31, 27, + 29, 16, 10, 11, 6, 3, 16, 28, 5, 9, 26, 16, 17, 8, 14, 3, 16, 37, + 11, 3, 26, 9, 16, 27, 9, 16, 33, 18, 15, 16, 23, 14, 18, 34, 26, 9, + 16, 36, 43, 9, 26, 9, 16, 31, 8, 32, 16, 37, 11, 3, 26, 9, 16, 33, + 25, 5, 16, 33, 18, 15, 16, 17, 18, 3, 16, 24, 27, 7, 26, 9, 16, 7, + 11, 14, 33, 4, 5, 26, 9, 16])} d ə n ə n WORD_BOUNDARY ɡ ɛ l d ə WORD_BOUNDARY aɪ n ə WORD_BOUNDARY a n ts a ː l WORD_BOUNDARY d ɛ ɐ z ɛ l b ə n WORD_BOUNDARY ʃ aɪ n ə WORD_BOUNDARY v i ː d ɐ WORD_BOUNDARY d i ː WORD_BOUNDARY ɛ ɐ WORD_BOUNDARY d e ː m WORD_BOUNDARY ɛ ɐ m ɔ ɐ d ə t ə n WORD_BOUNDARY ɪ n WORD_BOUNDARY ɛ t ə l b ɐ ʏ k WORD_BOUNDARY aʊ s ɡ ə h ɛ n d ɪ ç t WORD_BOUNDARY h a t ə WORD_BOUNDARY z o ː WORD_BOUNDARY f ɛ ɐ d ɪ ç t ə t ə WORD_BOUNDARY z ɪ ç WORD_BOUNDARY d ɛ ɐ WORD_BOUNDARY f ɛ ɐ d a x t WORD_BOUNDARY ɡ e ː ɡ ə n WORD_BOUNDARY d i ː WORD_BOUNDARY b aɪ d ə n WORD_BOUNDARY ɪ m ɐ WORD_BOUNDARY m e ː ɐ WORD_BOUNDARY ɪ m ɐ WORD_BOUNDARY m e ː ɐ WORD_BOUNDARY f ɛ ɐ v ɪ k ə l t ə n WORD_BOUNDARY z i ː WORD_BOUNDARY z ɪ ç WORD_BOUNDARY ɪ n WORD_BOUNDARY v i ː d ɐ ʃ p ɐ ʏ ç ə WORD_BOUNDARY n a ː x WORD_BOUNDARY m e ː ʀ ə ʀ ə n WORD_BOUNDARY v ɔ x ə n WORD_BOUNDARY l y ː ɡ ə n h a f t ɐ WORD_BOUNDARY a p ɐ e ː d ə WORD_BOUNDARY ɡ a ː b ə n WORD_BOUNDARY z i ː WORD_BOUNDARY ɛ n t l ɪ ç WORD_BOUNDARY d a s WORD_BOUNDARY f ɛ ɐ b ɐ ɛ ç ə n WORD_BOUNDARY ts u ː WORD_BOUNDARY ʊ n t WORD_BOUNDARY l e ː k t ə n WORD_BOUNDARY aɪ n WORD_BOUNDARY f ɔ l ʃ t ɛ n d ɪ ɡ ə s WORD_BOUNDARY ɡ ə ʃ t ɛ n d n ɪ s WORD_BOUNDARY a p WORD_BOUNDARY aʊ x WORD_BOUNDARY b ə k a n t ə n WORD_BOUNDARY z i ː WORD_BOUNDARY d i ː WORD_BOUNDARY t a ː t WORD_BOUNDARY m ɪ t WORD_BOUNDARY f o ː ɐ b ə d a x t WORD_BOUNDARY b ə ɡ a ŋ ə n WORD_BOUNDARY ts u ː WORD_BOUNDARY h a ː b ə n WORD_BOUNDARY ts v a ː ɐ WORD_BOUNDARY v ɛ ː ʀ ə WORD_BOUNDARY ɛ s WORD_BOUNDARY i ː n ə n WORD_BOUNDARY l i ː b ɐ WORD_BOUNDARY ɡ ə v e ː z ə n WORD_BOUNDARY z a ː ɡ t ə n WORD_BOUNDARY z i ː WORD_BOUNDARY v ɛ n WORD_BOUNDARY z i ː WORD_BOUNDARY d a s WORD_BOUNDARY ɡ ɛ l t WORD_BOUNDARY o ː n ə WORD_BOUNDARY m ɔ ɐ t WORD_BOUNDARY h ɛ t ə n WORD_BOUNDARY a n WORD_BOUNDARY z ɪ ç WORD_BOUNDARY b ɐ ɪ ŋ ə n WORD_BOUNDARY k œ n ə n WORD_BOUNDARY d ɔ x WORD_BOUNDARY h ɛ t ə n WORD_BOUNDARY z i ː WORD_BOUNDARY z ɪ ç WORD_BOUNDARY m ɪ t WORD_BOUNDARY v a f ə n WORD_BOUNDARY f ɛ ɐ z e ː ə n WORD_BOUNDARY +07/16/2026 10:07:56 - INFO - __main__ - [RANK 0] ***** Running training ***** +07/16/2026 10:07:56 - INFO - __main__ - [RANK 0] Num examples = 101723 +07/16/2026 10:07:56 - INFO - __main__ - [RANK 0] Num Training Steps = 47675 +07/16/2026 10:07:56 - INFO - __main__ - [RANK 0] Instantaneous batch size per device = 32 +07/16/2026 10:07:56 - INFO - __main__ - [RANK 0] Total train batch size (w. parallel, distributed & accumulation) = 32 +07/16/2026 10:07:56 - INFO - __main__ - [RANK 0] Gradient Accumulation steps = 1 +07/16/2026 10:07:56 - INFO - __main__ - [RANK 0] Total optimization steps = 47675 +07/16/2026 10:07:56 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/16/2026 10:07:56 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/16/2026 10:08:00 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/16/2026 10:08:00 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/16/2026 10:08:00 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/16/2026 10:08:08 - INFO - __main__ - [RANK 0] Starting epoch 0... +07/16/2026 10:08:09 - WARNING - transformers.modeling_utils - `loss_type=None` was set in the config but it is unrecognized. Using the default loss: `ForCausalLMLoss`. +07/16/2026 10:25:29 - INFO - __main__ - [RANK 0] Starting epoch 1... +07/16/2026 10:34:15 - INFO - __main__ - [RANK 0] perplexity: 3.0095933552027385 eval_loss: 1.1018049716949463 accuracy: 0.6686 +07/16/2026 10:42:57 - INFO - __main__ - [RANK 0] Starting epoch 2... +07/16/2026 10:56:39 - WARNING - asyncio - socket.send() raised exception. +07/16/2026 10:56:39 - WARNING - asyncio - socket.send() raised exception. +07/16/2026 10:56:39 - WARNING - asyncio - socket.send() raised exception. +07/16/2026 10:56:39 - WARNING - asyncio - socket.send() raised exception. +07/16/2026 11:04:18 - INFO - __main__ - [RANK 0] Distributed environment: DistributedType.NO +Num processes: 1 +Process index: 0 +Local process index: 0 +Device: cuda + +Mixed precision type: no + +07/16/2026 11:04:18 - INFO - __main__ - [RANK 0] Training args: +{'data_dir': 'data/phone/deu-eng/tokenized_sequential_interleaved', 'model_name_or_path': None, 'config_name': 'gpt_base_config.json', 'tokenizer_name': 'nikitastheo/phonelm-phone-deu-eng-tokenizer', 'per_device_train_batch_size': 32, 'per_device_eval_batch_size': 32, 'learning_rate': 0.0001, 'weight_decay': 0.01, 'max_steps': 47675, 'gradient_accumulation_steps': 1, 'lr_scheduler_type': , 'warmup_steps': 4767, 'output_dir': 'models/phonebabylm-phone-deu-eng-sequential_interleaved', 'seed': 42, 'model_type': None, 'block_size': None, 'push_to_hub': True, 'hub_model_id': 'nikitastheo/phonebabylm-phone-deu-eng-sequential_interleaved', 'hub_token': None, 'private': False, 'trust_remote_code': False, 'with_tracking': True, 'tracking_dir': None, 'report_to': 'wandb', 'restart_lr_at_switch': False, 'language_switch_epoch': 5, 'model_revision': 'main', 'config_overrides': None, 'use_slow_tokenizer': False, 'override_vocab_size': None, 'dtype': None, 'logging_steps': 10, 'cache_dir': None, 'eval_steps': 4767, 'max_grad_norm': 1.0, 'adam_epsilon': 1e-06, 'run_name': 'phonebabylm-phone-deu-eng-sequential_interleaved', 'max_train_samples': None, 'max_eval_samples': None} +07/16/2026 11:04:18 - INFO - transformers.configuration_utils - loading configuration file gpt_base_config.json +07/16/2026 11:04:18 - INFO - transformers.configuration_utils - Model config GPT2Config { + "activation_function": "gelu", + "architectures": [ + "GPT2LMHeadModel" + ], + "attn_pdrop": 0.1, + "bos_token_id": 50256, + "embd_pdrop": 0.1, + "eos_token_id": 50256, + "initializer_range": 0.02, + "layer_norm_epsilon": 1e-05, + "model_type": "gpt2", + "n_ctx": 512, + "n_embd": 768, + "n_head": 12, + "n_inner": 3072, + "n_layer": 12, + "n_positions": 512, + "reorder_and_upcast_attn": false, + "resid_pdrop": 0.1, + "scale_attn_by_inverse_layer_idx": false, + "scale_attn_weights": true, + "summary_activation": null, + "summary_first_dropout": 0.1, + "summary_proj_to_labels": true, + "summary_type": "cls_index", + "summary_use_proj": true, + "transformers_version": "4.57.6", + "use_cache": true, + "vocab_size": 1 +} + +07/16/2026 11:04:19 - INFO - transformers.tokenization_utils_base - loading file vocab.json from cache at /home/geofila/.cache/huggingface/hub/models--nikitastheo--phonelm-phone-deu-eng-tokenizer/snapshots/9bea9a4137e9f8a784365dd064e257186231952d/vocab.json +07/16/2026 11:04:19 - INFO - transformers.tokenization_utils_base - loading file merges.txt from cache at None +07/16/2026 11:04:19 - INFO - transformers.tokenization_utils_base - loading file tokenizer.json from cache at /home/geofila/.cache/huggingface/hub/models--nikitastheo--phonelm-phone-deu-eng-tokenizer/snapshots/9bea9a4137e9f8a784365dd064e257186231952d/tokenizer.json +07/16/2026 11:04:19 - INFO - transformers.tokenization_utils_base - loading file added_tokens.json from cache at None +07/16/2026 11:04:19 - INFO - transformers.tokenization_utils_base - loading file special_tokens_map.json from cache at /home/geofila/.cache/huggingface/hub/models--nikitastheo--phonelm-phone-deu-eng-tokenizer/snapshots/9bea9a4137e9f8a784365dd064e257186231952d/special_tokens_map.json +07/16/2026 11:04:19 - INFO - transformers.tokenization_utils_base - loading file tokenizer_config.json from cache at /home/geofila/.cache/huggingface/hub/models--nikitastheo--phonelm-phone-deu-eng-tokenizer/snapshots/9bea9a4137e9f8a784365dd064e257186231952d/tokenizer_config.json +07/16/2026 11:04:19 - INFO - transformers.tokenization_utils_base - loading file chat_template.jinja from cache at None +07/16/2026 11:04:20 - INFO - transformers.generation.configuration_utils - Generate config GenerationConfig { + "bos_token_id": 2, + "eos_token_id": 2, + "pad_token_id": 1 +} + +07/16/2026 11:04:21 - INFO - __main__ - [RANK 0] Model parameters: 85.50M total (85.50M trainable) +07/16/2026 11:04:32 - INFO - __main__ - [RANK 0] Sample 83810 of the training set: {'input_ids': tensor([ 3, 16, 18, 9, 16, 31, 11, 14, 16, 10, 27, 20, 27, 47, 26, 16, 2, 18, + 9, 24, 11, 29, 3, 18, 10, 27, 3, 28, 5, 14, 16, 24, 4, 5, 24, 4, + 5, 24, 4, 5, 16, 2, 17, 8, 3, 14, 16, 24, 4, 5, 24, 4, 5, 24, + 4, 5, 16, 2, 3, 27, 14, 10, 26, 3, 16, 12, 18, 6, 3, 16, 37, 25, + 5, 14, 16, 18, 29, 3, 16, 31, 27, 29, 16, 23, 6, 21, 26, 16, 21, 3, + 28, 5, 16, 2, 17, 8, 3, 14, 16, 24, 4, 5, 24, 4, 5, 24, 4, 5, + 16, 2, 3, 27, 14, 10, 26, 3, 16, 12, 18, 6, 3, 16, 19, 9, 16, 10, + 14, 22, 5, 9, 26, 9, 16, 21, 3, 28, 5, 16, 2, 3, 27, 14, 10, 26, + 3, 16, 12, 18, 6, 3, 16, 19, 9, 16, 19, 9, 16, 10, 14, 22, 5, 9, + 26, 29, 16, 21, 3, 28, 5, 16, 2, 3, 27, 14, 10, 26, 3, 16, 12, 18, + 6, 3, 16, 19, 9, 16, 10, 11, 6, 23, 26, 29, 16, 21, 3, 28, 5, 16, + 2, 3, 27, 14, 10, 26, 3, 16, 12, 18, 6, 3, 16, 20, 28, 5, 3, 26, + 29, 16, 21, 3, 28, 5, 16, 2, 3, 27, 14, 10, 26, 3, 16, 12, 18, 6, + 3, 16, 9, 4, 5, 16, 2, 17, 8, 3, 14, 16, 24, 4, 5, 24, 4, 5, + 24, 4, 5, 16, 2, 18, 9, 24, 11, 29, 3, 18, 10, 27, 3, 28, 5, 14, + 16, 24, 4, 5, 24, 4, 5, 24, 4, 5, 16, 2, 17, 8, 3, 14, 16, 24, + 4, 5, 24, 4, 5, 24, 4, 5, 16, 2, 3, 27, 14, 10, 26, 3, 16, 12, + 18, 6, 3, 16, 19, 9, 16, 10, 14, 22, 5, 9, 26, 29, 16, 21, 3, 28, + 5, 16, 2, 3, 27, 14, 10, 26, 3, 16, 12, 18, 6, 3, 16, 19, 9, 16, + 10, 11, 6, 23, 26, 29, 16, 21, 3, 28, 5, 16, 2, 3, 27, 14, 10, 26, + 3, 16, 12, 18, 6, 3, 16, 19, 9, 16, 20, 28, 5, 3, 26, 29, 16, 21, + 3, 28, 5, 16, 2, 3, 27, 14, 10, 26, 3, 16, 12, 18, 6, 3, 16, 19, + 9, 16, 23, 6, 21, 26, 29, 16, 21, 3, 28, 5, 16, 2, 18, 9, 24, 11, + 29, 3, 18, 10, 27, 3, 28, 5, 14, 16, 24, 4, 5, 24, 4, 5, 24, 4, + 5, 16, 2, 3, 27, 14, 10, 26, 3, 16, 12, 18, 6, 3, 16, 10, 30, 36, + 16, 17, 27, 5, 6, 16, 2, 3, 27, 14, 10, 26, 3, 16, 12, 18, 6, 3, + 16, 31, 39, 5, 16, 36, 14, 25, 5, 36, 29, 3, 16, 40, 11, 35, 3, 16, + 19, 9, 16, 10, 11, 6, 23, 26, 29, 16, 2, 3, 27, 14, 10, 26, 3, 16, + 12, 18, 6, 3, 16, 18, 15, 16])} t WORD_BOUNDARY ɪ n WORD_BOUNDARY d ɛ ɐ WORD_BOUNDARY ɡ a ʀ a ʒ ə WORD_BOUNDARY UTT_BOUNDARY ɪ n v ɛ s t ɪ ɡ a t o ː ɐ WORD_BOUNDARY v e ː v e ː v e ː WORD_BOUNDARY UTT_BOUNDARY m ɔ t ɐ WORD_BOUNDARY v e ː v e ː v e ː WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY h i ː ɐ WORD_BOUNDARY ɪ s t WORD_BOUNDARY d a s WORD_BOUNDARY b l aʊ ə WORD_BOUNDARY aʊ t o ː WORD_BOUNDARY UTT_BOUNDARY m ɔ t ɐ WORD_BOUNDARY v e ː v e ː v e ː WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY aɪ n WORD_BOUNDARY ɡ ɐ y ː n ə n WORD_BOUNDARY aʊ t o ː WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY aɪ n WORD_BOUNDARY aɪ n WORD_BOUNDARY ɡ ɐ y ː n ə s WORD_BOUNDARY aʊ t o ː WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY aɪ n WORD_BOUNDARY ɡ ɛ l b ə s WORD_BOUNDARY aʊ t o ː WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY ʀ o ː t ə s WORD_BOUNDARY aʊ t o ː WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY n e ː WORD_BOUNDARY UTT_BOUNDARY m ɔ t ɐ WORD_BOUNDARY v e ː v e ː v e ː WORD_BOUNDARY UTT_BOUNDARY ɪ n v ɛ s t ɪ ɡ a t o ː ɐ WORD_BOUNDARY v e ː v e ː v e ː WORD_BOUNDARY UTT_BOUNDARY m ɔ t ɐ WORD_BOUNDARY v e ː v e ː v e ː WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY aɪ n WORD_BOUNDARY ɡ ɐ y ː n ə s WORD_BOUNDARY aʊ t o ː WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY aɪ n WORD_BOUNDARY ɡ ɛ l b ə s WORD_BOUNDARY aʊ t o ː WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY aɪ n WORD_BOUNDARY ʀ o ː t ə s WORD_BOUNDARY aʊ t o ː WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY aɪ n WORD_BOUNDARY b l aʊ ə s WORD_BOUNDARY aʊ t o ː WORD_BOUNDARY UTT_BOUNDARY ɪ n v ɛ s t ɪ ɡ a t o ː ɐ WORD_BOUNDARY v e ː v e ː v e ː WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY ɡ ʊ k WORD_BOUNDARY m a ː l WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY d u ː WORD_BOUNDARY k ɐ i ː k s t WORD_BOUNDARY j ɛ ts t WORD_BOUNDARY aɪ n WORD_BOUNDARY ɡ ɛ l b ə s WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY ɪ ç WORD_BOUNDARY +07/16/2026 11:04:32 - INFO - __main__ - [RANK 0] Sample 14592 of the training set: {'input_ids': tensor([27, 29, 16, 18, 15, 16, 31, 27, 5, 16, 24, 27, 29, 16, 23, 21, 26, 9, + 16, 36, 27, 9, 16, 2, 18, 9, 24, 11, 29, 3, 18, 10, 27, 3, 28, 5, + 14, 16, 33, 28, 5, 16, 40, 11, 35, 3, 16, 31, 25, 5, 16, 23, 25, 5, + 9, 26, 16, 37, 28, 5, 6, 26, 9, 16, 2, 18, 9, 24, 11, 29, 3, 18, + 10, 27, 3, 28, 5, 14, 16, 31, 27, 9, 16, 36, 27, 9, 16, 17, 27, 9, + 16, 11, 29, 16, 27, 5, 23, 14, 16, 9, 18, 15, 3, 16, 31, 14, 30, 36, + 26, 9, 16, 24, 11, 9, 16, 31, 39, 5, 16, 31, 25, 5, 16, 23, 25, 5, + 9, 26, 16, 37, 28, 5, 6, 29, 3, 16, 24, 19, 29, 3, 16, 31, 39, 5, + 16, 31, 27, 29, 16, 2, 17, 4, 5, 31, 25, 5, 27, 5, 16, 17, 27, 32, + 26, 9, 16, 24, 25, 5, 14, 16, 19, 9, 16, 10, 14, 27, 17, 28, 5, 7, + 28, 5, 9, 16, 2, 3, 27, 14, 10, 26, 3, 16, 12, 18, 6, 3, 16, 7, + 11, 14, 10, 11, 29, 26, 9, 16, 17, 30, 29, 16, 18, 15, 16, 31, 27, 29, + 16, 4, 5, 23, 26, 9, 16, 33, 28, 5, 16, 31, 42, 15, 12, 13, 25, 5, + 6, 26, 9, 16, 2, 18, 9, 24, 11, 29, 3, 18, 10, 27, 3, 28, 5, 14, + 16, 40, 27, 5, 16, 31, 27, 9, 16, 36, 27, 9, 29, 3, 16, 31, 4, 5, + 16, 31, 25, 5, 14, 16, 31, 27, 9, 16, 24, 27, 29, 16, 21, 29, 33, 39, + 5, 32, 26, 9, 16, 35, 30, 17, 16, 31, 14, 30, 36, 26, 9, 16, 10, 11, + 6, 16, 2, 3, 27, 14, 10, 26, 3, 16, 12, 18, 6, 3, 16, 37, 26, 17, + 16, 2, 18, 9, 24, 11, 29, 3, 18, 10, 27, 3, 28, 5, 14, 16, 37, 26, + 17, 16, 2, 17, 4, 5, 31, 25, 5, 27, 5, 16, 31, 27, 29, 16, 3, 19, + 6, 16, 10, 26, 37, 43, 14, 3, 16, 31, 27, 5, 16, 9, 18, 15, 3, 16, + 37, 18, 9, 16, 2, 18, 9, 24, 11, 29, 3, 18, 10, 27, 3, 28, 5, 14, + 16, 37, 39, 5, 16, 18, 17, 14, 16, 17, 27, 5, 6, 16, 13, 27, 29, 25, + 5, 14, 3, 16, 31, 27, 29, 16, 10, 11, 6, 16, 2, 17, 4, 5, 31, 25, + 5, 27, 5, 16, 37, 11, 9, 12, 26, 9, 16, 13, 6, 30, 29, 16, 36, 6, + 19, 9, 16, 10, 18, 34, 16, 27, 6, 19, 9, 16, 18, 9, 16, 31, 25, 5, + 16, 2, 18, 9, 24, 11, 29, 3, 18, 10, 27, 3, 28, 5, 14, 16, 2, 3, + 27, 14, 10, 26, 3, 16, 12, 18, 6, 3, 16, 18, 9, 16, 31, 25, 5, 16, + 2, 17, 4, 5, 31, 25, 5, 27])} a s WORD_BOUNDARY ɪ ç WORD_BOUNDARY d a ː WORD_BOUNDARY v a s WORD_BOUNDARY b aʊ ə n WORD_BOUNDARY k a n WORD_BOUNDARY UTT_BOUNDARY ɪ n v ɛ s t ɪ ɡ a t o ː ɐ WORD_BOUNDARY z o ː WORD_BOUNDARY j ɛ ts t WORD_BOUNDARY d i ː WORD_BOUNDARY b i ː n ə WORD_BOUNDARY h o ː l ə n WORD_BOUNDARY UTT_BOUNDARY ɪ n v ɛ s t ɪ ɡ a t o ː ɐ WORD_BOUNDARY d a n WORD_BOUNDARY k a n WORD_BOUNDARY m a n WORD_BOUNDARY ɛ s WORD_BOUNDARY a ː b ɐ WORD_BOUNDARY n ɪ ç t WORD_BOUNDARY d ɐ ʊ k ə n WORD_BOUNDARY v ɛ n WORD_BOUNDARY d u ː WORD_BOUNDARY d i ː WORD_BOUNDARY b i ː n ə WORD_BOUNDARY h o ː l s t WORD_BOUNDARY v aɪ s t WORD_BOUNDARY d u ː WORD_BOUNDARY d a s WORD_BOUNDARY UTT_BOUNDARY m e ː d i ː a ː WORD_BOUNDARY m a x ə n WORD_BOUNDARY v i ː ɐ WORD_BOUNDARY aɪ n WORD_BOUNDARY ɡ ɐ a m o ː f o ː n WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY f ɛ ɐ ɡ ɛ s ə n WORD_BOUNDARY m ʊ s WORD_BOUNDARY ɪ ç WORD_BOUNDARY d a s WORD_BOUNDARY e ː b ə n WORD_BOUNDARY z o ː WORD_BOUNDARY d ʊɐ ç ʃ p i ː l ə n WORD_BOUNDARY UTT_BOUNDARY ɪ n v ɛ s t ɪ ɡ a t o ː ɐ WORD_BOUNDARY j a ː WORD_BOUNDARY d a n WORD_BOUNDARY k a n s t WORD_BOUNDARY d e ː WORD_BOUNDARY d i ː ɐ WORD_BOUNDARY d a n WORD_BOUNDARY v a s WORD_BOUNDARY aʊ s z u ː x ə n WORD_BOUNDARY ts ʊ m WORD_BOUNDARY d ɐ ʊ k ə n WORD_BOUNDARY ɡ ɛ l WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY h ə m WORD_BOUNDARY UTT_BOUNDARY ɪ n v ɛ s t ɪ ɡ a t o ː ɐ WORD_BOUNDARY h ə m WORD_BOUNDARY UTT_BOUNDARY m e ː d i ː a ː WORD_BOUNDARY d a s WORD_BOUNDARY t aɪ l WORD_BOUNDARY ɡ ə h œ ɐ t WORD_BOUNDARY d a ː WORD_BOUNDARY n ɪ ç t WORD_BOUNDARY h ɪ n WORD_BOUNDARY UTT_BOUNDARY ɪ n v ɛ s t ɪ ɡ a t o ː ɐ WORD_BOUNDARY h u ː WORD_BOUNDARY ɪ m ɐ WORD_BOUNDARY m a ː l WORD_BOUNDARY p a s i ː ɐ t WORD_BOUNDARY d a s WORD_BOUNDARY ɡ ɛ l WORD_BOUNDARY UTT_BOUNDARY m e ː d i ː a ː WORD_BOUNDARY h ɛ n ʃ ə n WORD_BOUNDARY p l ʊ s WORD_BOUNDARY k l aɪ n WORD_BOUNDARY ɡ ɪ ŋ WORD_BOUNDARY a l aɪ n WORD_BOUNDARY ɪ n WORD_BOUNDARY d i ː WORD_BOUNDARY UTT_BOUNDARY ɪ n v ɛ s t ɪ ɡ a t o ː ɐ WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY ɪ n WORD_BOUNDARY d i ː WORD_BOUNDARY UTT_BOUNDARY m e ː d i ː a +07/16/2026 11:04:32 - INFO - __main__ - [RANK 0] Sample 3278 of the training set: {'input_ids': tensor([31, 26, 9, 26, 9, 16, 10, 11, 6, 31, 26, 16, 19, 9, 26, 16, 27, 9, + 35, 27, 5, 6, 16, 31, 11, 14, 33, 11, 6, 23, 26, 9, 16, 12, 19, 9, + 26, 16, 24, 25, 5, 31, 14, 16, 31, 25, 5, 16, 11, 14, 16, 31, 4, 5, + 17, 16, 11, 14, 17, 8, 14, 31, 26, 3, 26, 9, 16, 18, 9, 16, 11, 3, + 26, 6, 23, 14, 41, 36, 16, 21, 29, 10, 26, 37, 11, 9, 31, 18, 15, 3, + 16, 37, 27, 3, 26, 16, 33, 28, 5, 16, 7, 11, 14, 31, 18, 15, 3, 26, + 3, 26, 16, 33, 18, 15, 16, 31, 11, 14, 16, 7, 11, 14, 31, 27, 32, 3, + 16, 10, 4, 5, 10, 26, 9, 16, 31, 25, 5, 16, 23, 19, 31, 26, 9, 16, + 18, 17, 14, 16, 17, 4, 5, 14, 16, 18, 17, 14, 16, 17, 4, 5, 14, 16, + 7, 11, 14, 24, 18, 36, 26, 6, 3, 26, 9, 16, 33, 25, 5, 16, 33, 18, + 15, 16, 18, 9, 16, 24, 25, 5, 31, 14, 12, 13, 14, 41, 15, 26, 16, 9, + 27, 5, 32, 16, 17, 4, 5, 20, 26, 20, 26, 9, 16, 24, 8, 32, 26, 9, + 16, 6, 22, 5, 10, 26, 9, 37, 27, 7, 3, 14, 16, 27, 13, 14, 4, 5, + 31, 26, 16, 10, 27, 5, 23, 26, 9, 16, 33, 25, 5, 16, 11, 9, 3, 6, + 18, 15, 16, 31, 27, 29, 16, 7, 11, 14, 23, 14, 11, 15, 26, 9, 16, 35, + 39, 5, 16, 30, 9, 3, 16, 6, 4, 5, 36, 3, 26, 9, 16, 19, 9, 16, + 7, 8, 6, 12, 3, 11, 9, 31, 18, 10, 26, 29, 16, 10, 26, 12, 3, 11, + 9, 31, 9, 18, 29, 16, 27, 13, 16, 21, 32, 16, 23, 26, 36, 27, 9, 3, + 26, 9, 16, 33, 25, 5, 16, 31, 25, 5, 16, 3, 27, 5, 3, 16, 17, 18, + 3, 16, 7, 28, 5, 14, 23, 26, 31, 27, 32, 3, 16, 23, 26, 10, 27, 34, + 26, 9, 16, 35, 39, 5, 16, 37, 27, 5, 23, 26, 9, 16, 35, 24, 27, 5, + 14, 16, 24, 11, 5, 20, 26, 16, 11, 29, 16, 25, 5, 9, 26, 9, 16, 6, + 25, 5, 23, 14, 16, 10, 26, 24, 4, 5, 33, 26, 9, 16, 33, 27, 5, 10, + 3, 26, 9, 16, 33, 25, 5, 16, 24, 11, 9, 16, 33, 25, 5, 16, 31, 27, + 29, 16, 10, 11, 6, 3, 16, 28, 5, 9, 26, 16, 17, 8, 14, 3, 16, 37, + 11, 3, 26, 9, 16, 27, 9, 16, 33, 18, 15, 16, 23, 14, 18, 34, 26, 9, + 16, 36, 43, 9, 26, 9, 16, 31, 8, 32, 16, 37, 11, 3, 26, 9, 16, 33, + 25, 5, 16, 33, 18, 15, 16, 17, 18, 3, 16, 24, 27, 7, 26, 9, 16, 7, + 11, 14, 33, 4, 5, 26, 9, 16])} d ə n ə n WORD_BOUNDARY ɡ ɛ l d ə WORD_BOUNDARY aɪ n ə WORD_BOUNDARY a n ts a ː l WORD_BOUNDARY d ɛ ɐ z ɛ l b ə n WORD_BOUNDARY ʃ aɪ n ə WORD_BOUNDARY v i ː d ɐ WORD_BOUNDARY d i ː WORD_BOUNDARY ɛ ɐ WORD_BOUNDARY d e ː m WORD_BOUNDARY ɛ ɐ m ɔ ɐ d ə t ə n WORD_BOUNDARY ɪ n WORD_BOUNDARY ɛ t ə l b ɐ ʏ k WORD_BOUNDARY aʊ s ɡ ə h ɛ n d ɪ ç t WORD_BOUNDARY h a t ə WORD_BOUNDARY z o ː WORD_BOUNDARY f ɛ ɐ d ɪ ç t ə t ə WORD_BOUNDARY z ɪ ç WORD_BOUNDARY d ɛ ɐ WORD_BOUNDARY f ɛ ɐ d a x t WORD_BOUNDARY ɡ e ː ɡ ə n WORD_BOUNDARY d i ː WORD_BOUNDARY b aɪ d ə n WORD_BOUNDARY ɪ m ɐ WORD_BOUNDARY m e ː ɐ WORD_BOUNDARY ɪ m ɐ WORD_BOUNDARY m e ː ɐ WORD_BOUNDARY f ɛ ɐ v ɪ k ə l t ə n WORD_BOUNDARY z i ː WORD_BOUNDARY z ɪ ç WORD_BOUNDARY ɪ n WORD_BOUNDARY v i ː d ɐ ʃ p ɐ ʏ ç ə WORD_BOUNDARY n a ː x WORD_BOUNDARY m e ː ʀ ə ʀ ə n WORD_BOUNDARY v ɔ x ə n WORD_BOUNDARY l y ː ɡ ə n h a f t ɐ WORD_BOUNDARY a p ɐ e ː d ə WORD_BOUNDARY ɡ a ː b ə n WORD_BOUNDARY z i ː WORD_BOUNDARY ɛ n t l ɪ ç WORD_BOUNDARY d a s WORD_BOUNDARY f ɛ ɐ b ɐ ɛ ç ə n WORD_BOUNDARY ts u ː WORD_BOUNDARY ʊ n t WORD_BOUNDARY l e ː k t ə n WORD_BOUNDARY aɪ n WORD_BOUNDARY f ɔ l ʃ t ɛ n d ɪ ɡ ə s WORD_BOUNDARY ɡ ə ʃ t ɛ n d n ɪ s WORD_BOUNDARY a p WORD_BOUNDARY aʊ x WORD_BOUNDARY b ə k a n t ə n WORD_BOUNDARY z i ː WORD_BOUNDARY d i ː WORD_BOUNDARY t a ː t WORD_BOUNDARY m ɪ t WORD_BOUNDARY f o ː ɐ b ə d a x t WORD_BOUNDARY b ə ɡ a ŋ ə n WORD_BOUNDARY ts u ː WORD_BOUNDARY h a ː b ə n WORD_BOUNDARY ts v a ː ɐ WORD_BOUNDARY v ɛ ː ʀ ə WORD_BOUNDARY ɛ s WORD_BOUNDARY i ː n ə n WORD_BOUNDARY l i ː b ɐ WORD_BOUNDARY ɡ ə v e ː z ə n WORD_BOUNDARY z a ː ɡ t ə n WORD_BOUNDARY z i ː WORD_BOUNDARY v ɛ n WORD_BOUNDARY z i ː WORD_BOUNDARY d a s WORD_BOUNDARY ɡ ɛ l t WORD_BOUNDARY o ː n ə WORD_BOUNDARY m ɔ ɐ t WORD_BOUNDARY h ɛ t ə n WORD_BOUNDARY a n WORD_BOUNDARY z ɪ ç WORD_BOUNDARY b ɐ ɪ ŋ ə n WORD_BOUNDARY k œ n ə n WORD_BOUNDARY d ɔ x WORD_BOUNDARY h ɛ t ə n WORD_BOUNDARY z i ː WORD_BOUNDARY z ɪ ç WORD_BOUNDARY m ɪ t WORD_BOUNDARY v a f ə n WORD_BOUNDARY f ɛ ɐ z e ː ə n WORD_BOUNDARY +07/16/2026 11:04:35 - INFO - __main__ - [RANK 0] ***** Running training ***** +07/16/2026 11:04:35 - INFO - __main__ - [RANK 0] Num examples = 101723 +07/16/2026 11:04:35 - INFO - __main__ - [RANK 0] Num Training Steps = 47675 +07/16/2026 11:04:35 - INFO - __main__ - [RANK 0] Instantaneous batch size per device = 32 +07/16/2026 11:04:35 - INFO - __main__ - [RANK 0] Total train batch size (w. parallel, distributed & accumulation) = 32 +07/16/2026 11:04:35 - INFO - __main__ - [RANK 0] Gradient Accumulation steps = 1 +07/16/2026 11:04:35 - INFO - __main__ - [RANK 0] Total optimization steps = 47675 +07/16/2026 11:04:35 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/16/2026 11:04:35 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/16/2026 11:04:38 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/16/2026 11:04:38 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/16/2026 11:04:38 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/16/2026 11:04:43 - INFO - __main__ - [RANK 0] Starting epoch 0... +07/16/2026 11:04:43 - WARNING - transformers.modeling_utils - `loss_type=None` was set in the config but it is unrecognized. Using the default loss: `ForCausalLMLoss`. +07/16/2026 11:22:04 - INFO - __main__ - [RANK 0] Starting epoch 1... +07/16/2026 11:30:49 - INFO - __main__ - [RANK 0] perplexity: 3.0095933552027385 eval_loss: 1.1018049716949463 accuracy: 0.6686 +07/16/2026 11:39:32 - INFO - __main__ - [RANK 0] Starting epoch 2... +07/16/2026 11:57:00 - INFO - __main__ - [RANK 0] perplexity: 2.659254185213257 eval_loss: 0.9780457019805908 accuracy: 0.7038 +07/16/2026 11:57:00 - INFO - __main__ - [RANK 0] Starting epoch 3... +07/16/2026 12:14:25 - INFO - __main__ - [RANK 0] Starting epoch 4... +07/16/2026 12:23:10 - INFO - __main__ - [RANK 0] perplexity: 2.5357842325294726 eval_loss: 0.9305029511451721 accuracy: 0.7177 +07/16/2026 12:31:54 - INFO - __main__ - [RANK 0] Starting epoch 5... +07/16/2026 12:31:54 - INFO - __main__ - [RANK 0] Switching language at epoch 5 (step 15895) +07/16/2026 12:32:07 - INFO - __main__ - [RANK 0] Sample 194393 of the training set: {'input_ids': tensor([53, 45, 33, 16, 13, 26, 33, 11, 29, 3, 16, 23, 19, 16, 45, 9, 16, 68, + 24, 26, 55, 17, 66, 29, 3, 26, 55, 55, 18, 34, 16, 29, 11, 9, 29, 16, + 45, 24, 16, 31, 39, 5, 3, 25, 16, 3, 26, 16, 37, 69, 5, 16, 7, 67, + 56, 26, 55, 55, 16, 66, 3, 16, 6, 66, 29, 3, 16, 53, 11, 9, 16, 12, + 25, 5, 16, 53, 45, 33, 16, 9, 18, 55, 6, 25, 16, 29, 11, 24, 26, 9, + 3, 25, 5, 9, 16, 23, 45, 3, 16, 18, 9, 16, 6, 30, 36, 29, 16, 66, + 9, 31, 16, 17, 19, 9, 31, 16, 68, 6, 31, 26, 55, 16, 56, 45, 9, 16, + 37, 69, 5, 16, 40, 18, 55, 33, 16, 12, 25, 5, 16, 58, 55, 11, 3, 26, + 9, 31, 16, 3, 26, 16, 29, 11, 3, 16, 7, 8, 55, 58, 16, 53, 18, 56, + 21, 3, 16, 45, 29, 18, 29, 3, 26, 9, 29, 16, 18, 7, 16, 37, 69, 5, + 55, 16, 66, 9, 3, 16, 55, 18, 7, 40, 39, 5, 33, 31, 16, 37, 69, 5, + 55, 16, 45, 29, 11, 9, 3, 16, 66, 9, 31, 16, 37, 11, 6, 13, 16, 37, + 66, 24, 18, 34, 16, 9, 68, 16, 8, 6, 3, 69, 5, 9, 26, 3, 18, 24, + 16, 56, 26, 16, 13, 30, 55, 16, 6, 50, 31, 25, 16, 40, 25, 5, 6, 31, + 18, 31, 16, 68, 9, 6, 25, 16, 29, 3, 18, 13, 40, 30, 6, 50, 3, 18, + 34, 16, 56, 66, 3, 16, 10, 66, 23, 55, 25, 5, 6, 33, 16, 68, 6, 31, + 16, 9, 69, 5, 29, 16, 12, 30, 31, 16, 45, 36, 45, 17, 13, 26, 9, 25, + 16, 37, 69, 5, 16, 7, 8, 55, 16, 29, 45, 17, 16, 17, 45, 9, 58, 29, + 16, 56, 50, 16, 24, 50, 9, 6, 25, 16, 3, 55, 19, 31, 16, 3, 26, 16, + 10, 11, 3, 16, 13, 26, 55, 17, 18, 12, 26, 9, 16, 7, 55, 45, 17, 16, + 29, 26, 9, 3, 16, 13, 25, 5, 3, 26, 55, 33, 23, 69, 5, 10, 16, 7, + 26, 55, 56, 26, 16, 10, 69, 5, 6, 16, 3, 26, 16, 48, 62, 9, 16, 37, + 69, 5, 16, 7, 67, 56, 26, 55, 55, 16, 18, 9, 56, 26, 16, 36, 50, 29, + 16, 45, 24, 16, 8, 55, 31, 18, 9, 11, 55, 25, 16, 36, 55, 18, 17, 18, + 9, 26, 6, 33, 16, 9, 68, 16, 31, 18, 7, 18, 36, 45, 6, 3, 25, 16, + 53, 45, 33, 16, 40, 39, 5, 47, 39, 5, 26, 6, 25, 16, 17, 50, 31, 16, + 18, 3, 16, 53, 45, 33, 16, 36, 6, 18, 55, 16, 56, 66, 3, 16, 66, 33, + 16, 37, 66, 13, 26, 9, 33, 16, 29, 68, 16, 8, 7, 26, 9, 16, 18, 9, + 16, 55, 45, 12, 26, 16, 56, 26])} w ʌ z WORD_BOUNDARY p ə z ɛ s t WORD_BOUNDARY b aɪ WORD_BOUNDARY ʌ n WORD_BOUNDARY oʊ v ə ɹ m æ s t ə ɹ ɹ ɪ ŋ WORD_BOUNDARY s ɛ n s WORD_BOUNDARY ʌ v WORD_BOUNDARY d u ː t i WORD_BOUNDARY t ə WORD_BOUNDARY h ɜ ː WORD_BOUNDARY f ɑ ð ə ɹ ɹ WORD_BOUNDARY æ t WORD_BOUNDARY l æ s t WORD_BOUNDARY w ɛ n WORD_BOUNDARY ʃ i ː WORD_BOUNDARY w ʌ z WORD_BOUNDARY n ɪ ɹ l i WORD_BOUNDARY s ɛ v ə n t i ː n WORD_BOUNDARY b ʌ t WORD_BOUNDARY ɪ n WORD_BOUNDARY l ʊ k s WORD_BOUNDARY æ n d WORD_BOUNDARY m aɪ n d WORD_BOUNDARY oʊ l d ə ɹ WORD_BOUNDARY ð ʌ n WORD_BOUNDARY h ɜ ː WORD_BOUNDARY j ɪ ɹ z WORD_BOUNDARY ʃ i ː WORD_BOUNDARY θ ɹ ɛ t ə n d WORD_BOUNDARY t ə WORD_BOUNDARY s ɛ t WORD_BOUNDARY f ɔ ɹ θ WORD_BOUNDARY w ɪ ð aʊ t WORD_BOUNDARY ʌ s ɪ s t ə n s WORD_BOUNDARY ɪ f WORD_BOUNDARY h ɜ ː ɹ WORD_BOUNDARY æ n t WORD_BOUNDARY ɹ ɪ f j u ː z d WORD_BOUNDARY h ɜ ː ɹ WORD_BOUNDARY ʌ s ɛ n t WORD_BOUNDARY æ n d WORD_BOUNDARY h ɛ l p WORD_BOUNDARY h æ v ɪ ŋ WORD_BOUNDARY n oʊ WORD_BOUNDARY ɔ l t ɜ ː n ə t ɪ v WORD_BOUNDARY ð ə WORD_BOUNDARY p ʊ ɹ WORD_BOUNDARY l eɪ d i WORD_BOUNDARY j i ː l d ɪ d WORD_BOUNDARY oʊ n l i WORD_BOUNDARY s t ɪ p j ʊ l eɪ t ɪ ŋ WORD_BOUNDARY ð æ t WORD_BOUNDARY ɡ æ b ɹ i ː l z WORD_BOUNDARY oʊ l d WORD_BOUNDARY n ɜ ː s WORD_BOUNDARY ʃ ʊ d WORD_BOUNDARY ʌ k ʌ m p ə n i WORD_BOUNDARY h ɜ ː WORD_BOUNDARY f ɔ ɹ WORD_BOUNDARY s ʌ m WORD_BOUNDARY m ʌ n θ s WORD_BOUNDARY ð eɪ WORD_BOUNDARY v eɪ n l i WORD_BOUNDARY t ɹ aɪ d WORD_BOUNDARY t ə WORD_BOUNDARY ɡ ɛ t WORD_BOUNDARY p ə ɹ m ɪ ʃ ə n WORD_BOUNDARY f ɹ ʌ m WORD_BOUNDARY s ə n t WORD_BOUNDARY p i ː t ə ɹ z b ɜ ː ɡ WORD_BOUNDARY f ə ɹ ð ə WORD_BOUNDARY ɡ ɜ ː l WORD_BOUNDARY t ə WORD_BOUNDARY d̠ʒ ɔɪ n WORD_BOUNDARY h ɜ ː WORD_BOUNDARY f ɑ ð ə ɹ ɹ WORD_BOUNDARY ɪ n ð ə WORD_BOUNDARY k eɪ s WORD_BOUNDARY ʌ v WORD_BOUNDARY ɔ ɹ d ɪ n ɛ ɹ i WORD_BOUNDARY k ɹ ɪ m ɪ n ə l z WORD_BOUNDARY n oʊ WORD_BOUNDARY d ɪ f ɪ k ʌ l t i WORD_BOUNDARY w ʌ z WORD_BOUNDARY j u ː ʒ u ː ə l i WORD_BOUNDARY m eɪ d WORD_BOUNDARY ɪ t WORD_BOUNDARY w ʌ z WORD_BOUNDARY k l ɪ ɹ WORD_BOUNDARY ð æ t WORD_BOUNDARY æ z WORD_BOUNDARY h æ p ə n z WORD_BOUNDARY s oʊ WORD_BOUNDARY ɔ f ə n WORD_BOUNDARY ɪ n WORD_BOUNDARY ɹ ʌ ʃ ə WORD_BOUNDARY ð ə +07/16/2026 12:32:07 - INFO - __main__ - [RANK 0] Sample 72097 of the training set: {'input_ids': tensor([ 2, 23, 66, 36, 16, 18, 9, 56, 26, 16, 17, 18, 31, 26, 6, 16, 53, 11, + 29, 3, 16, 11, 31, 53, 18, 9, 16, 66, 9, 31, 16, 29, 11, 6, 25, 5, + 9, 26, 16, 37, 45, 31, 23, 18, 9, 16, 17, 21, 9, 3, 18, 9, 33, 16, + 26, 55, 55, 19, 33, 18, 34, 16, 7, 55, 45, 17, 56, 26, 16, 13, 6, 50, + 9, 16, 66, 3, 16, 8, 6, 16, 13, 62, 9, 3, 29, 16, 23, 18, 40, 8, + 9, 31, 16, 10, 55, 11, 9, 18, 46, 16, 56, 50, 16, 53, 69, 5, 16, 29, + 18, 17, 13, 6, 25, 16, 3, 39, 5, 16, 45, 34, 36, 26, 9, 29, 18, 31, + 26, 55, 31, 16, 7, 55, 66, 10, 17, 26, 9, 3, 29, 16, 45, 24, 16, 7, + 8, 55, 26, 9, 16, 29, 45, 23, 29, 3, 26, 9, 29, 16, 2, 56, 26, 16, + 13, 66, 29, 13, 8, 55, 3, 16, 31, 18, 31, 9, 67, 3, 16, 29, 25, 5, + 17, 16, 3, 26, 16, 10, 11, 3, 16, 56, 11, 17, 16, 11, 9, 18, 58, 18, + 34, 16, 53, 19, 6, 16, 23, 25, 5, 18, 34, 16, 53, 8, 36, 3, 16, 26, + 13, 67, 9, 16, 23, 19, 16, 56, 26, 16, 37, 8, 3, 25, 16, 3, 25, 5, + 31, 55, 18, 34, 36, 26, 55, 33, 16, 56, 50, 16, 36, 30, 31, 16, 9, 67, + 3, 16, 36, 6, 50, 17, 16, 56, 26, 16, 13, 55, 26, 3, 11, 36, 12, 26, + 9, 16, 45, 24, 56, 18, 16, 45, 17, 11, 55, 18, 36, 26, 9, 16, 7, 6, + 66, 10, 16, 23, 18, 36, 45, 33, 16, 56, 50, 16, 31, 18, 31, 9, 3, 16, + 29, 25, 5, 16, 56, 26, 16, 29, 3, 67, 55, 25, 16, 23, 66, 9, 26, 55, + 55, 16, 66, 7, 3, 26, 55, 16, 6, 25, 5, 24, 18, 34, 16, 9, 39, 5, + 16, 40, 8, 55, 36, 16, 11, 36, 29, 11, 13, 3, 16, 18, 9, 16, 7, 55, + 45, 9, 3, 16, 26, 24, 26, 16, 7, 50, 36, 16, 8, 36, 12, 26, 9, 16, + 29, 50, 6, 16, 26, 55, 55, 50, 9, 48, 31, 16, 18, 29, 13, 11, 12, 26, + 6, 25, 16, 7, 8, 55, 16, 3, 30, 55, 55, 18, 29, 3, 29, 16, 2, 23, + 19, 16, 56, 26, 16, 3, 19, 17, 16, 56, 50, 16, 7, 21, 9, 31, 16, 56, + 11, 17, 29, 11, 6, 24, 33, 16, 18, 9, 16, 56, 66, 3, 16, 24, 66, 29, + 3, 16, 23, 50, 36, 45, 24, 26, 9, 16, 9, 68, 9, 16, 66, 33, 16, 18, + 9, 31, 25, 26, 16, 56, 50, 16, 53, 69, 5, 16, 23, 18, 9, 45, 17, 31, + 16, 66, 9, 31, 16, 29, 26, 23, 17, 18, 29, 18, 24, 16, 66, 9, 31, 16, + 37, 66, 31, 16, 29, 11, 3, 26])} UTT_BOUNDARY b æ k WORD_BOUNDARY ɪ n ð ə WORD_BOUNDARY m ɪ d ə l WORD_BOUNDARY w ɛ s t WORD_BOUNDARY ɛ d w ɪ n WORD_BOUNDARY æ n d WORD_BOUNDARY s ɛ l i ː n ə WORD_BOUNDARY h ʌ d b ɪ n WORD_BOUNDARY m aʊ n t ɪ n z WORD_BOUNDARY ə ɹ ɹ aɪ z ɪ ŋ WORD_BOUNDARY f ɹ ʌ m ð ə WORD_BOUNDARY p l eɪ n WORD_BOUNDARY æ t WORD_BOUNDARY ɔ l WORD_BOUNDARY p ɔɪ n t s WORD_BOUNDARY b ɪ j ɔ n d WORD_BOUNDARY ɡ ɹ ɛ n ɪ t̠ʃ WORD_BOUNDARY ð eɪ WORD_BOUNDARY w ɜ ː WORD_BOUNDARY s ɪ m p l i WORD_BOUNDARY t u ː WORD_BOUNDARY ʌ ŋ k ə n s ɪ d ə ɹ d WORD_BOUNDARY f ɹ æ ɡ m ə n t s WORD_BOUNDARY ʌ v WORD_BOUNDARY f ɔ ɹ ə n WORD_BOUNDARY s ʌ b s t ə n s WORD_BOUNDARY UTT_BOUNDARY ð ə WORD_BOUNDARY p æ s p ɔ ɹ t WORD_BOUNDARY d ɪ d n ɑ t WORD_BOUNDARY s i ː m WORD_BOUNDARY t ə WORD_BOUNDARY ɡ ɛ t WORD_BOUNDARY ð ɛ m WORD_BOUNDARY ɛ n ɪ θ ɪ ŋ WORD_BOUNDARY w aɪ l WORD_BOUNDARY b i ː ɪ ŋ WORD_BOUNDARY w ɔ k t WORD_BOUNDARY ə p ɑ n WORD_BOUNDARY b aɪ WORD_BOUNDARY ð ə WORD_BOUNDARY h ɔ t i WORD_BOUNDARY t i ː d ɹ ɪ ŋ k ə ɹ z WORD_BOUNDARY ð eɪ WORD_BOUNDARY k ʊ d WORD_BOUNDARY n ɑ t WORD_BOUNDARY k l eɪ m WORD_BOUNDARY ð ə WORD_BOUNDARY p ɹ ə t ɛ k ʃ ə n WORD_BOUNDARY ʌ v ð ɪ WORD_BOUNDARY ʌ m ɛ ɹ ɪ k ə n WORD_BOUNDARY f l æ ɡ WORD_BOUNDARY b ɪ k ʌ z WORD_BOUNDARY ð eɪ WORD_BOUNDARY d ɪ d n t WORD_BOUNDARY s i ː WORD_BOUNDARY ð ə WORD_BOUNDARY s t ɑ ɹ i WORD_BOUNDARY b æ n ə ɹ ɹ WORD_BOUNDARY æ f t ə ɹ WORD_BOUNDARY l i ː v ɪ ŋ WORD_BOUNDARY n u ː WORD_BOUNDARY j ɔ ɹ k WORD_BOUNDARY ɛ k s ɛ p t WORD_BOUNDARY ɪ n WORD_BOUNDARY f ɹ ʌ n t WORD_BOUNDARY ə v ə WORD_BOUNDARY f eɪ k WORD_BOUNDARY ɔ k ʃ ə n WORD_BOUNDARY s eɪ l WORD_BOUNDARY ə ɹ ɹ eɪ n d̠ʒ d WORD_BOUNDARY ɪ s p ɛ ʃ ə l i WORD_BOUNDARY f ɔ ɹ WORD_BOUNDARY t ʊ ɹ ɹ ɪ s t s WORD_BOUNDARY UTT_BOUNDARY b aɪ WORD_BOUNDARY ð ə WORD_BOUNDARY t aɪ m WORD_BOUNDARY ð eɪ WORD_BOUNDARY f aʊ n d WORD_BOUNDARY ð ɛ m s ɛ l v z WORD_BOUNDARY ɪ n WORD_BOUNDARY ð æ t WORD_BOUNDARY v æ s t WORD_BOUNDARY b eɪ k ʌ v ə n WORD_BOUNDARY n oʊ n WORD_BOUNDARY æ z WORD_BOUNDARY ɪ n d i ə WORD_BOUNDARY ð eɪ WORD_BOUNDARY w ɜ ː WORD_BOUNDARY b ɪ n ʌ m d WORD_BOUNDARY æ n d WORD_BOUNDARY s ə b m ɪ s ɪ v WORD_BOUNDARY æ n d WORD_BOUNDARY h æ d WORD_BOUNDARY s ɛ t ə +07/16/2026 12:32:07 - INFO - __main__ - [RANK 0] Sample 64196 of the training set: {'input_ids': tensor([ 6, 29, 3, 26, 16, 31, 27, 5, 16, 31, 42, 15, 16, 2, 18, 9, 24, 11, + 29, 3, 18, 10, 27, 3, 28, 5, 14, 16, 17, 27, 5, 36, 29, 26, 16, 7, + 11, 14, 12, 13, 11, 14, 3, 16, 31, 25, 5, 14, 16, 31, 4, 5, 9, 16, + 24, 4, 5, 36, 16, 2, 3, 27, 14, 10, 26, 3, 16, 12, 18, 6, 3, 16, + 21, 32, 16, 2, 18, 9, 24, 11, 29, 3, 18, 10, 27, 3, 28, 5, 14, 16, + 33, 8, 6, 16, 18, 15, 16, 21, 32, 16, 23, 6, 27, 5, 33, 26, 9, 16, + 2, 3, 27, 14, 10, 26, 3, 16, 12, 18, 6, 3, 16, 21, 32, 16, 2, 18, + 9, 24, 11, 29, 3, 18, 10, 27, 3, 28, 5, 14, 16, 40, 11, 35, 3, 16, + 18, 29, 3, 16, 10, 26, 9, 39, 5, 36, 16, 2, 18, 9, 24, 11, 29, 3, + 18, 10, 27, 3, 28, 5, 14, 16, 40, 11, 35, 3, 16, 18, 29, 3, 16, 10, + 26, 9, 39, 5, 36, 16, 2, 3, 27, 14, 10, 26, 3, 16, 12, 18, 6, 3, + 16, 17, 27, 17, 27, 5, 16, 21, 32, 16, 2, 3, 27, 14, 10, 26, 3, 16, + 12, 18, 6, 3, 16, 17, 27, 5, 17, 16, 2, 17, 8, 3, 14, 16, 36, 8, + 17, 16, 37, 4, 5, 14, 16, 2, 17, 8, 3, 14, 16, 24, 28, 5, 16, 37, + 27, 29, 3, 16, 11, 9, 16, 31, 39, 5, 16, 31, 19, 9, 26, 16, 24, 18, + 9, 31, 26, 6, 16, 37, 11, 34, 26, 9, 16, 2, 17, 8, 3, 14, 16, 36, + 8, 17, 16, 17, 27, 5, 6, 16, 37, 4, 5, 14, 16, 2, 3, 27, 14, 10, + 26, 3, 16, 12, 18, 6, 3, 16, 17, 27, 17, 27, 5, 16, 21, 32, 16, 17, + 27, 17, 27, 5, 16, 21, 32, 16, 17, 27, 17, 27, 5, 16, 21, 32, 16, 17, + 27, 17, 27, 5, 16, 21, 32, 16, 2, 17, 8, 3, 14, 16, 36, 8, 17, 16, + 17, 27, 5, 6, 16, 2, 17, 8, 3, 14, 16, 28, 5, 40, 26, 16, 2, 17, + 8, 3, 14, 16, 31, 39, 5, 16, 37, 27, 29, 3, 16, 7, 8, 14, 9, 26, + 16, 22, 5, 23, 14, 37, 21, 13, 3, 16, 9, 18, 36, 29, 16, 2, 17, 8, + 3, 14, 16, 31, 39, 5, 16, 2, 3, 27, 14, 10, 26, 3, 16, 12, 18, 6, + 3, 16, 17, 27, 17, 27, 5, 16, 21, 32, 16, 17, 27, 17, 27, 5, 16, 21, + 32, 16, 2, 18, 9, 24, 11, 29, 3, 18, 10, 27, 3, 28, 5, 14, 16, 31, + 27, 29, 16, 10, 4, 5, 3, 16, 18, 9, 16, 23, 41, 14, 10, 14, 6, 18, + 15, 26, 9, 16, 7, 11, 14, 37, 11, 6, 3, 9, 18, 29, 26, 9, 16, 9, + 18, 15, 3, 16, 36, 11, 14, 29])} l s t ə WORD_BOUNDARY d a ː WORD_BOUNDARY d ʊɐ ç WORD_BOUNDARY UTT_BOUNDARY ɪ n v ɛ s t ɪ ɡ a t o ː ɐ WORD_BOUNDARY m a ː k s ə WORD_BOUNDARY f ɛ ɐ ʃ p ɛ ɐ t WORD_BOUNDARY d i ː ɐ WORD_BOUNDARY d e ː n WORD_BOUNDARY v e ː k WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY aʊ x WORD_BOUNDARY UTT_BOUNDARY ɪ n v ɛ s t ɪ ɡ a t o ː ɐ WORD_BOUNDARY z ɔ l WORD_BOUNDARY ɪ ç WORD_BOUNDARY aʊ x WORD_BOUNDARY b l a ː z ə n WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY aʊ x WORD_BOUNDARY UTT_BOUNDARY ɪ n v ɛ s t ɪ ɡ a t o ː ɐ WORD_BOUNDARY j ɛ ts t WORD_BOUNDARY ɪ s t WORD_BOUNDARY ɡ ə n u ː k WORD_BOUNDARY UTT_BOUNDARY ɪ n v ɛ s t ɪ ɡ a t o ː ɐ WORD_BOUNDARY j ɛ ts t WORD_BOUNDARY ɪ s t WORD_BOUNDARY ɡ ə n u ː k WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY m a m a ː WORD_BOUNDARY aʊ x WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY m a ː m WORD_BOUNDARY UTT_BOUNDARY m ɔ t ɐ WORD_BOUNDARY k ɔ m WORD_BOUNDARY h e ː ɐ WORD_BOUNDARY UTT_BOUNDARY m ɔ t ɐ WORD_BOUNDARY v o ː WORD_BOUNDARY h a s t WORD_BOUNDARY ɛ n WORD_BOUNDARY d u ː WORD_BOUNDARY d aɪ n ə WORD_BOUNDARY v ɪ n d ə l WORD_BOUNDARY h ɛ ŋ ə n WORD_BOUNDARY UTT_BOUNDARY m ɔ t ɐ WORD_BOUNDARY k ɔ m WORD_BOUNDARY m a ː l WORD_BOUNDARY h e ː ɐ WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY m a m a ː WORD_BOUNDARY aʊ x WORD_BOUNDARY m a m a ː WORD_BOUNDARY aʊ x WORD_BOUNDARY m a m a ː WORD_BOUNDARY aʊ x WORD_BOUNDARY m a m a ː WORD_BOUNDARY aʊ x WORD_BOUNDARY UTT_BOUNDARY m ɔ t ɐ WORD_BOUNDARY k ɔ m WORD_BOUNDARY m a ː l WORD_BOUNDARY UTT_BOUNDARY m ɔ t ɐ WORD_BOUNDARY o ː j ə WORD_BOUNDARY UTT_BOUNDARY m ɔ t ɐ WORD_BOUNDARY d u ː WORD_BOUNDARY h a s t WORD_BOUNDARY f ɔ ɐ n ə WORD_BOUNDARY y ː b ɐ h aʊ p t WORD_BOUNDARY n ɪ k s WORD_BOUNDARY UTT_BOUNDARY m ɔ t ɐ WORD_BOUNDARY d u ː WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY m a m a ː WORD_BOUNDARY aʊ x WORD_BOUNDARY m a m a ː WORD_BOUNDARY aʊ x WORD_BOUNDARY UTT_BOUNDARY ɪ n v ɛ s t ɪ ɡ a t o ː ɐ WORD_BOUNDARY d a s WORD_BOUNDARY ɡ e ː t WORD_BOUNDARY ɪ n WORD_BOUNDARY b ʏ ɐ ɡ ɐ l ɪ ç ə n WORD_BOUNDARY f ɛ ɐ h ɛ l t n ɪ s ə n WORD_BOUNDARY n ɪ ç t WORD_BOUNDARY k ɛ ɐ s +07/16/2026 12:49:35 - INFO - __main__ - [RANK 0] perplexity: 2.706809656389217 eval_loss: 0.9957706928253174 accuracy: 0.7026 +07/16/2026 13:06:58 - INFO - __main__ - [RANK 0] Starting epoch 6... +07/16/2026 13:15:43 - INFO - __main__ - [RANK 0] perplexity: 2.6094868494192305 eval_loss: 0.9591535925865173 accuracy: 0.7123 +07/16/2026 13:41:52 - INFO - __main__ - [RANK 0] perplexity: 2.5569520695821653 eval_loss: 0.9388159513473511 accuracy: 0.7182 +07/16/2026 13:41:55 - INFO - __main__ - [RANK 0] Starting epoch 7... +07/16/2026 14:08:02 - INFO - __main__ - [RANK 0] perplexity: 2.528194706168988 eval_loss: 0.9275054931640625 accuracy: 0.7217 +07/16/2026 14:16:46 - INFO - __main__ - [RANK 0] Starting epoch 8... +07/16/2026 14:34:12 - INFO - __main__ - [RANK 0] perplexity: 2.5066965152271736 eval_loss: 0.918965756893158 accuracy: 0.7246 +07/16/2026 14:51:38 - INFO - __main__ - [RANK 0] Starting epoch 9... +07/16/2026 15:00:21 - INFO - __main__ - [RANK 0] perplexity: 2.4926945929262567 eval_loss: 0.9133642911911011 accuracy: 0.7265 +07/16/2026 15:26:31 - INFO - __main__ - [RANK 0] perplexity: 2.4800004363726162 eval_loss: 0.9082587361335754 accuracy: 0.7281 +07/16/2026 15:26:39 - INFO - __main__ - [RANK 0] perplexity: 2.4800004363726162 eval_loss: 0.9082587361335754 accuracy: 0.7281 +07/16/2026 16:48:36 - INFO - __main__ - [RANK 0] Distributed environment: DistributedType.NO +Num processes: 1 +Process index: 0 +Local process index: 0 +Device: cuda + +Mixed precision type: no + +07/16/2026 16:48:36 - INFO - __main__ - [RANK 0] Training args: +{'data_dir': 'data/phone/deu-eng/tokenized_sequential_interleaved', 'model_name_or_path': None, 'config_name': 'gpt_base_config.json', 'tokenizer_name': 'nikitastheo/phonelm-phone-deu-eng-tokenizer', 'per_device_train_batch_size': 32, 'per_device_eval_batch_size': 32, 'learning_rate': 0.0001, 'weight_decay': 0.01, 'max_steps': 32000, 'gradient_accumulation_steps': 1, 'lr_scheduler_type': , 'warmup_steps': 3200, 'output_dir': 'models/phonebabylm-phone-deu-eng-sequential_interleaved', 'seed': 42, 'model_type': None, 'block_size': None, 'push_to_hub': True, 'hub_model_id': 'nikitastheo/phonebabylm-phone-deu-eng-sequential_interleaved', 'hub_token': None, 'private': False, 'trust_remote_code': False, 'with_tracking': True, 'tracking_dir': None, 'report_to': 'wandb', 'restart_lr_at_switch': False, 'language_switch_steps': 8000, 'model_revision': 'main', 'config_overrides': None, 'use_slow_tokenizer': False, 'override_vocab_size': None, 'dtype': None, 'logging_steps': 10, 'cache_dir': None, 'eval_steps': 1600, 'max_grad_norm': 1.0, 'adam_epsilon': 1e-06, 'run_name': 'phonebabylm-phone-deu-eng-sequential_interleaved', 'max_train_samples': None, 'max_eval_samples': None} +07/16/2026 16:48:36 - INFO - transformers.configuration_utils - loading configuration file gpt_base_config.json +07/16/2026 16:48:36 - INFO - transformers.configuration_utils - Model config GPT2Config { + "activation_function": "gelu", + "architectures": [ + "GPT2LMHeadModel" + ], + "attn_pdrop": 0.1, + "bos_token_id": 50256, + "embd_pdrop": 0.1, + "eos_token_id": 50256, + "initializer_range": 0.02, + "layer_norm_epsilon": 1e-05, + "model_type": "gpt2", + "n_ctx": 512, + "n_embd": 768, + "n_head": 12, + "n_inner": 3072, + "n_layer": 12, + "n_positions": 512, + "reorder_and_upcast_attn": false, + "resid_pdrop": 0.1, + "scale_attn_by_inverse_layer_idx": false, + "scale_attn_weights": true, + "summary_activation": null, + "summary_first_dropout": 0.1, + "summary_proj_to_labels": true, + "summary_type": "cls_index", + "summary_use_proj": true, + "transformers_version": "4.57.6", + "use_cache": true, + "vocab_size": 1 +} + +07/16/2026 16:48:36 - INFO - transformers.tokenization_utils_base - loading file vocab.json from cache at /home/geofila/.cache/huggingface/hub/models--nikitastheo--phonelm-phone-deu-eng-tokenizer/snapshots/9bea9a4137e9f8a784365dd064e257186231952d/vocab.json +07/16/2026 16:48:36 - INFO - transformers.tokenization_utils_base - loading file merges.txt from cache at None +07/16/2026 16:48:36 - INFO - transformers.tokenization_utils_base - loading file tokenizer.json from cache at /home/geofila/.cache/huggingface/hub/models--nikitastheo--phonelm-phone-deu-eng-tokenizer/snapshots/9bea9a4137e9f8a784365dd064e257186231952d/tokenizer.json +07/16/2026 16:48:36 - INFO - transformers.tokenization_utils_base - loading file added_tokens.json from cache at None +07/16/2026 16:48:36 - INFO - transformers.tokenization_utils_base - loading file special_tokens_map.json from cache at /home/geofila/.cache/huggingface/hub/models--nikitastheo--phonelm-phone-deu-eng-tokenizer/snapshots/9bea9a4137e9f8a784365dd064e257186231952d/special_tokens_map.json +07/16/2026 16:48:36 - INFO - transformers.tokenization_utils_base - loading file tokenizer_config.json from cache at /home/geofila/.cache/huggingface/hub/models--nikitastheo--phonelm-phone-deu-eng-tokenizer/snapshots/9bea9a4137e9f8a784365dd064e257186231952d/tokenizer_config.json +07/16/2026 16:48:36 - INFO - transformers.tokenization_utils_base - loading file chat_template.jinja from cache at None +07/16/2026 16:48:38 - INFO - transformers.generation.configuration_utils - Generate config GenerationConfig { + "bos_token_id": 2, + "eos_token_id": 2, + "pad_token_id": 1 +} + +07/16/2026 16:48:40 - INFO - __main__ - [RANK 0] Model parameters: 85.50M total (85.50M trainable) +07/16/2026 16:48:50 - INFO - __main__ - [RANK 0] Sample 83810 of the training set: {'input_ids': tensor([ 3, 16, 18, 9, 16, 31, 11, 14, 16, 10, 27, 20, 27, 47, 26, 16, 2, 18, + 9, 24, 11, 29, 3, 18, 10, 27, 3, 28, 5, 14, 16, 24, 4, 5, 24, 4, + 5, 24, 4, 5, 16, 2, 17, 8, 3, 14, 16, 24, 4, 5, 24, 4, 5, 24, + 4, 5, 16, 2, 3, 27, 14, 10, 26, 3, 16, 12, 18, 6, 3, 16, 37, 25, + 5, 14, 16, 18, 29, 3, 16, 31, 27, 29, 16, 23, 6, 21, 26, 16, 21, 3, + 28, 5, 16, 2, 17, 8, 3, 14, 16, 24, 4, 5, 24, 4, 5, 24, 4, 5, + 16, 2, 3, 27, 14, 10, 26, 3, 16, 12, 18, 6, 3, 16, 19, 9, 16, 10, + 14, 22, 5, 9, 26, 9, 16, 21, 3, 28, 5, 16, 2, 3, 27, 14, 10, 26, + 3, 16, 12, 18, 6, 3, 16, 19, 9, 16, 19, 9, 16, 10, 14, 22, 5, 9, + 26, 29, 16, 21, 3, 28, 5, 16, 2, 3, 27, 14, 10, 26, 3, 16, 12, 18, + 6, 3, 16, 19, 9, 16, 10, 11, 6, 23, 26, 29, 16, 21, 3, 28, 5, 16, + 2, 3, 27, 14, 10, 26, 3, 16, 12, 18, 6, 3, 16, 20, 28, 5, 3, 26, + 29, 16, 21, 3, 28, 5, 16, 2, 3, 27, 14, 10, 26, 3, 16, 12, 18, 6, + 3, 16, 9, 4, 5, 16, 2, 17, 8, 3, 14, 16, 24, 4, 5, 24, 4, 5, + 24, 4, 5, 16, 2, 18, 9, 24, 11, 29, 3, 18, 10, 27, 3, 28, 5, 14, + 16, 24, 4, 5, 24, 4, 5, 24, 4, 5, 16, 2, 17, 8, 3, 14, 16, 24, + 4, 5, 24, 4, 5, 24, 4, 5, 16, 2, 3, 27, 14, 10, 26, 3, 16, 12, + 18, 6, 3, 16, 19, 9, 16, 10, 14, 22, 5, 9, 26, 29, 16, 21, 3, 28, + 5, 16, 2, 3, 27, 14, 10, 26, 3, 16, 12, 18, 6, 3, 16, 19, 9, 16, + 10, 11, 6, 23, 26, 29, 16, 21, 3, 28, 5, 16, 2, 3, 27, 14, 10, 26, + 3, 16, 12, 18, 6, 3, 16, 19, 9, 16, 20, 28, 5, 3, 26, 29, 16, 21, + 3, 28, 5, 16, 2, 3, 27, 14, 10, 26, 3, 16, 12, 18, 6, 3, 16, 19, + 9, 16, 23, 6, 21, 26, 29, 16, 21, 3, 28, 5, 16, 2, 18, 9, 24, 11, + 29, 3, 18, 10, 27, 3, 28, 5, 14, 16, 24, 4, 5, 24, 4, 5, 24, 4, + 5, 16, 2, 3, 27, 14, 10, 26, 3, 16, 12, 18, 6, 3, 16, 10, 30, 36, + 16, 17, 27, 5, 6, 16, 2, 3, 27, 14, 10, 26, 3, 16, 12, 18, 6, 3, + 16, 31, 39, 5, 16, 36, 14, 25, 5, 36, 29, 3, 16, 40, 11, 35, 3, 16, + 19, 9, 16, 10, 11, 6, 23, 26, 29, 16, 2, 3, 27, 14, 10, 26, 3, 16, + 12, 18, 6, 3, 16, 18, 15, 16])} t WORD_BOUNDARY ɪ n WORD_BOUNDARY d ɛ ɐ WORD_BOUNDARY ɡ a ʀ a ʒ ə WORD_BOUNDARY UTT_BOUNDARY ɪ n v ɛ s t ɪ ɡ a t o ː ɐ WORD_BOUNDARY v e ː v e ː v e ː WORD_BOUNDARY UTT_BOUNDARY m ɔ t ɐ WORD_BOUNDARY v e ː v e ː v e ː WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY h i ː ɐ WORD_BOUNDARY ɪ s t WORD_BOUNDARY d a s WORD_BOUNDARY b l aʊ ə WORD_BOUNDARY aʊ t o ː WORD_BOUNDARY UTT_BOUNDARY m ɔ t ɐ WORD_BOUNDARY v e ː v e ː v e ː WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY aɪ n WORD_BOUNDARY ɡ ɐ y ː n ə n WORD_BOUNDARY aʊ t o ː WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY aɪ n WORD_BOUNDARY aɪ n WORD_BOUNDARY ɡ ɐ y ː n ə s WORD_BOUNDARY aʊ t o ː WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY aɪ n WORD_BOUNDARY ɡ ɛ l b ə s WORD_BOUNDARY aʊ t o ː WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY ʀ o ː t ə s WORD_BOUNDARY aʊ t o ː WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY n e ː WORD_BOUNDARY UTT_BOUNDARY m ɔ t ɐ WORD_BOUNDARY v e ː v e ː v e ː WORD_BOUNDARY UTT_BOUNDARY ɪ n v ɛ s t ɪ ɡ a t o ː ɐ WORD_BOUNDARY v e ː v e ː v e ː WORD_BOUNDARY UTT_BOUNDARY m ɔ t ɐ WORD_BOUNDARY v e ː v e ː v e ː WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY aɪ n WORD_BOUNDARY ɡ ɐ y ː n ə s WORD_BOUNDARY aʊ t o ː WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY aɪ n WORD_BOUNDARY ɡ ɛ l b ə s WORD_BOUNDARY aʊ t o ː WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY aɪ n WORD_BOUNDARY ʀ o ː t ə s WORD_BOUNDARY aʊ t o ː WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY aɪ n WORD_BOUNDARY b l aʊ ə s WORD_BOUNDARY aʊ t o ː WORD_BOUNDARY UTT_BOUNDARY ɪ n v ɛ s t ɪ ɡ a t o ː ɐ WORD_BOUNDARY v e ː v e ː v e ː WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY ɡ ʊ k WORD_BOUNDARY m a ː l WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY d u ː WORD_BOUNDARY k ɐ i ː k s t WORD_BOUNDARY j ɛ ts t WORD_BOUNDARY aɪ n WORD_BOUNDARY ɡ ɛ l b ə s WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY ɪ ç WORD_BOUNDARY +07/16/2026 16:48:50 - INFO - __main__ - [RANK 0] Sample 14592 of the training set: {'input_ids': tensor([27, 29, 16, 18, 15, 16, 31, 27, 5, 16, 24, 27, 29, 16, 23, 21, 26, 9, + 16, 36, 27, 9, 16, 2, 18, 9, 24, 11, 29, 3, 18, 10, 27, 3, 28, 5, + 14, 16, 33, 28, 5, 16, 40, 11, 35, 3, 16, 31, 25, 5, 16, 23, 25, 5, + 9, 26, 16, 37, 28, 5, 6, 26, 9, 16, 2, 18, 9, 24, 11, 29, 3, 18, + 10, 27, 3, 28, 5, 14, 16, 31, 27, 9, 16, 36, 27, 9, 16, 17, 27, 9, + 16, 11, 29, 16, 27, 5, 23, 14, 16, 9, 18, 15, 3, 16, 31, 14, 30, 36, + 26, 9, 16, 24, 11, 9, 16, 31, 39, 5, 16, 31, 25, 5, 16, 23, 25, 5, + 9, 26, 16, 37, 28, 5, 6, 29, 3, 16, 24, 19, 29, 3, 16, 31, 39, 5, + 16, 31, 27, 29, 16, 2, 17, 4, 5, 31, 25, 5, 27, 5, 16, 17, 27, 32, + 26, 9, 16, 24, 25, 5, 14, 16, 19, 9, 16, 10, 14, 27, 17, 28, 5, 7, + 28, 5, 9, 16, 2, 3, 27, 14, 10, 26, 3, 16, 12, 18, 6, 3, 16, 7, + 11, 14, 10, 11, 29, 26, 9, 16, 17, 30, 29, 16, 18, 15, 16, 31, 27, 29, + 16, 4, 5, 23, 26, 9, 16, 33, 28, 5, 16, 31, 42, 15, 12, 13, 25, 5, + 6, 26, 9, 16, 2, 18, 9, 24, 11, 29, 3, 18, 10, 27, 3, 28, 5, 14, + 16, 40, 27, 5, 16, 31, 27, 9, 16, 36, 27, 9, 29, 3, 16, 31, 4, 5, + 16, 31, 25, 5, 14, 16, 31, 27, 9, 16, 24, 27, 29, 16, 21, 29, 33, 39, + 5, 32, 26, 9, 16, 35, 30, 17, 16, 31, 14, 30, 36, 26, 9, 16, 10, 11, + 6, 16, 2, 3, 27, 14, 10, 26, 3, 16, 12, 18, 6, 3, 16, 37, 26, 17, + 16, 2, 18, 9, 24, 11, 29, 3, 18, 10, 27, 3, 28, 5, 14, 16, 37, 26, + 17, 16, 2, 17, 4, 5, 31, 25, 5, 27, 5, 16, 31, 27, 29, 16, 3, 19, + 6, 16, 10, 26, 37, 43, 14, 3, 16, 31, 27, 5, 16, 9, 18, 15, 3, 16, + 37, 18, 9, 16, 2, 18, 9, 24, 11, 29, 3, 18, 10, 27, 3, 28, 5, 14, + 16, 37, 39, 5, 16, 18, 17, 14, 16, 17, 27, 5, 6, 16, 13, 27, 29, 25, + 5, 14, 3, 16, 31, 27, 29, 16, 10, 11, 6, 16, 2, 17, 4, 5, 31, 25, + 5, 27, 5, 16, 37, 11, 9, 12, 26, 9, 16, 13, 6, 30, 29, 16, 36, 6, + 19, 9, 16, 10, 18, 34, 16, 27, 6, 19, 9, 16, 18, 9, 16, 31, 25, 5, + 16, 2, 18, 9, 24, 11, 29, 3, 18, 10, 27, 3, 28, 5, 14, 16, 2, 3, + 27, 14, 10, 26, 3, 16, 12, 18, 6, 3, 16, 18, 9, 16, 31, 25, 5, 16, + 2, 17, 4, 5, 31, 25, 5, 27])} a s WORD_BOUNDARY ɪ ç WORD_BOUNDARY d a ː WORD_BOUNDARY v a s WORD_BOUNDARY b aʊ ə n WORD_BOUNDARY k a n WORD_BOUNDARY UTT_BOUNDARY ɪ n v ɛ s t ɪ ɡ a t o ː ɐ WORD_BOUNDARY z o ː WORD_BOUNDARY j ɛ ts t WORD_BOUNDARY d i ː WORD_BOUNDARY b i ː n ə WORD_BOUNDARY h o ː l ə n WORD_BOUNDARY UTT_BOUNDARY ɪ n v ɛ s t ɪ ɡ a t o ː ɐ WORD_BOUNDARY d a n WORD_BOUNDARY k a n WORD_BOUNDARY m a n WORD_BOUNDARY ɛ s WORD_BOUNDARY a ː b ɐ WORD_BOUNDARY n ɪ ç t WORD_BOUNDARY d ɐ ʊ k ə n WORD_BOUNDARY v ɛ n WORD_BOUNDARY d u ː WORD_BOUNDARY d i ː WORD_BOUNDARY b i ː n ə WORD_BOUNDARY h o ː l s t WORD_BOUNDARY v aɪ s t WORD_BOUNDARY d u ː WORD_BOUNDARY d a s WORD_BOUNDARY UTT_BOUNDARY m e ː d i ː a ː WORD_BOUNDARY m a x ə n WORD_BOUNDARY v i ː ɐ WORD_BOUNDARY aɪ n WORD_BOUNDARY ɡ ɐ a m o ː f o ː n WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY f ɛ ɐ ɡ ɛ s ə n WORD_BOUNDARY m ʊ s WORD_BOUNDARY ɪ ç WORD_BOUNDARY d a s WORD_BOUNDARY e ː b ə n WORD_BOUNDARY z o ː WORD_BOUNDARY d ʊɐ ç ʃ p i ː l ə n WORD_BOUNDARY UTT_BOUNDARY ɪ n v ɛ s t ɪ ɡ a t o ː ɐ WORD_BOUNDARY j a ː WORD_BOUNDARY d a n WORD_BOUNDARY k a n s t WORD_BOUNDARY d e ː WORD_BOUNDARY d i ː ɐ WORD_BOUNDARY d a n WORD_BOUNDARY v a s WORD_BOUNDARY aʊ s z u ː x ə n WORD_BOUNDARY ts ʊ m WORD_BOUNDARY d ɐ ʊ k ə n WORD_BOUNDARY ɡ ɛ l WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY h ə m WORD_BOUNDARY UTT_BOUNDARY ɪ n v ɛ s t ɪ ɡ a t o ː ɐ WORD_BOUNDARY h ə m WORD_BOUNDARY UTT_BOUNDARY m e ː d i ː a ː WORD_BOUNDARY d a s WORD_BOUNDARY t aɪ l WORD_BOUNDARY ɡ ə h œ ɐ t WORD_BOUNDARY d a ː WORD_BOUNDARY n ɪ ç t WORD_BOUNDARY h ɪ n WORD_BOUNDARY UTT_BOUNDARY ɪ n v ɛ s t ɪ ɡ a t o ː ɐ WORD_BOUNDARY h u ː WORD_BOUNDARY ɪ m ɐ WORD_BOUNDARY m a ː l WORD_BOUNDARY p a s i ː ɐ t WORD_BOUNDARY d a s WORD_BOUNDARY ɡ ɛ l WORD_BOUNDARY UTT_BOUNDARY m e ː d i ː a ː WORD_BOUNDARY h ɛ n ʃ ə n WORD_BOUNDARY p l ʊ s WORD_BOUNDARY k l aɪ n WORD_BOUNDARY ɡ ɪ ŋ WORD_BOUNDARY a l aɪ n WORD_BOUNDARY ɪ n WORD_BOUNDARY d i ː WORD_BOUNDARY UTT_BOUNDARY ɪ n v ɛ s t ɪ ɡ a t o ː ɐ WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY ɪ n WORD_BOUNDARY d i ː WORD_BOUNDARY UTT_BOUNDARY m e ː d i ː a +07/16/2026 16:48:50 - INFO - __main__ - [RANK 0] Sample 3278 of the training set: {'input_ids': tensor([31, 26, 9, 26, 9, 16, 10, 11, 6, 31, 26, 16, 19, 9, 26, 16, 27, 9, + 35, 27, 5, 6, 16, 31, 11, 14, 33, 11, 6, 23, 26, 9, 16, 12, 19, 9, + 26, 16, 24, 25, 5, 31, 14, 16, 31, 25, 5, 16, 11, 14, 16, 31, 4, 5, + 17, 16, 11, 14, 17, 8, 14, 31, 26, 3, 26, 9, 16, 18, 9, 16, 11, 3, + 26, 6, 23, 14, 41, 36, 16, 21, 29, 10, 26, 37, 11, 9, 31, 18, 15, 3, + 16, 37, 27, 3, 26, 16, 33, 28, 5, 16, 7, 11, 14, 31, 18, 15, 3, 26, + 3, 26, 16, 33, 18, 15, 16, 31, 11, 14, 16, 7, 11, 14, 31, 27, 32, 3, + 16, 10, 4, 5, 10, 26, 9, 16, 31, 25, 5, 16, 23, 19, 31, 26, 9, 16, + 18, 17, 14, 16, 17, 4, 5, 14, 16, 18, 17, 14, 16, 17, 4, 5, 14, 16, + 7, 11, 14, 24, 18, 36, 26, 6, 3, 26, 9, 16, 33, 25, 5, 16, 33, 18, + 15, 16, 18, 9, 16, 24, 25, 5, 31, 14, 12, 13, 14, 41, 15, 26, 16, 9, + 27, 5, 32, 16, 17, 4, 5, 20, 26, 20, 26, 9, 16, 24, 8, 32, 26, 9, + 16, 6, 22, 5, 10, 26, 9, 37, 27, 7, 3, 14, 16, 27, 13, 14, 4, 5, + 31, 26, 16, 10, 27, 5, 23, 26, 9, 16, 33, 25, 5, 16, 11, 9, 3, 6, + 18, 15, 16, 31, 27, 29, 16, 7, 11, 14, 23, 14, 11, 15, 26, 9, 16, 35, + 39, 5, 16, 30, 9, 3, 16, 6, 4, 5, 36, 3, 26, 9, 16, 19, 9, 16, + 7, 8, 6, 12, 3, 11, 9, 31, 18, 10, 26, 29, 16, 10, 26, 12, 3, 11, + 9, 31, 9, 18, 29, 16, 27, 13, 16, 21, 32, 16, 23, 26, 36, 27, 9, 3, + 26, 9, 16, 33, 25, 5, 16, 31, 25, 5, 16, 3, 27, 5, 3, 16, 17, 18, + 3, 16, 7, 28, 5, 14, 23, 26, 31, 27, 32, 3, 16, 23, 26, 10, 27, 34, + 26, 9, 16, 35, 39, 5, 16, 37, 27, 5, 23, 26, 9, 16, 35, 24, 27, 5, + 14, 16, 24, 11, 5, 20, 26, 16, 11, 29, 16, 25, 5, 9, 26, 9, 16, 6, + 25, 5, 23, 14, 16, 10, 26, 24, 4, 5, 33, 26, 9, 16, 33, 27, 5, 10, + 3, 26, 9, 16, 33, 25, 5, 16, 24, 11, 9, 16, 33, 25, 5, 16, 31, 27, + 29, 16, 10, 11, 6, 3, 16, 28, 5, 9, 26, 16, 17, 8, 14, 3, 16, 37, + 11, 3, 26, 9, 16, 27, 9, 16, 33, 18, 15, 16, 23, 14, 18, 34, 26, 9, + 16, 36, 43, 9, 26, 9, 16, 31, 8, 32, 16, 37, 11, 3, 26, 9, 16, 33, + 25, 5, 16, 33, 18, 15, 16, 17, 18, 3, 16, 24, 27, 7, 26, 9, 16, 7, + 11, 14, 33, 4, 5, 26, 9, 16])} d ə n ə n WORD_BOUNDARY ɡ ɛ l d ə WORD_BOUNDARY aɪ n ə WORD_BOUNDARY a n ts a ː l WORD_BOUNDARY d ɛ ɐ z ɛ l b ə n WORD_BOUNDARY ʃ aɪ n ə WORD_BOUNDARY v i ː d ɐ WORD_BOUNDARY d i ː WORD_BOUNDARY ɛ ɐ WORD_BOUNDARY d e ː m WORD_BOUNDARY ɛ ɐ m ɔ ɐ d ə t ə n WORD_BOUNDARY ɪ n WORD_BOUNDARY ɛ t ə l b ɐ ʏ k WORD_BOUNDARY aʊ s ɡ ə h ɛ n d ɪ ç t WORD_BOUNDARY h a t ə WORD_BOUNDARY z o ː WORD_BOUNDARY f ɛ ɐ d ɪ ç t ə t ə WORD_BOUNDARY z ɪ ç WORD_BOUNDARY d ɛ ɐ WORD_BOUNDARY f ɛ ɐ d a x t WORD_BOUNDARY ɡ e ː ɡ ə n WORD_BOUNDARY d i ː WORD_BOUNDARY b aɪ d ə n WORD_BOUNDARY ɪ m ɐ WORD_BOUNDARY m e ː ɐ WORD_BOUNDARY ɪ m ɐ WORD_BOUNDARY m e ː ɐ WORD_BOUNDARY f ɛ ɐ v ɪ k ə l t ə n WORD_BOUNDARY z i ː WORD_BOUNDARY z ɪ ç WORD_BOUNDARY ɪ n WORD_BOUNDARY v i ː d ɐ ʃ p ɐ ʏ ç ə WORD_BOUNDARY n a ː x WORD_BOUNDARY m e ː ʀ ə ʀ ə n WORD_BOUNDARY v ɔ x ə n WORD_BOUNDARY l y ː ɡ ə n h a f t ɐ WORD_BOUNDARY a p ɐ e ː d ə WORD_BOUNDARY ɡ a ː b ə n WORD_BOUNDARY z i ː WORD_BOUNDARY ɛ n t l ɪ ç WORD_BOUNDARY d a s WORD_BOUNDARY f ɛ ɐ b ɐ ɛ ç ə n WORD_BOUNDARY ts u ː WORD_BOUNDARY ʊ n t WORD_BOUNDARY l e ː k t ə n WORD_BOUNDARY aɪ n WORD_BOUNDARY f ɔ l ʃ t ɛ n d ɪ ɡ ə s WORD_BOUNDARY ɡ ə ʃ t ɛ n d n ɪ s WORD_BOUNDARY a p WORD_BOUNDARY aʊ x WORD_BOUNDARY b ə k a n t ə n WORD_BOUNDARY z i ː WORD_BOUNDARY d i ː WORD_BOUNDARY t a ː t WORD_BOUNDARY m ɪ t WORD_BOUNDARY f o ː ɐ b ə d a x t WORD_BOUNDARY b ə ɡ a ŋ ə n WORD_BOUNDARY ts u ː WORD_BOUNDARY h a ː b ə n WORD_BOUNDARY ts v a ː ɐ WORD_BOUNDARY v ɛ ː ʀ ə WORD_BOUNDARY ɛ s WORD_BOUNDARY i ː n ə n WORD_BOUNDARY l i ː b ɐ WORD_BOUNDARY ɡ ə v e ː z ə n WORD_BOUNDARY z a ː ɡ t ə n WORD_BOUNDARY z i ː WORD_BOUNDARY v ɛ n WORD_BOUNDARY z i ː WORD_BOUNDARY d a s WORD_BOUNDARY ɡ ɛ l t WORD_BOUNDARY o ː n ə WORD_BOUNDARY m ɔ ɐ t WORD_BOUNDARY h ɛ t ə n WORD_BOUNDARY a n WORD_BOUNDARY z ɪ ç WORD_BOUNDARY b ɐ ɪ ŋ ə n WORD_BOUNDARY k œ n ə n WORD_BOUNDARY d ɔ x WORD_BOUNDARY h ɛ t ə n WORD_BOUNDARY z i ː WORD_BOUNDARY z ɪ ç WORD_BOUNDARY m ɪ t WORD_BOUNDARY v a f ə n WORD_BOUNDARY f ɛ ɐ z e ː ə n WORD_BOUNDARY +07/16/2026 16:48:53 - INFO - __main__ - [RANK 0] ***** Running training ***** +07/16/2026 16:48:53 - INFO - __main__ - [RANK 0] Num examples = 101723 +07/16/2026 16:48:53 - INFO - __main__ - [RANK 0] Num Training Steps = 32000 +07/16/2026 16:48:53 - INFO - __main__ - [RANK 0] Instantaneous batch size per device = 32 +07/16/2026 16:48:53 - INFO - __main__ - [RANK 0] Total train batch size (w. parallel, distributed & accumulation) = 32 +07/16/2026 16:48:53 - INFO - __main__ - [RANK 0] Gradient Accumulation steps = 1 +07/16/2026 16:48:53 - INFO - __main__ - [RANK 0] Total optimization steps = 32000 +07/16/2026 16:48:53 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/16/2026 16:48:53 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/16/2026 16:48:57 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/16/2026 16:48:57 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/16/2026 16:48:57 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/16/2026 16:49:03 - INFO - __main__ - [RANK 0] Starting epoch 0... +07/16/2026 16:49:03 - WARNING - transformers.modeling_utils - `loss_type=None` was set in the config but it is unrecognized. Using the default loss: `ForCausalLMLoss`. +07/16/2026 16:57:49 - INFO - __main__ - [RANK 0] perplexity: 4.321081100637837 eval_loss: 1.4635056257247925 accuracy: 0.5634 +07/16/2026 17:06:28 - INFO - __main__ - [RANK 0] Starting epoch 1... +07/16/2026 17:06:38 - INFO - __main__ - [RANK 0] perplexity: 3.192933005613836 eval_loss: 1.1609399318695068 accuracy: 0.6509 +07/16/2026 17:15:28 - INFO - __main__ - [RANK 0] perplexity: 2.9154637588695076 eval_loss: 1.0700289011001587 accuracy: 0.6775 +07/16/2026 17:24:00 - INFO - __main__ - [RANK 0] Starting epoch 2... +07/16/2026 17:24:17 - INFO - __main__ - [RANK 0] perplexity: 2.7756626754262133 eval_loss: 1.0208895206451416 accuracy: 0.6916 +07/16/2026 17:35:00 - INFO - __main__ - [RANK 0] Distributed environment: DistributedType.NO +Num processes: 1 +Process index: 0 +Local process index: 0 +Device: cuda + +Mixed precision type: no + +07/16/2026 17:35:00 - INFO - __main__ - [RANK 0] Training args: +{'data_dir': 'data/phone/deu-eng/tokenized_sequential_interleaved', 'model_name_or_path': None, 'config_name': 'gpt_base_config.json', 'tokenizer_name': 'nikitastheo/phonelm-phone-deu-eng-tokenizer', 'per_device_train_batch_size': 32, 'per_device_eval_batch_size': 32, 'learning_rate': 0.0001, 'weight_decay': 0.01, 'max_steps': 24000, 'gradient_accumulation_steps': 1, 'lr_scheduler_type': , 'warmup_steps': 2400, 'output_dir': 'models/phonebabylm-phone-deu-eng-sequential_interleaved', 'seed': 42, 'model_type': None, 'block_size': None, 'push_to_hub': True, 'hub_model_id': 'nikitastheo/phonebabylm-phone-deu-eng-sequential_interleaved', 'hub_token': None, 'private': False, 'trust_remote_code': False, 'with_tracking': True, 'tracking_dir': None, 'report_to': 'wandb', 'restart_lr_at_switch': False, 'language_switch_steps': 8000, 'model_revision': 'main', 'config_overrides': None, 'use_slow_tokenizer': False, 'override_vocab_size': None, 'dtype': None, 'logging_steps': 10, 'cache_dir': None, 'eval_steps': 1200, 'max_grad_norm': 1.0, 'adam_epsilon': 1e-06, 'run_name': 'phonebabylm-phone-deu-eng-sequential_interleaved', 'max_train_samples': None, 'max_eval_samples': None} +07/16/2026 17:35:00 - INFO - transformers.configuration_utils - loading configuration file gpt_base_config.json +07/16/2026 17:35:00 - INFO - transformers.configuration_utils - Model config GPT2Config { + "activation_function": "gelu", + "architectures": [ + "GPT2LMHeadModel" + ], + "attn_pdrop": 0.1, + "bos_token_id": 50256, + "embd_pdrop": 0.1, + "eos_token_id": 50256, + "initializer_range": 0.02, + "layer_norm_epsilon": 1e-05, + "model_type": "gpt2", + "n_ctx": 512, + "n_embd": 768, + "n_head": 12, + "n_inner": 3072, + "n_layer": 12, + "n_positions": 512, + "reorder_and_upcast_attn": false, + "resid_pdrop": 0.1, + "scale_attn_by_inverse_layer_idx": false, + "scale_attn_weights": true, + "summary_activation": null, + "summary_first_dropout": 0.1, + "summary_proj_to_labels": true, + "summary_type": "cls_index", + "summary_use_proj": true, + "transformers_version": "4.57.6", + "use_cache": true, + "vocab_size": 1 +} + +07/16/2026 17:35:00 - INFO - transformers.tokenization_utils_base - loading file vocab.json from cache at /home/geofila/.cache/huggingface/hub/models--nikitastheo--phonelm-phone-deu-eng-tokenizer/snapshots/9bea9a4137e9f8a784365dd064e257186231952d/vocab.json +07/16/2026 17:35:00 - INFO - transformers.tokenization_utils_base - loading file merges.txt from cache at None +07/16/2026 17:35:00 - INFO - transformers.tokenization_utils_base - loading file tokenizer.json from cache at /home/geofila/.cache/huggingface/hub/models--nikitastheo--phonelm-phone-deu-eng-tokenizer/snapshots/9bea9a4137e9f8a784365dd064e257186231952d/tokenizer.json +07/16/2026 17:35:00 - INFO - transformers.tokenization_utils_base - loading file added_tokens.json from cache at None +07/16/2026 17:35:00 - INFO - transformers.tokenization_utils_base - loading file special_tokens_map.json from cache at /home/geofila/.cache/huggingface/hub/models--nikitastheo--phonelm-phone-deu-eng-tokenizer/snapshots/9bea9a4137e9f8a784365dd064e257186231952d/special_tokens_map.json +07/16/2026 17:35:00 - INFO - transformers.tokenization_utils_base - loading file tokenizer_config.json from cache at /home/geofila/.cache/huggingface/hub/models--nikitastheo--phonelm-phone-deu-eng-tokenizer/snapshots/9bea9a4137e9f8a784365dd064e257186231952d/tokenizer_config.json +07/16/2026 17:35:00 - INFO - transformers.tokenization_utils_base - loading file chat_template.jinja from cache at None +07/16/2026 17:35:02 - INFO - transformers.generation.configuration_utils - Generate config GenerationConfig { + "bos_token_id": 2, + "eos_token_id": 2, + "pad_token_id": 1 +} + +07/16/2026 17:35:03 - INFO - __main__ - [RANK 0] Model parameters: 85.50M total (85.50M trainable) +07/16/2026 17:35:14 - INFO - __main__ - [RANK 0] Sample 83810 of the training set: {'input_ids': tensor([ 3, 16, 18, 9, 16, 31, 11, 14, 16, 10, 27, 20, 27, 47, 26, 16, 2, 18, + 9, 24, 11, 29, 3, 18, 10, 27, 3, 28, 5, 14, 16, 24, 4, 5, 24, 4, + 5, 24, 4, 5, 16, 2, 17, 8, 3, 14, 16, 24, 4, 5, 24, 4, 5, 24, + 4, 5, 16, 2, 3, 27, 14, 10, 26, 3, 16, 12, 18, 6, 3, 16, 37, 25, + 5, 14, 16, 18, 29, 3, 16, 31, 27, 29, 16, 23, 6, 21, 26, 16, 21, 3, + 28, 5, 16, 2, 17, 8, 3, 14, 16, 24, 4, 5, 24, 4, 5, 24, 4, 5, + 16, 2, 3, 27, 14, 10, 26, 3, 16, 12, 18, 6, 3, 16, 19, 9, 16, 10, + 14, 22, 5, 9, 26, 9, 16, 21, 3, 28, 5, 16, 2, 3, 27, 14, 10, 26, + 3, 16, 12, 18, 6, 3, 16, 19, 9, 16, 19, 9, 16, 10, 14, 22, 5, 9, + 26, 29, 16, 21, 3, 28, 5, 16, 2, 3, 27, 14, 10, 26, 3, 16, 12, 18, + 6, 3, 16, 19, 9, 16, 10, 11, 6, 23, 26, 29, 16, 21, 3, 28, 5, 16, + 2, 3, 27, 14, 10, 26, 3, 16, 12, 18, 6, 3, 16, 20, 28, 5, 3, 26, + 29, 16, 21, 3, 28, 5, 16, 2, 3, 27, 14, 10, 26, 3, 16, 12, 18, 6, + 3, 16, 9, 4, 5, 16, 2, 17, 8, 3, 14, 16, 24, 4, 5, 24, 4, 5, + 24, 4, 5, 16, 2, 18, 9, 24, 11, 29, 3, 18, 10, 27, 3, 28, 5, 14, + 16, 24, 4, 5, 24, 4, 5, 24, 4, 5, 16, 2, 17, 8, 3, 14, 16, 24, + 4, 5, 24, 4, 5, 24, 4, 5, 16, 2, 3, 27, 14, 10, 26, 3, 16, 12, + 18, 6, 3, 16, 19, 9, 16, 10, 14, 22, 5, 9, 26, 29, 16, 21, 3, 28, + 5, 16, 2, 3, 27, 14, 10, 26, 3, 16, 12, 18, 6, 3, 16, 19, 9, 16, + 10, 11, 6, 23, 26, 29, 16, 21, 3, 28, 5, 16, 2, 3, 27, 14, 10, 26, + 3, 16, 12, 18, 6, 3, 16, 19, 9, 16, 20, 28, 5, 3, 26, 29, 16, 21, + 3, 28, 5, 16, 2, 3, 27, 14, 10, 26, 3, 16, 12, 18, 6, 3, 16, 19, + 9, 16, 23, 6, 21, 26, 29, 16, 21, 3, 28, 5, 16, 2, 18, 9, 24, 11, + 29, 3, 18, 10, 27, 3, 28, 5, 14, 16, 24, 4, 5, 24, 4, 5, 24, 4, + 5, 16, 2, 3, 27, 14, 10, 26, 3, 16, 12, 18, 6, 3, 16, 10, 30, 36, + 16, 17, 27, 5, 6, 16, 2, 3, 27, 14, 10, 26, 3, 16, 12, 18, 6, 3, + 16, 31, 39, 5, 16, 36, 14, 25, 5, 36, 29, 3, 16, 40, 11, 35, 3, 16, + 19, 9, 16, 10, 11, 6, 23, 26, 29, 16, 2, 3, 27, 14, 10, 26, 3, 16, + 12, 18, 6, 3, 16, 18, 15, 16])} t WORD_BOUNDARY ɪ n WORD_BOUNDARY d ɛ ɐ WORD_BOUNDARY ɡ a ʀ a ʒ ə WORD_BOUNDARY UTT_BOUNDARY ɪ n v ɛ s t ɪ ɡ a t o ː ɐ WORD_BOUNDARY v e ː v e ː v e ː WORD_BOUNDARY UTT_BOUNDARY m ɔ t ɐ WORD_BOUNDARY v e ː v e ː v e ː WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY h i ː ɐ WORD_BOUNDARY ɪ s t WORD_BOUNDARY d a s WORD_BOUNDARY b l aʊ ə WORD_BOUNDARY aʊ t o ː WORD_BOUNDARY UTT_BOUNDARY m ɔ t ɐ WORD_BOUNDARY v e ː v e ː v e ː WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY aɪ n WORD_BOUNDARY ɡ ɐ y ː n ə n WORD_BOUNDARY aʊ t o ː WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY aɪ n WORD_BOUNDARY aɪ n WORD_BOUNDARY ɡ ɐ y ː n ə s WORD_BOUNDARY aʊ t o ː WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY aɪ n WORD_BOUNDARY ɡ ɛ l b ə s WORD_BOUNDARY aʊ t o ː WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY ʀ o ː t ə s WORD_BOUNDARY aʊ t o ː WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY n e ː WORD_BOUNDARY UTT_BOUNDARY m ɔ t ɐ WORD_BOUNDARY v e ː v e ː v e ː WORD_BOUNDARY UTT_BOUNDARY ɪ n v ɛ s t ɪ ɡ a t o ː ɐ WORD_BOUNDARY v e ː v e ː v e ː WORD_BOUNDARY UTT_BOUNDARY m ɔ t ɐ WORD_BOUNDARY v e ː v e ː v e ː WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY aɪ n WORD_BOUNDARY ɡ ɐ y ː n ə s WORD_BOUNDARY aʊ t o ː WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY aɪ n WORD_BOUNDARY ɡ ɛ l b ə s WORD_BOUNDARY aʊ t o ː WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY aɪ n WORD_BOUNDARY ʀ o ː t ə s WORD_BOUNDARY aʊ t o ː WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY aɪ n WORD_BOUNDARY b l aʊ ə s WORD_BOUNDARY aʊ t o ː WORD_BOUNDARY UTT_BOUNDARY ɪ n v ɛ s t ɪ ɡ a t o ː ɐ WORD_BOUNDARY v e ː v e ː v e ː WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY ɡ ʊ k WORD_BOUNDARY m a ː l WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY d u ː WORD_BOUNDARY k ɐ i ː k s t WORD_BOUNDARY j ɛ ts t WORD_BOUNDARY aɪ n WORD_BOUNDARY ɡ ɛ l b ə s WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY ɪ ç WORD_BOUNDARY +07/16/2026 17:35:14 - INFO - __main__ - [RANK 0] Sample 14592 of the training set: {'input_ids': tensor([27, 29, 16, 18, 15, 16, 31, 27, 5, 16, 24, 27, 29, 16, 23, 21, 26, 9, + 16, 36, 27, 9, 16, 2, 18, 9, 24, 11, 29, 3, 18, 10, 27, 3, 28, 5, + 14, 16, 33, 28, 5, 16, 40, 11, 35, 3, 16, 31, 25, 5, 16, 23, 25, 5, + 9, 26, 16, 37, 28, 5, 6, 26, 9, 16, 2, 18, 9, 24, 11, 29, 3, 18, + 10, 27, 3, 28, 5, 14, 16, 31, 27, 9, 16, 36, 27, 9, 16, 17, 27, 9, + 16, 11, 29, 16, 27, 5, 23, 14, 16, 9, 18, 15, 3, 16, 31, 14, 30, 36, + 26, 9, 16, 24, 11, 9, 16, 31, 39, 5, 16, 31, 25, 5, 16, 23, 25, 5, + 9, 26, 16, 37, 28, 5, 6, 29, 3, 16, 24, 19, 29, 3, 16, 31, 39, 5, + 16, 31, 27, 29, 16, 2, 17, 4, 5, 31, 25, 5, 27, 5, 16, 17, 27, 32, + 26, 9, 16, 24, 25, 5, 14, 16, 19, 9, 16, 10, 14, 27, 17, 28, 5, 7, + 28, 5, 9, 16, 2, 3, 27, 14, 10, 26, 3, 16, 12, 18, 6, 3, 16, 7, + 11, 14, 10, 11, 29, 26, 9, 16, 17, 30, 29, 16, 18, 15, 16, 31, 27, 29, + 16, 4, 5, 23, 26, 9, 16, 33, 28, 5, 16, 31, 42, 15, 12, 13, 25, 5, + 6, 26, 9, 16, 2, 18, 9, 24, 11, 29, 3, 18, 10, 27, 3, 28, 5, 14, + 16, 40, 27, 5, 16, 31, 27, 9, 16, 36, 27, 9, 29, 3, 16, 31, 4, 5, + 16, 31, 25, 5, 14, 16, 31, 27, 9, 16, 24, 27, 29, 16, 21, 29, 33, 39, + 5, 32, 26, 9, 16, 35, 30, 17, 16, 31, 14, 30, 36, 26, 9, 16, 10, 11, + 6, 16, 2, 3, 27, 14, 10, 26, 3, 16, 12, 18, 6, 3, 16, 37, 26, 17, + 16, 2, 18, 9, 24, 11, 29, 3, 18, 10, 27, 3, 28, 5, 14, 16, 37, 26, + 17, 16, 2, 17, 4, 5, 31, 25, 5, 27, 5, 16, 31, 27, 29, 16, 3, 19, + 6, 16, 10, 26, 37, 43, 14, 3, 16, 31, 27, 5, 16, 9, 18, 15, 3, 16, + 37, 18, 9, 16, 2, 18, 9, 24, 11, 29, 3, 18, 10, 27, 3, 28, 5, 14, + 16, 37, 39, 5, 16, 18, 17, 14, 16, 17, 27, 5, 6, 16, 13, 27, 29, 25, + 5, 14, 3, 16, 31, 27, 29, 16, 10, 11, 6, 16, 2, 17, 4, 5, 31, 25, + 5, 27, 5, 16, 37, 11, 9, 12, 26, 9, 16, 13, 6, 30, 29, 16, 36, 6, + 19, 9, 16, 10, 18, 34, 16, 27, 6, 19, 9, 16, 18, 9, 16, 31, 25, 5, + 16, 2, 18, 9, 24, 11, 29, 3, 18, 10, 27, 3, 28, 5, 14, 16, 2, 3, + 27, 14, 10, 26, 3, 16, 12, 18, 6, 3, 16, 18, 9, 16, 31, 25, 5, 16, + 2, 17, 4, 5, 31, 25, 5, 27])} a s WORD_BOUNDARY ɪ ç WORD_BOUNDARY d a ː WORD_BOUNDARY v a s WORD_BOUNDARY b aʊ ə n WORD_BOUNDARY k a n WORD_BOUNDARY UTT_BOUNDARY ɪ n v ɛ s t ɪ ɡ a t o ː ɐ WORD_BOUNDARY z o ː WORD_BOUNDARY j ɛ ts t WORD_BOUNDARY d i ː WORD_BOUNDARY b i ː n ə WORD_BOUNDARY h o ː l ə n WORD_BOUNDARY UTT_BOUNDARY ɪ n v ɛ s t ɪ ɡ a t o ː ɐ WORD_BOUNDARY d a n WORD_BOUNDARY k a n WORD_BOUNDARY m a n WORD_BOUNDARY ɛ s WORD_BOUNDARY a ː b ɐ WORD_BOUNDARY n ɪ ç t WORD_BOUNDARY d ɐ ʊ k ə n WORD_BOUNDARY v ɛ n WORD_BOUNDARY d u ː WORD_BOUNDARY d i ː WORD_BOUNDARY b i ː n ə WORD_BOUNDARY h o ː l s t WORD_BOUNDARY v aɪ s t WORD_BOUNDARY d u ː WORD_BOUNDARY d a s WORD_BOUNDARY UTT_BOUNDARY m e ː d i ː a ː WORD_BOUNDARY m a x ə n WORD_BOUNDARY v i ː ɐ WORD_BOUNDARY aɪ n WORD_BOUNDARY ɡ ɐ a m o ː f o ː n WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY f ɛ ɐ ɡ ɛ s ə n WORD_BOUNDARY m ʊ s WORD_BOUNDARY ɪ ç WORD_BOUNDARY d a s WORD_BOUNDARY e ː b ə n WORD_BOUNDARY z o ː WORD_BOUNDARY d ʊɐ ç ʃ p i ː l ə n WORD_BOUNDARY UTT_BOUNDARY ɪ n v ɛ s t ɪ ɡ a t o ː ɐ WORD_BOUNDARY j a ː WORD_BOUNDARY d a n WORD_BOUNDARY k a n s t WORD_BOUNDARY d e ː WORD_BOUNDARY d i ː ɐ WORD_BOUNDARY d a n WORD_BOUNDARY v a s WORD_BOUNDARY aʊ s z u ː x ə n WORD_BOUNDARY ts ʊ m WORD_BOUNDARY d ɐ ʊ k ə n WORD_BOUNDARY ɡ ɛ l WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY h ə m WORD_BOUNDARY UTT_BOUNDARY ɪ n v ɛ s t ɪ ɡ a t o ː ɐ WORD_BOUNDARY h ə m WORD_BOUNDARY UTT_BOUNDARY m e ː d i ː a ː WORD_BOUNDARY d a s WORD_BOUNDARY t aɪ l WORD_BOUNDARY ɡ ə h œ ɐ t WORD_BOUNDARY d a ː WORD_BOUNDARY n ɪ ç t WORD_BOUNDARY h ɪ n WORD_BOUNDARY UTT_BOUNDARY ɪ n v ɛ s t ɪ ɡ a t o ː ɐ WORD_BOUNDARY h u ː WORD_BOUNDARY ɪ m ɐ WORD_BOUNDARY m a ː l WORD_BOUNDARY p a s i ː ɐ t WORD_BOUNDARY d a s WORD_BOUNDARY ɡ ɛ l WORD_BOUNDARY UTT_BOUNDARY m e ː d i ː a ː WORD_BOUNDARY h ɛ n ʃ ə n WORD_BOUNDARY p l ʊ s WORD_BOUNDARY k l aɪ n WORD_BOUNDARY ɡ ɪ ŋ WORD_BOUNDARY a l aɪ n WORD_BOUNDARY ɪ n WORD_BOUNDARY d i ː WORD_BOUNDARY UTT_BOUNDARY ɪ n v ɛ s t ɪ ɡ a t o ː ɐ WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY ɪ n WORD_BOUNDARY d i ː WORD_BOUNDARY UTT_BOUNDARY m e ː d i ː a +07/16/2026 17:35:14 - INFO - __main__ - [RANK 0] Sample 3278 of the training set: {'input_ids': tensor([31, 26, 9, 26, 9, 16, 10, 11, 6, 31, 26, 16, 19, 9, 26, 16, 27, 9, + 35, 27, 5, 6, 16, 31, 11, 14, 33, 11, 6, 23, 26, 9, 16, 12, 19, 9, + 26, 16, 24, 25, 5, 31, 14, 16, 31, 25, 5, 16, 11, 14, 16, 31, 4, 5, + 17, 16, 11, 14, 17, 8, 14, 31, 26, 3, 26, 9, 16, 18, 9, 16, 11, 3, + 26, 6, 23, 14, 41, 36, 16, 21, 29, 10, 26, 37, 11, 9, 31, 18, 15, 3, + 16, 37, 27, 3, 26, 16, 33, 28, 5, 16, 7, 11, 14, 31, 18, 15, 3, 26, + 3, 26, 16, 33, 18, 15, 16, 31, 11, 14, 16, 7, 11, 14, 31, 27, 32, 3, + 16, 10, 4, 5, 10, 26, 9, 16, 31, 25, 5, 16, 23, 19, 31, 26, 9, 16, + 18, 17, 14, 16, 17, 4, 5, 14, 16, 18, 17, 14, 16, 17, 4, 5, 14, 16, + 7, 11, 14, 24, 18, 36, 26, 6, 3, 26, 9, 16, 33, 25, 5, 16, 33, 18, + 15, 16, 18, 9, 16, 24, 25, 5, 31, 14, 12, 13, 14, 41, 15, 26, 16, 9, + 27, 5, 32, 16, 17, 4, 5, 20, 26, 20, 26, 9, 16, 24, 8, 32, 26, 9, + 16, 6, 22, 5, 10, 26, 9, 37, 27, 7, 3, 14, 16, 27, 13, 14, 4, 5, + 31, 26, 16, 10, 27, 5, 23, 26, 9, 16, 33, 25, 5, 16, 11, 9, 3, 6, + 18, 15, 16, 31, 27, 29, 16, 7, 11, 14, 23, 14, 11, 15, 26, 9, 16, 35, + 39, 5, 16, 30, 9, 3, 16, 6, 4, 5, 36, 3, 26, 9, 16, 19, 9, 16, + 7, 8, 6, 12, 3, 11, 9, 31, 18, 10, 26, 29, 16, 10, 26, 12, 3, 11, + 9, 31, 9, 18, 29, 16, 27, 13, 16, 21, 32, 16, 23, 26, 36, 27, 9, 3, + 26, 9, 16, 33, 25, 5, 16, 31, 25, 5, 16, 3, 27, 5, 3, 16, 17, 18, + 3, 16, 7, 28, 5, 14, 23, 26, 31, 27, 32, 3, 16, 23, 26, 10, 27, 34, + 26, 9, 16, 35, 39, 5, 16, 37, 27, 5, 23, 26, 9, 16, 35, 24, 27, 5, + 14, 16, 24, 11, 5, 20, 26, 16, 11, 29, 16, 25, 5, 9, 26, 9, 16, 6, + 25, 5, 23, 14, 16, 10, 26, 24, 4, 5, 33, 26, 9, 16, 33, 27, 5, 10, + 3, 26, 9, 16, 33, 25, 5, 16, 24, 11, 9, 16, 33, 25, 5, 16, 31, 27, + 29, 16, 10, 11, 6, 3, 16, 28, 5, 9, 26, 16, 17, 8, 14, 3, 16, 37, + 11, 3, 26, 9, 16, 27, 9, 16, 33, 18, 15, 16, 23, 14, 18, 34, 26, 9, + 16, 36, 43, 9, 26, 9, 16, 31, 8, 32, 16, 37, 11, 3, 26, 9, 16, 33, + 25, 5, 16, 33, 18, 15, 16, 17, 18, 3, 16, 24, 27, 7, 26, 9, 16, 7, + 11, 14, 33, 4, 5, 26, 9, 16])} d ə n ə n WORD_BOUNDARY ɡ ɛ l d ə WORD_BOUNDARY aɪ n ə WORD_BOUNDARY a n ts a ː l WORD_BOUNDARY d ɛ ɐ z ɛ l b ə n WORD_BOUNDARY ʃ aɪ n ə WORD_BOUNDARY v i ː d ɐ WORD_BOUNDARY d i ː WORD_BOUNDARY ɛ ɐ WORD_BOUNDARY d e ː m WORD_BOUNDARY ɛ ɐ m ɔ ɐ d ə t ə n WORD_BOUNDARY ɪ n WORD_BOUNDARY ɛ t ə l b ɐ ʏ k WORD_BOUNDARY aʊ s ɡ ə h ɛ n d ɪ ç t WORD_BOUNDARY h a t ə WORD_BOUNDARY z o ː WORD_BOUNDARY f ɛ ɐ d ɪ ç t ə t ə WORD_BOUNDARY z ɪ ç WORD_BOUNDARY d ɛ ɐ WORD_BOUNDARY f ɛ ɐ d a x t WORD_BOUNDARY ɡ e ː ɡ ə n WORD_BOUNDARY d i ː WORD_BOUNDARY b aɪ d ə n WORD_BOUNDARY ɪ m ɐ WORD_BOUNDARY m e ː ɐ WORD_BOUNDARY ɪ m ɐ WORD_BOUNDARY m e ː ɐ WORD_BOUNDARY f ɛ ɐ v ɪ k ə l t ə n WORD_BOUNDARY z i ː WORD_BOUNDARY z ɪ ç WORD_BOUNDARY ɪ n WORD_BOUNDARY v i ː d ɐ ʃ p ɐ ʏ ç ə WORD_BOUNDARY n a ː x WORD_BOUNDARY m e ː ʀ ə ʀ ə n WORD_BOUNDARY v ɔ x ə n WORD_BOUNDARY l y ː ɡ ə n h a f t ɐ WORD_BOUNDARY a p ɐ e ː d ə WORD_BOUNDARY ɡ a ː b ə n WORD_BOUNDARY z i ː WORD_BOUNDARY ɛ n t l ɪ ç WORD_BOUNDARY d a s WORD_BOUNDARY f ɛ ɐ b ɐ ɛ ç ə n WORD_BOUNDARY ts u ː WORD_BOUNDARY ʊ n t WORD_BOUNDARY l e ː k t ə n WORD_BOUNDARY aɪ n WORD_BOUNDARY f ɔ l ʃ t ɛ n d ɪ ɡ ə s WORD_BOUNDARY ɡ ə ʃ t ɛ n d n ɪ s WORD_BOUNDARY a p WORD_BOUNDARY aʊ x WORD_BOUNDARY b ə k a n t ə n WORD_BOUNDARY z i ː WORD_BOUNDARY d i ː WORD_BOUNDARY t a ː t WORD_BOUNDARY m ɪ t WORD_BOUNDARY f o ː ɐ b ə d a x t WORD_BOUNDARY b ə ɡ a ŋ ə n WORD_BOUNDARY ts u ː WORD_BOUNDARY h a ː b ə n WORD_BOUNDARY ts v a ː ɐ WORD_BOUNDARY v ɛ ː ʀ ə WORD_BOUNDARY ɛ s WORD_BOUNDARY i ː n ə n WORD_BOUNDARY l i ː b ɐ WORD_BOUNDARY ɡ ə v e ː z ə n WORD_BOUNDARY z a ː ɡ t ə n WORD_BOUNDARY z i ː WORD_BOUNDARY v ɛ n WORD_BOUNDARY z i ː WORD_BOUNDARY d a s WORD_BOUNDARY ɡ ɛ l t WORD_BOUNDARY o ː n ə WORD_BOUNDARY m ɔ ɐ t WORD_BOUNDARY h ɛ t ə n WORD_BOUNDARY a n WORD_BOUNDARY z ɪ ç WORD_BOUNDARY b ɐ ɪ ŋ ə n WORD_BOUNDARY k œ n ə n WORD_BOUNDARY d ɔ x WORD_BOUNDARY h ɛ t ə n WORD_BOUNDARY z i ː WORD_BOUNDARY z ɪ ç WORD_BOUNDARY m ɪ t WORD_BOUNDARY v a f ə n WORD_BOUNDARY f ɛ ɐ z e ː ə n WORD_BOUNDARY +07/16/2026 17:35:17 - INFO - __main__ - [RANK 0] ***** Running training ***** +07/16/2026 17:35:17 - INFO - __main__ - [RANK 0] Num examples = 101723 +07/16/2026 17:35:17 - INFO - __main__ - [RANK 0] Num Training Steps = 24000 +07/16/2026 17:35:17 - INFO - __main__ - [RANK 0] Instantaneous batch size per device = 32 +07/16/2026 17:35:17 - INFO - __main__ - [RANK 0] Total train batch size (w. parallel, distributed & accumulation) = 32 +07/16/2026 17:35:17 - INFO - __main__ - [RANK 0] Gradient Accumulation steps = 1 +07/16/2026 17:35:17 - INFO - __main__ - [RANK 0] Total optimization steps = 24000 +07/16/2026 17:35:17 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/16/2026 17:35:17 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/16/2026 17:35:20 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/16/2026 17:35:20 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/16/2026 17:35:20 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/16/2026 17:35:25 - INFO - __main__ - [RANK 0] Starting epoch 0... +07/16/2026 17:35:25 - WARNING - transformers.modeling_utils - `loss_type=None` was set in the config but it is unrecognized. Using the default loss: `ForCausalLMLoss`. +07/16/2026 17:42:01 - INFO - __main__ - [RANK 0] perplexity: 4.793493847832823 eval_loss: 1.5672595500946045 accuracy: 0.5374 +07/16/2026 17:48:39 - INFO - __main__ - [RANK 0] perplexity: 3.3856525870382113 eval_loss: 1.2195466756820679 accuracy: 0.6348 +07/16/2026 17:52:55 - INFO - __main__ - [RANK 0] Starting epoch 1... +07/16/2026 17:55:17 - INFO - __main__ - [RANK 0] perplexity: 3.0348656168706087 eval_loss: 1.110167145729065 accuracy: 0.6665 +07/16/2026 18:01:55 - INFO - __main__ - [RANK 0] perplexity: 2.8728437174130272 eval_loss: 1.055302381515503 accuracy: 0.6813 +07/16/2026 18:08:33 - INFO - __main__ - [RANK 0] perplexity: 2.775903239082145 eval_loss: 1.020976185798645 accuracy: 0.6910 +07/16/2026 18:10:31 - INFO - __main__ - [RANK 0] Starting epoch 2... +07/16/2026 18:15:11 - INFO - __main__ - [RANK 0] perplexity: 2.7108729459960044 eval_loss: 0.9972707033157349 accuracy: 0.6986 +07/16/2026 18:19:34 - INFO - __main__ - [RANK 0] Starting epoch 3... +07/16/2026 18:19:34 - INFO - __main__ - [RANK 0] Switching language at step 8000 +07/16/2026 18:19:48 - INFO - __main__ - [RANK 0] Sample 194393 of the training set: {'input_ids': tensor([53, 45, 33, 16, 13, 26, 33, 11, 29, 3, 16, 23, 19, 16, 45, 9, 16, 68, + 24, 26, 55, 17, 66, 29, 3, 26, 55, 55, 18, 34, 16, 29, 11, 9, 29, 16, + 45, 24, 16, 31, 39, 5, 3, 25, 16, 3, 26, 16, 37, 69, 5, 16, 7, 67, + 56, 26, 55, 55, 16, 66, 3, 16, 6, 66, 29, 3, 16, 53, 11, 9, 16, 12, + 25, 5, 16, 53, 45, 33, 16, 9, 18, 55, 6, 25, 16, 29, 11, 24, 26, 9, + 3, 25, 5, 9, 16, 23, 45, 3, 16, 18, 9, 16, 6, 30, 36, 29, 16, 66, + 9, 31, 16, 17, 19, 9, 31, 16, 68, 6, 31, 26, 55, 16, 56, 45, 9, 16, + 37, 69, 5, 16, 40, 18, 55, 33, 16, 12, 25, 5, 16, 58, 55, 11, 3, 26, + 9, 31, 16, 3, 26, 16, 29, 11, 3, 16, 7, 8, 55, 58, 16, 53, 18, 56, + 21, 3, 16, 45, 29, 18, 29, 3, 26, 9, 29, 16, 18, 7, 16, 37, 69, 5, + 55, 16, 66, 9, 3, 16, 55, 18, 7, 40, 39, 5, 33, 31, 16, 37, 69, 5, + 55, 16, 45, 29, 11, 9, 3, 16, 66, 9, 31, 16, 37, 11, 6, 13, 16, 37, + 66, 24, 18, 34, 16, 9, 68, 16, 8, 6, 3, 69, 5, 9, 26, 3, 18, 24, + 16, 56, 26, 16, 13, 30, 55, 16, 6, 50, 31, 25, 16, 40, 25, 5, 6, 31, + 18, 31, 16, 68, 9, 6, 25, 16, 29, 3, 18, 13, 40, 30, 6, 50, 3, 18, + 34, 16, 56, 66, 3, 16, 10, 66, 23, 55, 25, 5, 6, 33, 16, 68, 6, 31, + 16, 9, 69, 5, 29, 16, 12, 30, 31, 16, 45, 36, 45, 17, 13, 26, 9, 25, + 16, 37, 69, 5, 16, 7, 8, 55, 16, 29, 45, 17, 16, 17, 45, 9, 58, 29, + 16, 56, 50, 16, 24, 50, 9, 6, 25, 16, 3, 55, 19, 31, 16, 3, 26, 16, + 10, 11, 3, 16, 13, 26, 55, 17, 18, 12, 26, 9, 16, 7, 55, 45, 17, 16, + 29, 26, 9, 3, 16, 13, 25, 5, 3, 26, 55, 33, 23, 69, 5, 10, 16, 7, + 26, 55, 56, 26, 16, 10, 69, 5, 6, 16, 3, 26, 16, 48, 62, 9, 16, 37, + 69, 5, 16, 7, 67, 56, 26, 55, 55, 16, 18, 9, 56, 26, 16, 36, 50, 29, + 16, 45, 24, 16, 8, 55, 31, 18, 9, 11, 55, 25, 16, 36, 55, 18, 17, 18, + 9, 26, 6, 33, 16, 9, 68, 16, 31, 18, 7, 18, 36, 45, 6, 3, 25, 16, + 53, 45, 33, 16, 40, 39, 5, 47, 39, 5, 26, 6, 25, 16, 17, 50, 31, 16, + 18, 3, 16, 53, 45, 33, 16, 36, 6, 18, 55, 16, 56, 66, 3, 16, 66, 33, + 16, 37, 66, 13, 26, 9, 33, 16, 29, 68, 16, 8, 7, 26, 9, 16, 18, 9, + 16, 55, 45, 12, 26, 16, 56, 26])} w ʌ z WORD_BOUNDARY p ə z ɛ s t WORD_BOUNDARY b aɪ WORD_BOUNDARY ʌ n WORD_BOUNDARY oʊ v ə ɹ m æ s t ə ɹ ɹ ɪ ŋ WORD_BOUNDARY s ɛ n s WORD_BOUNDARY ʌ v WORD_BOUNDARY d u ː t i WORD_BOUNDARY t ə WORD_BOUNDARY h ɜ ː WORD_BOUNDARY f ɑ ð ə ɹ ɹ WORD_BOUNDARY æ t WORD_BOUNDARY l æ s t WORD_BOUNDARY w ɛ n WORD_BOUNDARY ʃ i ː WORD_BOUNDARY w ʌ z WORD_BOUNDARY n ɪ ɹ l i WORD_BOUNDARY s ɛ v ə n t i ː n WORD_BOUNDARY b ʌ t WORD_BOUNDARY ɪ n WORD_BOUNDARY l ʊ k s WORD_BOUNDARY æ n d WORD_BOUNDARY m aɪ n d WORD_BOUNDARY oʊ l d ə ɹ WORD_BOUNDARY ð ʌ n WORD_BOUNDARY h ɜ ː WORD_BOUNDARY j ɪ ɹ z WORD_BOUNDARY ʃ i ː WORD_BOUNDARY θ ɹ ɛ t ə n d WORD_BOUNDARY t ə WORD_BOUNDARY s ɛ t WORD_BOUNDARY f ɔ ɹ θ WORD_BOUNDARY w ɪ ð aʊ t WORD_BOUNDARY ʌ s ɪ s t ə n s WORD_BOUNDARY ɪ f WORD_BOUNDARY h ɜ ː ɹ WORD_BOUNDARY æ n t WORD_BOUNDARY ɹ ɪ f j u ː z d WORD_BOUNDARY h ɜ ː ɹ WORD_BOUNDARY ʌ s ɛ n t WORD_BOUNDARY æ n d WORD_BOUNDARY h ɛ l p WORD_BOUNDARY h æ v ɪ ŋ WORD_BOUNDARY n oʊ WORD_BOUNDARY ɔ l t ɜ ː n ə t ɪ v WORD_BOUNDARY ð ə WORD_BOUNDARY p ʊ ɹ WORD_BOUNDARY l eɪ d i WORD_BOUNDARY j i ː l d ɪ d WORD_BOUNDARY oʊ n l i WORD_BOUNDARY s t ɪ p j ʊ l eɪ t ɪ ŋ WORD_BOUNDARY ð æ t WORD_BOUNDARY ɡ æ b ɹ i ː l z WORD_BOUNDARY oʊ l d WORD_BOUNDARY n ɜ ː s WORD_BOUNDARY ʃ ʊ d WORD_BOUNDARY ʌ k ʌ m p ə n i WORD_BOUNDARY h ɜ ː WORD_BOUNDARY f ɔ ɹ WORD_BOUNDARY s ʌ m WORD_BOUNDARY m ʌ n θ s WORD_BOUNDARY ð eɪ WORD_BOUNDARY v eɪ n l i WORD_BOUNDARY t ɹ aɪ d WORD_BOUNDARY t ə WORD_BOUNDARY ɡ ɛ t WORD_BOUNDARY p ə ɹ m ɪ ʃ ə n WORD_BOUNDARY f ɹ ʌ m WORD_BOUNDARY s ə n t WORD_BOUNDARY p i ː t ə ɹ z b ɜ ː ɡ WORD_BOUNDARY f ə ɹ ð ə WORD_BOUNDARY ɡ ɜ ː l WORD_BOUNDARY t ə WORD_BOUNDARY d̠ʒ ɔɪ n WORD_BOUNDARY h ɜ ː WORD_BOUNDARY f ɑ ð ə ɹ ɹ WORD_BOUNDARY ɪ n ð ə WORD_BOUNDARY k eɪ s WORD_BOUNDARY ʌ v WORD_BOUNDARY ɔ ɹ d ɪ n ɛ ɹ i WORD_BOUNDARY k ɹ ɪ m ɪ n ə l z WORD_BOUNDARY n oʊ WORD_BOUNDARY d ɪ f ɪ k ʌ l t i WORD_BOUNDARY w ʌ z WORD_BOUNDARY j u ː ʒ u ː ə l i WORD_BOUNDARY m eɪ d WORD_BOUNDARY ɪ t WORD_BOUNDARY w ʌ z WORD_BOUNDARY k l ɪ ɹ WORD_BOUNDARY ð æ t WORD_BOUNDARY æ z WORD_BOUNDARY h æ p ə n z WORD_BOUNDARY s oʊ WORD_BOUNDARY ɔ f ə n WORD_BOUNDARY ɪ n WORD_BOUNDARY ɹ ʌ ʃ ə WORD_BOUNDARY ð ə +07/16/2026 18:19:48 - INFO - __main__ - [RANK 0] Sample 72097 of the training set: {'input_ids': tensor([ 2, 23, 66, 36, 16, 18, 9, 56, 26, 16, 17, 18, 31, 26, 6, 16, 53, 11, + 29, 3, 16, 11, 31, 53, 18, 9, 16, 66, 9, 31, 16, 29, 11, 6, 25, 5, + 9, 26, 16, 37, 45, 31, 23, 18, 9, 16, 17, 21, 9, 3, 18, 9, 33, 16, + 26, 55, 55, 19, 33, 18, 34, 16, 7, 55, 45, 17, 56, 26, 16, 13, 6, 50, + 9, 16, 66, 3, 16, 8, 6, 16, 13, 62, 9, 3, 29, 16, 23, 18, 40, 8, + 9, 31, 16, 10, 55, 11, 9, 18, 46, 16, 56, 50, 16, 53, 69, 5, 16, 29, + 18, 17, 13, 6, 25, 16, 3, 39, 5, 16, 45, 34, 36, 26, 9, 29, 18, 31, + 26, 55, 31, 16, 7, 55, 66, 10, 17, 26, 9, 3, 29, 16, 45, 24, 16, 7, + 8, 55, 26, 9, 16, 29, 45, 23, 29, 3, 26, 9, 29, 16, 2, 56, 26, 16, + 13, 66, 29, 13, 8, 55, 3, 16, 31, 18, 31, 9, 67, 3, 16, 29, 25, 5, + 17, 16, 3, 26, 16, 10, 11, 3, 16, 56, 11, 17, 16, 11, 9, 18, 58, 18, + 34, 16, 53, 19, 6, 16, 23, 25, 5, 18, 34, 16, 53, 8, 36, 3, 16, 26, + 13, 67, 9, 16, 23, 19, 16, 56, 26, 16, 37, 8, 3, 25, 16, 3, 25, 5, + 31, 55, 18, 34, 36, 26, 55, 33, 16, 56, 50, 16, 36, 30, 31, 16, 9, 67, + 3, 16, 36, 6, 50, 17, 16, 56, 26, 16, 13, 55, 26, 3, 11, 36, 12, 26, + 9, 16, 45, 24, 56, 18, 16, 45, 17, 11, 55, 18, 36, 26, 9, 16, 7, 6, + 66, 10, 16, 23, 18, 36, 45, 33, 16, 56, 50, 16, 31, 18, 31, 9, 3, 16, + 29, 25, 5, 16, 56, 26, 16, 29, 3, 67, 55, 25, 16, 23, 66, 9, 26, 55, + 55, 16, 66, 7, 3, 26, 55, 16, 6, 25, 5, 24, 18, 34, 16, 9, 39, 5, + 16, 40, 8, 55, 36, 16, 11, 36, 29, 11, 13, 3, 16, 18, 9, 16, 7, 55, + 45, 9, 3, 16, 26, 24, 26, 16, 7, 50, 36, 16, 8, 36, 12, 26, 9, 16, + 29, 50, 6, 16, 26, 55, 55, 50, 9, 48, 31, 16, 18, 29, 13, 11, 12, 26, + 6, 25, 16, 7, 8, 55, 16, 3, 30, 55, 55, 18, 29, 3, 29, 16, 2, 23, + 19, 16, 56, 26, 16, 3, 19, 17, 16, 56, 50, 16, 7, 21, 9, 31, 16, 56, + 11, 17, 29, 11, 6, 24, 33, 16, 18, 9, 16, 56, 66, 3, 16, 24, 66, 29, + 3, 16, 23, 50, 36, 45, 24, 26, 9, 16, 9, 68, 9, 16, 66, 33, 16, 18, + 9, 31, 25, 26, 16, 56, 50, 16, 53, 69, 5, 16, 23, 18, 9, 45, 17, 31, + 16, 66, 9, 31, 16, 29, 26, 23, 17, 18, 29, 18, 24, 16, 66, 9, 31, 16, + 37, 66, 31, 16, 29, 11, 3, 26])} UTT_BOUNDARY b æ k WORD_BOUNDARY ɪ n ð ə WORD_BOUNDARY m ɪ d ə l WORD_BOUNDARY w ɛ s t WORD_BOUNDARY ɛ d w ɪ n WORD_BOUNDARY æ n d WORD_BOUNDARY s ɛ l i ː n ə WORD_BOUNDARY h ʌ d b ɪ n WORD_BOUNDARY m aʊ n t ɪ n z WORD_BOUNDARY ə ɹ ɹ aɪ z ɪ ŋ WORD_BOUNDARY f ɹ ʌ m ð ə WORD_BOUNDARY p l eɪ n WORD_BOUNDARY æ t WORD_BOUNDARY ɔ l WORD_BOUNDARY p ɔɪ n t s WORD_BOUNDARY b ɪ j ɔ n d WORD_BOUNDARY ɡ ɹ ɛ n ɪ t̠ʃ WORD_BOUNDARY ð eɪ WORD_BOUNDARY w ɜ ː WORD_BOUNDARY s ɪ m p l i WORD_BOUNDARY t u ː WORD_BOUNDARY ʌ ŋ k ə n s ɪ d ə ɹ d WORD_BOUNDARY f ɹ æ ɡ m ə n t s WORD_BOUNDARY ʌ v WORD_BOUNDARY f ɔ ɹ ə n WORD_BOUNDARY s ʌ b s t ə n s WORD_BOUNDARY UTT_BOUNDARY ð ə WORD_BOUNDARY p æ s p ɔ ɹ t WORD_BOUNDARY d ɪ d n ɑ t WORD_BOUNDARY s i ː m WORD_BOUNDARY t ə WORD_BOUNDARY ɡ ɛ t WORD_BOUNDARY ð ɛ m WORD_BOUNDARY ɛ n ɪ θ ɪ ŋ WORD_BOUNDARY w aɪ l WORD_BOUNDARY b i ː ɪ ŋ WORD_BOUNDARY w ɔ k t WORD_BOUNDARY ə p ɑ n WORD_BOUNDARY b aɪ WORD_BOUNDARY ð ə WORD_BOUNDARY h ɔ t i WORD_BOUNDARY t i ː d ɹ ɪ ŋ k ə ɹ z WORD_BOUNDARY ð eɪ WORD_BOUNDARY k ʊ d WORD_BOUNDARY n ɑ t WORD_BOUNDARY k l eɪ m WORD_BOUNDARY ð ə WORD_BOUNDARY p ɹ ə t ɛ k ʃ ə n WORD_BOUNDARY ʌ v ð ɪ WORD_BOUNDARY ʌ m ɛ ɹ ɪ k ə n WORD_BOUNDARY f l æ ɡ WORD_BOUNDARY b ɪ k ʌ z WORD_BOUNDARY ð eɪ WORD_BOUNDARY d ɪ d n t WORD_BOUNDARY s i ː WORD_BOUNDARY ð ə WORD_BOUNDARY s t ɑ ɹ i WORD_BOUNDARY b æ n ə ɹ ɹ WORD_BOUNDARY æ f t ə ɹ WORD_BOUNDARY l i ː v ɪ ŋ WORD_BOUNDARY n u ː WORD_BOUNDARY j ɔ ɹ k WORD_BOUNDARY ɛ k s ɛ p t WORD_BOUNDARY ɪ n WORD_BOUNDARY f ɹ ʌ n t WORD_BOUNDARY ə v ə WORD_BOUNDARY f eɪ k WORD_BOUNDARY ɔ k ʃ ə n WORD_BOUNDARY s eɪ l WORD_BOUNDARY ə ɹ ɹ eɪ n d̠ʒ d WORD_BOUNDARY ɪ s p ɛ ʃ ə l i WORD_BOUNDARY f ɔ ɹ WORD_BOUNDARY t ʊ ɹ ɹ ɪ s t s WORD_BOUNDARY UTT_BOUNDARY b aɪ WORD_BOUNDARY ð ə WORD_BOUNDARY t aɪ m WORD_BOUNDARY ð eɪ WORD_BOUNDARY f aʊ n d WORD_BOUNDARY ð ɛ m s ɛ l v z WORD_BOUNDARY ɪ n WORD_BOUNDARY ð æ t WORD_BOUNDARY v æ s t WORD_BOUNDARY b eɪ k ʌ v ə n WORD_BOUNDARY n oʊ n WORD_BOUNDARY æ z WORD_BOUNDARY ɪ n d i ə WORD_BOUNDARY ð eɪ WORD_BOUNDARY w ɜ ː WORD_BOUNDARY b ɪ n ʌ m d WORD_BOUNDARY æ n d WORD_BOUNDARY s ə b m ɪ s ɪ v WORD_BOUNDARY æ n d WORD_BOUNDARY h æ d WORD_BOUNDARY s ɛ t ə +07/16/2026 18:19:48 - INFO - __main__ - [RANK 0] Sample 64196 of the training set: {'input_ids': tensor([ 6, 29, 3, 26, 16, 31, 27, 5, 16, 31, 42, 15, 16, 2, 18, 9, 24, 11, + 29, 3, 18, 10, 27, 3, 28, 5, 14, 16, 17, 27, 5, 36, 29, 26, 16, 7, + 11, 14, 12, 13, 11, 14, 3, 16, 31, 25, 5, 14, 16, 31, 4, 5, 9, 16, + 24, 4, 5, 36, 16, 2, 3, 27, 14, 10, 26, 3, 16, 12, 18, 6, 3, 16, + 21, 32, 16, 2, 18, 9, 24, 11, 29, 3, 18, 10, 27, 3, 28, 5, 14, 16, + 33, 8, 6, 16, 18, 15, 16, 21, 32, 16, 23, 6, 27, 5, 33, 26, 9, 16, + 2, 3, 27, 14, 10, 26, 3, 16, 12, 18, 6, 3, 16, 21, 32, 16, 2, 18, + 9, 24, 11, 29, 3, 18, 10, 27, 3, 28, 5, 14, 16, 40, 11, 35, 3, 16, + 18, 29, 3, 16, 10, 26, 9, 39, 5, 36, 16, 2, 18, 9, 24, 11, 29, 3, + 18, 10, 27, 3, 28, 5, 14, 16, 40, 11, 35, 3, 16, 18, 29, 3, 16, 10, + 26, 9, 39, 5, 36, 16, 2, 3, 27, 14, 10, 26, 3, 16, 12, 18, 6, 3, + 16, 17, 27, 17, 27, 5, 16, 21, 32, 16, 2, 3, 27, 14, 10, 26, 3, 16, + 12, 18, 6, 3, 16, 17, 27, 5, 17, 16, 2, 17, 8, 3, 14, 16, 36, 8, + 17, 16, 37, 4, 5, 14, 16, 2, 17, 8, 3, 14, 16, 24, 28, 5, 16, 37, + 27, 29, 3, 16, 11, 9, 16, 31, 39, 5, 16, 31, 19, 9, 26, 16, 24, 18, + 9, 31, 26, 6, 16, 37, 11, 34, 26, 9, 16, 2, 17, 8, 3, 14, 16, 36, + 8, 17, 16, 17, 27, 5, 6, 16, 37, 4, 5, 14, 16, 2, 3, 27, 14, 10, + 26, 3, 16, 12, 18, 6, 3, 16, 17, 27, 17, 27, 5, 16, 21, 32, 16, 17, + 27, 17, 27, 5, 16, 21, 32, 16, 17, 27, 17, 27, 5, 16, 21, 32, 16, 17, + 27, 17, 27, 5, 16, 21, 32, 16, 2, 17, 8, 3, 14, 16, 36, 8, 17, 16, + 17, 27, 5, 6, 16, 2, 17, 8, 3, 14, 16, 28, 5, 40, 26, 16, 2, 17, + 8, 3, 14, 16, 31, 39, 5, 16, 37, 27, 29, 3, 16, 7, 8, 14, 9, 26, + 16, 22, 5, 23, 14, 37, 21, 13, 3, 16, 9, 18, 36, 29, 16, 2, 17, 8, + 3, 14, 16, 31, 39, 5, 16, 2, 3, 27, 14, 10, 26, 3, 16, 12, 18, 6, + 3, 16, 17, 27, 17, 27, 5, 16, 21, 32, 16, 17, 27, 17, 27, 5, 16, 21, + 32, 16, 2, 18, 9, 24, 11, 29, 3, 18, 10, 27, 3, 28, 5, 14, 16, 31, + 27, 29, 16, 10, 4, 5, 3, 16, 18, 9, 16, 23, 41, 14, 10, 14, 6, 18, + 15, 26, 9, 16, 7, 11, 14, 37, 11, 6, 3, 9, 18, 29, 26, 9, 16, 9, + 18, 15, 3, 16, 36, 11, 14, 29])} l s t ə WORD_BOUNDARY d a ː WORD_BOUNDARY d ʊɐ ç WORD_BOUNDARY UTT_BOUNDARY ɪ n v ɛ s t ɪ ɡ a t o ː ɐ WORD_BOUNDARY m a ː k s ə WORD_BOUNDARY f ɛ ɐ ʃ p ɛ ɐ t WORD_BOUNDARY d i ː ɐ WORD_BOUNDARY d e ː n WORD_BOUNDARY v e ː k WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY aʊ x WORD_BOUNDARY UTT_BOUNDARY ɪ n v ɛ s t ɪ ɡ a t o ː ɐ WORD_BOUNDARY z ɔ l WORD_BOUNDARY ɪ ç WORD_BOUNDARY aʊ x WORD_BOUNDARY b l a ː z ə n WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY aʊ x WORD_BOUNDARY UTT_BOUNDARY ɪ n v ɛ s t ɪ ɡ a t o ː ɐ WORD_BOUNDARY j ɛ ts t WORD_BOUNDARY ɪ s t WORD_BOUNDARY ɡ ə n u ː k WORD_BOUNDARY UTT_BOUNDARY ɪ n v ɛ s t ɪ ɡ a t o ː ɐ WORD_BOUNDARY j ɛ ts t WORD_BOUNDARY ɪ s t WORD_BOUNDARY ɡ ə n u ː k WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY m a m a ː WORD_BOUNDARY aʊ x WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY m a ː m WORD_BOUNDARY UTT_BOUNDARY m ɔ t ɐ WORD_BOUNDARY k ɔ m WORD_BOUNDARY h e ː ɐ WORD_BOUNDARY UTT_BOUNDARY m ɔ t ɐ WORD_BOUNDARY v o ː WORD_BOUNDARY h a s t WORD_BOUNDARY ɛ n WORD_BOUNDARY d u ː WORD_BOUNDARY d aɪ n ə WORD_BOUNDARY v ɪ n d ə l WORD_BOUNDARY h ɛ ŋ ə n WORD_BOUNDARY UTT_BOUNDARY m ɔ t ɐ WORD_BOUNDARY k ɔ m WORD_BOUNDARY m a ː l WORD_BOUNDARY h e ː ɐ WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY m a m a ː WORD_BOUNDARY aʊ x WORD_BOUNDARY m a m a ː WORD_BOUNDARY aʊ x WORD_BOUNDARY m a m a ː WORD_BOUNDARY aʊ x WORD_BOUNDARY m a m a ː WORD_BOUNDARY aʊ x WORD_BOUNDARY UTT_BOUNDARY m ɔ t ɐ WORD_BOUNDARY k ɔ m WORD_BOUNDARY m a ː l WORD_BOUNDARY UTT_BOUNDARY m ɔ t ɐ WORD_BOUNDARY o ː j ə WORD_BOUNDARY UTT_BOUNDARY m ɔ t ɐ WORD_BOUNDARY d u ː WORD_BOUNDARY h a s t WORD_BOUNDARY f ɔ ɐ n ə WORD_BOUNDARY y ː b ɐ h aʊ p t WORD_BOUNDARY n ɪ k s WORD_BOUNDARY UTT_BOUNDARY m ɔ t ɐ WORD_BOUNDARY d u ː WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY m a m a ː WORD_BOUNDARY aʊ x WORD_BOUNDARY m a m a ː WORD_BOUNDARY aʊ x WORD_BOUNDARY UTT_BOUNDARY ɪ n v ɛ s t ɪ ɡ a t o ː ɐ WORD_BOUNDARY d a s WORD_BOUNDARY ɡ e ː t WORD_BOUNDARY ɪ n WORD_BOUNDARY b ʏ ɐ ɡ ɐ l ɪ ç ə n WORD_BOUNDARY f ɛ ɐ h ɛ l t n ɪ s ə n WORD_BOUNDARY n ɪ ç t WORD_BOUNDARY k ɛ ɐ s +07/16/2026 18:22:06 - INFO - __main__ - [RANK 0] perplexity: 3.0778281001221512 eval_loss: 1.1242241859436035 accuracy: 0.6677 +07/16/2026 18:28:47 - INFO - __main__ - [RANK 0] perplexity: 2.8814398696148675 eval_loss: 1.0582901239395142 accuracy: 0.6849 +07/16/2026 18:35:29 - INFO - __main__ - [RANK 0] perplexity: 2.811072931136782 eval_loss: 1.0335662364959717 accuracy: 0.6909 +07/16/2026 18:42:10 - INFO - __main__ - [RANK 0] perplexity: 2.763958548749346 eval_loss: 1.016663908958435 accuracy: 0.6955 +07/16/2026 18:48:51 - INFO - __main__ - [RANK 0] perplexity: 2.728964800296618 eval_loss: 1.0039223432540894 accuracy: 0.6991 +07/16/2026 18:55:12 - INFO - __main__ - [RANK 0] Starting epoch 4... +07/16/2026 18:55:32 - INFO - __main__ - [RANK 0] perplexity: 2.702445585589725 eval_loss: 0.9941571354866028 accuracy: 0.7021 +07/16/2026 19:02:13 - INFO - __main__ - [RANK 0] perplexity: 2.6785040812495846 eval_loss: 0.9852584600448608 accuracy: 0.7047 +07/16/2026 19:08:54 - INFO - __main__ - [RANK 0] perplexity: 2.6594497862205335 eval_loss: 0.9781192541122437 accuracy: 0.7069 +07/16/2026 19:15:36 - INFO - __main__ - [RANK 0] perplexity: 2.6409949022220234 eval_loss: 0.9711557030677795 accuracy: 0.7087 +07/16/2026 19:22:17 - INFO - __main__ - [RANK 0] perplexity: 2.6258159753673436 eval_loss: 0.9653916954994202 accuracy: 0.7103 +07/16/2026 19:28:59 - INFO - __main__ - [RANK 0] perplexity: 2.6094926043144344 eval_loss: 0.959155797958374 accuracy: 0.7121 +07/16/2026 19:30:42 - INFO - __main__ - [RANK 0] Starting epoch 5... +07/16/2026 19:35:40 - INFO - __main__ - [RANK 0] perplexity: 2.597885351525268 eval_loss: 0.9546977877616882 accuracy: 0.7133 +07/16/2026 19:42:21 - INFO - __main__ - [RANK 0] perplexity: 2.591033485214764 eval_loss: 0.9520568251609802 accuracy: 0.7143 +07/16/2026 19:49:03 - INFO - __main__ - [RANK 0] perplexity: 2.5850864410927508 eval_loss: 0.9497589468955994 accuracy: 0.7150 +07/17/2026 15:54:42 - INFO - __main__ - [RANK 0] Distributed environment: DistributedType.NO +Num processes: 1 +Process index: 0 +Local process index: 0 +Device: cuda + +Mixed precision type: no + +07/17/2026 15:54:42 - INFO - __main__ - [RANK 0] Training args: +{'data_dir': 'data/phone/deu-eng/tokenized_sequential_interleaved', 'model_name_or_path': None, 'config_name': 'gpt_base_config.json', 'tokenizer_name': 'nikitastheo/phonelm-phone-deu-eng-tokenizer', 'per_device_train_batch_size': 32, 'per_device_eval_batch_size': 32, 'learning_rate': 0.0001, 'weight_decay': 0.01, 'max_steps': 30000, 'gradient_accumulation_steps': 1, 'lr_scheduler_type': , 'warmup_steps': 3000, 'output_dir': 'models/phonebabylm-phone-deu-eng-sequential_interleaved', 'seed': 42, 'model_type': None, 'block_size': None, 'push_to_hub': True, 'hub_model_id': 'nikitastheo/phonebabylm-phone-deu-eng-sequential_interleaved', 'hub_token': None, 'private': False, 'trust_remote_code': False, 'with_tracking': True, 'tracking_dir': None, 'report_to': 'wandb', 'restart_lr_at_switch': False, 'language_switch_steps': 10000, 'model_revision': 'main', 'config_overrides': None, 'use_slow_tokenizer': False, 'override_vocab_size': None, 'dtype': None, 'logging_steps': 10, 'cache_dir': None, 'eval_steps': 750, 'max_grad_norm': 1.0, 'adam_epsilon': 1e-06, 'run_name': 'phonebabylm-phone-deu-eng-sequential_interleaved', 'max_train_samples': None, 'max_eval_samples': None} +07/17/2026 15:54:42 - INFO - transformers.configuration_utils - loading configuration file gpt_base_config.json +07/17/2026 15:54:42 - INFO - transformers.configuration_utils - Model config GPT2Config { + "activation_function": "gelu", + "architectures": [ + "GPT2LMHeadModel" + ], + "attn_pdrop": 0.1, + "bos_token_id": 50256, + "embd_pdrop": 0.1, + "eos_token_id": 50256, + "initializer_range": 0.02, + "layer_norm_epsilon": 1e-05, + "model_type": "gpt2", + "n_ctx": 512, + "n_embd": 768, + "n_head": 12, + "n_inner": 3072, + "n_layer": 12, + "n_positions": 512, + "reorder_and_upcast_attn": false, + "resid_pdrop": 0.1, + "scale_attn_by_inverse_layer_idx": false, + "scale_attn_weights": true, + "summary_activation": null, + "summary_first_dropout": 0.1, + "summary_proj_to_labels": true, + "summary_type": "cls_index", + "summary_use_proj": true, + "transformers_version": "4.57.6", + "use_cache": true, + "vocab_size": 1 +} + +07/17/2026 15:54:42 - INFO - transformers.tokenization_utils_base - loading file vocab.json from cache at /home/geofila/.cache/huggingface/hub/models--nikitastheo--phonelm-phone-deu-eng-tokenizer/snapshots/a9eb582bae549d2e78f9e6f7913343e704373e8a/vocab.json +07/17/2026 15:54:42 - INFO - transformers.tokenization_utils_base - loading file merges.txt from cache at None +07/17/2026 15:54:42 - INFO - transformers.tokenization_utils_base - loading file tokenizer.json from cache at /home/geofila/.cache/huggingface/hub/models--nikitastheo--phonelm-phone-deu-eng-tokenizer/snapshots/a9eb582bae549d2e78f9e6f7913343e704373e8a/tokenizer.json +07/17/2026 15:54:42 - INFO - transformers.tokenization_utils_base - loading file added_tokens.json from cache at None +07/17/2026 15:54:42 - INFO - transformers.tokenization_utils_base - loading file special_tokens_map.json from cache at /home/geofila/.cache/huggingface/hub/models--nikitastheo--phonelm-phone-deu-eng-tokenizer/snapshots/a9eb582bae549d2e78f9e6f7913343e704373e8a/special_tokens_map.json +07/17/2026 15:54:42 - INFO - transformers.tokenization_utils_base - loading file tokenizer_config.json from cache at /home/geofila/.cache/huggingface/hub/models--nikitastheo--phonelm-phone-deu-eng-tokenizer/snapshots/a9eb582bae549d2e78f9e6f7913343e704373e8a/tokenizer_config.json +07/17/2026 15:54:42 - INFO - transformers.tokenization_utils_base - loading file chat_template.jinja from cache at None +07/17/2026 15:54:44 - INFO - transformers.generation.configuration_utils - Generate config GenerationConfig { + "bos_token_id": 2, + "eos_token_id": 2, + "pad_token_id": 1 +} + +07/17/2026 15:54:46 - INFO - __main__ - [RANK 0] Model parameters: 85.50M total (85.50M trainable) +07/17/2026 15:54:55 - INFO - __main__ - [RANK 0] Sample 83810 of the training set: {'input_ids': tensor([ 9, 6, 34, 32, 39, 6, 21, 14, 15, 6, 8, 17, 13, 4, 15, 5, 6, 13, + 18, 39, 17, 18, 23, 11, 5, 6, 8, 9, 33, 8, 15, 12, 17, 6, 2, 13, + 30, 15, 9, 6, 13, 4, 15, 36, 11, 16, 6, 20, 18, 17, 6, 3, 22, 15, + 20, 6, 43, 4, 36, 9, 6, 7, 8, 9, 4, 35, 9, 22, 15, 3, 11, 17, + 6, 2, 21, 30, 15, 6, 21, 18, 16, 17, 6, 16, 18, 39, 17, 6, 23, 11, + 10, 34, 20, 11, 16, 6, 2, 13, 4, 5, 6, 18, 5, 17, 6, 24, 30, 15, + 9, 6, 12, 14, 15, 5, 6, 2, 31, 4, 9, 12, 11, 5, 6, 21, 34, 12, + 6, 8, 9, 20, 34, 9, 3, 11, 17, 6, 13, 34, 9, 3, 11, 16, 6, 21, + 19, 16, 6, 2, 12, 27, 15, 18, 5, 11, 6, 3, 30, 15, 21, 9, 6, 21, + 34, 12, 3, 4, 15, 17, 6, 18, 5, 17, 6, 16, 18, 39, 17, 6, 26, 19, + 6, 10, 12, 4, 15, 36, 11, 20, 6, 7, 8, 9, 31, 17, 4, 16, 17, 6, + 2, 30, 15, 9, 6, 21, 14, 15, 16, 6, 13, 44, 3, 11, 6, 16, 18, 39, + 17, 6, 8, 9, 20, 34, 9, 3, 11, 17, 6, 8, 9, 6, 18, 5, 17, 6, + 7, 8, 9, 20, 18, 5, 17, 6, 2, 3, 4, 5, 6, 31, 17, 22, 15, 17, + 6, 18, 16, 6, 4, 12, 11, 16, 6, 26, 11, 36, 18, 39, 17, 11, 16, 6, + 2, 25, 23, 29, 5, 17, 6, 24, 34, 12, 17, 6, 24, 25, 35, 17, 20, 4, + 16, 6, 26, 11, 25, 12, 30, 15, 27, 15, 6, 2, 13, 4, 5, 6, 21, 34, + 12, 6, 3, 8, 9, 6, 24, 25, 35, 17, 20, 4, 16, 6, 3, 4, 15, 20, + 18, 17, 6, 4, 16, 7, 4, 37, 11, 16, 6, 2, 31, 18, 10, 11, 16, 6, + 21, 30, 15, 6, 30, 15, 16, 6, 26, 8, 5, 9, 6, 23, 12, 19, 39, 6, + 7, 34, 9, 17, 6, 2, 22, 15, 9, 5, 17, 6, 21, 34, 12, 6, 7, 30, + 15, 12, 18, 35, 11, 6, 30, 15, 16, 6, 4, 16, 24, 41, 15, 36, 11, 16, + 6, 2, 10, 19, 16, 11, 6, 21, 34, 9, 23, 11, 6, 13, 30, 15, 9, 6, + 13, 8, 9, 3, 11, 16, 6, 30, 15, 16, 6, 31, 14, 15, 16, 6, 12, 14, + 15, 5, 6, 2, 21, 8, 12, 17, 21, 4, 15, 20, 11, 6, 23, 11, 31, 18, + 39, 17, 11, 6, 21, 34, 12, 3, 4, 15, 17, 6, 2, 13, 22, 15, 9, 6, + 24, 8, 17, 11, 6, 31, 4, 9, 12, 11, 5, 6, 17, 41, 15, 17, 11, 16, + 6, 13, 34, 12, 11, 16, 6, 2, 29, 16, 17, 6, 13, 4, 15, 36, 29, 20, + 6, 24, 4, 15, 26, 11, 16, 6])} ɐ WORD_BOUNDARY ɔ ʏ ç WORD_BOUNDARY z o ː WORD_BOUNDARY ɛ t v a ː s WORD_BOUNDARY v ɪ ç t ɪ ɡ ə s WORD_BOUNDARY ɛ ɐ ts ɛ ː l t WORD_BOUNDARY UTT_BOUNDARY v i ː ɐ WORD_BOUNDARY v a ː ʀ ə n WORD_BOUNDARY m ɪ t WORD_BOUNDARY d e ː m WORD_BOUNDARY pf a ʀ ɐ WORD_BOUNDARY f ɛ ɐ a p ɐ e ː d ə t WORD_BOUNDARY UTT_BOUNDARY z i ː WORD_BOUNDARY z ɪ n t WORD_BOUNDARY n ɪ ç t WORD_BOUNDARY ɡ ə k ɔ m ə n WORD_BOUNDARY UTT_BOUNDARY v a s WORD_BOUNDARY ɪ s t WORD_BOUNDARY h i ː ɐ WORD_BOUNDARY l o ː s WORD_BOUNDARY UTT_BOUNDARY ʃ a ɐ l ə s WORD_BOUNDARY z ɔ l WORD_BOUNDARY ɛ ɐ m ɔ ɐ d ə t WORD_BOUNDARY v ɔ ɐ d ə n WORD_BOUNDARY z aɪ n WORD_BOUNDARY UTT_BOUNDARY l u ː ɪ s ə WORD_BOUNDARY d i ː z ɐ WORD_BOUNDARY z ɔ l d a ː t WORD_BOUNDARY ɪ s t WORD_BOUNDARY n ɪ ç t WORD_BOUNDARY b aɪ WORD_BOUNDARY k l a ː ʀ ə m WORD_BOUNDARY f ɛ ɐ ʃ t a n t WORD_BOUNDARY UTT_BOUNDARY i ː ɐ WORD_BOUNDARY z o ː n WORD_BOUNDARY v ʊɐ d ə WORD_BOUNDARY n ɪ ç t WORD_BOUNDARY ɛ ɐ m ɔ ɐ d ə t WORD_BOUNDARY ɛ ɐ WORD_BOUNDARY ɪ s t WORD_BOUNDARY f ɛ ɐ m ɪ s t WORD_BOUNDARY UTT_BOUNDARY d a s WORD_BOUNDARY ʃ t e ː t WORD_BOUNDARY ɪ n WORD_BOUNDARY a l ə n WORD_BOUNDARY b ə ʀ ɪ ç t ə n WORD_BOUNDARY UTT_BOUNDARY aʊ ɡ ʊ s t WORD_BOUNDARY h ɔ l t WORD_BOUNDARY h aʊ p t m a n WORD_BOUNDARY b ə aʊ l i ː u ː WORD_BOUNDARY UTT_BOUNDARY v a s WORD_BOUNDARY z ɔ l WORD_BOUNDARY d ɛ ɐ WORD_BOUNDARY h aʊ p t m a n WORD_BOUNDARY d a ː m ɪ t WORD_BOUNDARY a n f a ŋ ə n WORD_BOUNDARY UTT_BOUNDARY ʃ ɪ k ə n WORD_BOUNDARY z i ː WORD_BOUNDARY i ː n WORD_BOUNDARY b ɛ s ɐ WORD_BOUNDARY ɡ l aɪ ç WORD_BOUNDARY f ɔ ɐ t WORD_BOUNDARY UTT_BOUNDARY e ː ɐ s t WORD_BOUNDARY z ɔ l WORD_BOUNDARY f i ː l ɪ p ə WORD_BOUNDARY i ː n WORD_BOUNDARY a n h ø ː ʀ ə n WORD_BOUNDARY UTT_BOUNDARY k aɪ n ə WORD_BOUNDARY z ɔ ɐ ɡ ə WORD_BOUNDARY v i ː ɐ WORD_BOUNDARY v ɛ ɐ d ə n WORD_BOUNDARY i ː n WORD_BOUNDARY ʃ o ː n WORD_BOUNDARY l o ː s WORD_BOUNDARY UTT_BOUNDARY z ɛ l t z a ː m ə WORD_BOUNDARY ɡ ə ʃ ɪ ç t ə WORD_BOUNDARY z ɔ l d a ː t WORD_BOUNDARY UTT_BOUNDARY v e ː ɐ WORD_BOUNDARY h ɛ t ə WORD_BOUNDARY ʃ a ɐ l ə s WORD_BOUNDARY t ø ː t ə n WORD_BOUNDARY v ɔ l ə n WORD_BOUNDARY UTT_BOUNDARY ʊ n t WORD_BOUNDARY v a ː ʀ ʊ m WORD_BOUNDARY h a ː b ə n WORD_BOUNDARY +07/17/2026 15:54:55 - INFO - __main__ - [RANK 0] Sample 14592 of the training set: {'input_ids': tensor([ 6, 17, 4, 31, 11, 6, 2, 17, 4, 9, 23, 11, 17, 6, 31, 18, 12, 17, + 6, 13, 22, 15, 9, 6, 2, 20, 34, 17, 9, 6, 19, 16, 6, 4, 16, 3, + 11, 36, 9, 6, 25, 5, 3, 9, 29, 10, 6, 7, 40, 15, 9, 6, 23, 8, + 12, 17, 6, 2, 20, 34, 17, 9, 6, 3, 4, 15, 6, 10, 4, 16, 6, 20, + 4, 16, 6, 25, 28, 6, 21, 4, 15, 23, 11, 16, 6, 10, 9, 41, 15, 17, + 11, 16, 6, 14, 15, 3, 9, 6, 7, 12, 34, 10, 11, 16, 6, 14, 15, 3, + 9, 6, 35, 22, 15, 16, 29, 16, 33, 11, 6, 2, 18, 16, 13, 8, 5, 17, + 18, 23, 4, 17, 14, 15, 9, 6, 24, 11, 20, 6, 2, 17, 4, 9, 23, 11, + 17, 6, 31, 18, 12, 17, 6, 13, 8, 10, 11, 16, 6, 2, 17, 4, 9, 23, + 11, 17, 6, 31, 18, 12, 17, 6, 14, 15, 3, 9, 6, 10, 9, 8, 15, 36, + 29, 16, 33, 11, 6, 2, 18, 16, 13, 8, 5, 17, 18, 23, 4, 17, 14, 15, + 9, 6, 14, 15, 3, 9, 6, 10, 14, 15, 12, 11, 6, 2, 20, 34, 17, 9, + 6, 10, 14, 15, 12, 11, 6, 23, 11, 16, 25, 6, 2, 17, 4, 9, 23, 11, + 17, 6, 31, 18, 12, 17, 6, 14, 15, 3, 9, 6, 7, 34, 32, 9, 6, 2, + 20, 34, 17, 9, 6, 24, 4, 5, 17, 6, 3, 27, 15, 6, 10, 14, 15, 12, + 11, 6, 2, 18, 16, 13, 8, 5, 17, 18, 23, 4, 17, 14, 15, 9, 6, 7, + 34, 32, 9, 6, 7, 34, 32, 9, 6, 35, 8, 9, 7, 8, 10, 17, 6, 2, + 17, 4, 9, 23, 11, 17, 6, 31, 18, 12, 17, 6, 14, 15, 3, 9, 6, 7, + 34, 32, 9, 6, 18, 16, 6, 3, 8, 9, 6, 17, 4, 31, 11, 6, 2, 20, + 34, 17, 9, 6, 24, 4, 5, 17, 6, 3, 27, 15, 6, 7, 34, 32, 9, 6, + 18, 16, 6, 3, 8, 9, 6, 17, 4, 31, 11, 6, 2, 20, 34, 17, 9, 6, + 10, 9, 30, 15, 23, 5, 17, 6, 4, 15, 26, 9, 6, 19, 16, 11, 16, 6, + 24, 19, 5, 11, 16, 6, 35, 14, 15, 6, 2, 17, 4, 9, 23, 11, 17, 6, + 31, 18, 12, 17, 6, 14, 15, 3, 9, 6, 4, 12, 11, 6, 2, 20, 34, 17, + 9, 6, 14, 15, 3, 9, 6, 13, 4, 5, 6, 2, 17, 4, 9, 23, 11, 17, + 6, 31, 18, 12, 17, 6, 14, 15, 3, 9, 6, 4, 12, 11, 6, 2, 18, 16, + 13, 8, 5, 17, 18, 23, 4, 17, 14, 15, 9, 6, 35, 25, 12, 30, 15, 16, + 11, 6, 23, 29, 10, 5, 17, 6, 3, 27, 15, 6, 20, 4, 15, 12, 6, 23, + 11, 16, 25, 6, 3, 4, 15, 20])} WORD_BOUNDARY t a ʃ ə WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY v e ː ɐ WORD_BOUNDARY UTT_BOUNDARY m ɔ t ɐ WORD_BOUNDARY aɪ n WORD_BOUNDARY a n d ə ʀ ɐ WORD_BOUNDARY aʊ s d ɐ ʊ k WORD_BOUNDARY f y ː ɐ WORD_BOUNDARY ɡ ɛ l t WORD_BOUNDARY UTT_BOUNDARY m ɔ t ɐ WORD_BOUNDARY d a ː WORD_BOUNDARY k a n WORD_BOUNDARY m a n WORD_BOUNDARY aʊ x WORD_BOUNDARY z a ː ɡ ə n WORD_BOUNDARY k ɐ ø ː t ə n WORD_BOUNDARY o ː d ɐ WORD_BOUNDARY f l ɔ k ə n WORD_BOUNDARY o ː d ɐ WORD_BOUNDARY p e ː n ʊ n ts ə WORD_BOUNDARY UTT_BOUNDARY ɪ n v ɛ s t ɪ ɡ a t o ː ɐ WORD_BOUNDARY h ə m WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY v ɛ k ə n WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY o ː d ɐ WORD_BOUNDARY k ɐ ɛ ː ʀ ʊ n ts ə WORD_BOUNDARY UTT_BOUNDARY ɪ n v ɛ s t ɪ ɡ a t o ː ɐ WORD_BOUNDARY o ː d ɐ WORD_BOUNDARY k o ː l ə WORD_BOUNDARY UTT_BOUNDARY m ɔ t ɐ WORD_BOUNDARY k o ː l ə WORD_BOUNDARY ɡ ə n aʊ WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY o ː d ɐ WORD_BOUNDARY f ɔ ʏ ɐ WORD_BOUNDARY UTT_BOUNDARY m ɔ t ɐ WORD_BOUNDARY h a s t WORD_BOUNDARY d u ː WORD_BOUNDARY k o ː l ə WORD_BOUNDARY UTT_BOUNDARY ɪ n v ɛ s t ɪ ɡ a t o ː ɐ WORD_BOUNDARY f ɔ ʏ ɐ WORD_BOUNDARY f ɔ ʏ ɐ WORD_BOUNDARY p ɛ ɐ f ɛ k t WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY o ː d ɐ WORD_BOUNDARY f ɔ ʏ ɐ WORD_BOUNDARY ɪ n WORD_BOUNDARY d ɛ ɐ WORD_BOUNDARY t a ʃ ə WORD_BOUNDARY UTT_BOUNDARY m ɔ t ɐ WORD_BOUNDARY h a s t WORD_BOUNDARY d u ː WORD_BOUNDARY f ɔ ʏ ɐ WORD_BOUNDARY ɪ n WORD_BOUNDARY d ɛ ɐ WORD_BOUNDARY t a ʃ ə WORD_BOUNDARY UTT_BOUNDARY m ɔ t ɐ WORD_BOUNDARY k ɐ i ː ɡ s t WORD_BOUNDARY a ː b ɐ WORD_BOUNDARY aɪ n ə n WORD_BOUNDARY h aɪ s ə n WORD_BOUNDARY p o ː WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY o ː d ɐ WORD_BOUNDARY a l ə WORD_BOUNDARY UTT_BOUNDARY m ɔ t ɐ WORD_BOUNDARY o ː d ɐ WORD_BOUNDARY v a s WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY o ː d ɐ WORD_BOUNDARY a l ə WORD_BOUNDARY UTT_BOUNDARY ɪ n v ɛ s t ɪ ɡ a t o ː ɐ WORD_BOUNDARY p aʊ l i ː n ə WORD_BOUNDARY ɡ ʊ k s t WORD_BOUNDARY d u ː WORD_BOUNDARY m a ː l WORD_BOUNDARY ɡ ə n aʊ WORD_BOUNDARY d a ː m +07/17/2026 15:54:55 - INFO - __main__ - [RANK 0] Sample 3278 of the training set: {'input_ids': tensor([16, 6, 10, 4, 16, 6, 2, 42, 4, 15, 6, 2, 42, 4, 15, 6, 18, 39, + 6, 13, 18, 12, 6, 8, 5, 6, 14, 15, 3, 9, 6, 42, 4, 15, 6, 18, + 39, 6, 10, 9, 30, 15, 23, 11, 6, 8, 5, 6, 2, 42, 4, 15, 6, 13, + 4, 5, 6, 20, 4, 16, 6, 40, 15, 26, 9, 6, 21, 30, 15, 6, 21, 4, + 15, 10, 17, 6, 31, 17, 18, 20, 17, 6, 2, 31, 22, 15, 36, 11, 16, 6, + 21, 30, 15, 6, 21, 18, 39, 6, 33, 27, 15, 36, 32, 10, 6, 16, 4, 15, + 28, 6, 26, 8, 9, 12, 30, 15, 16, 6, 2, 18, 39, 6, 20, 4, 15, 10, + 6, 26, 8, 9, 12, 30, 15, 16, 6, 16, 18, 39, 17, 6, 4, 12, 33, 27, + 15, 6, 21, 22, 15, 9, 6, 2, 42, 4, 13, 14, 15, 12, 6, 2, 13, 8, + 16, 6, 21, 30, 15, 6, 13, 34, 12, 11, 16, 6, 12, 4, 5, 11, 16, 6, + 21, 30, 15, 6, 26, 4, 35, 10, 34, 10, 6, 19, 16, 11, 16, 6, 35, 18, + 5, 17, 14, 15, 12, 8, 16, 4, 16, 17, 9, 4, 15, 10, 6, 19, 16, 36, + 19, 39, 11, 16, 6, 2, 36, 8, 39, 17, 6, 24, 8, 9, 33, 12, 18, 39, + 11, 16, 6, 3, 4, 37, 10, 6, 21, 30, 15, 9, 6, 2, 26, 4, 35, 10, + 34, 10, 6, 2, 8, 9, 6, 13, 18, 12, 6, 19, 16, 11, 16, 6, 13, 4, + 7, 11, 16, 4, 16, 17, 9, 4, 15, 10, 6, 2, 13, 8, 16, 6, 8, 9, + 6, 3, 8, 37, 10, 17, 6, 18, 39, 6, 26, 9, 25, 28, 11, 6, 19, 16, + 11, 6, 35, 18, 5, 17, 14, 15, 12, 11, 6, 26, 9, 25, 28, 11, 6, 18, + 39, 6, 19, 16, 11, 16, 6, 21, 4, 9, 10, 6, 2, 18, 39, 6, 31, 17, + 8, 12, 11, 6, 3, 30, 15, 9, 6, 19, 16, 11, 16, 6, 25, 5, 6, 2, + 13, 22, 15, 9, 6, 18, 5, 17, 6, 21, 4, 20, 4, 16, 17, 4, 15, 6, + 31, 17, 22, 15, 12, 6, 2, 17, 22, 15, 6, 19, 16, 24, 29, 16, 3, 9, + 17, 6, 7, 32, 16, 7, 6, 8, 5, 6, 21, 18, 16, 17, 6, 23, 11, 24, + 19, 20, 6, 2, 13, 19, 5, 17, 6, 3, 27, 15, 6, 3, 4, 5, 6, 8, + 17, 13, 4, 15, 6, 16, 18, 39, 17, 6, 2, 18, 39, 6, 24, 4, 15, 26, + 11, 6, 24, 34, 32, 17, 11, 6, 4, 15, 26, 11, 16, 17, 6, 33, 19, 17, + 6, 2, 16, 19, 16, 6, 21, 30, 15, 6, 7, 12, 30, 15, 23, 11, 16, 6, + 29, 20, 6, 8, 12, 7, 6, 16, 29, 12, 6, 16, 29, 12, 6, 27, 15, 9, + 6, 33, 27, 15, 36, 32, 10, 6])} n WORD_BOUNDARY k a n WORD_BOUNDARY UTT_BOUNDARY j a ː WORD_BOUNDARY UTT_BOUNDARY j a ː WORD_BOUNDARY ɪ ç WORD_BOUNDARY v ɪ l WORD_BOUNDARY ɛ s WORD_BOUNDARY o ː d ɐ WORD_BOUNDARY j a ː WORD_BOUNDARY ɪ ç WORD_BOUNDARY k ɐ i ː ɡ ə WORD_BOUNDARY ɛ s WORD_BOUNDARY UTT_BOUNDARY j a ː WORD_BOUNDARY v a s WORD_BOUNDARY m a n WORD_BOUNDARY y ː b ɐ WORD_BOUNDARY z i ː WORD_BOUNDARY z a ː k t WORD_BOUNDARY ʃ t ɪ m t WORD_BOUNDARY UTT_BOUNDARY ʃ e ː ʀ ə n WORD_BOUNDARY z i ː WORD_BOUNDARY z ɪ ç WORD_BOUNDARY ts u ː ʀ ʏ k WORD_BOUNDARY n a ː x WORD_BOUNDARY b ɛ ɐ l i ː n WORD_BOUNDARY UTT_BOUNDARY ɪ ç WORD_BOUNDARY m a ː k WORD_BOUNDARY b ɛ ɐ l i ː n WORD_BOUNDARY n ɪ ç t WORD_BOUNDARY a l ts u ː WORD_BOUNDARY z e ː ɐ WORD_BOUNDARY UTT_BOUNDARY j a v o ː l WORD_BOUNDARY UTT_BOUNDARY v ɛ n WORD_BOUNDARY z i ː WORD_BOUNDARY v ɔ l ə n WORD_BOUNDARY l a s ə n WORD_BOUNDARY z i ː WORD_BOUNDARY b a p k ɔ k WORD_BOUNDARY aɪ n ə n WORD_BOUNDARY p ɪ s t o ː l ɛ n a n t ɐ a ː k WORD_BOUNDARY aɪ n ʀ aɪ ç ə n WORD_BOUNDARY UTT_BOUNDARY ʀ ɛ ç t WORD_BOUNDARY h ɛ ɐ ts l ɪ ç ə n WORD_BOUNDARY d a ŋ k WORD_BOUNDARY z i ː ɐ WORD_BOUNDARY UTT_BOUNDARY b a p k ɔ k WORD_BOUNDARY UTT_BOUNDARY ɛ ɐ WORD_BOUNDARY v ɪ l WORD_BOUNDARY aɪ n ə n WORD_BOUNDARY v a f ə n a n t ɐ a ː k WORD_BOUNDARY UTT_BOUNDARY v ɛ n WORD_BOUNDARY ɛ ɐ WORD_BOUNDARY d ɛ ŋ k t WORD_BOUNDARY ɪ ç WORD_BOUNDARY b ɐ aʊ x ə WORD_BOUNDARY aɪ n ə WORD_BOUNDARY p ɪ s t o ː l ə WORD_BOUNDARY b ɐ aʊ x ə WORD_BOUNDARY ɪ ç WORD_BOUNDARY aɪ n ə n WORD_BOUNDARY z a ɐ k WORD_BOUNDARY UTT_BOUNDARY ɪ ç WORD_BOUNDARY ʃ t ɛ l ə WORD_BOUNDARY d i ː ɐ WORD_BOUNDARY aɪ n ə n WORD_BOUNDARY aʊ s WORD_BOUNDARY UTT_BOUNDARY v e ː ɐ WORD_BOUNDARY ɪ s t WORD_BOUNDARY z a m a n t a ː WORD_BOUNDARY ʃ t e ː l WORD_BOUNDARY UTT_BOUNDARY t e ː WORD_BOUNDARY aɪ n h ʊ n d ɐ t WORD_BOUNDARY f ʏ n f WORD_BOUNDARY ɛ s WORD_BOUNDARY z ɪ n t WORD_BOUNDARY ɡ ə h aɪ m WORD_BOUNDARY UTT_BOUNDARY v aɪ s t WORD_BOUNDARY d u ː WORD_BOUNDARY d a s WORD_BOUNDARY ɛ t v a ː WORD_BOUNDARY n ɪ ç t WORD_BOUNDARY UTT_BOUNDARY ɪ ç WORD_BOUNDARY h a ː b ə WORD_BOUNDARY h ɔ ʏ t ə WORD_BOUNDARY a ː b ə n t WORD_BOUNDARY ts aɪ t WORD_BOUNDARY UTT_BOUNDARY n aɪ n WORD_BOUNDARY z i ː WORD_BOUNDARY f l i ː ɡ ə n WORD_BOUNDARY ʊ m WORD_BOUNDARY ɛ l f WORD_BOUNDARY n ʊ l WORD_BOUNDARY n ʊ l WORD_BOUNDARY u ː ɐ WORD_BOUNDARY ts u ː ʀ ʏ k WORD_BOUNDARY +07/17/2026 15:54:58 - INFO - __main__ - [RANK 0] ***** Running training ***** +07/17/2026 15:54:58 - INFO - __main__ - [RANK 0] Num examples = 101825 +07/17/2026 15:54:58 - INFO - __main__ - [RANK 0] Num Training Steps = 30000 +07/17/2026 15:54:58 - INFO - __main__ - [RANK 0] Instantaneous batch size per device = 32 +07/17/2026 15:54:58 - INFO - __main__ - [RANK 0] Total train batch size (w. parallel, distributed & accumulation) = 32 +07/17/2026 15:54:58 - INFO - __main__ - [RANK 0] Gradient Accumulation steps = 1 +07/17/2026 15:54:58 - INFO - __main__ - [RANK 0] Total optimization steps = 30000 +07/17/2026 15:54:58 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/17/2026 15:54:58 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/17/2026 15:55:03 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/17/2026 15:55:03 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/17/2026 15:55:03 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/17/2026 15:55:09 - INFO - __main__ - [RANK 0] Starting epoch 0... +07/17/2026 17:23:04 - INFO - __main__ - [RANK 0] Distributed environment: DistributedType.NO +Num processes: 1 +Process index: 0 +Local process index: 0 +Device: cuda + +Mixed precision type: no + +07/17/2026 17:23:04 - INFO - __main__ - [RANK 0] Training args: +{'data_dir': 'data/phone/deu-eng/tokenized_sequential_interleaved', 'model_name_or_path': None, 'config_name': 'gpt_base_config.json', 'tokenizer_name': 'nikitastheo/phonelm-phone-deu-eng-tokenizer', 'per_device_train_batch_size': 32, 'per_device_eval_batch_size': 32, 'learning_rate': 0.0001, 'weight_decay': 0.01, 'max_steps': 30000, 'gradient_accumulation_steps': 1, 'lr_scheduler_type': , 'warmup_steps': 3000, 'output_dir': 'models/phonebabylm-phone-deu-eng-sequential_interleaved', 'seed': 42, 'model_type': None, 'block_size': None, 'push_to_hub': True, 'hub_model_id': 'nikitastheo/phonebabylm-phone-deu-eng-sequential_interleaved', 'hub_token': None, 'private': False, 'trust_remote_code': False, 'with_tracking': True, 'tracking_dir': None, 'report_to': 'wandb', 'restart_lr_at_switch': False, 'language_switch_steps': 10000, 'model_revision': 'main', 'config_overrides': None, 'use_slow_tokenizer': False, 'override_vocab_size': None, 'dtype': None, 'logging_steps': 10, 'cache_dir': None, 'eval_steps': 750, 'max_grad_norm': 1.0, 'adam_epsilon': 1e-06, 'run_name': 'phonebabylm-phone-deu-eng-sequential_interleaved', 'max_train_samples': None, 'max_eval_samples': None} +07/17/2026 17:23:04 - INFO - transformers.configuration_utils - loading configuration file gpt_base_config.json +07/17/2026 17:23:04 - INFO - transformers.configuration_utils - Model config GPT2Config { + "activation_function": "gelu", + "architectures": [ + "GPT2LMHeadModel" + ], + "attn_pdrop": 0.1, + "bos_token_id": 50256, + "embd_pdrop": 0.1, + "eos_token_id": 50256, + "initializer_range": 0.02, + "layer_norm_epsilon": 1e-05, + "model_type": "gpt2", + "n_ctx": 512, + "n_embd": 768, + "n_head": 12, + "n_inner": 3072, + "n_layer": 12, + "n_positions": 512, + "reorder_and_upcast_attn": false, + "resid_pdrop": 0.1, + "scale_attn_by_inverse_layer_idx": false, + "scale_attn_weights": true, + "summary_activation": null, + "summary_first_dropout": 0.1, + "summary_proj_to_labels": true, + "summary_type": "cls_index", + "summary_use_proj": true, + "transformers_version": "4.57.6", + "use_cache": true, + "vocab_size": 1 +} + +07/17/2026 17:23:05 - INFO - transformers.tokenization_utils_base - loading file vocab.json from cache at /home/geofila/.cache/huggingface/hub/models--nikitastheo--phonelm-phone-deu-eng-tokenizer/snapshots/a9eb582bae549d2e78f9e6f7913343e704373e8a/vocab.json +07/17/2026 17:23:05 - INFO - transformers.tokenization_utils_base - loading file merges.txt from cache at None +07/17/2026 17:23:05 - INFO - transformers.tokenization_utils_base - loading file tokenizer.json from cache at /home/geofila/.cache/huggingface/hub/models--nikitastheo--phonelm-phone-deu-eng-tokenizer/snapshots/a9eb582bae549d2e78f9e6f7913343e704373e8a/tokenizer.json +07/17/2026 17:23:05 - INFO - transformers.tokenization_utils_base - loading file added_tokens.json from cache at None +07/17/2026 17:23:05 - INFO - transformers.tokenization_utils_base - loading file special_tokens_map.json from cache at /home/geofila/.cache/huggingface/hub/models--nikitastheo--phonelm-phone-deu-eng-tokenizer/snapshots/a9eb582bae549d2e78f9e6f7913343e704373e8a/special_tokens_map.json +07/17/2026 17:23:05 - INFO - transformers.tokenization_utils_base - loading file tokenizer_config.json from cache at /home/geofila/.cache/huggingface/hub/models--nikitastheo--phonelm-phone-deu-eng-tokenizer/snapshots/a9eb582bae549d2e78f9e6f7913343e704373e8a/tokenizer_config.json +07/17/2026 17:23:05 - INFO - transformers.tokenization_utils_base - loading file chat_template.jinja from cache at None +07/17/2026 17:23:06 - INFO - transformers.generation.configuration_utils - Generate config GenerationConfig { + "bos_token_id": 2, + "eos_token_id": 2, + "pad_token_id": 1 +} + +07/17/2026 17:23:07 - INFO - __main__ - [RANK 0] Model parameters: 85.50M total (85.50M trainable) +07/17/2026 17:23:14 - INFO - __main__ - [RANK 0] Sample 83810 of the training set: {'input_ids': tensor([ 9, 6, 34, 32, 39, 6, 21, 14, 15, 6, 8, 17, 13, 4, 15, 5, 6, 13, + 18, 39, 17, 18, 23, 11, 5, 6, 8, 9, 33, 8, 15, 12, 17, 6, 2, 13, + 30, 15, 9, 6, 13, 4, 15, 36, 11, 16, 6, 20, 18, 17, 6, 3, 22, 15, + 20, 6, 43, 4, 36, 9, 6, 7, 8, 9, 4, 35, 9, 22, 15, 3, 11, 17, + 6, 2, 21, 30, 15, 6, 21, 18, 16, 17, 6, 16, 18, 39, 17, 6, 23, 11, + 10, 34, 20, 11, 16, 6, 2, 13, 4, 5, 6, 18, 5, 17, 6, 24, 30, 15, + 9, 6, 12, 14, 15, 5, 6, 2, 31, 4, 9, 12, 11, 5, 6, 21, 34, 12, + 6, 8, 9, 20, 34, 9, 3, 11, 17, 6, 13, 34, 9, 3, 11, 16, 6, 21, + 19, 16, 6, 2, 12, 27, 15, 18, 5, 11, 6, 3, 30, 15, 21, 9, 6, 21, + 34, 12, 3, 4, 15, 17, 6, 18, 5, 17, 6, 16, 18, 39, 17, 6, 26, 19, + 6, 10, 12, 4, 15, 36, 11, 20, 6, 7, 8, 9, 31, 17, 4, 16, 17, 6, + 2, 30, 15, 9, 6, 21, 14, 15, 16, 6, 13, 44, 3, 11, 6, 16, 18, 39, + 17, 6, 8, 9, 20, 34, 9, 3, 11, 17, 6, 8, 9, 6, 18, 5, 17, 6, + 7, 8, 9, 20, 18, 5, 17, 6, 2, 3, 4, 5, 6, 31, 17, 22, 15, 17, + 6, 18, 16, 6, 4, 12, 11, 16, 6, 26, 11, 36, 18, 39, 17, 11, 16, 6, + 2, 25, 23, 29, 5, 17, 6, 24, 34, 12, 17, 6, 24, 25, 35, 17, 20, 4, + 16, 6, 26, 11, 25, 12, 30, 15, 27, 15, 6, 2, 13, 4, 5, 6, 21, 34, + 12, 6, 3, 8, 9, 6, 24, 25, 35, 17, 20, 4, 16, 6, 3, 4, 15, 20, + 18, 17, 6, 4, 16, 7, 4, 37, 11, 16, 6, 2, 31, 18, 10, 11, 16, 6, + 21, 30, 15, 6, 30, 15, 16, 6, 26, 8, 5, 9, 6, 23, 12, 19, 39, 6, + 7, 34, 9, 17, 6, 2, 22, 15, 9, 5, 17, 6, 21, 34, 12, 6, 7, 30, + 15, 12, 18, 35, 11, 6, 30, 15, 16, 6, 4, 16, 24, 41, 15, 36, 11, 16, + 6, 2, 10, 19, 16, 11, 6, 21, 34, 9, 23, 11, 6, 13, 30, 15, 9, 6, + 13, 8, 9, 3, 11, 16, 6, 30, 15, 16, 6, 31, 14, 15, 16, 6, 12, 14, + 15, 5, 6, 2, 21, 8, 12, 17, 21, 4, 15, 20, 11, 6, 23, 11, 31, 18, + 39, 17, 11, 6, 21, 34, 12, 3, 4, 15, 17, 6, 2, 13, 22, 15, 9, 6, + 24, 8, 17, 11, 6, 31, 4, 9, 12, 11, 5, 6, 17, 41, 15, 17, 11, 16, + 6, 13, 34, 12, 11, 16, 6, 2, 29, 16, 17, 6, 13, 4, 15, 36, 29, 20, + 6, 24, 4, 15, 26, 11, 16, 6])} ɐ WORD_BOUNDARY ɔ ʏ ç WORD_BOUNDARY z o ː WORD_BOUNDARY ɛ t v a ː s WORD_BOUNDARY v ɪ ç t ɪ ɡ ə s WORD_BOUNDARY ɛ ɐ ts ɛ ː l t WORD_BOUNDARY UTT_BOUNDARY v i ː ɐ WORD_BOUNDARY v a ː ʀ ə n WORD_BOUNDARY m ɪ t WORD_BOUNDARY d e ː m WORD_BOUNDARY pf a ʀ ɐ WORD_BOUNDARY f ɛ ɐ a p ɐ e ː d ə t WORD_BOUNDARY UTT_BOUNDARY z i ː WORD_BOUNDARY z ɪ n t WORD_BOUNDARY n ɪ ç t WORD_BOUNDARY ɡ ə k ɔ m ə n WORD_BOUNDARY UTT_BOUNDARY v a s WORD_BOUNDARY ɪ s t WORD_BOUNDARY h i ː ɐ WORD_BOUNDARY l o ː s WORD_BOUNDARY UTT_BOUNDARY ʃ a ɐ l ə s WORD_BOUNDARY z ɔ l WORD_BOUNDARY ɛ ɐ m ɔ ɐ d ə t WORD_BOUNDARY v ɔ ɐ d ə n WORD_BOUNDARY z aɪ n WORD_BOUNDARY UTT_BOUNDARY l u ː ɪ s ə WORD_BOUNDARY d i ː z ɐ WORD_BOUNDARY z ɔ l d a ː t WORD_BOUNDARY ɪ s t WORD_BOUNDARY n ɪ ç t WORD_BOUNDARY b aɪ WORD_BOUNDARY k l a ː ʀ ə m WORD_BOUNDARY f ɛ ɐ ʃ t a n t WORD_BOUNDARY UTT_BOUNDARY i ː ɐ WORD_BOUNDARY z o ː n WORD_BOUNDARY v ʊɐ d ə WORD_BOUNDARY n ɪ ç t WORD_BOUNDARY ɛ ɐ m ɔ ɐ d ə t WORD_BOUNDARY ɛ ɐ WORD_BOUNDARY ɪ s t WORD_BOUNDARY f ɛ ɐ m ɪ s t WORD_BOUNDARY UTT_BOUNDARY d a s WORD_BOUNDARY ʃ t e ː t WORD_BOUNDARY ɪ n WORD_BOUNDARY a l ə n WORD_BOUNDARY b ə ʀ ɪ ç t ə n WORD_BOUNDARY UTT_BOUNDARY aʊ ɡ ʊ s t WORD_BOUNDARY h ɔ l t WORD_BOUNDARY h aʊ p t m a n WORD_BOUNDARY b ə aʊ l i ː u ː WORD_BOUNDARY UTT_BOUNDARY v a s WORD_BOUNDARY z ɔ l WORD_BOUNDARY d ɛ ɐ WORD_BOUNDARY h aʊ p t m a n WORD_BOUNDARY d a ː m ɪ t WORD_BOUNDARY a n f a ŋ ə n WORD_BOUNDARY UTT_BOUNDARY ʃ ɪ k ə n WORD_BOUNDARY z i ː WORD_BOUNDARY i ː n WORD_BOUNDARY b ɛ s ɐ WORD_BOUNDARY ɡ l aɪ ç WORD_BOUNDARY f ɔ ɐ t WORD_BOUNDARY UTT_BOUNDARY e ː ɐ s t WORD_BOUNDARY z ɔ l WORD_BOUNDARY f i ː l ɪ p ə WORD_BOUNDARY i ː n WORD_BOUNDARY a n h ø ː ʀ ə n WORD_BOUNDARY UTT_BOUNDARY k aɪ n ə WORD_BOUNDARY z ɔ ɐ ɡ ə WORD_BOUNDARY v i ː ɐ WORD_BOUNDARY v ɛ ɐ d ə n WORD_BOUNDARY i ː n WORD_BOUNDARY ʃ o ː n WORD_BOUNDARY l o ː s WORD_BOUNDARY UTT_BOUNDARY z ɛ l t z a ː m ə WORD_BOUNDARY ɡ ə ʃ ɪ ç t ə WORD_BOUNDARY z ɔ l d a ː t WORD_BOUNDARY UTT_BOUNDARY v e ː ɐ WORD_BOUNDARY h ɛ t ə WORD_BOUNDARY ʃ a ɐ l ə s WORD_BOUNDARY t ø ː t ə n WORD_BOUNDARY v ɔ l ə n WORD_BOUNDARY UTT_BOUNDARY ʊ n t WORD_BOUNDARY v a ː ʀ ʊ m WORD_BOUNDARY h a ː b ə n WORD_BOUNDARY +07/17/2026 17:23:14 - INFO - __main__ - [RANK 0] Sample 14592 of the training set: {'input_ids': tensor([ 6, 17, 4, 31, 11, 6, 2, 17, 4, 9, 23, 11, 17, 6, 31, 18, 12, 17, + 6, 13, 22, 15, 9, 6, 2, 20, 34, 17, 9, 6, 19, 16, 6, 4, 16, 3, + 11, 36, 9, 6, 25, 5, 3, 9, 29, 10, 6, 7, 40, 15, 9, 6, 23, 8, + 12, 17, 6, 2, 20, 34, 17, 9, 6, 3, 4, 15, 6, 10, 4, 16, 6, 20, + 4, 16, 6, 25, 28, 6, 21, 4, 15, 23, 11, 16, 6, 10, 9, 41, 15, 17, + 11, 16, 6, 14, 15, 3, 9, 6, 7, 12, 34, 10, 11, 16, 6, 14, 15, 3, + 9, 6, 35, 22, 15, 16, 29, 16, 33, 11, 6, 2, 18, 16, 13, 8, 5, 17, + 18, 23, 4, 17, 14, 15, 9, 6, 24, 11, 20, 6, 2, 17, 4, 9, 23, 11, + 17, 6, 31, 18, 12, 17, 6, 13, 8, 10, 11, 16, 6, 2, 17, 4, 9, 23, + 11, 17, 6, 31, 18, 12, 17, 6, 14, 15, 3, 9, 6, 10, 9, 8, 15, 36, + 29, 16, 33, 11, 6, 2, 18, 16, 13, 8, 5, 17, 18, 23, 4, 17, 14, 15, + 9, 6, 14, 15, 3, 9, 6, 10, 14, 15, 12, 11, 6, 2, 20, 34, 17, 9, + 6, 10, 14, 15, 12, 11, 6, 23, 11, 16, 25, 6, 2, 17, 4, 9, 23, 11, + 17, 6, 31, 18, 12, 17, 6, 14, 15, 3, 9, 6, 7, 34, 32, 9, 6, 2, + 20, 34, 17, 9, 6, 24, 4, 5, 17, 6, 3, 27, 15, 6, 10, 14, 15, 12, + 11, 6, 2, 18, 16, 13, 8, 5, 17, 18, 23, 4, 17, 14, 15, 9, 6, 7, + 34, 32, 9, 6, 7, 34, 32, 9, 6, 35, 8, 9, 7, 8, 10, 17, 6, 2, + 17, 4, 9, 23, 11, 17, 6, 31, 18, 12, 17, 6, 14, 15, 3, 9, 6, 7, + 34, 32, 9, 6, 18, 16, 6, 3, 8, 9, 6, 17, 4, 31, 11, 6, 2, 20, + 34, 17, 9, 6, 24, 4, 5, 17, 6, 3, 27, 15, 6, 7, 34, 32, 9, 6, + 18, 16, 6, 3, 8, 9, 6, 17, 4, 31, 11, 6, 2, 20, 34, 17, 9, 6, + 10, 9, 30, 15, 23, 5, 17, 6, 4, 15, 26, 9, 6, 19, 16, 11, 16, 6, + 24, 19, 5, 11, 16, 6, 35, 14, 15, 6, 2, 17, 4, 9, 23, 11, 17, 6, + 31, 18, 12, 17, 6, 14, 15, 3, 9, 6, 4, 12, 11, 6, 2, 20, 34, 17, + 9, 6, 14, 15, 3, 9, 6, 13, 4, 5, 6, 2, 17, 4, 9, 23, 11, 17, + 6, 31, 18, 12, 17, 6, 14, 15, 3, 9, 6, 4, 12, 11, 6, 2, 18, 16, + 13, 8, 5, 17, 18, 23, 4, 17, 14, 15, 9, 6, 35, 25, 12, 30, 15, 16, + 11, 6, 23, 29, 10, 5, 17, 6, 3, 27, 15, 6, 20, 4, 15, 12, 6, 23, + 11, 16, 25, 6, 3, 4, 15, 20])} WORD_BOUNDARY t a ʃ ə WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY v e ː ɐ WORD_BOUNDARY UTT_BOUNDARY m ɔ t ɐ WORD_BOUNDARY aɪ n WORD_BOUNDARY a n d ə ʀ ɐ WORD_BOUNDARY aʊ s d ɐ ʊ k WORD_BOUNDARY f y ː ɐ WORD_BOUNDARY ɡ ɛ l t WORD_BOUNDARY UTT_BOUNDARY m ɔ t ɐ WORD_BOUNDARY d a ː WORD_BOUNDARY k a n WORD_BOUNDARY m a n WORD_BOUNDARY aʊ x WORD_BOUNDARY z a ː ɡ ə n WORD_BOUNDARY k ɐ ø ː t ə n WORD_BOUNDARY o ː d ɐ WORD_BOUNDARY f l ɔ k ə n WORD_BOUNDARY o ː d ɐ WORD_BOUNDARY p e ː n ʊ n ts ə WORD_BOUNDARY UTT_BOUNDARY ɪ n v ɛ s t ɪ ɡ a t o ː ɐ WORD_BOUNDARY h ə m WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY v ɛ k ə n WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY o ː d ɐ WORD_BOUNDARY k ɐ ɛ ː ʀ ʊ n ts ə WORD_BOUNDARY UTT_BOUNDARY ɪ n v ɛ s t ɪ ɡ a t o ː ɐ WORD_BOUNDARY o ː d ɐ WORD_BOUNDARY k o ː l ə WORD_BOUNDARY UTT_BOUNDARY m ɔ t ɐ WORD_BOUNDARY k o ː l ə WORD_BOUNDARY ɡ ə n aʊ WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY o ː d ɐ WORD_BOUNDARY f ɔ ʏ ɐ WORD_BOUNDARY UTT_BOUNDARY m ɔ t ɐ WORD_BOUNDARY h a s t WORD_BOUNDARY d u ː WORD_BOUNDARY k o ː l ə WORD_BOUNDARY UTT_BOUNDARY ɪ n v ɛ s t ɪ ɡ a t o ː ɐ WORD_BOUNDARY f ɔ ʏ ɐ WORD_BOUNDARY f ɔ ʏ ɐ WORD_BOUNDARY p ɛ ɐ f ɛ k t WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY o ː d ɐ WORD_BOUNDARY f ɔ ʏ ɐ WORD_BOUNDARY ɪ n WORD_BOUNDARY d ɛ ɐ WORD_BOUNDARY t a ʃ ə WORD_BOUNDARY UTT_BOUNDARY m ɔ t ɐ WORD_BOUNDARY h a s t WORD_BOUNDARY d u ː WORD_BOUNDARY f ɔ ʏ ɐ WORD_BOUNDARY ɪ n WORD_BOUNDARY d ɛ ɐ WORD_BOUNDARY t a ʃ ə WORD_BOUNDARY UTT_BOUNDARY m ɔ t ɐ WORD_BOUNDARY k ɐ i ː ɡ s t WORD_BOUNDARY a ː b ɐ WORD_BOUNDARY aɪ n ə n WORD_BOUNDARY h aɪ s ə n WORD_BOUNDARY p o ː WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY o ː d ɐ WORD_BOUNDARY a l ə WORD_BOUNDARY UTT_BOUNDARY m ɔ t ɐ WORD_BOUNDARY o ː d ɐ WORD_BOUNDARY v a s WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY o ː d ɐ WORD_BOUNDARY a l ə WORD_BOUNDARY UTT_BOUNDARY ɪ n v ɛ s t ɪ ɡ a t o ː ɐ WORD_BOUNDARY p aʊ l i ː n ə WORD_BOUNDARY ɡ ʊ k s t WORD_BOUNDARY d u ː WORD_BOUNDARY m a ː l WORD_BOUNDARY ɡ ə n aʊ WORD_BOUNDARY d a ː m +07/17/2026 17:23:14 - INFO - __main__ - [RANK 0] Sample 3278 of the training set: {'input_ids': tensor([16, 6, 10, 4, 16, 6, 2, 42, 4, 15, 6, 2, 42, 4, 15, 6, 18, 39, + 6, 13, 18, 12, 6, 8, 5, 6, 14, 15, 3, 9, 6, 42, 4, 15, 6, 18, + 39, 6, 10, 9, 30, 15, 23, 11, 6, 8, 5, 6, 2, 42, 4, 15, 6, 13, + 4, 5, 6, 20, 4, 16, 6, 40, 15, 26, 9, 6, 21, 30, 15, 6, 21, 4, + 15, 10, 17, 6, 31, 17, 18, 20, 17, 6, 2, 31, 22, 15, 36, 11, 16, 6, + 21, 30, 15, 6, 21, 18, 39, 6, 33, 27, 15, 36, 32, 10, 6, 16, 4, 15, + 28, 6, 26, 8, 9, 12, 30, 15, 16, 6, 2, 18, 39, 6, 20, 4, 15, 10, + 6, 26, 8, 9, 12, 30, 15, 16, 6, 16, 18, 39, 17, 6, 4, 12, 33, 27, + 15, 6, 21, 22, 15, 9, 6, 2, 42, 4, 13, 14, 15, 12, 6, 2, 13, 8, + 16, 6, 21, 30, 15, 6, 13, 34, 12, 11, 16, 6, 12, 4, 5, 11, 16, 6, + 21, 30, 15, 6, 26, 4, 35, 10, 34, 10, 6, 19, 16, 11, 16, 6, 35, 18, + 5, 17, 14, 15, 12, 8, 16, 4, 16, 17, 9, 4, 15, 10, 6, 19, 16, 36, + 19, 39, 11, 16, 6, 2, 36, 8, 39, 17, 6, 24, 8, 9, 33, 12, 18, 39, + 11, 16, 6, 3, 4, 37, 10, 6, 21, 30, 15, 9, 6, 2, 26, 4, 35, 10, + 34, 10, 6, 2, 8, 9, 6, 13, 18, 12, 6, 19, 16, 11, 16, 6, 13, 4, + 7, 11, 16, 4, 16, 17, 9, 4, 15, 10, 6, 2, 13, 8, 16, 6, 8, 9, + 6, 3, 8, 37, 10, 17, 6, 18, 39, 6, 26, 9, 25, 28, 11, 6, 19, 16, + 11, 6, 35, 18, 5, 17, 14, 15, 12, 11, 6, 26, 9, 25, 28, 11, 6, 18, + 39, 6, 19, 16, 11, 16, 6, 21, 4, 9, 10, 6, 2, 18, 39, 6, 31, 17, + 8, 12, 11, 6, 3, 30, 15, 9, 6, 19, 16, 11, 16, 6, 25, 5, 6, 2, + 13, 22, 15, 9, 6, 18, 5, 17, 6, 21, 4, 20, 4, 16, 17, 4, 15, 6, + 31, 17, 22, 15, 12, 6, 2, 17, 22, 15, 6, 19, 16, 24, 29, 16, 3, 9, + 17, 6, 7, 32, 16, 7, 6, 8, 5, 6, 21, 18, 16, 17, 6, 23, 11, 24, + 19, 20, 6, 2, 13, 19, 5, 17, 6, 3, 27, 15, 6, 3, 4, 5, 6, 8, + 17, 13, 4, 15, 6, 16, 18, 39, 17, 6, 2, 18, 39, 6, 24, 4, 15, 26, + 11, 6, 24, 34, 32, 17, 11, 6, 4, 15, 26, 11, 16, 17, 6, 33, 19, 17, + 6, 2, 16, 19, 16, 6, 21, 30, 15, 6, 7, 12, 30, 15, 23, 11, 16, 6, + 29, 20, 6, 8, 12, 7, 6, 16, 29, 12, 6, 16, 29, 12, 6, 27, 15, 9, + 6, 33, 27, 15, 36, 32, 10, 6])} n WORD_BOUNDARY k a n WORD_BOUNDARY UTT_BOUNDARY j a ː WORD_BOUNDARY UTT_BOUNDARY j a ː WORD_BOUNDARY ɪ ç WORD_BOUNDARY v ɪ l WORD_BOUNDARY ɛ s WORD_BOUNDARY o ː d ɐ WORD_BOUNDARY j a ː WORD_BOUNDARY ɪ ç WORD_BOUNDARY k ɐ i ː ɡ ə WORD_BOUNDARY ɛ s WORD_BOUNDARY UTT_BOUNDARY j a ː WORD_BOUNDARY v a s WORD_BOUNDARY m a n WORD_BOUNDARY y ː b ɐ WORD_BOUNDARY z i ː WORD_BOUNDARY z a ː k t WORD_BOUNDARY ʃ t ɪ m t WORD_BOUNDARY UTT_BOUNDARY ʃ e ː ʀ ə n WORD_BOUNDARY z i ː WORD_BOUNDARY z ɪ ç WORD_BOUNDARY ts u ː ʀ ʏ k WORD_BOUNDARY n a ː x WORD_BOUNDARY b ɛ ɐ l i ː n WORD_BOUNDARY UTT_BOUNDARY ɪ ç WORD_BOUNDARY m a ː k WORD_BOUNDARY b ɛ ɐ l i ː n WORD_BOUNDARY n ɪ ç t WORD_BOUNDARY a l ts u ː WORD_BOUNDARY z e ː ɐ WORD_BOUNDARY UTT_BOUNDARY j a v o ː l WORD_BOUNDARY UTT_BOUNDARY v ɛ n WORD_BOUNDARY z i ː WORD_BOUNDARY v ɔ l ə n WORD_BOUNDARY l a s ə n WORD_BOUNDARY z i ː WORD_BOUNDARY b a p k ɔ k WORD_BOUNDARY aɪ n ə n WORD_BOUNDARY p ɪ s t o ː l ɛ n a n t ɐ a ː k WORD_BOUNDARY aɪ n ʀ aɪ ç ə n WORD_BOUNDARY UTT_BOUNDARY ʀ ɛ ç t WORD_BOUNDARY h ɛ ɐ ts l ɪ ç ə n WORD_BOUNDARY d a ŋ k WORD_BOUNDARY z i ː ɐ WORD_BOUNDARY UTT_BOUNDARY b a p k ɔ k WORD_BOUNDARY UTT_BOUNDARY ɛ ɐ WORD_BOUNDARY v ɪ l WORD_BOUNDARY aɪ n ə n WORD_BOUNDARY v a f ə n a n t ɐ a ː k WORD_BOUNDARY UTT_BOUNDARY v ɛ n WORD_BOUNDARY ɛ ɐ WORD_BOUNDARY d ɛ ŋ k t WORD_BOUNDARY ɪ ç WORD_BOUNDARY b ɐ aʊ x ə WORD_BOUNDARY aɪ n ə WORD_BOUNDARY p ɪ s t o ː l ə WORD_BOUNDARY b ɐ aʊ x ə WORD_BOUNDARY ɪ ç WORD_BOUNDARY aɪ n ə n WORD_BOUNDARY z a ɐ k WORD_BOUNDARY UTT_BOUNDARY ɪ ç WORD_BOUNDARY ʃ t ɛ l ə WORD_BOUNDARY d i ː ɐ WORD_BOUNDARY aɪ n ə n WORD_BOUNDARY aʊ s WORD_BOUNDARY UTT_BOUNDARY v e ː ɐ WORD_BOUNDARY ɪ s t WORD_BOUNDARY z a m a n t a ː WORD_BOUNDARY ʃ t e ː l WORD_BOUNDARY UTT_BOUNDARY t e ː WORD_BOUNDARY aɪ n h ʊ n d ɐ t WORD_BOUNDARY f ʏ n f WORD_BOUNDARY ɛ s WORD_BOUNDARY z ɪ n t WORD_BOUNDARY ɡ ə h aɪ m WORD_BOUNDARY UTT_BOUNDARY v aɪ s t WORD_BOUNDARY d u ː WORD_BOUNDARY d a s WORD_BOUNDARY ɛ t v a ː WORD_BOUNDARY n ɪ ç t WORD_BOUNDARY UTT_BOUNDARY ɪ ç WORD_BOUNDARY h a ː b ə WORD_BOUNDARY h ɔ ʏ t ə WORD_BOUNDARY a ː b ə n t WORD_BOUNDARY ts aɪ t WORD_BOUNDARY UTT_BOUNDARY n aɪ n WORD_BOUNDARY z i ː WORD_BOUNDARY f l i ː ɡ ə n WORD_BOUNDARY ʊ m WORD_BOUNDARY ɛ l f WORD_BOUNDARY n ʊ l WORD_BOUNDARY n ʊ l WORD_BOUNDARY u ː ɐ WORD_BOUNDARY ts u ː ʀ ʏ k WORD_BOUNDARY +07/17/2026 17:23:17 - INFO - __main__ - [RANK 0] ***** Running training ***** +07/17/2026 17:23:17 - INFO - __main__ - [RANK 0] Num examples = 101825 +07/17/2026 17:23:17 - INFO - __main__ - [RANK 0] Num Training Steps = 30000 +07/17/2026 17:23:17 - INFO - __main__ - [RANK 0] Instantaneous batch size per device = 32 +07/17/2026 17:23:17 - INFO - __main__ - [RANK 0] Total train batch size (w. parallel, distributed & accumulation) = 32 +07/17/2026 17:23:17 - INFO - __main__ - [RANK 0] Gradient Accumulation steps = 1 +07/17/2026 17:23:17 - INFO - __main__ - [RANK 0] Total optimization steps = 30000 +07/17/2026 17:23:17 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/17/2026 17:23:17 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/17/2026 17:23:20 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/17/2026 17:23:20 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/17/2026 17:23:20 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/17/2026 17:23:26 - INFO - __main__ - [RANK 0] Starting epoch 0... +07/17/2026 17:23:26 - WARNING - transformers.modeling_utils - `loss_type=None` was set in the config but it is unrecognized. Using the default loss: `ForCausalLMLoss`. +07/17/2026 17:27:32 - INFO - __main__ - [RANK 0] perplexity: 7.909086097933963 eval_loss: 2.068012237548828 accuracy: 0.4003 +07/17/2026 17:31:42 - INFO - __main__ - [RANK 0] perplexity: 4.531085486765549 eval_loss: 1.5109615325927734 accuracy: 0.5508 +07/17/2026 17:35:52 - INFO - __main__ - [RANK 0] perplexity: 3.763946708277698 eval_loss: 1.3254680633544922 accuracy: 0.6023 +07/17/2026 17:40:02 - INFO - __main__ - [RANK 0] perplexity: 3.2925302794646143 eval_loss: 1.1916563510894775 accuracy: 0.6412 +07/17/2026 17:41:02 - INFO - __main__ - [RANK 0] Starting epoch 1... +07/17/2026 17:44:11 - INFO - __main__ - [RANK 0] perplexity: 3.1018616790820457 eval_loss: 1.1320024728775024 accuracy: 0.6589 +07/17/2026 17:48:21 - INFO - __main__ - [RANK 0] perplexity: 2.9848321959746005 eval_loss: 1.093543529510498 accuracy: 0.6699 +07/17/2026 17:52:31 - INFO - __main__ - [RANK 0] perplexity: 2.9038398848998557 eval_loss: 1.066033959388733 accuracy: 0.6778 +07/17/2026 17:56:41 - INFO - __main__ - [RANK 0] perplexity: 2.8438977328136237 eval_loss: 1.045175552368164 accuracy: 0.6837 +07/17/2026 17:58:41 - INFO - __main__ - [RANK 0] Starting epoch 2... +07/17/2026 18:00:50 - INFO - __main__ - [RANK 0] perplexity: 2.7927952750158576 eval_loss: 1.0270429849624634 accuracy: 0.6894 +07/17/2026 18:05:00 - INFO - __main__ - [RANK 0] perplexity: 2.754699620188895 eval_loss: 1.0133084058761597 accuracy: 0.6932 +07/17/2026 18:09:10 - INFO - __main__ - [RANK 0] perplexity: 2.720248192942417 eval_loss: 1.000723123550415 accuracy: 0.6967 +07/17/2026 18:13:19 - INFO - __main__ - [RANK 0] perplexity: 2.6901476184602613 eval_loss: 0.9895960688591003 accuracy: 0.7000 +07/17/2026 18:16:20 - INFO - __main__ - [RANK 0] Starting epoch 3... +07/17/2026 18:17:29 - INFO - __main__ - [RANK 0] perplexity: 2.6656972363789504 eval_loss: 0.9804656505584717 accuracy: 0.7028 +07/17/2026 18:18:51 - INFO - __main__ - [RANK 0] Starting epoch 4... +07/17/2026 18:18:51 - INFO - __main__ - [RANK 0] Switching language at step 10000 +07/17/2026 18:19:05 - INFO - __main__ - [RANK 0] Sample 194393 of the training set: {'input_ids': tensor([53, 11, 6, 24, 25, 5, 6, 67, 16, 3, 6, 67, 16, 3, 6, 53, 45, 6, + 12, 68, 10, 17, 6, 12, 68, 10, 17, 6, 12, 68, 10, 17, 6, 53, 11, 6, + 2, 70, 6, 53, 67, 17, 5, 6, 55, 19, 17, 6, 42, 8, 24, 6, 23, 55, + 67, 16, 3, 35, 68, 6, 3, 18, 3, 6, 53, 67, 17, 6, 2, 24, 30, 15, + 6, 31, 57, 17, 6, 53, 11, 6, 3, 34, 55, 6, 42, 8, 24, 6, 2, 53, + 11, 6, 3, 34, 55, 6, 2, 5, 17, 68, 55, 17, 18, 3, 6, 67, 21, 6, + 53, 45, 6, 20, 8, 16, 17, 6, 17, 11, 6, 23, 70, 6, 34, 16, 6, 7, + 11, 55, 53, 11, 6, 24, 68, 12, 18, 3, 45, 6, 2, 8, 20, 8, 20, 6, + 2, 26, 69, 15, 23, 12, 11, 55, 6, 26, 18, 12, 6, 2, 42, 8, 24, 6, + 2, 19, 6, 3, 70, 16, 17, 6, 55, 18, 20, 8, 20, 26, 11, 55, 6, 53, + 67, 17, 6, 2, 10, 12, 19, 20, 3, 6, 54, 55, 27, 15, 6, 53, 11, 6, + 48, 18, 16, 3, 70, 6, 53, 11, 6, 16, 8, 10, 5, 17, 6, 3, 34, 55, + 6, 16, 45, 26, 11, 55, 6, 7, 70, 16, 3, 6, 57, 35, 6, 23, 68, 17, + 6, 57, 6, 5, 35, 8, 55, 6, 10, 30, 15, 6, 2, 18, 20, 67, 46, 18, + 16, 45, 31, 11, 16, 6, 48, 69, 15, 10, 18, 37, 6, 70, 13, 11, 55, 17, + 19, 20, 6, 53, 8, 55, 6, 2, 8, 20, 8, 20, 6, 2, 19, 6, 42, 27, + 15, 21, 3, 6, 17, 11, 6, 12, 19, 10, 6, 53, 11, 6, 10, 68, 55, 6, + 26, 27, 15, 17, 6, 5, 45, 12, 21, 6, 25, 17, 6, 24, 18, 55, 6, 53, + 45, 6, 48, 69, 15, 6, 23, 29, 3, 6, 2, 26, 57, 17, 6, 53, 45, 6, + 5, 17, 68, 35, 17, 6, 53, 8, 20, 6, 12, 67, 5, 17, 6, 42, 18, 55, + 6, 3, 18, 3, 16, 17, 6, 53, 45, 6, 2, 19, 6, 54, 18, 37, 10, 6, + 5, 70, 6, 2, 53, 45, 6, 53, 45, 6, 5, 17, 68, 35, 17, 6, 24, 67, + 13, 18, 37, 6, 53, 8, 20, 6, 34, 16, 53, 11, 6, 53, 11, 6, 5, 30, + 15, 7, 55, 57, 16, 17, 6, 8, 16, 18, 48, 45, 6, 2, 19, 6, 42, 27, + 15, 21, 3, 6, 17, 11, 6, 12, 19, 10, 6, 53, 8, 20, 6, 57, 12, 34, + 37, 6, 53, 8, 55, 6, 18, 17, 6, 48, 57, 21, 6, 16, 19, 5, 6, 10, + 68, 5, 6, 42, 27, 15, 6, 10, 29, 3, 6, 23, 70, 6, 10, 29, 3, 6, + 10, 11, 20, 26, 19, 16, 6, 57, 6, 57, 6, 48, 34, 10, 6, 26, 19, 6, + 53, 11, 6, 5, 30, 15, 6, 48])} ð ə WORD_BOUNDARY h aʊ s WORD_BOUNDARY æ n d WORD_BOUNDARY æ n d WORD_BOUNDARY ð eɪ WORD_BOUNDARY l ɑ k t WORD_BOUNDARY l ɑ k t WORD_BOUNDARY l ɑ k t WORD_BOUNDARY ð ə WORD_BOUNDARY UTT_BOUNDARY oʊ WORD_BOUNDARY ð æ t s WORD_BOUNDARY ɹ aɪ t WORD_BOUNDARY j ɛ h WORD_BOUNDARY ɡ ɹ æ n d p ɑ WORD_BOUNDARY d ɪ d WORD_BOUNDARY ð æ t WORD_BOUNDARY UTT_BOUNDARY h i ː WORD_BOUNDARY ʃ ʌ t WORD_BOUNDARY ð ə WORD_BOUNDARY d ɔ ɹ WORD_BOUNDARY j ɛ h WORD_BOUNDARY UTT_BOUNDARY ð ə WORD_BOUNDARY d ɔ ɹ WORD_BOUNDARY UTT_BOUNDARY s t ɑ ɹ t ɪ d WORD_BOUNDARY æ z WORD_BOUNDARY ð eɪ WORD_BOUNDARY m ɛ n t WORD_BOUNDARY t ə WORD_BOUNDARY ɡ oʊ WORD_BOUNDARY ɔ n WORD_BOUNDARY f ə ɹ ð ə WORD_BOUNDARY h ɑ l ɪ d eɪ WORD_BOUNDARY UTT_BOUNDARY ɛ m ɛ m WORD_BOUNDARY UTT_BOUNDARY b ɜ ː ɡ l ə ɹ WORD_BOUNDARY b ɪ l WORD_BOUNDARY UTT_BOUNDARY j ɛ h WORD_BOUNDARY UTT_BOUNDARY aɪ WORD_BOUNDARY d oʊ n t WORD_BOUNDARY ɹ ɪ m ɛ m b ə ɹ WORD_BOUNDARY ð æ t WORD_BOUNDARY UTT_BOUNDARY k l aɪ m d WORD_BOUNDARY θ ɹ u ː WORD_BOUNDARY ð ə WORD_BOUNDARY w ɪ n d oʊ WORD_BOUNDARY ð ə WORD_BOUNDARY n ɛ k s t WORD_BOUNDARY d ɔ ɹ WORD_BOUNDARY n eɪ b ə ɹ WORD_BOUNDARY f oʊ n d WORD_BOUNDARY ʌ p WORD_BOUNDARY ɡ ɑ t WORD_BOUNDARY ʌ WORD_BOUNDARY s p ɛ ɹ WORD_BOUNDARY k i ː WORD_BOUNDARY UTT_BOUNDARY ɪ m æ d̠ʒ ɪ n eɪ ʃ ə n WORD_BOUNDARY w ɜ ː k ɪ ŋ WORD_BOUNDARY oʊ v ə ɹ t aɪ m WORD_BOUNDARY ð ɛ ɹ WORD_BOUNDARY UTT_BOUNDARY ɛ m ɛ m WORD_BOUNDARY UTT_BOUNDARY aɪ WORD_BOUNDARY j u ː z d WORD_BOUNDARY t ə WORD_BOUNDARY l aɪ k WORD_BOUNDARY ð ə WORD_BOUNDARY k ɑ ɹ WORD_BOUNDARY b u ː t WORD_BOUNDARY s eɪ l z WORD_BOUNDARY aʊ t WORD_BOUNDARY h ɪ ɹ WORD_BOUNDARY ð eɪ WORD_BOUNDARY w ɜ ː WORD_BOUNDARY ɡ ʊ d WORD_BOUNDARY UTT_BOUNDARY b ʌ t WORD_BOUNDARY ð eɪ WORD_BOUNDARY s t ɑ p t WORD_BOUNDARY ð ɛ m WORD_BOUNDARY l æ s t WORD_BOUNDARY j ɪ ɹ WORD_BOUNDARY d ɪ d n t WORD_BOUNDARY ð eɪ WORD_BOUNDARY UTT_BOUNDARY aɪ WORD_BOUNDARY θ ɪ ŋ k WORD_BOUNDARY s oʊ WORD_BOUNDARY UTT_BOUNDARY ð eɪ WORD_BOUNDARY ð eɪ WORD_BOUNDARY s t ɑ p t WORD_BOUNDARY h æ v ɪ ŋ WORD_BOUNDARY ð ɛ m WORD_BOUNDARY ɔ n ð ə WORD_BOUNDARY ð ə WORD_BOUNDARY s i ː f ɹ ʌ n t WORD_BOUNDARY ɛ n ɪ w eɪ WORD_BOUNDARY UTT_BOUNDARY aɪ WORD_BOUNDARY j u ː z d WORD_BOUNDARY t ə WORD_BOUNDARY l aɪ k WORD_BOUNDARY ð ɛ m WORD_BOUNDARY ʌ l ɔ ŋ WORD_BOUNDARY ð ɛ ɹ WORD_BOUNDARY ɪ t WORD_BOUNDARY w ʌ z WORD_BOUNDARY n aɪ s WORD_BOUNDARY k ɑ s WORD_BOUNDARY j u ː WORD_BOUNDARY k ʊ d WORD_BOUNDARY ɡ oʊ WORD_BOUNDARY k ʊ d WORD_BOUNDARY k ə m b aɪ n WORD_BOUNDARY ʌ WORD_BOUNDARY ʌ WORD_BOUNDARY w ɔ k WORD_BOUNDARY b aɪ WORD_BOUNDARY ð ə WORD_BOUNDARY s i ː WORD_BOUNDARY w +07/17/2026 18:19:05 - INFO - __main__ - [RANK 0] Sample 72097 of the training set: {'input_ids': tensor([35, 55, 8, 5, 18, 37, 6, 46, 68, 26, 2, 35, 12, 45, 55, 27, 15, 20, + 6, 13, 18, 21, 18, 17, 6, 2, 67, 5, 17, 11, 55, 55, 18, 5, 10, 6, + 8, 20, 70, 17, 30, 15, 6, 34, 12, 55, 19, 17, 6, 5, 70, 6, 12, 8, + 17, 6, 20, 30, 15, 6, 35, 29, 17, 6, 3, 25, 16, 6, 20, 19, 6, 5, + 17, 57, 7, 6, 2, 67, 5, 17, 11, 55, 55, 18, 5, 10, 6, 8, 20, 70, + 17, 30, 15, 6, 3, 18, 3, 6, 42, 27, 15, 6, 24, 18, 55, 6, 48, 57, + 17, 6, 46, 67, 16, 18, 17, 6, 48, 57, 21, 6, 17, 8, 12, 18, 37, 6, + 42, 27, 15, 6, 2, 67, 5, 17, 11, 55, 55, 18, 5, 10, 6, 10, 19, 6, + 48, 18, 51, 6, 2, 67, 5, 17, 11, 55, 55, 18, 5, 10, 6, 20, 68, 17, + 6, 24, 57, 6, 2, 67, 5, 17, 11, 55, 55, 18, 5, 10, 6, 8, 20, 70, + 17, 30, 15, 6, 31, 30, 15, 6, 48, 57, 21, 6, 17, 8, 12, 18, 37, 6, + 20, 30, 15, 6, 2, 67, 5, 17, 11, 55, 55, 18, 5, 10, 6, 8, 20, 70, + 17, 30, 15, 6, 48, 8, 12, 6, 53, 8, 55, 68, 55, 6, 54, 55, 30, 15, + 6, 3, 18, 7, 55, 11, 16, 17, 6, 54, 18, 37, 21, 6, 17, 11, 6, 3, + 27, 15, 6, 70, 10, 45, 6, 2, 67, 5, 17, 11, 55, 55, 18, 5, 10, 6, + 8, 20, 70, 17, 30, 15, 6, 53, 8, 55, 21, 6, 57, 6, 26, 29, 10, 6, + 17, 11, 6, 12, 29, 10, 6, 67, 17, 6, 53, 8, 55, 21, 6, 57, 6, 17, + 45, 10, 6, 57, 35, 68, 55, 17, 6, 10, 68, 55, 6, 67, 16, 3, 6, 53, + 8, 55, 21, 6, 57, 6, 57, 6, 5, 17, 34, 55, 6, 5, 34, 55, 17, 6, + 57, 13, 6, 12, 19, 10, 6, 46, 68, 31, 6, 24, 57, 21, 6, 67, 17, 6, + 24, 18, 21, 6, 24, 25, 5, 6, 2, 67, 5, 17, 11, 55, 55, 18, 5, 10, + 6, 8, 20, 70, 17, 30, 15, 6, 5, 70, 6, 3, 27, 15, 6, 42, 27, 15, + 6, 16, 70, 6, 48, 18, 51, 6, 48, 57, 16, 6, 42, 27, 15, 6, 48, 29, + 3, 6, 12, 19, 10, 6, 17, 11, 6, 3, 27, 15, 6, 7, 69, 15, 5, 17, + 6, 2, 67, 5, 17, 11, 55, 55, 18, 5, 10, 6, 8, 20, 70, 17, 30, 15, + 6, 53, 11, 6, 17, 45, 10, 6, 57, 35, 68, 55, 17, 6, 10, 68, 55, 6, + 2, 67, 5, 17, 11, 55, 55, 18, 5, 10, 6, 8, 20, 70, 17, 30, 15, 6, + 12, 8, 17, 5, 6, 17, 45, 10, 6, 18, 17, 6, 3, 25, 16, 6, 2, 67, + 5, 17, 11, 55, 55, 18, 5, 10])} p ɹ ɛ s ɪ ŋ WORD_BOUNDARY d̠ʒ ɑ b UTT_BOUNDARY p l eɪ ɹ u ː m WORD_BOUNDARY v ɪ z ɪ t WORD_BOUNDARY UTT_BOUNDARY æ s t ə ɹ ɹ ɪ s k WORD_BOUNDARY ɛ m oʊ t i ː WORD_BOUNDARY ɔ l ɹ aɪ t WORD_BOUNDARY s oʊ WORD_BOUNDARY l ɛ t WORD_BOUNDARY m i ː WORD_BOUNDARY p ʊ t WORD_BOUNDARY d aʊ n WORD_BOUNDARY m aɪ WORD_BOUNDARY s t ʌ f WORD_BOUNDARY UTT_BOUNDARY æ s t ə ɹ ɹ ɪ s k WORD_BOUNDARY ɛ m oʊ t i ː WORD_BOUNDARY d ɪ d WORD_BOUNDARY j u ː WORD_BOUNDARY h ɪ ɹ WORD_BOUNDARY w ʌ t WORD_BOUNDARY d̠ʒ æ n ɪ t WORD_BOUNDARY w ʌ z WORD_BOUNDARY t ɛ l ɪ ŋ WORD_BOUNDARY j u ː WORD_BOUNDARY UTT_BOUNDARY æ s t ə ɹ ɹ ɪ s k WORD_BOUNDARY k aɪ WORD_BOUNDARY w ɪ t̠ʃ WORD_BOUNDARY UTT_BOUNDARY æ s t ə ɹ ɹ ɪ s k WORD_BOUNDARY m ɑ t WORD_BOUNDARY h ʌ WORD_BOUNDARY UTT_BOUNDARY æ s t ə ɹ ɹ ɪ s k WORD_BOUNDARY ɛ m oʊ t i ː WORD_BOUNDARY ʃ i ː WORD_BOUNDARY w ʌ z WORD_BOUNDARY t ɛ l ɪ ŋ WORD_BOUNDARY m i ː WORD_BOUNDARY UTT_BOUNDARY æ s t ə ɹ ɹ ɪ s k WORD_BOUNDARY ɛ m oʊ t i ː WORD_BOUNDARY w ɛ l WORD_BOUNDARY ð ɛ ɹ ɑ ɹ WORD_BOUNDARY θ ɹ i ː WORD_BOUNDARY d ɪ f ɹ ə n t WORD_BOUNDARY θ ɪ ŋ z WORD_BOUNDARY t ə WORD_BOUNDARY d u ː WORD_BOUNDARY oʊ k eɪ WORD_BOUNDARY UTT_BOUNDARY æ s t ə ɹ ɹ ɪ s k WORD_BOUNDARY ɛ m oʊ t i ː WORD_BOUNDARY ð ɛ ɹ z WORD_BOUNDARY ʌ WORD_BOUNDARY b ʊ k WORD_BOUNDARY t ə WORD_BOUNDARY l ʊ k WORD_BOUNDARY æ t WORD_BOUNDARY ð ɛ ɹ z WORD_BOUNDARY ʌ WORD_BOUNDARY t eɪ k WORD_BOUNDARY ʌ p ɑ ɹ t WORD_BOUNDARY k ɑ ɹ WORD_BOUNDARY æ n d WORD_BOUNDARY ð ɛ ɹ z WORD_BOUNDARY ʌ WORD_BOUNDARY ʌ WORD_BOUNDARY s t ɔ ɹ WORD_BOUNDARY s ɔ ɹ t WORD_BOUNDARY ʌ v WORD_BOUNDARY l aɪ k WORD_BOUNDARY d̠ʒ ɑ ʃ WORD_BOUNDARY h ʌ z WORD_BOUNDARY æ t WORD_BOUNDARY h ɪ z WORD_BOUNDARY h aʊ s WORD_BOUNDARY UTT_BOUNDARY æ s t ə ɹ ɹ ɪ s k WORD_BOUNDARY ɛ m oʊ t i ː WORD_BOUNDARY s oʊ WORD_BOUNDARY d u ː WORD_BOUNDARY j u ː WORD_BOUNDARY n oʊ WORD_BOUNDARY w ɪ t̠ʃ WORD_BOUNDARY w ʌ n WORD_BOUNDARY j u ː WORD_BOUNDARY w ʊ d WORD_BOUNDARY l aɪ k WORD_BOUNDARY t ə WORD_BOUNDARY d u ː WORD_BOUNDARY f ɜ ː s t WORD_BOUNDARY UTT_BOUNDARY æ s t ə ɹ ɹ ɪ s k WORD_BOUNDARY ɛ m oʊ t i ː WORD_BOUNDARY ð ə WORD_BOUNDARY t eɪ k WORD_BOUNDARY ʌ p ɑ ɹ t WORD_BOUNDARY k ɑ ɹ WORD_BOUNDARY UTT_BOUNDARY æ s t ə ɹ ɹ ɪ s k WORD_BOUNDARY ɛ m oʊ t i ː WORD_BOUNDARY l ɛ t s WORD_BOUNDARY t eɪ k WORD_BOUNDARY ɪ t WORD_BOUNDARY d aʊ n WORD_BOUNDARY UTT_BOUNDARY æ s t ə ɹ ɹ ɪ s k +07/17/2026 18:19:05 - INFO - __main__ - [RANK 0] Sample 64196 of the training set: {'input_ids': tensor([28, 17, 6, 2, 20, 34, 17, 9, 6, 19, 16, 6, 26, 18, 5, 39, 11, 16, + 6, 20, 22, 15, 9, 6, 8, 9, 16, 5, 17, 6, 24, 30, 15, 9, 6, 26, + 18, 17, 11, 6, 2, 7, 4, 17, 9, 6, 36, 41, 15, 20, 18, 31, 6, 3, + 9, 19, 5, 18, 39, 6, 2, 7, 4, 17, 9, 6, 19, 39, 30, 15, 6, 24, + 4, 17, 6, 3, 22, 15, 16, 6, 36, 41, 15, 20, 18, 31, 6, 3, 9, 19, + 5, 18, 39, 6, 4, 16, 23, 11, 10, 16, 4, 26, 9, 17, 6, 2, 20, 34, + 17, 9, 6, 19, 16, 6, 10, 12, 19, 16, 11, 5, 6, 12, 30, 15, 26, 11, + 5, 6, 21, 25, 36, 30, 15, 9, 13, 19, 26, 39, 11, 16, 6, 18, 5, 17, + 6, 3, 4, 15, 6, 23, 8, 5, 17, 9, 16, 6, 16, 4, 28, 17, 6, 7, + 14, 15, 9, 26, 19, 23, 11, 12, 25, 7, 11, 16, 6, 36, 41, 15, 20, 18, + 31, 6, 3, 9, 19, 5, 18, 39, 6, 2, 7, 4, 17, 9, 6, 18, 16, 6, + 3, 22, 15, 20, 6, 23, 4, 16, 33, 11, 16, 6, 36, 41, 15, 20, 18, 31, + 6, 3, 9, 19, 5, 18, 39, 6, 24, 30, 15, 9, 6, 12, 8, 33, 17, 11, + 6, 16, 4, 28, 17, 6, 2, 17, 4, 9, 23, 11, 17, 6, 31, 18, 12, 17, + 6, 3, 4, 15, 6, 4, 28, 33, 22, 15, 16, 6, 23, 12, 19, 21, 11, 6, + 2, 17, 4, 9, 23, 11, 17, 6, 31, 18, 12, 17, 6, 3, 4, 15, 6, 4, + 28, 33, 22, 15, 16, 6, 23, 12, 19, 5, 6, 2, 7, 4, 17, 9, 6, 4, + 28, 6, 21, 14, 15, 6, 2, 17, 4, 9, 23, 11, 17, 6, 31, 18, 12, 17, + 6, 19, 16, 6, 26, 29, 5, 6, 10, 34, 20, 17, 6, 24, 18, 16, 17, 9, + 24, 9, 6, 2, 7, 4, 17, 9, 6, 19, 16, 6, 26, 29, 5, 6, 10, 34, + 20, 17, 6, 16, 34, 28, 6, 24, 18, 16, 17, 9, 24, 9, 6, 2, 17, 4, + 9, 23, 11, 17, 6, 31, 18, 12, 17, 6, 42, 4, 15, 6, 2, 7, 4, 17, + 9, 6, 34, 28, 6, 12, 22, 15, 14, 15, 6, 2, 7, 4, 17, 9, 6, 13, + 30, 15, 9, 6, 21, 34, 12, 17, 11, 16, 6, 16, 18, 39, 17, 6, 21, 8, + 20, 17, 12, 18, 39, 11, 6, 12, 14, 15, 10, 14, 15, 20, 14, 15, 17, 30, + 15, 13, 11, 16, 6, 29, 16, 17, 6, 33, 40, 15, 23, 11, 6, 24, 30, 15, + 9, 6, 25, 7, 26, 25, 11, 16, 6, 2, 7, 4, 17, 9, 6, 3, 27, 15, + 6, 3, 4, 5, 6, 13, 18, 9, 17, 6, 3, 4, 16, 6, 13, 18, 9, 10, + 12, 18, 39, 6, 19, 16, 6, 26])} x t WORD_BOUNDARY UTT_BOUNDARY m ɔ t ɐ WORD_BOUNDARY aɪ n WORD_BOUNDARY b ɪ s ç ə n WORD_BOUNDARY m e ː ɐ WORD_BOUNDARY ɛ ɐ n s t WORD_BOUNDARY h i ː ɐ WORD_BOUNDARY b ɪ t ə WORD_BOUNDARY UTT_BOUNDARY f a t ɐ WORD_BOUNDARY ʀ ø ː m ɪ ʃ WORD_BOUNDARY d ɐ aɪ s ɪ ç WORD_BOUNDARY UTT_BOUNDARY f a t ɐ WORD_BOUNDARY aɪ ç i ː WORD_BOUNDARY h a t WORD_BOUNDARY d e ː n WORD_BOUNDARY ʀ ø ː m ɪ ʃ WORD_BOUNDARY d ɐ aɪ s ɪ ç WORD_BOUNDARY a n ɡ ə k n a b ɐ t WORD_BOUNDARY UTT_BOUNDARY m ɔ t ɐ WORD_BOUNDARY aɪ n WORD_BOUNDARY k l aɪ n ə s WORD_BOUNDARY l i ː b ə s WORD_BOUNDARY z aʊ ʀ i ː ɐ v aɪ b ç ə n WORD_BOUNDARY ɪ s t WORD_BOUNDARY d a ː WORD_BOUNDARY ɡ ɛ s t ɐ n WORD_BOUNDARY n a x t WORD_BOUNDARY f o ː ɐ b aɪ ɡ ə l aʊ f ə n WORD_BOUNDARY ʀ ø ː m ɪ ʃ WORD_BOUNDARY d ɐ aɪ s ɪ ç WORD_BOUNDARY UTT_BOUNDARY f a t ɐ WORD_BOUNDARY ɪ n WORD_BOUNDARY d e ː m WORD_BOUNDARY ɡ a n ts ə n WORD_BOUNDARY ʀ ø ː m ɪ ʃ WORD_BOUNDARY d ɐ aɪ s ɪ ç WORD_BOUNDARY h i ː ɐ WORD_BOUNDARY l ɛ ts t ə WORD_BOUNDARY n a x t WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY d a ː WORD_BOUNDARY a x ts e ː n WORD_BOUNDARY ɡ l aɪ z ə WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY d a ː WORD_BOUNDARY a x ts e ː n WORD_BOUNDARY ɡ l aɪ s WORD_BOUNDARY UTT_BOUNDARY f a t ɐ WORD_BOUNDARY a x WORD_BOUNDARY z o ː WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY aɪ n WORD_BOUNDARY b ʊ s WORD_BOUNDARY k ɔ m t WORD_BOUNDARY h ɪ n t ɐ h ɐ WORD_BOUNDARY UTT_BOUNDARY f a t ɐ WORD_BOUNDARY aɪ n WORD_BOUNDARY b ʊ s WORD_BOUNDARY k ɔ m t WORD_BOUNDARY n ɔ x WORD_BOUNDARY h ɪ n t ɐ h ɐ WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY j a ː WORD_BOUNDARY UTT_BOUNDARY f a t ɐ WORD_BOUNDARY ɔ x WORD_BOUNDARY l e ː o ː WORD_BOUNDARY UTT_BOUNDARY f a t ɐ WORD_BOUNDARY v i ː ɐ WORD_BOUNDARY z ɔ l t ə n WORD_BOUNDARY n ɪ ç t WORD_BOUNDARY z ɛ m t l ɪ ç ə WORD_BOUNDARY l o ː k o ː m o ː t i ː v ə n WORD_BOUNDARY ʊ n t WORD_BOUNDARY ts y ː ɡ ə WORD_BOUNDARY h i ː ɐ WORD_BOUNDARY aʊ f b aʊ ə n WORD_BOUNDARY UTT_BOUNDARY f a t ɐ WORD_BOUNDARY d u ː WORD_BOUNDARY d a s WORD_BOUNDARY v ɪ ɐ t WORD_BOUNDARY d a n WORD_BOUNDARY v ɪ ɐ k l ɪ ç WORD_BOUNDARY aɪ n WORD_BOUNDARY b +07/17/2026 18:21:56 - INFO - __main__ - [RANK 0] perplexity: 3.0484192847352145 eval_loss: 1.1146231889724731 accuracy: 0.6692 +07/17/2026 18:26:09 - INFO - __main__ - [RANK 0] perplexity: 2.918761442567045 eval_loss: 1.0711593627929688 accuracy: 0.6814 +07/17/2026 18:30:22 - INFO - __main__ - [RANK 0] perplexity: 2.859224235205469 eval_loss: 1.0505503416061401 accuracy: 0.6865 +07/17/2026 18:34:35 - INFO - __main__ - [RANK 0] perplexity: 2.813755057927187 eval_loss: 1.034519910812378 accuracy: 0.6909 +07/17/2026 18:38:48 - INFO - __main__ - [RANK 0] perplexity: 2.7864847898100327 eval_loss: 1.0247808694839478 accuracy: 0.6934 +07/17/2026 18:43:01 - INFO - __main__ - [RANK 0] perplexity: 2.764500612181717 eval_loss: 1.016860008239746 accuracy: 0.6957 +07/17/2026 18:47:14 - INFO - __main__ - [RANK 0] perplexity: 2.7417712931540006 eval_loss: 1.0086041688919067 accuracy: 0.6980 +07/17/2026 18:51:27 - INFO - __main__ - [RANK 0] perplexity: 2.7246291571910985 eval_loss: 1.002332329750061 accuracy: 0.6998 +07/17/2026 18:55:13 - INFO - __main__ - [RANK 0] Starting epoch 5... +07/17/2026 18:55:40 - INFO - __main__ - [RANK 0] perplexity: 2.7098060775772677 eval_loss: 0.9968770742416382 accuracy: 0.7014 +07/17/2026 18:59:53 - INFO - __main__ - [RANK 0] perplexity: 2.6975715684966413 eval_loss: 0.9923519492149353 accuracy: 0.7030 +07/17/2026 19:04:06 - INFO - __main__ - [RANK 0] perplexity: 2.6870244633754528 eval_loss: 0.9884344339370728 accuracy: 0.7036 +07/17/2026 19:08:19 - INFO - __main__ - [RANK 0] perplexity: 2.6716521655713357 eval_loss: 0.982697069644928 accuracy: 0.7056 +07/17/2026 19:12:32 - INFO - __main__ - [RANK 0] perplexity: 2.6608852148278723 eval_loss: 0.9786588549613953 accuracy: 0.7066 +07/17/2026 19:16:45 - INFO - __main__ - [RANK 0] perplexity: 2.6509383103696127 eval_loss: 0.9749136567115784 accuracy: 0.7073 +07/17/2026 19:20:58 - INFO - __main__ - [RANK 0] perplexity: 2.6409390202883816 eval_loss: 0.9711345434188843 accuracy: 0.7082 +07/17/2026 19:25:11 - INFO - __main__ - [RANK 0] perplexity: 2.632221939372491 eval_loss: 0.9678283333778381 accuracy: 0.7095 +07/17/2026 19:29:24 - INFO - __main__ - [RANK 0] perplexity: 2.625493426362093 eval_loss: 0.9652688503265381 accuracy: 0.7106 +07/17/2026 19:31:27 - INFO - __main__ - [RANK 0] Starting epoch 6... +07/17/2026 19:33:38 - INFO - __main__ - [RANK 0] perplexity: 2.6188832621420537 eval_loss: 0.9627479910850525 accuracy: 0.7111 +07/17/2026 19:37:51 - INFO - __main__ - [RANK 0] perplexity: 2.610899356870117 eval_loss: 0.9596947431564331 accuracy: 0.7122 +07/17/2026 19:42:04 - INFO - __main__ - [RANK 0] perplexity: 2.6061475749225473 eval_loss: 0.9578731060028076 accuracy: 0.7125 +07/17/2026 19:46:17 - INFO - __main__ - [RANK 0] perplexity: 2.599512518477179 eval_loss: 0.9553239345550537 accuracy: 0.7132 +07/17/2026 19:50:29 - INFO - __main__ - [RANK 0] perplexity: 2.5936871550638263 eval_loss: 0.9530804753303528 accuracy: 0.7142 +07/17/2026 19:54:42 - INFO - __main__ - [RANK 0] perplexity: 2.587028926508626 eval_loss: 0.9505100846290588 accuracy: 0.7145 +07/17/2026 19:58:55 - INFO - __main__ - [RANK 0] perplexity: 2.5835993501697985 eval_loss: 0.9491835236549377 accuracy: 0.7150 +07/17/2026 20:03:08 - INFO - __main__ - [RANK 0] perplexity: 2.5794144002582415 eval_loss: 0.9475623965263367 accuracy: 0.7156 +07/17/2026 20:07:21 - INFO - __main__ - [RANK 0] perplexity: 2.5754363958021846 eval_loss: 0.9460189938545227 accuracy: 0.7159 +07/17/2026 20:07:41 - INFO - __main__ - [RANK 0] Starting epoch 7... +07/17/2026 20:11:28 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/17/2026 20:11:28 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/17/2026 20:11:31 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/17/2026 20:11:31 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/17/2026 20:11:31 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/17/2026 20:12:44 - INFO - __main__ - [RANK 0] perplexity: 2.576217716361442 eval_loss: 0.9463223218917847 accuracy: 0.7161 +07/17/2026 20:12:44 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/17/2026 20:12:44 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/17/2026 20:12:47 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/17/2026 20:12:47 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/17/2026 20:12:47 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 09:47:49 - INFO - __main__ - [RANK 0] Distributed environment: DistributedType.NO +Num processes: 1 +Process index: 0 +Local process index: 0 +Device: cuda + +Mixed precision type: no + +07/18/2026 09:47:49 - INFO - __main__ - [RANK 0] Training args: +{'data_dir': 'data/phone/deu-eng/tokenized_sequential_interleaved', 'model_name_or_path': None, 'config_name': 'gpt_base_config.json', 'tokenizer_name': 'nikitastheo/phonelm-phone-deu-eng-tokenizer', 'per_device_train_batch_size': 32, 'per_device_eval_batch_size': 32, 'learning_rate': 0.0001, 'weight_decay': 0.01, 'max_steps': 30000, 'gradient_accumulation_steps': 1, 'lr_scheduler_type': , 'warmup_steps': 3000, 'output_dir': 'models/phonebabylm-phone-deu-eng-sequential_interleaved', 'seed': 42, 'model_type': None, 'block_size': None, 'push_to_hub': True, 'hub_model_id': 'nikitastheo/phonebabylm-phone-deu-eng-sequential_interleaved', 'hub_token': None, 'private': False, 'trust_remote_code': False, 'with_tracking': True, 'tracking_dir': None, 'report_to': 'wandb', 'restart_lr_at_switch': False, 'language_switch_steps': 10000, 'model_revision': 'main', 'config_overrides': None, 'use_slow_tokenizer': False, 'override_vocab_size': None, 'dtype': None, 'logging_steps': 10, 'cache_dir': None, 'eval_steps': 750, 'max_grad_norm': 1.0, 'adam_epsilon': 1e-06, 'run_name': 'phonebabylm-phone-deu-eng-sequential_interleaved', 'max_train_samples': None, 'max_eval_samples': None} +07/18/2026 09:47:49 - INFO - transformers.configuration_utils - loading configuration file gpt_base_config.json +07/18/2026 09:47:49 - INFO - transformers.configuration_utils - Model config GPT2Config { + "activation_function": "gelu", + "architectures": [ + "GPT2LMHeadModel" + ], + "attn_pdrop": 0.1, + "bos_token_id": 50256, + "embd_pdrop": 0.1, + "eos_token_id": 50256, + "initializer_range": 0.02, + "layer_norm_epsilon": 1e-05, + "model_type": "gpt2", + "n_ctx": 512, + "n_embd": 768, + "n_head": 12, + "n_inner": 3072, + "n_layer": 12, + "n_positions": 512, + "reorder_and_upcast_attn": false, + "resid_pdrop": 0.1, + "scale_attn_by_inverse_layer_idx": false, + "scale_attn_weights": true, + "summary_activation": null, + "summary_first_dropout": 0.1, + "summary_proj_to_labels": true, + "summary_type": "cls_index", + "summary_use_proj": true, + "transformers_version": "4.57.6", + "use_cache": true, + "vocab_size": 1 +} + +07/18/2026 09:47:49 - INFO - transformers.tokenization_utils_base - loading file vocab.json from cache at /home/geofila/.cache/huggingface/hub/models--nikitastheo--phonelm-phone-deu-eng-tokenizer/snapshots/a9eb582bae549d2e78f9e6f7913343e704373e8a/vocab.json +07/18/2026 09:47:49 - INFO - transformers.tokenization_utils_base - loading file merges.txt from cache at None +07/18/2026 09:47:49 - INFO - transformers.tokenization_utils_base - loading file tokenizer.json from cache at /home/geofila/.cache/huggingface/hub/models--nikitastheo--phonelm-phone-deu-eng-tokenizer/snapshots/a9eb582bae549d2e78f9e6f7913343e704373e8a/tokenizer.json +07/18/2026 09:47:49 - INFO - transformers.tokenization_utils_base - loading file added_tokens.json from cache at None +07/18/2026 09:47:49 - INFO - transformers.tokenization_utils_base - loading file special_tokens_map.json from cache at /home/geofila/.cache/huggingface/hub/models--nikitastheo--phonelm-phone-deu-eng-tokenizer/snapshots/a9eb582bae549d2e78f9e6f7913343e704373e8a/special_tokens_map.json +07/18/2026 09:47:49 - INFO - transformers.tokenization_utils_base - loading file tokenizer_config.json from cache at /home/geofila/.cache/huggingface/hub/models--nikitastheo--phonelm-phone-deu-eng-tokenizer/snapshots/a9eb582bae549d2e78f9e6f7913343e704373e8a/tokenizer_config.json +07/18/2026 09:47:49 - INFO - transformers.tokenization_utils_base - loading file chat_template.jinja from cache at None +07/18/2026 09:47:51 - INFO - transformers.generation.configuration_utils - Generate config GenerationConfig { + "bos_token_id": 2, + "eos_token_id": 2, + "pad_token_id": 1 +} + +07/18/2026 09:47:52 - INFO - __main__ - [RANK 0] Model parameters: 85.50M total (85.50M trainable) +07/18/2026 09:48:03 - INFO - __main__ - [RANK 0] Sample 83810 of the training set: {'input_ids': tensor([ 9, 6, 34, 32, 39, 6, 21, 14, 15, 6, 8, 17, 13, 4, 15, 5, 6, 13, + 18, 39, 17, 18, 23, 11, 5, 6, 8, 9, 33, 8, 15, 12, 17, 6, 2, 13, + 30, 15, 9, 6, 13, 4, 15, 36, 11, 16, 6, 20, 18, 17, 6, 3, 22, 15, + 20, 6, 43, 4, 36, 9, 6, 7, 8, 9, 4, 35, 9, 22, 15, 3, 11, 17, + 6, 2, 21, 30, 15, 6, 21, 18, 16, 17, 6, 16, 18, 39, 17, 6, 23, 11, + 10, 34, 20, 11, 16, 6, 2, 13, 4, 5, 6, 18, 5, 17, 6, 24, 30, 15, + 9, 6, 12, 14, 15, 5, 6, 2, 31, 4, 9, 12, 11, 5, 6, 21, 34, 12, + 6, 8, 9, 20, 34, 9, 3, 11, 17, 6, 13, 34, 9, 3, 11, 16, 6, 21, + 19, 16, 6, 2, 12, 27, 15, 18, 5, 11, 6, 3, 30, 15, 21, 9, 6, 21, + 34, 12, 3, 4, 15, 17, 6, 18, 5, 17, 6, 16, 18, 39, 17, 6, 26, 19, + 6, 10, 12, 4, 15, 36, 11, 20, 6, 7, 8, 9, 31, 17, 4, 16, 17, 6, + 2, 30, 15, 9, 6, 21, 14, 15, 16, 6, 13, 44, 3, 11, 6, 16, 18, 39, + 17, 6, 8, 9, 20, 34, 9, 3, 11, 17, 6, 8, 9, 6, 18, 5, 17, 6, + 7, 8, 9, 20, 18, 5, 17, 6, 2, 3, 4, 5, 6, 31, 17, 22, 15, 17, + 6, 18, 16, 6, 4, 12, 11, 16, 6, 26, 11, 36, 18, 39, 17, 11, 16, 6, + 2, 25, 23, 29, 5, 17, 6, 24, 34, 12, 17, 6, 24, 25, 35, 17, 20, 4, + 16, 6, 26, 11, 25, 12, 30, 15, 27, 15, 6, 2, 13, 4, 5, 6, 21, 34, + 12, 6, 3, 8, 9, 6, 24, 25, 35, 17, 20, 4, 16, 6, 3, 4, 15, 20, + 18, 17, 6, 4, 16, 7, 4, 37, 11, 16, 6, 2, 31, 18, 10, 11, 16, 6, + 21, 30, 15, 6, 30, 15, 16, 6, 26, 8, 5, 9, 6, 23, 12, 19, 39, 6, + 7, 34, 9, 17, 6, 2, 22, 15, 9, 5, 17, 6, 21, 34, 12, 6, 7, 30, + 15, 12, 18, 35, 11, 6, 30, 15, 16, 6, 4, 16, 24, 41, 15, 36, 11, 16, + 6, 2, 10, 19, 16, 11, 6, 21, 34, 9, 23, 11, 6, 13, 30, 15, 9, 6, + 13, 8, 9, 3, 11, 16, 6, 30, 15, 16, 6, 31, 14, 15, 16, 6, 12, 14, + 15, 5, 6, 2, 21, 8, 12, 17, 21, 4, 15, 20, 11, 6, 23, 11, 31, 18, + 39, 17, 11, 6, 21, 34, 12, 3, 4, 15, 17, 6, 2, 13, 22, 15, 9, 6, + 24, 8, 17, 11, 6, 31, 4, 9, 12, 11, 5, 6, 17, 41, 15, 17, 11, 16, + 6, 13, 34, 12, 11, 16, 6, 2, 29, 16, 17, 6, 13, 4, 15, 36, 29, 20, + 6, 24, 4, 15, 26, 11, 16, 6])} ɐ WORD_BOUNDARY ɔ ʏ ç WORD_BOUNDARY z o ː WORD_BOUNDARY ɛ t v a ː s WORD_BOUNDARY v ɪ ç t ɪ ɡ ə s WORD_BOUNDARY ɛ ɐ ts ɛ ː l t WORD_BOUNDARY UTT_BOUNDARY v i ː ɐ WORD_BOUNDARY v a ː ʀ ə n WORD_BOUNDARY m ɪ t WORD_BOUNDARY d e ː m WORD_BOUNDARY pf a ʀ ɐ WORD_BOUNDARY f ɛ ɐ a p ɐ e ː d ə t WORD_BOUNDARY UTT_BOUNDARY z i ː WORD_BOUNDARY z ɪ n t WORD_BOUNDARY n ɪ ç t WORD_BOUNDARY ɡ ə k ɔ m ə n WORD_BOUNDARY UTT_BOUNDARY v a s WORD_BOUNDARY ɪ s t WORD_BOUNDARY h i ː ɐ WORD_BOUNDARY l o ː s WORD_BOUNDARY UTT_BOUNDARY ʃ a ɐ l ə s WORD_BOUNDARY z ɔ l WORD_BOUNDARY ɛ ɐ m ɔ ɐ d ə t WORD_BOUNDARY v ɔ ɐ d ə n WORD_BOUNDARY z aɪ n WORD_BOUNDARY UTT_BOUNDARY l u ː ɪ s ə WORD_BOUNDARY d i ː z ɐ WORD_BOUNDARY z ɔ l d a ː t WORD_BOUNDARY ɪ s t WORD_BOUNDARY n ɪ ç t WORD_BOUNDARY b aɪ WORD_BOUNDARY k l a ː ʀ ə m WORD_BOUNDARY f ɛ ɐ ʃ t a n t WORD_BOUNDARY UTT_BOUNDARY i ː ɐ WORD_BOUNDARY z o ː n WORD_BOUNDARY v ʊɐ d ə WORD_BOUNDARY n ɪ ç t WORD_BOUNDARY ɛ ɐ m ɔ ɐ d ə t WORD_BOUNDARY ɛ ɐ WORD_BOUNDARY ɪ s t WORD_BOUNDARY f ɛ ɐ m ɪ s t WORD_BOUNDARY UTT_BOUNDARY d a s WORD_BOUNDARY ʃ t e ː t WORD_BOUNDARY ɪ n WORD_BOUNDARY a l ə n WORD_BOUNDARY b ə ʀ ɪ ç t ə n WORD_BOUNDARY UTT_BOUNDARY aʊ ɡ ʊ s t WORD_BOUNDARY h ɔ l t WORD_BOUNDARY h aʊ p t m a n WORD_BOUNDARY b ə aʊ l i ː u ː WORD_BOUNDARY UTT_BOUNDARY v a s WORD_BOUNDARY z ɔ l WORD_BOUNDARY d ɛ ɐ WORD_BOUNDARY h aʊ p t m a n WORD_BOUNDARY d a ː m ɪ t WORD_BOUNDARY a n f a ŋ ə n WORD_BOUNDARY UTT_BOUNDARY ʃ ɪ k ə n WORD_BOUNDARY z i ː WORD_BOUNDARY i ː n WORD_BOUNDARY b ɛ s ɐ WORD_BOUNDARY ɡ l aɪ ç WORD_BOUNDARY f ɔ ɐ t WORD_BOUNDARY UTT_BOUNDARY e ː ɐ s t WORD_BOUNDARY z ɔ l WORD_BOUNDARY f i ː l ɪ p ə WORD_BOUNDARY i ː n WORD_BOUNDARY a n h ø ː ʀ ə n WORD_BOUNDARY UTT_BOUNDARY k aɪ n ə WORD_BOUNDARY z ɔ ɐ ɡ ə WORD_BOUNDARY v i ː ɐ WORD_BOUNDARY v ɛ ɐ d ə n WORD_BOUNDARY i ː n WORD_BOUNDARY ʃ o ː n WORD_BOUNDARY l o ː s WORD_BOUNDARY UTT_BOUNDARY z ɛ l t z a ː m ə WORD_BOUNDARY ɡ ə ʃ ɪ ç t ə WORD_BOUNDARY z ɔ l d a ː t WORD_BOUNDARY UTT_BOUNDARY v e ː ɐ WORD_BOUNDARY h ɛ t ə WORD_BOUNDARY ʃ a ɐ l ə s WORD_BOUNDARY t ø ː t ə n WORD_BOUNDARY v ɔ l ə n WORD_BOUNDARY UTT_BOUNDARY ʊ n t WORD_BOUNDARY v a ː ʀ ʊ m WORD_BOUNDARY h a ː b ə n WORD_BOUNDARY +07/18/2026 09:48:03 - INFO - __main__ - [RANK 0] Sample 14592 of the training set: {'input_ids': tensor([ 6, 17, 4, 31, 11, 6, 2, 17, 4, 9, 23, 11, 17, 6, 31, 18, 12, 17, + 6, 13, 22, 15, 9, 6, 2, 20, 34, 17, 9, 6, 19, 16, 6, 4, 16, 3, + 11, 36, 9, 6, 25, 5, 3, 9, 29, 10, 6, 7, 40, 15, 9, 6, 23, 8, + 12, 17, 6, 2, 20, 34, 17, 9, 6, 3, 4, 15, 6, 10, 4, 16, 6, 20, + 4, 16, 6, 25, 28, 6, 21, 4, 15, 23, 11, 16, 6, 10, 9, 41, 15, 17, + 11, 16, 6, 14, 15, 3, 9, 6, 7, 12, 34, 10, 11, 16, 6, 14, 15, 3, + 9, 6, 35, 22, 15, 16, 29, 16, 33, 11, 6, 2, 18, 16, 13, 8, 5, 17, + 18, 23, 4, 17, 14, 15, 9, 6, 24, 11, 20, 6, 2, 17, 4, 9, 23, 11, + 17, 6, 31, 18, 12, 17, 6, 13, 8, 10, 11, 16, 6, 2, 17, 4, 9, 23, + 11, 17, 6, 31, 18, 12, 17, 6, 14, 15, 3, 9, 6, 10, 9, 8, 15, 36, + 29, 16, 33, 11, 6, 2, 18, 16, 13, 8, 5, 17, 18, 23, 4, 17, 14, 15, + 9, 6, 14, 15, 3, 9, 6, 10, 14, 15, 12, 11, 6, 2, 20, 34, 17, 9, + 6, 10, 14, 15, 12, 11, 6, 23, 11, 16, 25, 6, 2, 17, 4, 9, 23, 11, + 17, 6, 31, 18, 12, 17, 6, 14, 15, 3, 9, 6, 7, 34, 32, 9, 6, 2, + 20, 34, 17, 9, 6, 24, 4, 5, 17, 6, 3, 27, 15, 6, 10, 14, 15, 12, + 11, 6, 2, 18, 16, 13, 8, 5, 17, 18, 23, 4, 17, 14, 15, 9, 6, 7, + 34, 32, 9, 6, 7, 34, 32, 9, 6, 35, 8, 9, 7, 8, 10, 17, 6, 2, + 17, 4, 9, 23, 11, 17, 6, 31, 18, 12, 17, 6, 14, 15, 3, 9, 6, 7, + 34, 32, 9, 6, 18, 16, 6, 3, 8, 9, 6, 17, 4, 31, 11, 6, 2, 20, + 34, 17, 9, 6, 24, 4, 5, 17, 6, 3, 27, 15, 6, 7, 34, 32, 9, 6, + 18, 16, 6, 3, 8, 9, 6, 17, 4, 31, 11, 6, 2, 20, 34, 17, 9, 6, + 10, 9, 30, 15, 23, 5, 17, 6, 4, 15, 26, 9, 6, 19, 16, 11, 16, 6, + 24, 19, 5, 11, 16, 6, 35, 14, 15, 6, 2, 17, 4, 9, 23, 11, 17, 6, + 31, 18, 12, 17, 6, 14, 15, 3, 9, 6, 4, 12, 11, 6, 2, 20, 34, 17, + 9, 6, 14, 15, 3, 9, 6, 13, 4, 5, 6, 2, 17, 4, 9, 23, 11, 17, + 6, 31, 18, 12, 17, 6, 14, 15, 3, 9, 6, 4, 12, 11, 6, 2, 18, 16, + 13, 8, 5, 17, 18, 23, 4, 17, 14, 15, 9, 6, 35, 25, 12, 30, 15, 16, + 11, 6, 23, 29, 10, 5, 17, 6, 3, 27, 15, 6, 20, 4, 15, 12, 6, 23, + 11, 16, 25, 6, 3, 4, 15, 20])} WORD_BOUNDARY t a ʃ ə WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY v e ː ɐ WORD_BOUNDARY UTT_BOUNDARY m ɔ t ɐ WORD_BOUNDARY aɪ n WORD_BOUNDARY a n d ə ʀ ɐ WORD_BOUNDARY aʊ s d ɐ ʊ k WORD_BOUNDARY f y ː ɐ WORD_BOUNDARY ɡ ɛ l t WORD_BOUNDARY UTT_BOUNDARY m ɔ t ɐ WORD_BOUNDARY d a ː WORD_BOUNDARY k a n WORD_BOUNDARY m a n WORD_BOUNDARY aʊ x WORD_BOUNDARY z a ː ɡ ə n WORD_BOUNDARY k ɐ ø ː t ə n WORD_BOUNDARY o ː d ɐ WORD_BOUNDARY f l ɔ k ə n WORD_BOUNDARY o ː d ɐ WORD_BOUNDARY p e ː n ʊ n ts ə WORD_BOUNDARY UTT_BOUNDARY ɪ n v ɛ s t ɪ ɡ a t o ː ɐ WORD_BOUNDARY h ə m WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY v ɛ k ə n WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY o ː d ɐ WORD_BOUNDARY k ɐ ɛ ː ʀ ʊ n ts ə WORD_BOUNDARY UTT_BOUNDARY ɪ n v ɛ s t ɪ ɡ a t o ː ɐ WORD_BOUNDARY o ː d ɐ WORD_BOUNDARY k o ː l ə WORD_BOUNDARY UTT_BOUNDARY m ɔ t ɐ WORD_BOUNDARY k o ː l ə WORD_BOUNDARY ɡ ə n aʊ WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY o ː d ɐ WORD_BOUNDARY f ɔ ʏ ɐ WORD_BOUNDARY UTT_BOUNDARY m ɔ t ɐ WORD_BOUNDARY h a s t WORD_BOUNDARY d u ː WORD_BOUNDARY k o ː l ə WORD_BOUNDARY UTT_BOUNDARY ɪ n v ɛ s t ɪ ɡ a t o ː ɐ WORD_BOUNDARY f ɔ ʏ ɐ WORD_BOUNDARY f ɔ ʏ ɐ WORD_BOUNDARY p ɛ ɐ f ɛ k t WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY o ː d ɐ WORD_BOUNDARY f ɔ ʏ ɐ WORD_BOUNDARY ɪ n WORD_BOUNDARY d ɛ ɐ WORD_BOUNDARY t a ʃ ə WORD_BOUNDARY UTT_BOUNDARY m ɔ t ɐ WORD_BOUNDARY h a s t WORD_BOUNDARY d u ː WORD_BOUNDARY f ɔ ʏ ɐ WORD_BOUNDARY ɪ n WORD_BOUNDARY d ɛ ɐ WORD_BOUNDARY t a ʃ ə WORD_BOUNDARY UTT_BOUNDARY m ɔ t ɐ WORD_BOUNDARY k ɐ i ː ɡ s t WORD_BOUNDARY a ː b ɐ WORD_BOUNDARY aɪ n ə n WORD_BOUNDARY h aɪ s ə n WORD_BOUNDARY p o ː WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY o ː d ɐ WORD_BOUNDARY a l ə WORD_BOUNDARY UTT_BOUNDARY m ɔ t ɐ WORD_BOUNDARY o ː d ɐ WORD_BOUNDARY v a s WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY o ː d ɐ WORD_BOUNDARY a l ə WORD_BOUNDARY UTT_BOUNDARY ɪ n v ɛ s t ɪ ɡ a t o ː ɐ WORD_BOUNDARY p aʊ l i ː n ə WORD_BOUNDARY ɡ ʊ k s t WORD_BOUNDARY d u ː WORD_BOUNDARY m a ː l WORD_BOUNDARY ɡ ə n aʊ WORD_BOUNDARY d a ː m +07/18/2026 09:48:03 - INFO - __main__ - [RANK 0] Sample 3278 of the training set: {'input_ids': tensor([16, 6, 10, 4, 16, 6, 2, 42, 4, 15, 6, 2, 42, 4, 15, 6, 18, 39, + 6, 13, 18, 12, 6, 8, 5, 6, 14, 15, 3, 9, 6, 42, 4, 15, 6, 18, + 39, 6, 10, 9, 30, 15, 23, 11, 6, 8, 5, 6, 2, 42, 4, 15, 6, 13, + 4, 5, 6, 20, 4, 16, 6, 40, 15, 26, 9, 6, 21, 30, 15, 6, 21, 4, + 15, 10, 17, 6, 31, 17, 18, 20, 17, 6, 2, 31, 22, 15, 36, 11, 16, 6, + 21, 30, 15, 6, 21, 18, 39, 6, 33, 27, 15, 36, 32, 10, 6, 16, 4, 15, + 28, 6, 26, 8, 9, 12, 30, 15, 16, 6, 2, 18, 39, 6, 20, 4, 15, 10, + 6, 26, 8, 9, 12, 30, 15, 16, 6, 16, 18, 39, 17, 6, 4, 12, 33, 27, + 15, 6, 21, 22, 15, 9, 6, 2, 42, 4, 13, 14, 15, 12, 6, 2, 13, 8, + 16, 6, 21, 30, 15, 6, 13, 34, 12, 11, 16, 6, 12, 4, 5, 11, 16, 6, + 21, 30, 15, 6, 26, 4, 35, 10, 34, 10, 6, 19, 16, 11, 16, 6, 35, 18, + 5, 17, 14, 15, 12, 8, 16, 4, 16, 17, 9, 4, 15, 10, 6, 19, 16, 36, + 19, 39, 11, 16, 6, 2, 36, 8, 39, 17, 6, 24, 8, 9, 33, 12, 18, 39, + 11, 16, 6, 3, 4, 37, 10, 6, 21, 30, 15, 9, 6, 2, 26, 4, 35, 10, + 34, 10, 6, 2, 8, 9, 6, 13, 18, 12, 6, 19, 16, 11, 16, 6, 13, 4, + 7, 11, 16, 4, 16, 17, 9, 4, 15, 10, 6, 2, 13, 8, 16, 6, 8, 9, + 6, 3, 8, 37, 10, 17, 6, 18, 39, 6, 26, 9, 25, 28, 11, 6, 19, 16, + 11, 6, 35, 18, 5, 17, 14, 15, 12, 11, 6, 26, 9, 25, 28, 11, 6, 18, + 39, 6, 19, 16, 11, 16, 6, 21, 4, 9, 10, 6, 2, 18, 39, 6, 31, 17, + 8, 12, 11, 6, 3, 30, 15, 9, 6, 19, 16, 11, 16, 6, 25, 5, 6, 2, + 13, 22, 15, 9, 6, 18, 5, 17, 6, 21, 4, 20, 4, 16, 17, 4, 15, 6, + 31, 17, 22, 15, 12, 6, 2, 17, 22, 15, 6, 19, 16, 24, 29, 16, 3, 9, + 17, 6, 7, 32, 16, 7, 6, 8, 5, 6, 21, 18, 16, 17, 6, 23, 11, 24, + 19, 20, 6, 2, 13, 19, 5, 17, 6, 3, 27, 15, 6, 3, 4, 5, 6, 8, + 17, 13, 4, 15, 6, 16, 18, 39, 17, 6, 2, 18, 39, 6, 24, 4, 15, 26, + 11, 6, 24, 34, 32, 17, 11, 6, 4, 15, 26, 11, 16, 17, 6, 33, 19, 17, + 6, 2, 16, 19, 16, 6, 21, 30, 15, 6, 7, 12, 30, 15, 23, 11, 16, 6, + 29, 20, 6, 8, 12, 7, 6, 16, 29, 12, 6, 16, 29, 12, 6, 27, 15, 9, + 6, 33, 27, 15, 36, 32, 10, 6])} n WORD_BOUNDARY k a n WORD_BOUNDARY UTT_BOUNDARY j a ː WORD_BOUNDARY UTT_BOUNDARY j a ː WORD_BOUNDARY ɪ ç WORD_BOUNDARY v ɪ l WORD_BOUNDARY ɛ s WORD_BOUNDARY o ː d ɐ WORD_BOUNDARY j a ː WORD_BOUNDARY ɪ ç WORD_BOUNDARY k ɐ i ː ɡ ə WORD_BOUNDARY ɛ s WORD_BOUNDARY UTT_BOUNDARY j a ː WORD_BOUNDARY v a s WORD_BOUNDARY m a n WORD_BOUNDARY y ː b ɐ WORD_BOUNDARY z i ː WORD_BOUNDARY z a ː k t WORD_BOUNDARY ʃ t ɪ m t WORD_BOUNDARY UTT_BOUNDARY ʃ e ː ʀ ə n WORD_BOUNDARY z i ː WORD_BOUNDARY z ɪ ç WORD_BOUNDARY ts u ː ʀ ʏ k WORD_BOUNDARY n a ː x WORD_BOUNDARY b ɛ ɐ l i ː n WORD_BOUNDARY UTT_BOUNDARY ɪ ç WORD_BOUNDARY m a ː k WORD_BOUNDARY b ɛ ɐ l i ː n WORD_BOUNDARY n ɪ ç t WORD_BOUNDARY a l ts u ː WORD_BOUNDARY z e ː ɐ WORD_BOUNDARY UTT_BOUNDARY j a v o ː l WORD_BOUNDARY UTT_BOUNDARY v ɛ n WORD_BOUNDARY z i ː WORD_BOUNDARY v ɔ l ə n WORD_BOUNDARY l a s ə n WORD_BOUNDARY z i ː WORD_BOUNDARY b a p k ɔ k WORD_BOUNDARY aɪ n ə n WORD_BOUNDARY p ɪ s t o ː l ɛ n a n t ɐ a ː k WORD_BOUNDARY aɪ n ʀ aɪ ç ə n WORD_BOUNDARY UTT_BOUNDARY ʀ ɛ ç t WORD_BOUNDARY h ɛ ɐ ts l ɪ ç ə n WORD_BOUNDARY d a ŋ k WORD_BOUNDARY z i ː ɐ WORD_BOUNDARY UTT_BOUNDARY b a p k ɔ k WORD_BOUNDARY UTT_BOUNDARY ɛ ɐ WORD_BOUNDARY v ɪ l WORD_BOUNDARY aɪ n ə n WORD_BOUNDARY v a f ə n a n t ɐ a ː k WORD_BOUNDARY UTT_BOUNDARY v ɛ n WORD_BOUNDARY ɛ ɐ WORD_BOUNDARY d ɛ ŋ k t WORD_BOUNDARY ɪ ç WORD_BOUNDARY b ɐ aʊ x ə WORD_BOUNDARY aɪ n ə WORD_BOUNDARY p ɪ s t o ː l ə WORD_BOUNDARY b ɐ aʊ x ə WORD_BOUNDARY ɪ ç WORD_BOUNDARY aɪ n ə n WORD_BOUNDARY z a ɐ k WORD_BOUNDARY UTT_BOUNDARY ɪ ç WORD_BOUNDARY ʃ t ɛ l ə WORD_BOUNDARY d i ː ɐ WORD_BOUNDARY aɪ n ə n WORD_BOUNDARY aʊ s WORD_BOUNDARY UTT_BOUNDARY v e ː ɐ WORD_BOUNDARY ɪ s t WORD_BOUNDARY z a m a n t a ː WORD_BOUNDARY ʃ t e ː l WORD_BOUNDARY UTT_BOUNDARY t e ː WORD_BOUNDARY aɪ n h ʊ n d ɐ t WORD_BOUNDARY f ʏ n f WORD_BOUNDARY ɛ s WORD_BOUNDARY z ɪ n t WORD_BOUNDARY ɡ ə h aɪ m WORD_BOUNDARY UTT_BOUNDARY v aɪ s t WORD_BOUNDARY d u ː WORD_BOUNDARY d a s WORD_BOUNDARY ɛ t v a ː WORD_BOUNDARY n ɪ ç t WORD_BOUNDARY UTT_BOUNDARY ɪ ç WORD_BOUNDARY h a ː b ə WORD_BOUNDARY h ɔ ʏ t ə WORD_BOUNDARY a ː b ə n t WORD_BOUNDARY ts aɪ t WORD_BOUNDARY UTT_BOUNDARY n aɪ n WORD_BOUNDARY z i ː WORD_BOUNDARY f l i ː ɡ ə n WORD_BOUNDARY ʊ m WORD_BOUNDARY ɛ l f WORD_BOUNDARY n ʊ l WORD_BOUNDARY n ʊ l WORD_BOUNDARY u ː ɐ WORD_BOUNDARY ts u ː ʀ ʏ k WORD_BOUNDARY +07/18/2026 09:48:07 - INFO - __main__ - [RANK 0] ***** Running training ***** +07/18/2026 09:48:07 - INFO - __main__ - [RANK 0] Num examples = 101825 +07/18/2026 09:48:07 - INFO - __main__ - [RANK 0] Num Training Steps = 30000 +07/18/2026 09:48:07 - INFO - __main__ - [RANK 0] Instantaneous batch size per device = 32 +07/18/2026 09:48:07 - INFO - __main__ - [RANK 0] Total train batch size (w. parallel, distributed & accumulation) = 32 +07/18/2026 09:48:07 - INFO - __main__ - [RANK 0] Gradient Accumulation steps = 1 +07/18/2026 09:48:07 - INFO - __main__ - [RANK 0] Total optimization steps = 30000 +07/18/2026 09:48:07 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 09:48:07 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 09:48:10 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 09:48:10 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 09:48:10 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 09:48:17 - INFO - __main__ - [RANK 0] Starting epoch 0... +07/18/2026 09:48:17 - WARNING - transformers.modeling_utils - `loss_type=None` was set in the config but it is unrecognized. Using the default loss: `ForCausalLMLoss`. +07/18/2026 09:48:18 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 09:48:18 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 09:48:22 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 09:48:22 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 09:48:22 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 09:48:27 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 09:48:27 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 09:48:31 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 09:48:31 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 09:48:31 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 09:49:38 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 09:49:38 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 09:49:41 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 09:49:41 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 09:49:41 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 09:50:52 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 09:50:52 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 09:50:56 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 09:50:56 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 09:50:56 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 09:52:15 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 09:52:15 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 09:52:18 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 09:52:18 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 09:52:18 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 09:53:16 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 09:53:16 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 09:53:20 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 09:53:20 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 09:53:20 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 09:54:35 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 09:54:35 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 09:54:39 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 09:54:39 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 09:54:39 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 09:56:27 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 09:56:27 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 09:56:31 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 09:56:31 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 09:56:31 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 09:58:18 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 09:58:18 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 09:58:22 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 09:58:22 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 09:58:22 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 10:00:45 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 10:00:45 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 10:00:49 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 10:00:49 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 10:00:49 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 10:03:01 - INFO - __main__ - [RANK 0] perplexity: 7.909086097933963 eval_loss: 2.068012237548828 accuracy: 0.4003 +07/18/2026 10:04:23 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 10:04:23 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 10:04:27 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 10:04:27 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 10:04:27 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 10:08:12 - INFO - __main__ - [RANK 0] perplexity: 4.531085486765549 eval_loss: 1.5109615325927734 accuracy: 0.5508 +07/18/2026 10:10:56 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 10:10:56 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 10:11:00 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 10:11:00 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 10:11:00 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 10:13:36 - INFO - __main__ - [RANK 0] perplexity: 3.763946708277698 eval_loss: 1.3254680633544922 accuracy: 0.6023 +07/18/2026 10:17:43 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 10:17:43 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 10:17:48 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 10:17:48 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 10:17:48 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 10:19:54 - INFO - __main__ - [RANK 0] perplexity: 3.2925302794646143 eval_loss: 1.1916563510894775 accuracy: 0.6412 +07/18/2026 10:20:53 - INFO - __main__ - [RANK 0] Starting epoch 1... +07/18/2026 10:24:02 - INFO - __main__ - [RANK 0] perplexity: 3.1018616790820457 eval_loss: 1.1320024728775024 accuracy: 0.6589 +07/18/2026 10:25:24 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 10:25:24 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 10:25:28 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 10:25:28 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 10:25:28 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 10:29:17 - INFO - __main__ - [RANK 0] perplexity: 2.9848321959746005 eval_loss: 1.093543529510498 accuracy: 0.6699 +07/18/2026 10:32:01 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 10:32:01 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 10:32:08 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 10:32:08 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 10:32:08 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 10:35:11 - INFO - __main__ - [RANK 0] perplexity: 2.9038398848998557 eval_loss: 1.066033959388733 accuracy: 0.6778 +07/18/2026 10:39:18 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 10:39:18 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 10:39:21 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 10:39:21 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 10:39:21 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 10:40:55 - INFO - __main__ - [RANK 0] perplexity: 2.8438977328136237 eval_loss: 1.045175552368164 accuracy: 0.6837 +07/18/2026 10:42:54 - INFO - __main__ - [RANK 0] Starting epoch 2... +07/18/2026 10:45:04 - INFO - __main__ - [RANK 0] perplexity: 2.7927952750158576 eval_loss: 1.0270429849624634 accuracy: 0.6894 +07/18/2026 10:46:26 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 10:46:26 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 10:46:29 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 10:46:29 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 10:46:29 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 10:50:06 - INFO - __main__ - [RANK 0] perplexity: 2.7546999485746992 eval_loss: 1.0133085250854492 accuracy: 0.6932 +07/18/2026 10:52:50 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 10:52:50 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 10:52:54 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 10:52:54 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 10:52:54 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 10:55:08 - INFO - __main__ - [RANK 0] perplexity: 2.720248517221291 eval_loss: 1.0007232427597046 accuracy: 0.6967 +07/18/2026 10:59:14 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 10:59:14 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 10:59:18 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 10:59:18 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 10:59:18 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 11:00:10 - INFO - __main__ - [RANK 0] perplexity: 2.6901474581149727 eval_loss: 0.9895960092544556 accuracy: 0.7000 +07/18/2026 11:03:09 - INFO - __main__ - [RANK 0] Starting epoch 3... +07/18/2026 11:04:19 - INFO - __main__ - [RANK 0] perplexity: 2.6656972363789504 eval_loss: 0.9804656505584717 accuracy: 0.7028 +07/18/2026 11:05:41 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 11:05:41 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 11:05:45 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 11:05:45 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 11:05:45 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 11:06:45 - INFO - __main__ - [RANK 0] Starting epoch 4... +07/18/2026 11:06:45 - INFO - __main__ - [RANK 0] Switching language at step 10000 +07/18/2026 11:07:00 - INFO - __main__ - [RANK 0] Sample 194393 of the training set: {'input_ids': tensor([53, 11, 6, 24, 25, 5, 6, 67, 16, 3, 6, 67, 16, 3, 6, 53, 45, 6, + 12, 68, 10, 17, 6, 12, 68, 10, 17, 6, 12, 68, 10, 17, 6, 53, 11, 6, + 2, 70, 6, 53, 67, 17, 5, 6, 55, 19, 17, 6, 42, 8, 24, 6, 23, 55, + 67, 16, 3, 35, 68, 6, 3, 18, 3, 6, 53, 67, 17, 6, 2, 24, 30, 15, + 6, 31, 57, 17, 6, 53, 11, 6, 3, 34, 55, 6, 42, 8, 24, 6, 2, 53, + 11, 6, 3, 34, 55, 6, 2, 5, 17, 68, 55, 17, 18, 3, 6, 67, 21, 6, + 53, 45, 6, 20, 8, 16, 17, 6, 17, 11, 6, 23, 70, 6, 34, 16, 6, 7, + 11, 55, 53, 11, 6, 24, 68, 12, 18, 3, 45, 6, 2, 8, 20, 8, 20, 6, + 2, 26, 69, 15, 23, 12, 11, 55, 6, 26, 18, 12, 6, 2, 42, 8, 24, 6, + 2, 19, 6, 3, 70, 16, 17, 6, 55, 18, 20, 8, 20, 26, 11, 55, 6, 53, + 67, 17, 6, 2, 10, 12, 19, 20, 3, 6, 54, 55, 27, 15, 6, 53, 11, 6, + 48, 18, 16, 3, 70, 6, 53, 11, 6, 16, 8, 10, 5, 17, 6, 3, 34, 55, + 6, 16, 45, 26, 11, 55, 6, 7, 70, 16, 3, 6, 57, 35, 6, 23, 68, 17, + 6, 57, 6, 5, 35, 8, 55, 6, 10, 30, 15, 6, 2, 18, 20, 67, 46, 18, + 16, 45, 31, 11, 16, 6, 48, 69, 15, 10, 18, 37, 6, 70, 13, 11, 55, 17, + 19, 20, 6, 53, 8, 55, 6, 2, 8, 20, 8, 20, 6, 2, 19, 6, 42, 27, + 15, 21, 3, 6, 17, 11, 6, 12, 19, 10, 6, 53, 11, 6, 10, 68, 55, 6, + 26, 27, 15, 17, 6, 5, 45, 12, 21, 6, 25, 17, 6, 24, 18, 55, 6, 53, + 45, 6, 48, 69, 15, 6, 23, 29, 3, 6, 2, 26, 57, 17, 6, 53, 45, 6, + 5, 17, 68, 35, 17, 6, 53, 8, 20, 6, 12, 67, 5, 17, 6, 42, 18, 55, + 6, 3, 18, 3, 16, 17, 6, 53, 45, 6, 2, 19, 6, 54, 18, 37, 10, 6, + 5, 70, 6, 2, 53, 45, 6, 53, 45, 6, 5, 17, 68, 35, 17, 6, 24, 67, + 13, 18, 37, 6, 53, 8, 20, 6, 34, 16, 53, 11, 6, 53, 11, 6, 5, 30, + 15, 7, 55, 57, 16, 17, 6, 8, 16, 18, 48, 45, 6, 2, 19, 6, 42, 27, + 15, 21, 3, 6, 17, 11, 6, 12, 19, 10, 6, 53, 8, 20, 6, 57, 12, 34, + 37, 6, 53, 8, 55, 6, 18, 17, 6, 48, 57, 21, 6, 16, 19, 5, 6, 10, + 68, 5, 6, 42, 27, 15, 6, 10, 29, 3, 6, 23, 70, 6, 10, 29, 3, 6, + 10, 11, 20, 26, 19, 16, 6, 57, 6, 57, 6, 48, 34, 10, 6, 26, 19, 6, + 53, 11, 6, 5, 30, 15, 6, 48])} ð ə WORD_BOUNDARY h aʊ s WORD_BOUNDARY æ n d WORD_BOUNDARY æ n d WORD_BOUNDARY ð eɪ WORD_BOUNDARY l ɑ k t WORD_BOUNDARY l ɑ k t WORD_BOUNDARY l ɑ k t WORD_BOUNDARY ð ə WORD_BOUNDARY UTT_BOUNDARY oʊ WORD_BOUNDARY ð æ t s WORD_BOUNDARY ɹ aɪ t WORD_BOUNDARY j ɛ h WORD_BOUNDARY ɡ ɹ æ n d p ɑ WORD_BOUNDARY d ɪ d WORD_BOUNDARY ð æ t WORD_BOUNDARY UTT_BOUNDARY h i ː WORD_BOUNDARY ʃ ʌ t WORD_BOUNDARY ð ə WORD_BOUNDARY d ɔ ɹ WORD_BOUNDARY j ɛ h WORD_BOUNDARY UTT_BOUNDARY ð ə WORD_BOUNDARY d ɔ ɹ WORD_BOUNDARY UTT_BOUNDARY s t ɑ ɹ t ɪ d WORD_BOUNDARY æ z WORD_BOUNDARY ð eɪ WORD_BOUNDARY m ɛ n t WORD_BOUNDARY t ə WORD_BOUNDARY ɡ oʊ WORD_BOUNDARY ɔ n WORD_BOUNDARY f ə ɹ ð ə WORD_BOUNDARY h ɑ l ɪ d eɪ WORD_BOUNDARY UTT_BOUNDARY ɛ m ɛ m WORD_BOUNDARY UTT_BOUNDARY b ɜ ː ɡ l ə ɹ WORD_BOUNDARY b ɪ l WORD_BOUNDARY UTT_BOUNDARY j ɛ h WORD_BOUNDARY UTT_BOUNDARY aɪ WORD_BOUNDARY d oʊ n t WORD_BOUNDARY ɹ ɪ m ɛ m b ə ɹ WORD_BOUNDARY ð æ t WORD_BOUNDARY UTT_BOUNDARY k l aɪ m d WORD_BOUNDARY θ ɹ u ː WORD_BOUNDARY ð ə WORD_BOUNDARY w ɪ n d oʊ WORD_BOUNDARY ð ə WORD_BOUNDARY n ɛ k s t WORD_BOUNDARY d ɔ ɹ WORD_BOUNDARY n eɪ b ə ɹ WORD_BOUNDARY f oʊ n d WORD_BOUNDARY ʌ p WORD_BOUNDARY ɡ ɑ t WORD_BOUNDARY ʌ WORD_BOUNDARY s p ɛ ɹ WORD_BOUNDARY k i ː WORD_BOUNDARY UTT_BOUNDARY ɪ m æ d̠ʒ ɪ n eɪ ʃ ə n WORD_BOUNDARY w ɜ ː k ɪ ŋ WORD_BOUNDARY oʊ v ə ɹ t aɪ m WORD_BOUNDARY ð ɛ ɹ WORD_BOUNDARY UTT_BOUNDARY ɛ m ɛ m WORD_BOUNDARY UTT_BOUNDARY aɪ WORD_BOUNDARY j u ː z d WORD_BOUNDARY t ə WORD_BOUNDARY l aɪ k WORD_BOUNDARY ð ə WORD_BOUNDARY k ɑ ɹ WORD_BOUNDARY b u ː t WORD_BOUNDARY s eɪ l z WORD_BOUNDARY aʊ t WORD_BOUNDARY h ɪ ɹ WORD_BOUNDARY ð eɪ WORD_BOUNDARY w ɜ ː WORD_BOUNDARY ɡ ʊ d WORD_BOUNDARY UTT_BOUNDARY b ʌ t WORD_BOUNDARY ð eɪ WORD_BOUNDARY s t ɑ p t WORD_BOUNDARY ð ɛ m WORD_BOUNDARY l æ s t WORD_BOUNDARY j ɪ ɹ WORD_BOUNDARY d ɪ d n t WORD_BOUNDARY ð eɪ WORD_BOUNDARY UTT_BOUNDARY aɪ WORD_BOUNDARY θ ɪ ŋ k WORD_BOUNDARY s oʊ WORD_BOUNDARY UTT_BOUNDARY ð eɪ WORD_BOUNDARY ð eɪ WORD_BOUNDARY s t ɑ p t WORD_BOUNDARY h æ v ɪ ŋ WORD_BOUNDARY ð ɛ m WORD_BOUNDARY ɔ n ð ə WORD_BOUNDARY ð ə WORD_BOUNDARY s i ː f ɹ ʌ n t WORD_BOUNDARY ɛ n ɪ w eɪ WORD_BOUNDARY UTT_BOUNDARY aɪ WORD_BOUNDARY j u ː z d WORD_BOUNDARY t ə WORD_BOUNDARY l aɪ k WORD_BOUNDARY ð ɛ m WORD_BOUNDARY ʌ l ɔ ŋ WORD_BOUNDARY ð ɛ ɹ WORD_BOUNDARY ɪ t WORD_BOUNDARY w ʌ z WORD_BOUNDARY n aɪ s WORD_BOUNDARY k ɑ s WORD_BOUNDARY j u ː WORD_BOUNDARY k ʊ d WORD_BOUNDARY ɡ oʊ WORD_BOUNDARY k ʊ d WORD_BOUNDARY k ə m b aɪ n WORD_BOUNDARY ʌ WORD_BOUNDARY ʌ WORD_BOUNDARY w ɔ k WORD_BOUNDARY b aɪ WORD_BOUNDARY ð ə WORD_BOUNDARY s i ː WORD_BOUNDARY w +07/18/2026 11:07:00 - INFO - __main__ - [RANK 0] Sample 72097 of the training set: {'input_ids': tensor([35, 55, 8, 5, 18, 37, 6, 46, 68, 26, 2, 35, 12, 45, 55, 27, 15, 20, + 6, 13, 18, 21, 18, 17, 6, 2, 67, 5, 17, 11, 55, 55, 18, 5, 10, 6, + 8, 20, 70, 17, 30, 15, 6, 34, 12, 55, 19, 17, 6, 5, 70, 6, 12, 8, + 17, 6, 20, 30, 15, 6, 35, 29, 17, 6, 3, 25, 16, 6, 20, 19, 6, 5, + 17, 57, 7, 6, 2, 67, 5, 17, 11, 55, 55, 18, 5, 10, 6, 8, 20, 70, + 17, 30, 15, 6, 3, 18, 3, 6, 42, 27, 15, 6, 24, 18, 55, 6, 48, 57, + 17, 6, 46, 67, 16, 18, 17, 6, 48, 57, 21, 6, 17, 8, 12, 18, 37, 6, + 42, 27, 15, 6, 2, 67, 5, 17, 11, 55, 55, 18, 5, 10, 6, 10, 19, 6, + 48, 18, 51, 6, 2, 67, 5, 17, 11, 55, 55, 18, 5, 10, 6, 20, 68, 17, + 6, 24, 57, 6, 2, 67, 5, 17, 11, 55, 55, 18, 5, 10, 6, 8, 20, 70, + 17, 30, 15, 6, 31, 30, 15, 6, 48, 57, 21, 6, 17, 8, 12, 18, 37, 6, + 20, 30, 15, 6, 2, 67, 5, 17, 11, 55, 55, 18, 5, 10, 6, 8, 20, 70, + 17, 30, 15, 6, 48, 8, 12, 6, 53, 8, 55, 68, 55, 6, 54, 55, 30, 15, + 6, 3, 18, 7, 55, 11, 16, 17, 6, 54, 18, 37, 21, 6, 17, 11, 6, 3, + 27, 15, 6, 70, 10, 45, 6, 2, 67, 5, 17, 11, 55, 55, 18, 5, 10, 6, + 8, 20, 70, 17, 30, 15, 6, 53, 8, 55, 21, 6, 57, 6, 26, 29, 10, 6, + 17, 11, 6, 12, 29, 10, 6, 67, 17, 6, 53, 8, 55, 21, 6, 57, 6, 17, + 45, 10, 6, 57, 35, 68, 55, 17, 6, 10, 68, 55, 6, 67, 16, 3, 6, 53, + 8, 55, 21, 6, 57, 6, 57, 6, 5, 17, 34, 55, 6, 5, 34, 55, 17, 6, + 57, 13, 6, 12, 19, 10, 6, 46, 68, 31, 6, 24, 57, 21, 6, 67, 17, 6, + 24, 18, 21, 6, 24, 25, 5, 6, 2, 67, 5, 17, 11, 55, 55, 18, 5, 10, + 6, 8, 20, 70, 17, 30, 15, 6, 5, 70, 6, 3, 27, 15, 6, 42, 27, 15, + 6, 16, 70, 6, 48, 18, 51, 6, 48, 57, 16, 6, 42, 27, 15, 6, 48, 29, + 3, 6, 12, 19, 10, 6, 17, 11, 6, 3, 27, 15, 6, 7, 69, 15, 5, 17, + 6, 2, 67, 5, 17, 11, 55, 55, 18, 5, 10, 6, 8, 20, 70, 17, 30, 15, + 6, 53, 11, 6, 17, 45, 10, 6, 57, 35, 68, 55, 17, 6, 10, 68, 55, 6, + 2, 67, 5, 17, 11, 55, 55, 18, 5, 10, 6, 8, 20, 70, 17, 30, 15, 6, + 12, 8, 17, 5, 6, 17, 45, 10, 6, 18, 17, 6, 3, 25, 16, 6, 2, 67, + 5, 17, 11, 55, 55, 18, 5, 10])} p ɹ ɛ s ɪ ŋ WORD_BOUNDARY d̠ʒ ɑ b UTT_BOUNDARY p l eɪ ɹ u ː m WORD_BOUNDARY v ɪ z ɪ t WORD_BOUNDARY UTT_BOUNDARY æ s t ə ɹ ɹ ɪ s k WORD_BOUNDARY ɛ m oʊ t i ː WORD_BOUNDARY ɔ l ɹ aɪ t WORD_BOUNDARY s oʊ WORD_BOUNDARY l ɛ t WORD_BOUNDARY m i ː WORD_BOUNDARY p ʊ t WORD_BOUNDARY d aʊ n WORD_BOUNDARY m aɪ WORD_BOUNDARY s t ʌ f WORD_BOUNDARY UTT_BOUNDARY æ s t ə ɹ ɹ ɪ s k WORD_BOUNDARY ɛ m oʊ t i ː WORD_BOUNDARY d ɪ d WORD_BOUNDARY j u ː WORD_BOUNDARY h ɪ ɹ WORD_BOUNDARY w ʌ t WORD_BOUNDARY d̠ʒ æ n ɪ t WORD_BOUNDARY w ʌ z WORD_BOUNDARY t ɛ l ɪ ŋ WORD_BOUNDARY j u ː WORD_BOUNDARY UTT_BOUNDARY æ s t ə ɹ ɹ ɪ s k WORD_BOUNDARY k aɪ WORD_BOUNDARY w ɪ t̠ʃ WORD_BOUNDARY UTT_BOUNDARY æ s t ə ɹ ɹ ɪ s k WORD_BOUNDARY m ɑ t WORD_BOUNDARY h ʌ WORD_BOUNDARY UTT_BOUNDARY æ s t ə ɹ ɹ ɪ s k WORD_BOUNDARY ɛ m oʊ t i ː WORD_BOUNDARY ʃ i ː WORD_BOUNDARY w ʌ z WORD_BOUNDARY t ɛ l ɪ ŋ WORD_BOUNDARY m i ː WORD_BOUNDARY UTT_BOUNDARY æ s t ə ɹ ɹ ɪ s k WORD_BOUNDARY ɛ m oʊ t i ː WORD_BOUNDARY w ɛ l WORD_BOUNDARY ð ɛ ɹ ɑ ɹ WORD_BOUNDARY θ ɹ i ː WORD_BOUNDARY d ɪ f ɹ ə n t WORD_BOUNDARY θ ɪ ŋ z WORD_BOUNDARY t ə WORD_BOUNDARY d u ː WORD_BOUNDARY oʊ k eɪ WORD_BOUNDARY UTT_BOUNDARY æ s t ə ɹ ɹ ɪ s k WORD_BOUNDARY ɛ m oʊ t i ː WORD_BOUNDARY ð ɛ ɹ z WORD_BOUNDARY ʌ WORD_BOUNDARY b ʊ k WORD_BOUNDARY t ə WORD_BOUNDARY l ʊ k WORD_BOUNDARY æ t WORD_BOUNDARY ð ɛ ɹ z WORD_BOUNDARY ʌ WORD_BOUNDARY t eɪ k WORD_BOUNDARY ʌ p ɑ ɹ t WORD_BOUNDARY k ɑ ɹ WORD_BOUNDARY æ n d WORD_BOUNDARY ð ɛ ɹ z WORD_BOUNDARY ʌ WORD_BOUNDARY ʌ WORD_BOUNDARY s t ɔ ɹ WORD_BOUNDARY s ɔ ɹ t WORD_BOUNDARY ʌ v WORD_BOUNDARY l aɪ k WORD_BOUNDARY d̠ʒ ɑ ʃ WORD_BOUNDARY h ʌ z WORD_BOUNDARY æ t WORD_BOUNDARY h ɪ z WORD_BOUNDARY h aʊ s WORD_BOUNDARY UTT_BOUNDARY æ s t ə ɹ ɹ ɪ s k WORD_BOUNDARY ɛ m oʊ t i ː WORD_BOUNDARY s oʊ WORD_BOUNDARY d u ː WORD_BOUNDARY j u ː WORD_BOUNDARY n oʊ WORD_BOUNDARY w ɪ t̠ʃ WORD_BOUNDARY w ʌ n WORD_BOUNDARY j u ː WORD_BOUNDARY w ʊ d WORD_BOUNDARY l aɪ k WORD_BOUNDARY t ə WORD_BOUNDARY d u ː WORD_BOUNDARY f ɜ ː s t WORD_BOUNDARY UTT_BOUNDARY æ s t ə ɹ ɹ ɪ s k WORD_BOUNDARY ɛ m oʊ t i ː WORD_BOUNDARY ð ə WORD_BOUNDARY t eɪ k WORD_BOUNDARY ʌ p ɑ ɹ t WORD_BOUNDARY k ɑ ɹ WORD_BOUNDARY UTT_BOUNDARY æ s t ə ɹ ɹ ɪ s k WORD_BOUNDARY ɛ m oʊ t i ː WORD_BOUNDARY l ɛ t s WORD_BOUNDARY t eɪ k WORD_BOUNDARY ɪ t WORD_BOUNDARY d aʊ n WORD_BOUNDARY UTT_BOUNDARY æ s t ə ɹ ɹ ɪ s k +07/18/2026 11:07:00 - INFO - __main__ - [RANK 0] Sample 64196 of the training set: {'input_ids': tensor([28, 17, 6, 2, 20, 34, 17, 9, 6, 19, 16, 6, 26, 18, 5, 39, 11, 16, + 6, 20, 22, 15, 9, 6, 8, 9, 16, 5, 17, 6, 24, 30, 15, 9, 6, 26, + 18, 17, 11, 6, 2, 7, 4, 17, 9, 6, 36, 41, 15, 20, 18, 31, 6, 3, + 9, 19, 5, 18, 39, 6, 2, 7, 4, 17, 9, 6, 19, 39, 30, 15, 6, 24, + 4, 17, 6, 3, 22, 15, 16, 6, 36, 41, 15, 20, 18, 31, 6, 3, 9, 19, + 5, 18, 39, 6, 4, 16, 23, 11, 10, 16, 4, 26, 9, 17, 6, 2, 20, 34, + 17, 9, 6, 19, 16, 6, 10, 12, 19, 16, 11, 5, 6, 12, 30, 15, 26, 11, + 5, 6, 21, 25, 36, 30, 15, 9, 13, 19, 26, 39, 11, 16, 6, 18, 5, 17, + 6, 3, 4, 15, 6, 23, 8, 5, 17, 9, 16, 6, 16, 4, 28, 17, 6, 7, + 14, 15, 9, 26, 19, 23, 11, 12, 25, 7, 11, 16, 6, 36, 41, 15, 20, 18, + 31, 6, 3, 9, 19, 5, 18, 39, 6, 2, 7, 4, 17, 9, 6, 18, 16, 6, + 3, 22, 15, 20, 6, 23, 4, 16, 33, 11, 16, 6, 36, 41, 15, 20, 18, 31, + 6, 3, 9, 19, 5, 18, 39, 6, 24, 30, 15, 9, 6, 12, 8, 33, 17, 11, + 6, 16, 4, 28, 17, 6, 2, 17, 4, 9, 23, 11, 17, 6, 31, 18, 12, 17, + 6, 3, 4, 15, 6, 4, 28, 33, 22, 15, 16, 6, 23, 12, 19, 21, 11, 6, + 2, 17, 4, 9, 23, 11, 17, 6, 31, 18, 12, 17, 6, 3, 4, 15, 6, 4, + 28, 33, 22, 15, 16, 6, 23, 12, 19, 5, 6, 2, 7, 4, 17, 9, 6, 4, + 28, 6, 21, 14, 15, 6, 2, 17, 4, 9, 23, 11, 17, 6, 31, 18, 12, 17, + 6, 19, 16, 6, 26, 29, 5, 6, 10, 34, 20, 17, 6, 24, 18, 16, 17, 9, + 24, 9, 6, 2, 7, 4, 17, 9, 6, 19, 16, 6, 26, 29, 5, 6, 10, 34, + 20, 17, 6, 16, 34, 28, 6, 24, 18, 16, 17, 9, 24, 9, 6, 2, 17, 4, + 9, 23, 11, 17, 6, 31, 18, 12, 17, 6, 42, 4, 15, 6, 2, 7, 4, 17, + 9, 6, 34, 28, 6, 12, 22, 15, 14, 15, 6, 2, 7, 4, 17, 9, 6, 13, + 30, 15, 9, 6, 21, 34, 12, 17, 11, 16, 6, 16, 18, 39, 17, 6, 21, 8, + 20, 17, 12, 18, 39, 11, 6, 12, 14, 15, 10, 14, 15, 20, 14, 15, 17, 30, + 15, 13, 11, 16, 6, 29, 16, 17, 6, 33, 40, 15, 23, 11, 6, 24, 30, 15, + 9, 6, 25, 7, 26, 25, 11, 16, 6, 2, 7, 4, 17, 9, 6, 3, 27, 15, + 6, 3, 4, 5, 6, 13, 18, 9, 17, 6, 3, 4, 16, 6, 13, 18, 9, 10, + 12, 18, 39, 6, 19, 16, 6, 26])} x t WORD_BOUNDARY UTT_BOUNDARY m ɔ t ɐ WORD_BOUNDARY aɪ n WORD_BOUNDARY b ɪ s ç ə n WORD_BOUNDARY m e ː ɐ WORD_BOUNDARY ɛ ɐ n s t WORD_BOUNDARY h i ː ɐ WORD_BOUNDARY b ɪ t ə WORD_BOUNDARY UTT_BOUNDARY f a t ɐ WORD_BOUNDARY ʀ ø ː m ɪ ʃ WORD_BOUNDARY d ɐ aɪ s ɪ ç WORD_BOUNDARY UTT_BOUNDARY f a t ɐ WORD_BOUNDARY aɪ ç i ː WORD_BOUNDARY h a t WORD_BOUNDARY d e ː n WORD_BOUNDARY ʀ ø ː m ɪ ʃ WORD_BOUNDARY d ɐ aɪ s ɪ ç WORD_BOUNDARY a n ɡ ə k n a b ɐ t WORD_BOUNDARY UTT_BOUNDARY m ɔ t ɐ WORD_BOUNDARY aɪ n WORD_BOUNDARY k l aɪ n ə s WORD_BOUNDARY l i ː b ə s WORD_BOUNDARY z aʊ ʀ i ː ɐ v aɪ b ç ə n WORD_BOUNDARY ɪ s t WORD_BOUNDARY d a ː WORD_BOUNDARY ɡ ɛ s t ɐ n WORD_BOUNDARY n a x t WORD_BOUNDARY f o ː ɐ b aɪ ɡ ə l aʊ f ə n WORD_BOUNDARY ʀ ø ː m ɪ ʃ WORD_BOUNDARY d ɐ aɪ s ɪ ç WORD_BOUNDARY UTT_BOUNDARY f a t ɐ WORD_BOUNDARY ɪ n WORD_BOUNDARY d e ː m WORD_BOUNDARY ɡ a n ts ə n WORD_BOUNDARY ʀ ø ː m ɪ ʃ WORD_BOUNDARY d ɐ aɪ s ɪ ç WORD_BOUNDARY h i ː ɐ WORD_BOUNDARY l ɛ ts t ə WORD_BOUNDARY n a x t WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY d a ː WORD_BOUNDARY a x ts e ː n WORD_BOUNDARY ɡ l aɪ z ə WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY d a ː WORD_BOUNDARY a x ts e ː n WORD_BOUNDARY ɡ l aɪ s WORD_BOUNDARY UTT_BOUNDARY f a t ɐ WORD_BOUNDARY a x WORD_BOUNDARY z o ː WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY aɪ n WORD_BOUNDARY b ʊ s WORD_BOUNDARY k ɔ m t WORD_BOUNDARY h ɪ n t ɐ h ɐ WORD_BOUNDARY UTT_BOUNDARY f a t ɐ WORD_BOUNDARY aɪ n WORD_BOUNDARY b ʊ s WORD_BOUNDARY k ɔ m t WORD_BOUNDARY n ɔ x WORD_BOUNDARY h ɪ n t ɐ h ɐ WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY j a ː WORD_BOUNDARY UTT_BOUNDARY f a t ɐ WORD_BOUNDARY ɔ x WORD_BOUNDARY l e ː o ː WORD_BOUNDARY UTT_BOUNDARY f a t ɐ WORD_BOUNDARY v i ː ɐ WORD_BOUNDARY z ɔ l t ə n WORD_BOUNDARY n ɪ ç t WORD_BOUNDARY z ɛ m t l ɪ ç ə WORD_BOUNDARY l o ː k o ː m o ː t i ː v ə n WORD_BOUNDARY ʊ n t WORD_BOUNDARY ts y ː ɡ ə WORD_BOUNDARY h i ː ɐ WORD_BOUNDARY aʊ f b aʊ ə n WORD_BOUNDARY UTT_BOUNDARY f a t ɐ WORD_BOUNDARY d u ː WORD_BOUNDARY d a s WORD_BOUNDARY v ɪ ɐ t WORD_BOUNDARY d a n WORD_BOUNDARY v ɪ ɐ k l ɪ ç WORD_BOUNDARY aɪ n WORD_BOUNDARY b +07/18/2026 11:07:00 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 11:07:00 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 11:07:03 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 11:07:03 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 11:07:03 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 11:08:12 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 11:08:12 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 11:08:16 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 11:08:16 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 11:08:16 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 11:09:07 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 11:09:07 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 11:09:10 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 11:09:10 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 11:09:10 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 11:10:02 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 11:10:02 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 11:10:06 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 11:10:06 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 11:10:06 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 11:10:59 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 11:10:59 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 11:11:03 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 11:11:03 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 11:11:03 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 11:12:20 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 11:12:20 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 11:12:23 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 11:12:23 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 11:12:23 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 11:13:26 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 11:13:26 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 11:13:30 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 11:13:30 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 11:13:30 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 11:15:10 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 11:15:10 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 11:15:14 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 11:15:14 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 11:15:14 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 11:16:44 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 11:16:44 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 11:16:48 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 11:16:48 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 11:16:48 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 11:19:03 - INFO - __main__ - [RANK 0] perplexity: 3.0484232821367048 eval_loss: 1.1146245002746582 accuracy: 0.6692 +07/18/2026 11:19:07 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 11:19:07 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 11:19:10 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 11:19:10 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 11:19:10 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 11:22:36 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 11:22:36 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 11:22:40 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 11:22:40 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 11:22:40 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 11:25:13 - INFO - __main__ - [RANK 0] perplexity: 2.9187663137798014 eval_loss: 1.0711610317230225 accuracy: 0.6814 +07/18/2026 11:29:19 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 11:29:19 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 11:29:23 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 11:29:23 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 11:29:23 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 11:31:02 - INFO - __main__ - [RANK 0] perplexity: 2.859235823996006 eval_loss: 1.0505543947219849 accuracy: 0.6865 +07/18/2026 11:35:14 - INFO - __main__ - [RANK 0] perplexity: 2.813762437303175 eval_loss: 1.034522533416748 accuracy: 0.6909 +07/18/2026 11:36:36 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 11:36:36 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 11:36:40 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 11:36:40 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 11:36:40 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 11:41:06 - INFO - __main__ - [RANK 0] perplexity: 2.7864668524246716 eval_loss: 1.024774432182312 accuracy: 0.6934 +07/18/2026 11:43:50 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 11:43:50 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 11:43:54 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 11:43:54 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 11:43:54 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 11:46:29 - INFO - __main__ - [RANK 0] perplexity: 2.76449468021331 eval_loss: 1.0168578624725342 accuracy: 0.6957 +07/18/2026 11:50:36 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 11:50:36 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 11:50:40 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 11:50:40 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 11:50:40 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 11:52:05 - INFO - __main__ - [RANK 0] perplexity: 2.7417673710215102 eval_loss: 1.0086027383804321 accuracy: 0.6980 +07/18/2026 11:56:18 - INFO - __main__ - [RANK 0] perplexity: 2.7246489701306094 eval_loss: 1.0023396015167236 accuracy: 0.6998 +07/18/2026 11:57:40 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 11:57:40 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 11:57:44 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 11:57:44 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 11:57:44 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 12:01:00 - INFO - __main__ - [RANK 0] Starting epoch 5... +07/18/2026 12:01:27 - INFO - __main__ - [RANK 0] perplexity: 2.7098047854413463 eval_loss: 0.99687659740448 accuracy: 0.7014 +07/18/2026 12:04:12 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 12:04:12 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 12:04:15 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 12:04:15 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 12:04:15 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 12:06:55 - INFO - __main__ - [RANK 0] perplexity: 2.697566423292105 eval_loss: 0.9923500418663025 accuracy: 0.7030 +07/18/2026 12:11:01 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 12:11:01 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 12:11:05 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 12:11:05 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 12:11:05 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 12:12:29 - INFO - __main__ - [RANK 0] perplexity: 2.6870174163825933 eval_loss: 0.9884318113327026 accuracy: 0.7036 +07/18/2026 12:16:42 - INFO - __main__ - [RANK 0] perplexity: 2.671657898321105 eval_loss: 0.9826992154121399 accuracy: 0.7056 +07/18/2026 12:18:03 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 12:18:03 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 12:18:07 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 12:18:07 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 12:18:07 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 12:21:48 - INFO - __main__ - [RANK 0] perplexity: 2.660904088427851 eval_loss: 0.9786659479141235 accuracy: 0.7066 +07/18/2026 12:24:32 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 12:24:32 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 12:24:36 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 12:24:36 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 12:24:36 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 12:27:32 - INFO - __main__ - [RANK 0] perplexity: 2.650937520328549 eval_loss: 0.9749133586883545 accuracy: 0.7073 +07/18/2026 12:31:39 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 12:31:39 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 12:31:42 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 12:31:42 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 12:31:42 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 12:32:52 - INFO - __main__ - [RANK 0] perplexity: 2.6409431130095893 eval_loss: 0.9711360931396484 accuracy: 0.7082 +07/18/2026 12:37:04 - INFO - __main__ - [RANK 0] perplexity: 2.6322275875140826 eval_loss: 0.96783047914505 accuracy: 0.7095 +07/18/2026 12:38:26 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 12:38:26 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 12:38:30 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 12:38:30 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 12:38:30 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 12:42:11 - INFO - __main__ - [RANK 0] perplexity: 2.6254932698704945 eval_loss: 0.9652687907218933 accuracy: 0.7106 +07/18/2026 12:44:13 - INFO - __main__ - [RANK 0] Starting epoch 6... +07/18/2026 12:44:55 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 12:44:55 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 12:44:59 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 12:44:59 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 12:44:59 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 12:50:47 - INFO - __main__ - [RANK 0] perplexity: 2.6188930963097303 eval_loss: 0.9627517461776733 accuracy: 0.7111 +07/18/2026 21:56:23 - INFO - __main__ - [RANK 0] Distributed environment: DistributedType.NO +Num processes: 1 +Process index: 0 +Local process index: 0 +Device: cuda + +Mixed precision type: no + +07/18/2026 21:56:23 - INFO - __main__ - [RANK 0] Training args: +{'data_dir': 'data/phone/deu-eng/tokenized_sequential_interleaved', 'model_name_or_path': None, 'config_name': 'gpt_base_config.json', 'tokenizer_name': 'nikitastheo/phonelm-phone-deu-eng-tokenizer', 'per_device_train_batch_size': 32, 'per_device_eval_batch_size': 32, 'learning_rate': 0.0001, 'weight_decay': 0.01, 'max_steps': 30000, 'gradient_accumulation_steps': 1, 'lr_scheduler_type': , 'warmup_steps': 3000, 'output_dir': 'models/phonebabylm-phone-deu-eng-sequential_interleaved', 'seed': 42, 'model_type': None, 'block_size': None, 'push_to_hub': True, 'hub_model_id': 'nikitastheo/phonebabylm-phone-deu-eng-sequential_interleaved', 'hub_token': None, 'private': False, 'trust_remote_code': False, 'with_tracking': True, 'tracking_dir': None, 'report_to': 'wandb', 'restart_lr_at_switch': False, 'language_switch_steps': 10000, 'model_revision': 'main', 'config_overrides': None, 'use_slow_tokenizer': False, 'override_vocab_size': None, 'dtype': None, 'logging_steps': 10, 'cache_dir': None, 'eval_steps': 750, 'max_grad_norm': 1.0, 'adam_epsilon': 1e-06, 'run_name': 'phonebabylm-phone-deu-eng-sequential_interleaved', 'max_train_samples': None, 'max_eval_samples': None} +07/18/2026 21:56:23 - INFO - transformers.configuration_utils - loading configuration file gpt_base_config.json +07/18/2026 21:56:23 - INFO - transformers.configuration_utils - Model config GPT2Config { + "activation_function": "gelu", + "architectures": [ + "GPT2LMHeadModel" + ], + "attn_pdrop": 0.1, + "bos_token_id": 50256, + "embd_pdrop": 0.1, + "eos_token_id": 50256, + "initializer_range": 0.02, + "layer_norm_epsilon": 1e-05, + "model_type": "gpt2", + "n_ctx": 512, + "n_embd": 768, + "n_head": 12, + "n_inner": 3072, + "n_layer": 12, + "n_positions": 512, + "reorder_and_upcast_attn": false, + "resid_pdrop": 0.1, + "scale_attn_by_inverse_layer_idx": false, + "scale_attn_weights": true, + "summary_activation": null, + "summary_first_dropout": 0.1, + "summary_proj_to_labels": true, + "summary_type": "cls_index", + "summary_use_proj": true, + "transformers_version": "4.57.6", + "use_cache": true, + "vocab_size": 1 +} + +07/18/2026 21:56:23 - INFO - transformers.tokenization_utils_base - loading file vocab.json from cache at /home/geofila/.cache/huggingface/hub/models--nikitastheo--phonelm-phone-deu-eng-tokenizer/snapshots/a9eb582bae549d2e78f9e6f7913343e704373e8a/vocab.json +07/18/2026 21:56:23 - INFO - transformers.tokenization_utils_base - loading file merges.txt from cache at None +07/18/2026 21:56:23 - INFO - transformers.tokenization_utils_base - loading file tokenizer.json from cache at /home/geofila/.cache/huggingface/hub/models--nikitastheo--phonelm-phone-deu-eng-tokenizer/snapshots/a9eb582bae549d2e78f9e6f7913343e704373e8a/tokenizer.json +07/18/2026 21:56:23 - INFO - transformers.tokenization_utils_base - loading file added_tokens.json from cache at None +07/18/2026 21:56:23 - INFO - transformers.tokenization_utils_base - loading file special_tokens_map.json from cache at /home/geofila/.cache/huggingface/hub/models--nikitastheo--phonelm-phone-deu-eng-tokenizer/snapshots/a9eb582bae549d2e78f9e6f7913343e704373e8a/special_tokens_map.json +07/18/2026 21:56:23 - INFO - transformers.tokenization_utils_base - loading file tokenizer_config.json from cache at /home/geofila/.cache/huggingface/hub/models--nikitastheo--phonelm-phone-deu-eng-tokenizer/snapshots/a9eb582bae549d2e78f9e6f7913343e704373e8a/tokenizer_config.json +07/18/2026 21:56:23 - INFO - transformers.tokenization_utils_base - loading file chat_template.jinja from cache at None +07/18/2026 21:56:25 - INFO - transformers.generation.configuration_utils - Generate config GenerationConfig { + "bos_token_id": 2, + "eos_token_id": 2, + "pad_token_id": 1 +} + +07/18/2026 21:56:26 - INFO - __main__ - [RANK 0] Model parameters: 85.50M total (85.50M trainable) +07/18/2026 21:56:38 - INFO - __main__ - [RANK 0] Sample 83810 of the training set: {'input_ids': tensor([ 9, 6, 34, 32, 39, 6, 21, 14, 15, 6, 8, 17, 13, 4, 15, 5, 6, 13, + 18, 39, 17, 18, 23, 11, 5, 6, 8, 9, 33, 8, 15, 12, 17, 6, 2, 13, + 30, 15, 9, 6, 13, 4, 15, 36, 11, 16, 6, 20, 18, 17, 6, 3, 22, 15, + 20, 6, 43, 4, 36, 9, 6, 7, 8, 9, 4, 35, 9, 22, 15, 3, 11, 17, + 6, 2, 21, 30, 15, 6, 21, 18, 16, 17, 6, 16, 18, 39, 17, 6, 23, 11, + 10, 34, 20, 11, 16, 6, 2, 13, 4, 5, 6, 18, 5, 17, 6, 24, 30, 15, + 9, 6, 12, 14, 15, 5, 6, 2, 31, 4, 9, 12, 11, 5, 6, 21, 34, 12, + 6, 8, 9, 20, 34, 9, 3, 11, 17, 6, 13, 34, 9, 3, 11, 16, 6, 21, + 19, 16, 6, 2, 12, 27, 15, 18, 5, 11, 6, 3, 30, 15, 21, 9, 6, 21, + 34, 12, 3, 4, 15, 17, 6, 18, 5, 17, 6, 16, 18, 39, 17, 6, 26, 19, + 6, 10, 12, 4, 15, 36, 11, 20, 6, 7, 8, 9, 31, 17, 4, 16, 17, 6, + 2, 30, 15, 9, 6, 21, 14, 15, 16, 6, 13, 44, 3, 11, 6, 16, 18, 39, + 17, 6, 8, 9, 20, 34, 9, 3, 11, 17, 6, 8, 9, 6, 18, 5, 17, 6, + 7, 8, 9, 20, 18, 5, 17, 6, 2, 3, 4, 5, 6, 31, 17, 22, 15, 17, + 6, 18, 16, 6, 4, 12, 11, 16, 6, 26, 11, 36, 18, 39, 17, 11, 16, 6, + 2, 25, 23, 29, 5, 17, 6, 24, 34, 12, 17, 6, 24, 25, 35, 17, 20, 4, + 16, 6, 26, 11, 25, 12, 30, 15, 27, 15, 6, 2, 13, 4, 5, 6, 21, 34, + 12, 6, 3, 8, 9, 6, 24, 25, 35, 17, 20, 4, 16, 6, 3, 4, 15, 20, + 18, 17, 6, 4, 16, 7, 4, 37, 11, 16, 6, 2, 31, 18, 10, 11, 16, 6, + 21, 30, 15, 6, 30, 15, 16, 6, 26, 8, 5, 9, 6, 23, 12, 19, 39, 6, + 7, 34, 9, 17, 6, 2, 22, 15, 9, 5, 17, 6, 21, 34, 12, 6, 7, 30, + 15, 12, 18, 35, 11, 6, 30, 15, 16, 6, 4, 16, 24, 41, 15, 36, 11, 16, + 6, 2, 10, 19, 16, 11, 6, 21, 34, 9, 23, 11, 6, 13, 30, 15, 9, 6, + 13, 8, 9, 3, 11, 16, 6, 30, 15, 16, 6, 31, 14, 15, 16, 6, 12, 14, + 15, 5, 6, 2, 21, 8, 12, 17, 21, 4, 15, 20, 11, 6, 23, 11, 31, 18, + 39, 17, 11, 6, 21, 34, 12, 3, 4, 15, 17, 6, 2, 13, 22, 15, 9, 6, + 24, 8, 17, 11, 6, 31, 4, 9, 12, 11, 5, 6, 17, 41, 15, 17, 11, 16, + 6, 13, 34, 12, 11, 16, 6, 2, 29, 16, 17, 6, 13, 4, 15, 36, 29, 20, + 6, 24, 4, 15, 26, 11, 16, 6])} ɐ WORD_BOUNDARY ɔ ʏ ç WORD_BOUNDARY z o ː WORD_BOUNDARY ɛ t v a ː s WORD_BOUNDARY v ɪ ç t ɪ ɡ ə s WORD_BOUNDARY ɛ ɐ ts ɛ ː l t WORD_BOUNDARY UTT_BOUNDARY v i ː ɐ WORD_BOUNDARY v a ː ʀ ə n WORD_BOUNDARY m ɪ t WORD_BOUNDARY d e ː m WORD_BOUNDARY pf a ʀ ɐ WORD_BOUNDARY f ɛ ɐ a p ɐ e ː d ə t WORD_BOUNDARY UTT_BOUNDARY z i ː WORD_BOUNDARY z ɪ n t WORD_BOUNDARY n ɪ ç t WORD_BOUNDARY ɡ ə k ɔ m ə n WORD_BOUNDARY UTT_BOUNDARY v a s WORD_BOUNDARY ɪ s t WORD_BOUNDARY h i ː ɐ WORD_BOUNDARY l o ː s WORD_BOUNDARY UTT_BOUNDARY ʃ a ɐ l ə s WORD_BOUNDARY z ɔ l WORD_BOUNDARY ɛ ɐ m ɔ ɐ d ə t WORD_BOUNDARY v ɔ ɐ d ə n WORD_BOUNDARY z aɪ n WORD_BOUNDARY UTT_BOUNDARY l u ː ɪ s ə WORD_BOUNDARY d i ː z ɐ WORD_BOUNDARY z ɔ l d a ː t WORD_BOUNDARY ɪ s t WORD_BOUNDARY n ɪ ç t WORD_BOUNDARY b aɪ WORD_BOUNDARY k l a ː ʀ ə m WORD_BOUNDARY f ɛ ɐ ʃ t a n t WORD_BOUNDARY UTT_BOUNDARY i ː ɐ WORD_BOUNDARY z o ː n WORD_BOUNDARY v ʊɐ d ə WORD_BOUNDARY n ɪ ç t WORD_BOUNDARY ɛ ɐ m ɔ ɐ d ə t WORD_BOUNDARY ɛ ɐ WORD_BOUNDARY ɪ s t WORD_BOUNDARY f ɛ ɐ m ɪ s t WORD_BOUNDARY UTT_BOUNDARY d a s WORD_BOUNDARY ʃ t e ː t WORD_BOUNDARY ɪ n WORD_BOUNDARY a l ə n WORD_BOUNDARY b ə ʀ ɪ ç t ə n WORD_BOUNDARY UTT_BOUNDARY aʊ ɡ ʊ s t WORD_BOUNDARY h ɔ l t WORD_BOUNDARY h aʊ p t m a n WORD_BOUNDARY b ə aʊ l i ː u ː WORD_BOUNDARY UTT_BOUNDARY v a s WORD_BOUNDARY z ɔ l WORD_BOUNDARY d ɛ ɐ WORD_BOUNDARY h aʊ p t m a n WORD_BOUNDARY d a ː m ɪ t WORD_BOUNDARY a n f a ŋ ə n WORD_BOUNDARY UTT_BOUNDARY ʃ ɪ k ə n WORD_BOUNDARY z i ː WORD_BOUNDARY i ː n WORD_BOUNDARY b ɛ s ɐ WORD_BOUNDARY ɡ l aɪ ç WORD_BOUNDARY f ɔ ɐ t WORD_BOUNDARY UTT_BOUNDARY e ː ɐ s t WORD_BOUNDARY z ɔ l WORD_BOUNDARY f i ː l ɪ p ə WORD_BOUNDARY i ː n WORD_BOUNDARY a n h ø ː ʀ ə n WORD_BOUNDARY UTT_BOUNDARY k aɪ n ə WORD_BOUNDARY z ɔ ɐ ɡ ə WORD_BOUNDARY v i ː ɐ WORD_BOUNDARY v ɛ ɐ d ə n WORD_BOUNDARY i ː n WORD_BOUNDARY ʃ o ː n WORD_BOUNDARY l o ː s WORD_BOUNDARY UTT_BOUNDARY z ɛ l t z a ː m ə WORD_BOUNDARY ɡ ə ʃ ɪ ç t ə WORD_BOUNDARY z ɔ l d a ː t WORD_BOUNDARY UTT_BOUNDARY v e ː ɐ WORD_BOUNDARY h ɛ t ə WORD_BOUNDARY ʃ a ɐ l ə s WORD_BOUNDARY t ø ː t ə n WORD_BOUNDARY v ɔ l ə n WORD_BOUNDARY UTT_BOUNDARY ʊ n t WORD_BOUNDARY v a ː ʀ ʊ m WORD_BOUNDARY h a ː b ə n WORD_BOUNDARY +07/18/2026 21:56:38 - INFO - __main__ - [RANK 0] Sample 14592 of the training set: {'input_ids': tensor([ 6, 17, 4, 31, 11, 6, 2, 17, 4, 9, 23, 11, 17, 6, 31, 18, 12, 17, + 6, 13, 22, 15, 9, 6, 2, 20, 34, 17, 9, 6, 19, 16, 6, 4, 16, 3, + 11, 36, 9, 6, 25, 5, 3, 9, 29, 10, 6, 7, 40, 15, 9, 6, 23, 8, + 12, 17, 6, 2, 20, 34, 17, 9, 6, 3, 4, 15, 6, 10, 4, 16, 6, 20, + 4, 16, 6, 25, 28, 6, 21, 4, 15, 23, 11, 16, 6, 10, 9, 41, 15, 17, + 11, 16, 6, 14, 15, 3, 9, 6, 7, 12, 34, 10, 11, 16, 6, 14, 15, 3, + 9, 6, 35, 22, 15, 16, 29, 16, 33, 11, 6, 2, 18, 16, 13, 8, 5, 17, + 18, 23, 4, 17, 14, 15, 9, 6, 24, 11, 20, 6, 2, 17, 4, 9, 23, 11, + 17, 6, 31, 18, 12, 17, 6, 13, 8, 10, 11, 16, 6, 2, 17, 4, 9, 23, + 11, 17, 6, 31, 18, 12, 17, 6, 14, 15, 3, 9, 6, 10, 9, 8, 15, 36, + 29, 16, 33, 11, 6, 2, 18, 16, 13, 8, 5, 17, 18, 23, 4, 17, 14, 15, + 9, 6, 14, 15, 3, 9, 6, 10, 14, 15, 12, 11, 6, 2, 20, 34, 17, 9, + 6, 10, 14, 15, 12, 11, 6, 23, 11, 16, 25, 6, 2, 17, 4, 9, 23, 11, + 17, 6, 31, 18, 12, 17, 6, 14, 15, 3, 9, 6, 7, 34, 32, 9, 6, 2, + 20, 34, 17, 9, 6, 24, 4, 5, 17, 6, 3, 27, 15, 6, 10, 14, 15, 12, + 11, 6, 2, 18, 16, 13, 8, 5, 17, 18, 23, 4, 17, 14, 15, 9, 6, 7, + 34, 32, 9, 6, 7, 34, 32, 9, 6, 35, 8, 9, 7, 8, 10, 17, 6, 2, + 17, 4, 9, 23, 11, 17, 6, 31, 18, 12, 17, 6, 14, 15, 3, 9, 6, 7, + 34, 32, 9, 6, 18, 16, 6, 3, 8, 9, 6, 17, 4, 31, 11, 6, 2, 20, + 34, 17, 9, 6, 24, 4, 5, 17, 6, 3, 27, 15, 6, 7, 34, 32, 9, 6, + 18, 16, 6, 3, 8, 9, 6, 17, 4, 31, 11, 6, 2, 20, 34, 17, 9, 6, + 10, 9, 30, 15, 23, 5, 17, 6, 4, 15, 26, 9, 6, 19, 16, 11, 16, 6, + 24, 19, 5, 11, 16, 6, 35, 14, 15, 6, 2, 17, 4, 9, 23, 11, 17, 6, + 31, 18, 12, 17, 6, 14, 15, 3, 9, 6, 4, 12, 11, 6, 2, 20, 34, 17, + 9, 6, 14, 15, 3, 9, 6, 13, 4, 5, 6, 2, 17, 4, 9, 23, 11, 17, + 6, 31, 18, 12, 17, 6, 14, 15, 3, 9, 6, 4, 12, 11, 6, 2, 18, 16, + 13, 8, 5, 17, 18, 23, 4, 17, 14, 15, 9, 6, 35, 25, 12, 30, 15, 16, + 11, 6, 23, 29, 10, 5, 17, 6, 3, 27, 15, 6, 20, 4, 15, 12, 6, 23, + 11, 16, 25, 6, 3, 4, 15, 20])} WORD_BOUNDARY t a ʃ ə WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY v e ː ɐ WORD_BOUNDARY UTT_BOUNDARY m ɔ t ɐ WORD_BOUNDARY aɪ n WORD_BOUNDARY a n d ə ʀ ɐ WORD_BOUNDARY aʊ s d ɐ ʊ k WORD_BOUNDARY f y ː ɐ WORD_BOUNDARY ɡ ɛ l t WORD_BOUNDARY UTT_BOUNDARY m ɔ t ɐ WORD_BOUNDARY d a ː WORD_BOUNDARY k a n WORD_BOUNDARY m a n WORD_BOUNDARY aʊ x WORD_BOUNDARY z a ː ɡ ə n WORD_BOUNDARY k ɐ ø ː t ə n WORD_BOUNDARY o ː d ɐ WORD_BOUNDARY f l ɔ k ə n WORD_BOUNDARY o ː d ɐ WORD_BOUNDARY p e ː n ʊ n ts ə WORD_BOUNDARY UTT_BOUNDARY ɪ n v ɛ s t ɪ ɡ a t o ː ɐ WORD_BOUNDARY h ə m WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY v ɛ k ə n WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY o ː d ɐ WORD_BOUNDARY k ɐ ɛ ː ʀ ʊ n ts ə WORD_BOUNDARY UTT_BOUNDARY ɪ n v ɛ s t ɪ ɡ a t o ː ɐ WORD_BOUNDARY o ː d ɐ WORD_BOUNDARY k o ː l ə WORD_BOUNDARY UTT_BOUNDARY m ɔ t ɐ WORD_BOUNDARY k o ː l ə WORD_BOUNDARY ɡ ə n aʊ WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY o ː d ɐ WORD_BOUNDARY f ɔ ʏ ɐ WORD_BOUNDARY UTT_BOUNDARY m ɔ t ɐ WORD_BOUNDARY h a s t WORD_BOUNDARY d u ː WORD_BOUNDARY k o ː l ə WORD_BOUNDARY UTT_BOUNDARY ɪ n v ɛ s t ɪ ɡ a t o ː ɐ WORD_BOUNDARY f ɔ ʏ ɐ WORD_BOUNDARY f ɔ ʏ ɐ WORD_BOUNDARY p ɛ ɐ f ɛ k t WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY o ː d ɐ WORD_BOUNDARY f ɔ ʏ ɐ WORD_BOUNDARY ɪ n WORD_BOUNDARY d ɛ ɐ WORD_BOUNDARY t a ʃ ə WORD_BOUNDARY UTT_BOUNDARY m ɔ t ɐ WORD_BOUNDARY h a s t WORD_BOUNDARY d u ː WORD_BOUNDARY f ɔ ʏ ɐ WORD_BOUNDARY ɪ n WORD_BOUNDARY d ɛ ɐ WORD_BOUNDARY t a ʃ ə WORD_BOUNDARY UTT_BOUNDARY m ɔ t ɐ WORD_BOUNDARY k ɐ i ː ɡ s t WORD_BOUNDARY a ː b ɐ WORD_BOUNDARY aɪ n ə n WORD_BOUNDARY h aɪ s ə n WORD_BOUNDARY p o ː WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY o ː d ɐ WORD_BOUNDARY a l ə WORD_BOUNDARY UTT_BOUNDARY m ɔ t ɐ WORD_BOUNDARY o ː d ɐ WORD_BOUNDARY v a s WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY o ː d ɐ WORD_BOUNDARY a l ə WORD_BOUNDARY UTT_BOUNDARY ɪ n v ɛ s t ɪ ɡ a t o ː ɐ WORD_BOUNDARY p aʊ l i ː n ə WORD_BOUNDARY ɡ ʊ k s t WORD_BOUNDARY d u ː WORD_BOUNDARY m a ː l WORD_BOUNDARY ɡ ə n aʊ WORD_BOUNDARY d a ː m +07/18/2026 21:56:38 - INFO - __main__ - [RANK 0] Sample 3278 of the training set: {'input_ids': tensor([16, 6, 10, 4, 16, 6, 2, 42, 4, 15, 6, 2, 42, 4, 15, 6, 18, 39, + 6, 13, 18, 12, 6, 8, 5, 6, 14, 15, 3, 9, 6, 42, 4, 15, 6, 18, + 39, 6, 10, 9, 30, 15, 23, 11, 6, 8, 5, 6, 2, 42, 4, 15, 6, 13, + 4, 5, 6, 20, 4, 16, 6, 40, 15, 26, 9, 6, 21, 30, 15, 6, 21, 4, + 15, 10, 17, 6, 31, 17, 18, 20, 17, 6, 2, 31, 22, 15, 36, 11, 16, 6, + 21, 30, 15, 6, 21, 18, 39, 6, 33, 27, 15, 36, 32, 10, 6, 16, 4, 15, + 28, 6, 26, 8, 9, 12, 30, 15, 16, 6, 2, 18, 39, 6, 20, 4, 15, 10, + 6, 26, 8, 9, 12, 30, 15, 16, 6, 16, 18, 39, 17, 6, 4, 12, 33, 27, + 15, 6, 21, 22, 15, 9, 6, 2, 42, 4, 13, 14, 15, 12, 6, 2, 13, 8, + 16, 6, 21, 30, 15, 6, 13, 34, 12, 11, 16, 6, 12, 4, 5, 11, 16, 6, + 21, 30, 15, 6, 26, 4, 35, 10, 34, 10, 6, 19, 16, 11, 16, 6, 35, 18, + 5, 17, 14, 15, 12, 8, 16, 4, 16, 17, 9, 4, 15, 10, 6, 19, 16, 36, + 19, 39, 11, 16, 6, 2, 36, 8, 39, 17, 6, 24, 8, 9, 33, 12, 18, 39, + 11, 16, 6, 3, 4, 37, 10, 6, 21, 30, 15, 9, 6, 2, 26, 4, 35, 10, + 34, 10, 6, 2, 8, 9, 6, 13, 18, 12, 6, 19, 16, 11, 16, 6, 13, 4, + 7, 11, 16, 4, 16, 17, 9, 4, 15, 10, 6, 2, 13, 8, 16, 6, 8, 9, + 6, 3, 8, 37, 10, 17, 6, 18, 39, 6, 26, 9, 25, 28, 11, 6, 19, 16, + 11, 6, 35, 18, 5, 17, 14, 15, 12, 11, 6, 26, 9, 25, 28, 11, 6, 18, + 39, 6, 19, 16, 11, 16, 6, 21, 4, 9, 10, 6, 2, 18, 39, 6, 31, 17, + 8, 12, 11, 6, 3, 30, 15, 9, 6, 19, 16, 11, 16, 6, 25, 5, 6, 2, + 13, 22, 15, 9, 6, 18, 5, 17, 6, 21, 4, 20, 4, 16, 17, 4, 15, 6, + 31, 17, 22, 15, 12, 6, 2, 17, 22, 15, 6, 19, 16, 24, 29, 16, 3, 9, + 17, 6, 7, 32, 16, 7, 6, 8, 5, 6, 21, 18, 16, 17, 6, 23, 11, 24, + 19, 20, 6, 2, 13, 19, 5, 17, 6, 3, 27, 15, 6, 3, 4, 5, 6, 8, + 17, 13, 4, 15, 6, 16, 18, 39, 17, 6, 2, 18, 39, 6, 24, 4, 15, 26, + 11, 6, 24, 34, 32, 17, 11, 6, 4, 15, 26, 11, 16, 17, 6, 33, 19, 17, + 6, 2, 16, 19, 16, 6, 21, 30, 15, 6, 7, 12, 30, 15, 23, 11, 16, 6, + 29, 20, 6, 8, 12, 7, 6, 16, 29, 12, 6, 16, 29, 12, 6, 27, 15, 9, + 6, 33, 27, 15, 36, 32, 10, 6])} n WORD_BOUNDARY k a n WORD_BOUNDARY UTT_BOUNDARY j a ː WORD_BOUNDARY UTT_BOUNDARY j a ː WORD_BOUNDARY ɪ ç WORD_BOUNDARY v ɪ l WORD_BOUNDARY ɛ s WORD_BOUNDARY o ː d ɐ WORD_BOUNDARY j a ː WORD_BOUNDARY ɪ ç WORD_BOUNDARY k ɐ i ː ɡ ə WORD_BOUNDARY ɛ s WORD_BOUNDARY UTT_BOUNDARY j a ː WORD_BOUNDARY v a s WORD_BOUNDARY m a n WORD_BOUNDARY y ː b ɐ WORD_BOUNDARY z i ː WORD_BOUNDARY z a ː k t WORD_BOUNDARY ʃ t ɪ m t WORD_BOUNDARY UTT_BOUNDARY ʃ e ː ʀ ə n WORD_BOUNDARY z i ː WORD_BOUNDARY z ɪ ç WORD_BOUNDARY ts u ː ʀ ʏ k WORD_BOUNDARY n a ː x WORD_BOUNDARY b ɛ ɐ l i ː n WORD_BOUNDARY UTT_BOUNDARY ɪ ç WORD_BOUNDARY m a ː k WORD_BOUNDARY b ɛ ɐ l i ː n WORD_BOUNDARY n ɪ ç t WORD_BOUNDARY a l ts u ː WORD_BOUNDARY z e ː ɐ WORD_BOUNDARY UTT_BOUNDARY j a v o ː l WORD_BOUNDARY UTT_BOUNDARY v ɛ n WORD_BOUNDARY z i ː WORD_BOUNDARY v ɔ l ə n WORD_BOUNDARY l a s ə n WORD_BOUNDARY z i ː WORD_BOUNDARY b a p k ɔ k WORD_BOUNDARY aɪ n ə n WORD_BOUNDARY p ɪ s t o ː l ɛ n a n t ɐ a ː k WORD_BOUNDARY aɪ n ʀ aɪ ç ə n WORD_BOUNDARY UTT_BOUNDARY ʀ ɛ ç t WORD_BOUNDARY h ɛ ɐ ts l ɪ ç ə n WORD_BOUNDARY d a ŋ k WORD_BOUNDARY z i ː ɐ WORD_BOUNDARY UTT_BOUNDARY b a p k ɔ k WORD_BOUNDARY UTT_BOUNDARY ɛ ɐ WORD_BOUNDARY v ɪ l WORD_BOUNDARY aɪ n ə n WORD_BOUNDARY v a f ə n a n t ɐ a ː k WORD_BOUNDARY UTT_BOUNDARY v ɛ n WORD_BOUNDARY ɛ ɐ WORD_BOUNDARY d ɛ ŋ k t WORD_BOUNDARY ɪ ç WORD_BOUNDARY b ɐ aʊ x ə WORD_BOUNDARY aɪ n ə WORD_BOUNDARY p ɪ s t o ː l ə WORD_BOUNDARY b ɐ aʊ x ə WORD_BOUNDARY ɪ ç WORD_BOUNDARY aɪ n ə n WORD_BOUNDARY z a ɐ k WORD_BOUNDARY UTT_BOUNDARY ɪ ç WORD_BOUNDARY ʃ t ɛ l ə WORD_BOUNDARY d i ː ɐ WORD_BOUNDARY aɪ n ə n WORD_BOUNDARY aʊ s WORD_BOUNDARY UTT_BOUNDARY v e ː ɐ WORD_BOUNDARY ɪ s t WORD_BOUNDARY z a m a n t a ː WORD_BOUNDARY ʃ t e ː l WORD_BOUNDARY UTT_BOUNDARY t e ː WORD_BOUNDARY aɪ n h ʊ n d ɐ t WORD_BOUNDARY f ʏ n f WORD_BOUNDARY ɛ s WORD_BOUNDARY z ɪ n t WORD_BOUNDARY ɡ ə h aɪ m WORD_BOUNDARY UTT_BOUNDARY v aɪ s t WORD_BOUNDARY d u ː WORD_BOUNDARY d a s WORD_BOUNDARY ɛ t v a ː WORD_BOUNDARY n ɪ ç t WORD_BOUNDARY UTT_BOUNDARY ɪ ç WORD_BOUNDARY h a ː b ə WORD_BOUNDARY h ɔ ʏ t ə WORD_BOUNDARY a ː b ə n t WORD_BOUNDARY ts aɪ t WORD_BOUNDARY UTT_BOUNDARY n aɪ n WORD_BOUNDARY z i ː WORD_BOUNDARY f l i ː ɡ ə n WORD_BOUNDARY ʊ m WORD_BOUNDARY ɛ l f WORD_BOUNDARY n ʊ l WORD_BOUNDARY n ʊ l WORD_BOUNDARY u ː ɐ WORD_BOUNDARY ts u ː ʀ ʏ k WORD_BOUNDARY +07/18/2026 21:56:41 - INFO - __main__ - [RANK 0] ***** Running training ***** +07/18/2026 21:56:41 - INFO - __main__ - [RANK 0] Num examples = 101825 +07/18/2026 21:56:41 - INFO - __main__ - [RANK 0] Num Training Steps = 30000 +07/18/2026 21:56:41 - INFO - __main__ - [RANK 0] Instantaneous batch size per device = 32 +07/18/2026 21:56:41 - INFO - __main__ - [RANK 0] Total train batch size (w. parallel, distributed & accumulation) = 32 +07/18/2026 21:56:41 - INFO - __main__ - [RANK 0] Gradient Accumulation steps = 1 +07/18/2026 21:56:41 - INFO - __main__ - [RANK 0] Total optimization steps = 30000 +07/18/2026 21:56:41 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 21:56:41 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 21:56:45 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 21:56:45 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 21:56:45 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 21:56:54 - INFO - __main__ - [RANK 0] Starting epoch 0... +07/18/2026 21:56:54 - WARNING - transformers.modeling_utils - `loss_type=None` was set in the config but it is unrecognized. Using the default loss: `ForCausalLMLoss`. +07/18/2026 21:56:55 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 21:56:55 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 21:56:58 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 21:56:58 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 21:56:58 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 21:57:05 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 21:57:05 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 21:57:09 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 21:57:09 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 21:57:09 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 21:57:14 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 21:57:14 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 21:57:18 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 21:57:18 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 21:57:18 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 21:58:16 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 21:58:16 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 21:58:20 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 21:58:20 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 21:58:20 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 21:59:26 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 21:59:26 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 21:59:30 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 21:59:30 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 21:59:30 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 22:00:57 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 22:00:57 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 22:01:01 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 22:01:01 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 22:01:01 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 22:02:32 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 22:02:32 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 22:02:36 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 22:02:36 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 22:02:36 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 22:04:27 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 22:04:27 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 22:04:31 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 22:04:31 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 22:04:31 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 22:06:26 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 22:06:26 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 22:06:30 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 22:06:30 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 22:06:30 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 22:09:09 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 22:09:09 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 22:09:13 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 22:09:13 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 22:09:13 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 22:11:26 - INFO - __main__ - [RANK 0] perplexity: 7.909086097933963 eval_loss: 2.068012237548828 accuracy: 0.4003 +07/18/2026 22:12:48 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 22:12:48 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 22:12:51 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 22:12:51 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 22:12:51 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 22:16:29 - INFO - __main__ - [RANK 0] perplexity: 4.531085486765549 eval_loss: 1.5109615325927734 accuracy: 0.5508 +07/18/2026 22:19:12 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 22:19:12 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 22:19:16 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 22:19:16 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 22:19:16 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 22:21:30 - INFO - __main__ - [RANK 0] perplexity: 3.763946708277698 eval_loss: 1.3254680633544922 accuracy: 0.6023 +07/18/2026 22:25:36 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 22:25:36 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 22:25:39 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 22:25:39 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 22:25:39 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 22:27:23 - INFO - __main__ - [RANK 0] perplexity: 3.2925302794646143 eval_loss: 1.1916563510894775 accuracy: 0.6412 +07/18/2026 22:28:21 - INFO - __main__ - [RANK 0] Starting epoch 1... +07/18/2026 22:31:31 - INFO - __main__ - [RANK 0] perplexity: 3.1018616790820457 eval_loss: 1.1320024728775024 accuracy: 0.6589 +07/18/2026 22:32:53 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 22:32:53 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 22:32:56 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 22:32:56 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 22:32:56 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 22:36:32 - INFO - __main__ - [RANK 0] perplexity: 2.9848321959746005 eval_loss: 1.093543529510498 accuracy: 0.6699 +07/18/2026 22:39:16 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 22:39:16 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 22:39:19 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 22:39:19 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 22:39:19 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 22:41:52 - INFO - __main__ - [RANK 0] perplexity: 2.9038398848998557 eval_loss: 1.066033959388733 accuracy: 0.6778 +07/18/2026 22:45:58 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 22:45:58 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 22:46:02 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 22:46:02 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 22:46:02 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 22:46:56 - INFO - __main__ - [RANK 0] perplexity: 2.8438977328136237 eval_loss: 1.045175552368164 accuracy: 0.6837 +07/18/2026 22:48:55 - INFO - __main__ - [RANK 0] Starting epoch 2... +07/18/2026 22:51:05 - INFO - __main__ - [RANK 0] perplexity: 2.7927952750158576 eval_loss: 1.0270429849624634 accuracy: 0.6894 +07/18/2026 22:52:27 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 22:52:27 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 22:52:30 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 22:52:30 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 22:52:30 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 22:56:29 - INFO - __main__ - [RANK 0] perplexity: 2.75469929180313 eval_loss: 1.0133082866668701 accuracy: 0.6932 +07/18/2026 22:59:13 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 22:59:13 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 22:59:17 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 22:59:17 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 22:59:17 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 23:01:32 - INFO - __main__ - [RANK 0] perplexity: 2.720248192942417 eval_loss: 1.000723123550415 accuracy: 0.6967 +07/18/2026 23:05:38 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 23:05:38 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 23:05:42 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 23:05:42 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 23:05:42 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 23:06:40 - INFO - __main__ - [RANK 0] perplexity: 2.6901474581149727 eval_loss: 0.9895960092544556 accuracy: 0.7000 +07/18/2026 23:09:40 - INFO - __main__ - [RANK 0] Starting epoch 3... +07/18/2026 23:10:49 - INFO - __main__ - [RANK 0] perplexity: 2.6656972363789504 eval_loss: 0.9804656505584717 accuracy: 0.7028 +07/18/2026 23:12:11 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 23:12:11 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 23:12:15 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 23:12:15 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 23:12:15 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 23:13:16 - INFO - __main__ - [RANK 0] Starting epoch 4... +07/18/2026 23:13:16 - INFO - __main__ - [RANK 0] Switching language at step 10000 +07/18/2026 23:13:32 - INFO - __main__ - [RANK 0] Sample 194393 of the training set: {'input_ids': tensor([53, 11, 6, 24, 25, 5, 6, 67, 16, 3, 6, 67, 16, 3, 6, 53, 45, 6, + 12, 68, 10, 17, 6, 12, 68, 10, 17, 6, 12, 68, 10, 17, 6, 53, 11, 6, + 2, 70, 6, 53, 67, 17, 5, 6, 55, 19, 17, 6, 42, 8, 24, 6, 23, 55, + 67, 16, 3, 35, 68, 6, 3, 18, 3, 6, 53, 67, 17, 6, 2, 24, 30, 15, + 6, 31, 57, 17, 6, 53, 11, 6, 3, 34, 55, 6, 42, 8, 24, 6, 2, 53, + 11, 6, 3, 34, 55, 6, 2, 5, 17, 68, 55, 17, 18, 3, 6, 67, 21, 6, + 53, 45, 6, 20, 8, 16, 17, 6, 17, 11, 6, 23, 70, 6, 34, 16, 6, 7, + 11, 55, 53, 11, 6, 24, 68, 12, 18, 3, 45, 6, 2, 8, 20, 8, 20, 6, + 2, 26, 69, 15, 23, 12, 11, 55, 6, 26, 18, 12, 6, 2, 42, 8, 24, 6, + 2, 19, 6, 3, 70, 16, 17, 6, 55, 18, 20, 8, 20, 26, 11, 55, 6, 53, + 67, 17, 6, 2, 10, 12, 19, 20, 3, 6, 54, 55, 27, 15, 6, 53, 11, 6, + 48, 18, 16, 3, 70, 6, 53, 11, 6, 16, 8, 10, 5, 17, 6, 3, 34, 55, + 6, 16, 45, 26, 11, 55, 6, 7, 70, 16, 3, 6, 57, 35, 6, 23, 68, 17, + 6, 57, 6, 5, 35, 8, 55, 6, 10, 30, 15, 6, 2, 18, 20, 67, 46, 18, + 16, 45, 31, 11, 16, 6, 48, 69, 15, 10, 18, 37, 6, 70, 13, 11, 55, 17, + 19, 20, 6, 53, 8, 55, 6, 2, 8, 20, 8, 20, 6, 2, 19, 6, 42, 27, + 15, 21, 3, 6, 17, 11, 6, 12, 19, 10, 6, 53, 11, 6, 10, 68, 55, 6, + 26, 27, 15, 17, 6, 5, 45, 12, 21, 6, 25, 17, 6, 24, 18, 55, 6, 53, + 45, 6, 48, 69, 15, 6, 23, 29, 3, 6, 2, 26, 57, 17, 6, 53, 45, 6, + 5, 17, 68, 35, 17, 6, 53, 8, 20, 6, 12, 67, 5, 17, 6, 42, 18, 55, + 6, 3, 18, 3, 16, 17, 6, 53, 45, 6, 2, 19, 6, 54, 18, 37, 10, 6, + 5, 70, 6, 2, 53, 45, 6, 53, 45, 6, 5, 17, 68, 35, 17, 6, 24, 67, + 13, 18, 37, 6, 53, 8, 20, 6, 34, 16, 53, 11, 6, 53, 11, 6, 5, 30, + 15, 7, 55, 57, 16, 17, 6, 8, 16, 18, 48, 45, 6, 2, 19, 6, 42, 27, + 15, 21, 3, 6, 17, 11, 6, 12, 19, 10, 6, 53, 8, 20, 6, 57, 12, 34, + 37, 6, 53, 8, 55, 6, 18, 17, 6, 48, 57, 21, 6, 16, 19, 5, 6, 10, + 68, 5, 6, 42, 27, 15, 6, 10, 29, 3, 6, 23, 70, 6, 10, 29, 3, 6, + 10, 11, 20, 26, 19, 16, 6, 57, 6, 57, 6, 48, 34, 10, 6, 26, 19, 6, + 53, 11, 6, 5, 30, 15, 6, 48])} ð ə WORD_BOUNDARY h aʊ s WORD_BOUNDARY æ n d WORD_BOUNDARY æ n d WORD_BOUNDARY ð eɪ WORD_BOUNDARY l ɑ k t WORD_BOUNDARY l ɑ k t WORD_BOUNDARY l ɑ k t WORD_BOUNDARY ð ə WORD_BOUNDARY UTT_BOUNDARY oʊ WORD_BOUNDARY ð æ t s WORD_BOUNDARY ɹ aɪ t WORD_BOUNDARY j ɛ h WORD_BOUNDARY ɡ ɹ æ n d p ɑ WORD_BOUNDARY d ɪ d WORD_BOUNDARY ð æ t WORD_BOUNDARY UTT_BOUNDARY h i ː WORD_BOUNDARY ʃ ʌ t WORD_BOUNDARY ð ə WORD_BOUNDARY d ɔ ɹ WORD_BOUNDARY j ɛ h WORD_BOUNDARY UTT_BOUNDARY ð ə WORD_BOUNDARY d ɔ ɹ WORD_BOUNDARY UTT_BOUNDARY s t ɑ ɹ t ɪ d WORD_BOUNDARY æ z WORD_BOUNDARY ð eɪ WORD_BOUNDARY m ɛ n t WORD_BOUNDARY t ə WORD_BOUNDARY ɡ oʊ WORD_BOUNDARY ɔ n WORD_BOUNDARY f ə ɹ ð ə WORD_BOUNDARY h ɑ l ɪ d eɪ WORD_BOUNDARY UTT_BOUNDARY ɛ m ɛ m WORD_BOUNDARY UTT_BOUNDARY b ɜ ː ɡ l ə ɹ WORD_BOUNDARY b ɪ l WORD_BOUNDARY UTT_BOUNDARY j ɛ h WORD_BOUNDARY UTT_BOUNDARY aɪ WORD_BOUNDARY d oʊ n t WORD_BOUNDARY ɹ ɪ m ɛ m b ə ɹ WORD_BOUNDARY ð æ t WORD_BOUNDARY UTT_BOUNDARY k l aɪ m d WORD_BOUNDARY θ ɹ u ː WORD_BOUNDARY ð ə WORD_BOUNDARY w ɪ n d oʊ WORD_BOUNDARY ð ə WORD_BOUNDARY n ɛ k s t WORD_BOUNDARY d ɔ ɹ WORD_BOUNDARY n eɪ b ə ɹ WORD_BOUNDARY f oʊ n d WORD_BOUNDARY ʌ p WORD_BOUNDARY ɡ ɑ t WORD_BOUNDARY ʌ WORD_BOUNDARY s p ɛ ɹ WORD_BOUNDARY k i ː WORD_BOUNDARY UTT_BOUNDARY ɪ m æ d̠ʒ ɪ n eɪ ʃ ə n WORD_BOUNDARY w ɜ ː k ɪ ŋ WORD_BOUNDARY oʊ v ə ɹ t aɪ m WORD_BOUNDARY ð ɛ ɹ WORD_BOUNDARY UTT_BOUNDARY ɛ m ɛ m WORD_BOUNDARY UTT_BOUNDARY aɪ WORD_BOUNDARY j u ː z d WORD_BOUNDARY t ə WORD_BOUNDARY l aɪ k WORD_BOUNDARY ð ə WORD_BOUNDARY k ɑ ɹ WORD_BOUNDARY b u ː t WORD_BOUNDARY s eɪ l z WORD_BOUNDARY aʊ t WORD_BOUNDARY h ɪ ɹ WORD_BOUNDARY ð eɪ WORD_BOUNDARY w ɜ ː WORD_BOUNDARY ɡ ʊ d WORD_BOUNDARY UTT_BOUNDARY b ʌ t WORD_BOUNDARY ð eɪ WORD_BOUNDARY s t ɑ p t WORD_BOUNDARY ð ɛ m WORD_BOUNDARY l æ s t WORD_BOUNDARY j ɪ ɹ WORD_BOUNDARY d ɪ d n t WORD_BOUNDARY ð eɪ WORD_BOUNDARY UTT_BOUNDARY aɪ WORD_BOUNDARY θ ɪ ŋ k WORD_BOUNDARY s oʊ WORD_BOUNDARY UTT_BOUNDARY ð eɪ WORD_BOUNDARY ð eɪ WORD_BOUNDARY s t ɑ p t WORD_BOUNDARY h æ v ɪ ŋ WORD_BOUNDARY ð ɛ m WORD_BOUNDARY ɔ n ð ə WORD_BOUNDARY ð ə WORD_BOUNDARY s i ː f ɹ ʌ n t WORD_BOUNDARY ɛ n ɪ w eɪ WORD_BOUNDARY UTT_BOUNDARY aɪ WORD_BOUNDARY j u ː z d WORD_BOUNDARY t ə WORD_BOUNDARY l aɪ k WORD_BOUNDARY ð ɛ m WORD_BOUNDARY ʌ l ɔ ŋ WORD_BOUNDARY ð ɛ ɹ WORD_BOUNDARY ɪ t WORD_BOUNDARY w ʌ z WORD_BOUNDARY n aɪ s WORD_BOUNDARY k ɑ s WORD_BOUNDARY j u ː WORD_BOUNDARY k ʊ d WORD_BOUNDARY ɡ oʊ WORD_BOUNDARY k ʊ d WORD_BOUNDARY k ə m b aɪ n WORD_BOUNDARY ʌ WORD_BOUNDARY ʌ WORD_BOUNDARY w ɔ k WORD_BOUNDARY b aɪ WORD_BOUNDARY ð ə WORD_BOUNDARY s i ː WORD_BOUNDARY w +07/18/2026 23:13:32 - INFO - __main__ - [RANK 0] Sample 72097 of the training set: {'input_ids': tensor([35, 55, 8, 5, 18, 37, 6, 46, 68, 26, 2, 35, 12, 45, 55, 27, 15, 20, + 6, 13, 18, 21, 18, 17, 6, 2, 67, 5, 17, 11, 55, 55, 18, 5, 10, 6, + 8, 20, 70, 17, 30, 15, 6, 34, 12, 55, 19, 17, 6, 5, 70, 6, 12, 8, + 17, 6, 20, 30, 15, 6, 35, 29, 17, 6, 3, 25, 16, 6, 20, 19, 6, 5, + 17, 57, 7, 6, 2, 67, 5, 17, 11, 55, 55, 18, 5, 10, 6, 8, 20, 70, + 17, 30, 15, 6, 3, 18, 3, 6, 42, 27, 15, 6, 24, 18, 55, 6, 48, 57, + 17, 6, 46, 67, 16, 18, 17, 6, 48, 57, 21, 6, 17, 8, 12, 18, 37, 6, + 42, 27, 15, 6, 2, 67, 5, 17, 11, 55, 55, 18, 5, 10, 6, 10, 19, 6, + 48, 18, 51, 6, 2, 67, 5, 17, 11, 55, 55, 18, 5, 10, 6, 20, 68, 17, + 6, 24, 57, 6, 2, 67, 5, 17, 11, 55, 55, 18, 5, 10, 6, 8, 20, 70, + 17, 30, 15, 6, 31, 30, 15, 6, 48, 57, 21, 6, 17, 8, 12, 18, 37, 6, + 20, 30, 15, 6, 2, 67, 5, 17, 11, 55, 55, 18, 5, 10, 6, 8, 20, 70, + 17, 30, 15, 6, 48, 8, 12, 6, 53, 8, 55, 68, 55, 6, 54, 55, 30, 15, + 6, 3, 18, 7, 55, 11, 16, 17, 6, 54, 18, 37, 21, 6, 17, 11, 6, 3, + 27, 15, 6, 70, 10, 45, 6, 2, 67, 5, 17, 11, 55, 55, 18, 5, 10, 6, + 8, 20, 70, 17, 30, 15, 6, 53, 8, 55, 21, 6, 57, 6, 26, 29, 10, 6, + 17, 11, 6, 12, 29, 10, 6, 67, 17, 6, 53, 8, 55, 21, 6, 57, 6, 17, + 45, 10, 6, 57, 35, 68, 55, 17, 6, 10, 68, 55, 6, 67, 16, 3, 6, 53, + 8, 55, 21, 6, 57, 6, 57, 6, 5, 17, 34, 55, 6, 5, 34, 55, 17, 6, + 57, 13, 6, 12, 19, 10, 6, 46, 68, 31, 6, 24, 57, 21, 6, 67, 17, 6, + 24, 18, 21, 6, 24, 25, 5, 6, 2, 67, 5, 17, 11, 55, 55, 18, 5, 10, + 6, 8, 20, 70, 17, 30, 15, 6, 5, 70, 6, 3, 27, 15, 6, 42, 27, 15, + 6, 16, 70, 6, 48, 18, 51, 6, 48, 57, 16, 6, 42, 27, 15, 6, 48, 29, + 3, 6, 12, 19, 10, 6, 17, 11, 6, 3, 27, 15, 6, 7, 69, 15, 5, 17, + 6, 2, 67, 5, 17, 11, 55, 55, 18, 5, 10, 6, 8, 20, 70, 17, 30, 15, + 6, 53, 11, 6, 17, 45, 10, 6, 57, 35, 68, 55, 17, 6, 10, 68, 55, 6, + 2, 67, 5, 17, 11, 55, 55, 18, 5, 10, 6, 8, 20, 70, 17, 30, 15, 6, + 12, 8, 17, 5, 6, 17, 45, 10, 6, 18, 17, 6, 3, 25, 16, 6, 2, 67, + 5, 17, 11, 55, 55, 18, 5, 10])} p ɹ ɛ s ɪ ŋ WORD_BOUNDARY d̠ʒ ɑ b UTT_BOUNDARY p l eɪ ɹ u ː m WORD_BOUNDARY v ɪ z ɪ t WORD_BOUNDARY UTT_BOUNDARY æ s t ə ɹ ɹ ɪ s k WORD_BOUNDARY ɛ m oʊ t i ː WORD_BOUNDARY ɔ l ɹ aɪ t WORD_BOUNDARY s oʊ WORD_BOUNDARY l ɛ t WORD_BOUNDARY m i ː WORD_BOUNDARY p ʊ t WORD_BOUNDARY d aʊ n WORD_BOUNDARY m aɪ WORD_BOUNDARY s t ʌ f WORD_BOUNDARY UTT_BOUNDARY æ s t ə ɹ ɹ ɪ s k WORD_BOUNDARY ɛ m oʊ t i ː WORD_BOUNDARY d ɪ d WORD_BOUNDARY j u ː WORD_BOUNDARY h ɪ ɹ WORD_BOUNDARY w ʌ t WORD_BOUNDARY d̠ʒ æ n ɪ t WORD_BOUNDARY w ʌ z WORD_BOUNDARY t ɛ l ɪ ŋ WORD_BOUNDARY j u ː WORD_BOUNDARY UTT_BOUNDARY æ s t ə ɹ ɹ ɪ s k WORD_BOUNDARY k aɪ WORD_BOUNDARY w ɪ t̠ʃ WORD_BOUNDARY UTT_BOUNDARY æ s t ə ɹ ɹ ɪ s k WORD_BOUNDARY m ɑ t WORD_BOUNDARY h ʌ WORD_BOUNDARY UTT_BOUNDARY æ s t ə ɹ ɹ ɪ s k WORD_BOUNDARY ɛ m oʊ t i ː WORD_BOUNDARY ʃ i ː WORD_BOUNDARY w ʌ z WORD_BOUNDARY t ɛ l ɪ ŋ WORD_BOUNDARY m i ː WORD_BOUNDARY UTT_BOUNDARY æ s t ə ɹ ɹ ɪ s k WORD_BOUNDARY ɛ m oʊ t i ː WORD_BOUNDARY w ɛ l WORD_BOUNDARY ð ɛ ɹ ɑ ɹ WORD_BOUNDARY θ ɹ i ː WORD_BOUNDARY d ɪ f ɹ ə n t WORD_BOUNDARY θ ɪ ŋ z WORD_BOUNDARY t ə WORD_BOUNDARY d u ː WORD_BOUNDARY oʊ k eɪ WORD_BOUNDARY UTT_BOUNDARY æ s t ə ɹ ɹ ɪ s k WORD_BOUNDARY ɛ m oʊ t i ː WORD_BOUNDARY ð ɛ ɹ z WORD_BOUNDARY ʌ WORD_BOUNDARY b ʊ k WORD_BOUNDARY t ə WORD_BOUNDARY l ʊ k WORD_BOUNDARY æ t WORD_BOUNDARY ð ɛ ɹ z WORD_BOUNDARY ʌ WORD_BOUNDARY t eɪ k WORD_BOUNDARY ʌ p ɑ ɹ t WORD_BOUNDARY k ɑ ɹ WORD_BOUNDARY æ n d WORD_BOUNDARY ð ɛ ɹ z WORD_BOUNDARY ʌ WORD_BOUNDARY ʌ WORD_BOUNDARY s t ɔ ɹ WORD_BOUNDARY s ɔ ɹ t WORD_BOUNDARY ʌ v WORD_BOUNDARY l aɪ k WORD_BOUNDARY d̠ʒ ɑ ʃ WORD_BOUNDARY h ʌ z WORD_BOUNDARY æ t WORD_BOUNDARY h ɪ z WORD_BOUNDARY h aʊ s WORD_BOUNDARY UTT_BOUNDARY æ s t ə ɹ ɹ ɪ s k WORD_BOUNDARY ɛ m oʊ t i ː WORD_BOUNDARY s oʊ WORD_BOUNDARY d u ː WORD_BOUNDARY j u ː WORD_BOUNDARY n oʊ WORD_BOUNDARY w ɪ t̠ʃ WORD_BOUNDARY w ʌ n WORD_BOUNDARY j u ː WORD_BOUNDARY w ʊ d WORD_BOUNDARY l aɪ k WORD_BOUNDARY t ə WORD_BOUNDARY d u ː WORD_BOUNDARY f ɜ ː s t WORD_BOUNDARY UTT_BOUNDARY æ s t ə ɹ ɹ ɪ s k WORD_BOUNDARY ɛ m oʊ t i ː WORD_BOUNDARY ð ə WORD_BOUNDARY t eɪ k WORD_BOUNDARY ʌ p ɑ ɹ t WORD_BOUNDARY k ɑ ɹ WORD_BOUNDARY UTT_BOUNDARY æ s t ə ɹ ɹ ɪ s k WORD_BOUNDARY ɛ m oʊ t i ː WORD_BOUNDARY l ɛ t s WORD_BOUNDARY t eɪ k WORD_BOUNDARY ɪ t WORD_BOUNDARY d aʊ n WORD_BOUNDARY UTT_BOUNDARY æ s t ə ɹ ɹ ɪ s k +07/18/2026 23:13:32 - INFO - __main__ - [RANK 0] Sample 64196 of the training set: {'input_ids': tensor([28, 17, 6, 2, 20, 34, 17, 9, 6, 19, 16, 6, 26, 18, 5, 39, 11, 16, + 6, 20, 22, 15, 9, 6, 8, 9, 16, 5, 17, 6, 24, 30, 15, 9, 6, 26, + 18, 17, 11, 6, 2, 7, 4, 17, 9, 6, 36, 41, 15, 20, 18, 31, 6, 3, + 9, 19, 5, 18, 39, 6, 2, 7, 4, 17, 9, 6, 19, 39, 30, 15, 6, 24, + 4, 17, 6, 3, 22, 15, 16, 6, 36, 41, 15, 20, 18, 31, 6, 3, 9, 19, + 5, 18, 39, 6, 4, 16, 23, 11, 10, 16, 4, 26, 9, 17, 6, 2, 20, 34, + 17, 9, 6, 19, 16, 6, 10, 12, 19, 16, 11, 5, 6, 12, 30, 15, 26, 11, + 5, 6, 21, 25, 36, 30, 15, 9, 13, 19, 26, 39, 11, 16, 6, 18, 5, 17, + 6, 3, 4, 15, 6, 23, 8, 5, 17, 9, 16, 6, 16, 4, 28, 17, 6, 7, + 14, 15, 9, 26, 19, 23, 11, 12, 25, 7, 11, 16, 6, 36, 41, 15, 20, 18, + 31, 6, 3, 9, 19, 5, 18, 39, 6, 2, 7, 4, 17, 9, 6, 18, 16, 6, + 3, 22, 15, 20, 6, 23, 4, 16, 33, 11, 16, 6, 36, 41, 15, 20, 18, 31, + 6, 3, 9, 19, 5, 18, 39, 6, 24, 30, 15, 9, 6, 12, 8, 33, 17, 11, + 6, 16, 4, 28, 17, 6, 2, 17, 4, 9, 23, 11, 17, 6, 31, 18, 12, 17, + 6, 3, 4, 15, 6, 4, 28, 33, 22, 15, 16, 6, 23, 12, 19, 21, 11, 6, + 2, 17, 4, 9, 23, 11, 17, 6, 31, 18, 12, 17, 6, 3, 4, 15, 6, 4, + 28, 33, 22, 15, 16, 6, 23, 12, 19, 5, 6, 2, 7, 4, 17, 9, 6, 4, + 28, 6, 21, 14, 15, 6, 2, 17, 4, 9, 23, 11, 17, 6, 31, 18, 12, 17, + 6, 19, 16, 6, 26, 29, 5, 6, 10, 34, 20, 17, 6, 24, 18, 16, 17, 9, + 24, 9, 6, 2, 7, 4, 17, 9, 6, 19, 16, 6, 26, 29, 5, 6, 10, 34, + 20, 17, 6, 16, 34, 28, 6, 24, 18, 16, 17, 9, 24, 9, 6, 2, 17, 4, + 9, 23, 11, 17, 6, 31, 18, 12, 17, 6, 42, 4, 15, 6, 2, 7, 4, 17, + 9, 6, 34, 28, 6, 12, 22, 15, 14, 15, 6, 2, 7, 4, 17, 9, 6, 13, + 30, 15, 9, 6, 21, 34, 12, 17, 11, 16, 6, 16, 18, 39, 17, 6, 21, 8, + 20, 17, 12, 18, 39, 11, 6, 12, 14, 15, 10, 14, 15, 20, 14, 15, 17, 30, + 15, 13, 11, 16, 6, 29, 16, 17, 6, 33, 40, 15, 23, 11, 6, 24, 30, 15, + 9, 6, 25, 7, 26, 25, 11, 16, 6, 2, 7, 4, 17, 9, 6, 3, 27, 15, + 6, 3, 4, 5, 6, 13, 18, 9, 17, 6, 3, 4, 16, 6, 13, 18, 9, 10, + 12, 18, 39, 6, 19, 16, 6, 26])} x t WORD_BOUNDARY UTT_BOUNDARY m ɔ t ɐ WORD_BOUNDARY aɪ n WORD_BOUNDARY b ɪ s ç ə n WORD_BOUNDARY m e ː ɐ WORD_BOUNDARY ɛ ɐ n s t WORD_BOUNDARY h i ː ɐ WORD_BOUNDARY b ɪ t ə WORD_BOUNDARY UTT_BOUNDARY f a t ɐ WORD_BOUNDARY ʀ ø ː m ɪ ʃ WORD_BOUNDARY d ɐ aɪ s ɪ ç WORD_BOUNDARY UTT_BOUNDARY f a t ɐ WORD_BOUNDARY aɪ ç i ː WORD_BOUNDARY h a t WORD_BOUNDARY d e ː n WORD_BOUNDARY ʀ ø ː m ɪ ʃ WORD_BOUNDARY d ɐ aɪ s ɪ ç WORD_BOUNDARY a n ɡ ə k n a b ɐ t WORD_BOUNDARY UTT_BOUNDARY m ɔ t ɐ WORD_BOUNDARY aɪ n WORD_BOUNDARY k l aɪ n ə s WORD_BOUNDARY l i ː b ə s WORD_BOUNDARY z aʊ ʀ i ː ɐ v aɪ b ç ə n WORD_BOUNDARY ɪ s t WORD_BOUNDARY d a ː WORD_BOUNDARY ɡ ɛ s t ɐ n WORD_BOUNDARY n a x t WORD_BOUNDARY f o ː ɐ b aɪ ɡ ə l aʊ f ə n WORD_BOUNDARY ʀ ø ː m ɪ ʃ WORD_BOUNDARY d ɐ aɪ s ɪ ç WORD_BOUNDARY UTT_BOUNDARY f a t ɐ WORD_BOUNDARY ɪ n WORD_BOUNDARY d e ː m WORD_BOUNDARY ɡ a n ts ə n WORD_BOUNDARY ʀ ø ː m ɪ ʃ WORD_BOUNDARY d ɐ aɪ s ɪ ç WORD_BOUNDARY h i ː ɐ WORD_BOUNDARY l ɛ ts t ə WORD_BOUNDARY n a x t WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY d a ː WORD_BOUNDARY a x ts e ː n WORD_BOUNDARY ɡ l aɪ z ə WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY d a ː WORD_BOUNDARY a x ts e ː n WORD_BOUNDARY ɡ l aɪ s WORD_BOUNDARY UTT_BOUNDARY f a t ɐ WORD_BOUNDARY a x WORD_BOUNDARY z o ː WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY aɪ n WORD_BOUNDARY b ʊ s WORD_BOUNDARY k ɔ m t WORD_BOUNDARY h ɪ n t ɐ h ɐ WORD_BOUNDARY UTT_BOUNDARY f a t ɐ WORD_BOUNDARY aɪ n WORD_BOUNDARY b ʊ s WORD_BOUNDARY k ɔ m t WORD_BOUNDARY n ɔ x WORD_BOUNDARY h ɪ n t ɐ h ɐ WORD_BOUNDARY UTT_BOUNDARY t a ɐ ɡ ə t WORD_BOUNDARY ʃ ɪ l t WORD_BOUNDARY j a ː WORD_BOUNDARY UTT_BOUNDARY f a t ɐ WORD_BOUNDARY ɔ x WORD_BOUNDARY l e ː o ː WORD_BOUNDARY UTT_BOUNDARY f a t ɐ WORD_BOUNDARY v i ː ɐ WORD_BOUNDARY z ɔ l t ə n WORD_BOUNDARY n ɪ ç t WORD_BOUNDARY z ɛ m t l ɪ ç ə WORD_BOUNDARY l o ː k o ː m o ː t i ː v ə n WORD_BOUNDARY ʊ n t WORD_BOUNDARY ts y ː ɡ ə WORD_BOUNDARY h i ː ɐ WORD_BOUNDARY aʊ f b aʊ ə n WORD_BOUNDARY UTT_BOUNDARY f a t ɐ WORD_BOUNDARY d u ː WORD_BOUNDARY d a s WORD_BOUNDARY v ɪ ɐ t WORD_BOUNDARY d a n WORD_BOUNDARY v ɪ ɐ k l ɪ ç WORD_BOUNDARY aɪ n WORD_BOUNDARY b +07/18/2026 23:13:32 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 23:13:32 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 23:13:35 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 23:13:35 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 23:13:35 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 23:15:23 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 23:15:23 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 23:15:26 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 23:15:26 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 23:15:26 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 23:16:18 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 23:16:18 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 23:16:21 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 23:16:21 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 23:16:21 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 23:17:12 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 23:17:12 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 23:17:16 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 23:17:16 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 23:17:16 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 23:18:04 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 23:18:04 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 23:18:08 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 23:18:08 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 23:18:08 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 23:19:04 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 23:19:04 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 23:19:08 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 23:19:08 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 23:19:08 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 23:20:13 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 23:20:13 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 23:20:17 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 23:20:17 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 23:20:17 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 23:21:52 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 23:21:52 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 23:21:56 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 23:21:56 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 23:21:56 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 23:24:41 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 23:24:41 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 23:24:45 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 23:24:45 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 23:24:45 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 23:27:02 - INFO - __main__ - [RANK 0] perplexity: 3.0484109265490376 eval_loss: 1.1146204471588135 accuracy: 0.6692 +07/18/2026 23:27:06 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 23:27:06 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 23:27:09 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 23:27:09 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 23:27:09 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 23:30:38 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 23:30:38 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 23:30:42 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 23:30:42 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 23:30:42 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 23:33:28 - INFO - __main__ - [RANK 0] perplexity: 2.9187844069269286 eval_loss: 1.071167230606079 accuracy: 0.6814 +07/18/2026 23:37:34 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 23:37:34 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 23:37:37 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 23:37:37 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 23:37:37 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 23:38:44 - INFO - __main__ - [RANK 0] perplexity: 2.8592508213240726 eval_loss: 1.050559639930725 accuracy: 0.6865 +07/18/2026 23:42:56 - INFO - __main__ - [RANK 0] perplexity: 2.813762437303175 eval_loss: 1.034522533416748 accuracy: 0.6909 +07/18/2026 23:44:18 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 23:44:18 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 23:44:21 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 23:44:21 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 23:44:21 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 23:48:00 - INFO - __main__ - [RANK 0] perplexity: 2.7864598768059925 eval_loss: 1.0247719287872314 accuracy: 0.6934 +07/18/2026 23:50:44 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 23:50:44 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 23:50:48 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 23:50:48 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 23:50:48 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 23:53:06 - INFO - __main__ - [RANK 0] perplexity: 2.7645032486162053 eval_loss: 1.0168609619140625 accuracy: 0.6957 +07/18/2026 23:57:12 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/18/2026 23:57:12 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/18/2026 23:57:16 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/18/2026 23:57:16 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/18/2026 23:57:16 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/18/2026 23:58:10 - INFO - __main__ - [RANK 0] perplexity: 2.7417788105902896 eval_loss: 1.0086069107055664 accuracy: 0.6980 +07/19/2026 00:02:23 - INFO - __main__ - [RANK 0] perplexity: 2.7245989508556705 eval_loss: 1.0023212432861328 accuracy: 0.6998 +07/19/2026 00:03:44 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/19/2026 00:03:44 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/19/2026 00:03:48 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/19/2026 00:03:48 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/19/2026 00:03:48 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/19/2026 00:07:21 - INFO - __main__ - [RANK 0] Starting epoch 5... +07/19/2026 00:07:48 - INFO - __main__ - [RANK 0] perplexity: 2.7098122152313078 eval_loss: 0.9968793392181396 accuracy: 0.7014 +07/19/2026 00:10:32 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/19/2026 00:10:32 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/19/2026 00:10:36 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/19/2026 00:10:36 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/19/2026 00:10:36 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/19/2026 00:12:54 - INFO - __main__ - [RANK 0] perplexity: 2.697568513530264 eval_loss: 0.9923508167266846 accuracy: 0.7029 +07/19/2026 00:17:00 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/19/2026 00:17:00 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/19/2026 00:17:03 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/19/2026 00:17:03 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/19/2026 00:17:03 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/19/2026 00:18:01 - INFO - __main__ - [RANK 0] perplexity: 2.687025584489657 eval_loss: 0.9884348511695862 accuracy: 0.7036 +07/19/2026 00:22:14 - INFO - __main__ - [RANK 0] perplexity: 2.671650891628613 eval_loss: 0.9826965928077698 accuracy: 0.7056 +07/19/2026 00:23:35 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/19/2026 00:23:35 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/19/2026 00:23:39 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/19/2026 00:23:39 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/19/2026 00:23:39 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/19/2026 00:27:35 - INFO - __main__ - [RANK 0] perplexity: 2.66090281961021 eval_loss: 0.9786654710769653 accuracy: 0.7066 +07/19/2026 00:30:18 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/19/2026 00:30:18 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/19/2026 00:30:22 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/19/2026 00:30:22 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/19/2026 00:30:22 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/19/2026 00:32:42 - INFO - __main__ - [RANK 0] perplexity: 2.650939100410912 eval_loss: 0.9749139547348022 accuracy: 0.7073 +07/19/2026 00:36:48 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/19/2026 00:36:48 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/19/2026 00:36:52 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/19/2026 00:36:52 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/19/2026 00:36:52 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/19/2026 00:37:48 - INFO - __main__ - [RANK 0] perplexity: 2.6409590117175332 eval_loss: 0.9711421132087708 accuracy: 0.7082 +07/19/2026 00:42:01 - INFO - __main__ - [RANK 0] perplexity: 2.632241551027258 eval_loss: 0.9678357839584351 accuracy: 0.7095 +07/19/2026 00:43:22 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/19/2026 00:43:23 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/19/2026 00:43:26 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/19/2026 00:43:26 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/19/2026 00:43:26 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/19/2026 00:47:34 - INFO - __main__ - [RANK 0] perplexity: 2.6254989035739125 eval_loss: 0.9652709364891052 accuracy: 0.7106 +07/19/2026 00:49:37 - INFO - __main__ - [RANK 0] Starting epoch 6... +07/19/2026 00:50:18 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/19/2026 00:50:18 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/19/2026 00:50:22 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/19/2026 00:50:22 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/19/2026 00:50:22 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/19/2026 00:52:38 - INFO - __main__ - [RANK 0] perplexity: 2.618889662151742 eval_loss: 0.9627504348754883 accuracy: 0.7112 +07/19/2026 00:56:44 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/19/2026 00:56:44 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/19/2026 00:56:47 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/19/2026 00:56:47 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/19/2026 00:56:47 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/19/2026 00:57:43 - INFO - __main__ - [RANK 0] perplexity: 2.610897178166824 eval_loss: 0.9596939086914062 accuracy: 0.7122 +07/19/2026 01:01:55 - INFO - __main__ - [RANK 0] perplexity: 2.6061491283080147 eval_loss: 0.9578737020492554 accuracy: 0.7125 +07/19/2026 01:03:17 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/19/2026 01:03:17 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/19/2026 01:03:20 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/19/2026 01:03:20 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/19/2026 01:03:20 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/19/2026 01:07:00 - INFO - __main__ - [RANK 0] perplexity: 2.5995157722826403 eval_loss: 0.955325186252594 accuracy: 0.7132 +07/19/2026 01:09:44 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/19/2026 01:09:44 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/19/2026 01:09:47 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/19/2026 01:09:47 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/19/2026 01:09:47 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/19/2026 01:12:05 - INFO - __main__ - [RANK 0] perplexity: 2.5936876188512725 eval_loss: 0.9530806541442871 accuracy: 0.7142 +07/19/2026 01:16:12 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/19/2026 01:16:12 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/19/2026 01:16:16 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/19/2026 01:16:16 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/19/2026 01:16:16 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/19/2026 01:17:35 - INFO - __main__ - [RANK 0] perplexity: 2.5870301601004417 eval_loss: 0.950510561466217 accuracy: 0.7145 +07/19/2026 01:21:47 - INFO - __main__ - [RANK 0] perplexity: 2.583599966147958 eval_loss: 0.9491837620735168 accuracy: 0.7149 +07/19/2026 01:23:09 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/19/2026 01:23:09 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/19/2026 01:23:13 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/19/2026 01:23:13 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/19/2026 01:23:13 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/19/2026 01:26:54 - INFO - __main__ - [RANK 0] perplexity: 2.579420242577862 eval_loss: 0.9475646615028381 accuracy: 0.7156 +07/19/2026 01:29:38 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/19/2026 01:29:38 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/19/2026 01:29:42 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/19/2026 01:29:42 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/19/2026 01:29:42 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/19/2026 01:32:23 - INFO - __main__ - [RANK 0] perplexity: 2.5754383914065877 eval_loss: 0.9460197687149048 accuracy: 0.7159 +07/19/2026 01:32:43 - INFO - __main__ - [RANK 0] Starting epoch 7... +07/19/2026 01:36:29 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/19/2026 01:36:29 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/19/2026 01:36:33 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/19/2026 01:36:33 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/19/2026 01:36:33 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json +07/19/2026 01:37:50 - INFO - __main__ - [RANK 0] perplexity: 2.5762201732352827 eval_loss: 0.9463232755661011 accuracy: 0.7161 +07/19/2026 01:37:50 - INFO - transformers.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/config.json +07/19/2026 01:37:50 - INFO - transformers.generation.configuration_utils - Configuration saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/generation_config.json +07/19/2026 01:37:53 - INFO - transformers.modeling_utils - Model weights saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/model.safetensors +07/19/2026 01:37:53 - INFO - transformers.tokenization_utils_base - tokenizer config file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/tokenizer_config.json +07/19/2026 01:37:53 - INFO - transformers.tokenization_utils_base - Special tokens file saved in models/phonebabylm-phone-deu-eng-sequential_interleaved/special_tokens_map.json diff --git a/training_metadata.json b/training_metadata.json new file mode 100644 index 0000000..b236f8f --- /dev/null +++ b/training_metadata.json @@ -0,0 +1,74 @@ +{ + "input_config": { + "model_type": "gpt2", + "architectures": [ + "GPT2LMHeadModel" + ], + "activation_function": "gelu", + "attn_pdrop": 0.1, + "embd_pdrop": 0.1, + "resid_pdrop": 0.1, + "initializer_range": 0.02, + "layer_norm_epsilon": 1e-05, + "n_embd": 768, + "n_inner": 3072, + "n_head": 12, + "n_layer": 12, + "vocab_size": 1, + "n_ctx": 512, + "n_positions": 512 + }, + "num_steps_switch": 10000, + "save_steps": [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512, + 1000, + 2000, + 3000, + 4000, + 5000, + 6000, + 7000, + 8000, + 9000, + 10000, + 10001, + 10002, + 10004, + 10008, + 10016, + 10032, + 10064, + 10128, + 10256, + 10512, + 11000, + 12000, + 13000, + 14000, + 15000, + 16000, + 17000, + 18000, + 19000, + 20000, + 21000, + 22000, + 23000, + 24000, + 25000, + 26000, + 27000, + 28000, + 29000, + 30000 + ] +} \ No newline at end of file diff --git a/vocab.json b/vocab.json new file mode 100644 index 0000000..3455a0a --- /dev/null +++ b/vocab.json @@ -0,0 +1 @@ +{"UNK":0,"PAD":1,"UTT_BOUNDARY":2,"d":3,"a":4,"s":5,"WORD_BOUNDARY":6,"f":7,"ɛ":8,"ɐ":9,"k":10,"ə":11,"l":12,"v":13,"o":14,"ː":15,"n":16,"t":17,"ɪ":18,"aɪ":19,"m":20,"z":21,"e":22,"ɡ":23,"h":24,"aʊ":25,"b":26,"u":27,"x":28,"ʊ":29,"i":30,"ʃ":31,"ʏ":32,"ts":33,"ɔ":34,"p":35,"ʀ":36,"ŋ":37,"œ":38,"ç":39,"y":40,"ø":41,"j":42,"pf":43,"ʊɐ":44,"eɪ":45,"d̠ʒ":46,"ɛɪ":47,"w":48,"ɒ":49,"ã":50,"t̠ʃ":51,"tɕ":52,"ð":53,"θ":54,"ɹ":55,"əʊ":56,"ʌ":57,"eə":58,"ɔɪ":59,"l̩":60,"ʒ":61,"ʊə":62,"ɔ̃":63,"iə":64,"œ̃":65,"ʔ":66,"æ":67,"ɑ":68,"ɜ":69,"oʊ":70} \ No newline at end of file