From ff21151abf6e3e9b87f048a06f1229b0705a9712 Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Mon, 13 Jul 2026 11:45:11 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: AetherModels/AetherCuTF-Ko-3-shared-Seed1234 Source: Original Platform --- .gitattributes | 35 + README.md | 58 ++ aether.py | 315 ++++++++ aethertokenizers.py | 467 ++++++++++++ all_results.json | 9 + bpe_utils.py | 64 ++ config.json | 32 + consts.py | 57 ++ generation_config.json | 11 + model.safetensors | 3 + tokenizer_config.json | 35 + train_results.json | 9 + trainer_state.json | 1569 ++++++++++++++++++++++++++++++++++++++++ training_args.bin | 3 + 14 files changed, 2667 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 aether.py create mode 100644 aethertokenizers.py create mode 100644 all_results.json create mode 100644 bpe_utils.py create mode 100644 config.json create mode 100644 consts.py create mode 100644 generation_config.json create mode 100644 model.safetensors create mode 100644 tokenizer_config.json create mode 100644 train_results.json create mode 100644 trainer_state.json create mode 100644 training_args.bin diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..a6344aa --- /dev/null +++ b/.gitattributes @@ -0,0 +1,35 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..1d2245f --- /dev/null +++ b/README.md @@ -0,0 +1,58 @@ +--- +library_name: transformers +base_model: llama3_gptsize_config_dtypefix.json +tags: +- generated_from_trainer +model-index: +- name: AetherCuTF-Ko-3-shared-V7-Seed1234-LR5e-4 + results: [] +--- + + + +# AetherCuTF-Ko-3-shared-V7-Seed1234-LR5e-4 + +This model is a fine-tuned version of [llama3_gptsize_config_dtypefix.json](https://huggingface.co/llama3_gptsize_config_dtypefix.json) on the None dataset. + +## Model description + +More information needed + +## Intended uses & limitations + +More information needed + +## Training and evaluation data + +More information needed + +## Training procedure + +### Training hyperparameters + +The following hyperparameters were used during training: +- learning_rate: 0.0005 +- train_batch_size: 8 +- eval_batch_size: 8 +- seed: 1234 +- distributed_type: multi-GPU +- num_devices: 2 +- gradient_accumulation_steps: 4 +- total_train_batch_size: 64 +- total_eval_batch_size: 16 +- optimizer: Use OptimizerNames.ADAMW_TORCH_FUSED with betas=(0.9,0.95) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments +- lr_scheduler_type: linear +- lr_scheduler_warmup_steps: 0.1 +- num_epochs: 1.0 + +### Training results + + + +### Framework versions + +- Transformers 5.0.0 +- Pytorch 2.10.0+cu128 +- Datasets 4.8.2 +- Tokenizers 0.22.2 diff --git a/aether.py b/aether.py new file mode 100644 index 0000000..5055eef --- /dev/null +++ b/aether.py @@ -0,0 +1,315 @@ +import regex +from unicodedata import normalize +from .bpe_utils import bytes_to_unicode_special +from random import shuffle, seed +from math import ceil + +def pretok_with_targeted_handling(text_data, split_regex, target_regex, target_process, nontarget_process): + if split_regex: + # 1st Regex: The general split regex + split_regex = regex.compile(split_regex) + split_regexed = split_regex.findall(text_data) + else: + split_regexed = [text_data] + + # 2nd Regex: Splits a string into target and non-target chunks + final_processed_list = [] + for chunk in split_regexed: + if not chunk: + continue # Skip empty tokens if any + + # Split the token into target/non-target parts + processed_sub_tokens = "" + for match in regex.finditer(target_regex, chunk): + # Check if the chunk is target to decide which process to use + label = match.lastgroup + content = match.group() + if label == 'target': + processed_sub_tokens+=(target_process(content)) + else: + processed_sub_tokens+=(nontarget_process(content)) + + final_processed_list.append(processed_sub_tokens) + + return final_processed_list + +def pretok_with_targeted_handling_list(text_data, split_regex, target_regex, target_process, nontarget_process): + if split_regex: + # 1st Regex: The general split regex + split_regex = regex.compile(split_regex) + split_regexed = split_regex.findall(text_data) + else: + split_regexed = [text_data] + # 2nd Regex: Splits a string into target and non-target chunks + final_processed_list = [] + for chunk in split_regexed: + if not chunk: + continue # Skip empty tokens if any + # Split the token into target/non-target parts + for match in regex.finditer(target_regex, chunk): + # Check if the chunk is target to decide which process to use + label = match.lastgroup + content = match.group() + if label == 'target': + final_processed_list+=target_process(content) + else: + final_processed_list+=nontarget_process(content) + return final_processed_list + + +# String "좋아" -> String "ƴƸƯŠ" +def apply_encoding(input_string, encoding_type, mapping, offset=0, shuffle=False, offset_mapping=None, unicode_start=None): + if "Jamo" in encoding_type: + varlen = "varlen" in encoding_type + final_string = "" + for char in input_string: + if char == " ": + final_string += mapping[ord(char)] + continue + normalized = normalize("NFKD", char) + if len(normalized)==1: # Single Jamo can be used, but if not usable need fallback + if normalized not in mapping: + final_string += apply_encoding(normalized, "utf-8", mapping) + continue + final_string += normalized + if not varlen and len(normalized)==2: # Hangul syllable decomposed into 2 Jamo + final_string += '∅' # Placeholder for missing Jamo + return final_string + if shuffle: + # For each character, calculate offset and map to shuffled offset + input_string = "".join([chr(offset_mapping[ord(c) - unicode_start] + unicode_start) for c in input_string]) + if encoding_type == "nybbles": + return apply_nybble_encoding(input_string, mapping, offset) + encoded = input_string.encode(encoding_type) + return "".join([mapping.get(x + offset) if x + offset in mapping else mapping.get(x) for x in encoded]) + +# String "ƴƸƯŠ" -> String "좋아" +def apply_decoding(input_string, encoding_type, mapping, offset=0): + antimap = {value: key for key, value in mapping.items()} + return b"".join([antimap[a+offset].to_bytes() for a in input_string]).decode(encoding_type) + +def apply_character_decoding(input_string, mapping): + antimap = {value: key for key, value in mapping.items()} + final_string = "" + temp_string = b"" + char_mode = False + for symbol in input_string: + # For token in charmode + if symbol not in antimap: + # If we are in the normal case, resolve the current temp_string + if not char_mode: + final_string += temp_string.decode("utf8", errors='replace') + temp_string = "" + temp_string += symbol + char_mode = True + # For tokens in regular case + else: + # If we are in charmode, resolve the current temp_string + if char_mode: + final_string += temp_string + temp_string = b"" + temp_string += (antimap[symbol]).to_bytes() + char_mode = False + # Resolve final piece + if char_mode: + final_string += temp_string + else: + final_string += temp_string.decode('utf8', errors='replace') + + return final_string + +# String "è¿Ļæĺ¯ƴƸƯŠ" -> String "这是좋아" +def apply_specialized_decoding(input_string, encoding_type, mapping, offset=0, shuffle=False, offset_mapping=None, unicode_start=None): + if shuffle: + def unshuffle_char(c): + shuffled_offset = ord(c) - unicode_start + original_offset = offset_mapping[shuffled_offset] + return chr(original_offset + unicode_start) + + antimap = {value: key for key, value in mapping.items()} + final_string = "" + temp_string = b"" + special_mode = False + for symbol in input_string: + byte_id = antimap[symbol] + # For token in special case + if byte_id >= offset: + # If we are in the normal case, resolve the current temp_string + if not special_mode: + final_string += temp_string.decode("utf8", errors='replace') + temp_string = b"" + temp_string += (byte_id - offset).to_bytes() + special_mode = True + # For tokens in regular case + else: + # If we are in the special case, resolve the current temp_string + if special_mode: + temp_add = temp_string.decode(encoding_type, errors='replace') + if shuffle: + temp_add = "".join([unshuffle_char(c) for c in temp_add]) + final_string += temp_add + temp_string = b"" + temp_string += (byte_id).to_bytes() + special_mode = False + # Resolve final piece + if special_mode: + temp_add = temp_string.decode(encoding_type, errors='replace') + if shuffle: + temp_add = "".join([unshuffle_char(c) for c in temp_add]) + final_string += temp_add + else: + final_string += temp_string.decode('utf8', errors='replace') + + return final_string + +def apply_nybble_decoding(input_string, mapping, offset=0): + def decode_char(temp_string): + res = [] + for i in range(0, len(temp_string), 4): + try: + n0, n1, n2, n3 = [t for t in temp_string[i:i+4]] + codepoint = (n0 << 12) | (n1 << 8) | (n2 << 4) | n3 + res.append(chr(codepoint)) + except: + # In case of any error (e.g., invalid character), append a replacement character + res.append('�') + return "".join(res) + + antimap = {value: key for key, value in mapping.items()} + final_string = "" + temp_string = b"" + special_mode = False + for symbol in input_string: + byte_id = antimap[symbol] + # For token in special case + if byte_id >= offset: + # If we are in the normal case, resolve the current temp_string + if not special_mode: + final_string += temp_string.decode("utf8", errors='replace') + temp_string = b"" + temp_string += (byte_id - offset).to_bytes() + special_mode = True + # For tokens in regular case + else: + # If we are in the special case, resolve the current temp_string + if special_mode: + final_string += decode_char(temp_string) + temp_string = b"" + temp_string += (byte_id).to_bytes() + special_mode = False + # Resolve final piece + if special_mode: + final_string += decode_char(temp_string) + else: + final_string += temp_string.decode('utf8', errors='replace') + + return final_string + +def apply_Jamo_decoding(input_string, mapping): + antimap = {value: key for key, value in mapping.items()} + final_string = "" + current_type = type(input_string[0]) + current_pieces = [] + for char in input_string: + char = antimap[char] + if type(char) == current_type: + current_pieces.append(char) + else: + # Process current pieces + if current_type == str: + # Jamo pieces + syllables = "".join((current_pieces)).replace("∅", "") + # Put it into unicode format + final_string += normalize("NFKC", syllables) + else: + # Byte pieces + byte_string = b"".join([c.to_bytes() for c in current_pieces]).decode('utf-8', errors='replace') + final_string += byte_string + current_type = type(char) + current_pieces = [char] + # Final flush + if current_type == str: + # Jamo pieces + syllables = "".join((current_pieces)).replace("∅", "") + final_string += syllables + else: + # Byte pieces + byte_string = b"".join([c.to_bytes() for c in current_pieces]).decode('utf-8', errors='replace') + final_string += byte_string + return final_string + + + +def apply_nybble_encoding(input_string, mapping, offset,n_nybbles=4): + def encode_char(c): + o = ord(c) + return "".join(mapping[offset + ((o >> (4 * i)) & 0xF)] for i in range(n_nybbles - 1, -1, -1)) + + return "".join(encode_char(c) for c in input_string) + +# Given a range of unicode, we shuffle the points +# For performance this can be stored as a list, a mapping of offset2newoffset +def create_shuffled_unicode_mapping(range_start, range_end, given_seed=6767): + unicode_points = list(range(range_end - range_start + 1)) + seed(given_seed) + shuffle(unicode_points) + # to use, calculate offset = ord(char) - range_start, then map to unicode_points[offset] + range_start + return unicode_points + +# Given a range of unicode, provide a Customized UTF with given parameter properties +def create_CuTF_encoding(range_start, range_end, encode_length): + target_space_len = range_end - range_start + 1 + num_indices = ceil(target_space_len**(1/encode_length)) + # Plan is to do the math only here and save the mapping for efficiency + CuTF_mapping = {} + for codepoint in range(range_start, range_end+1): + offset = codepoint-range_start + CuTF_rep = [] + for i in range(encode_length-1): + rep_index = offset//(num_indices**(encode_length-i-1)) + CuTF_rep.append(rep_index) + offset -= (num_indices**(encode_length-i-1))*rep_index + CuTF_rep.append(offset) + CuTF_mapping[codepoint] = CuTF_rep + + return CuTF_mapping, num_indices + + + +def apply_CuTF_decoding(input_string, encode_length, num_index, range_start, indexing_strategy="shared"): + def decode_target_chars(sequence): + outpieces = "" + for i in range(0, len(sequence), encode_length): + piece = sequence[i:i+encode_length] + offset = 0 + for j, byte in enumerate(piece): + if indexing_strategy == "shared": + offset += (byte-256) * (num_index**(encode_length-j-1)) + elif indexing_strategy == "unique": + offset += (byte - (256 + num_index*(encode_length-j-1))) * (num_index**(encode_length-j-1)) + outpieces += chr(range_start + offset) + return outpieces + final_string = "" + temp_pieces = [] + is_target = False + for token in input_string: + if token >= 256: + if not is_target: + # Flush non-target tokens + final_string += bytes(temp_pieces).decode("utf8", errors='replace') + temp_pieces = [] + temp_pieces.append(token) + is_target = True + else: + if is_target: + # Flush target tokens + final_string += decode_target_chars(temp_pieces) + temp_pieces = [] + temp_pieces.append(token) + is_target = False + # Flush any remaining tokens + if is_target: + final_string += decode_target_chars(temp_pieces) + else: + final_string += bytes(temp_pieces).decode("utf8", errors='replace') + return final_string \ No newline at end of file diff --git a/aethertokenizers.py b/aethertokenizers.py new file mode 100644 index 0000000..ec1c6e3 --- /dev/null +++ b/aethertokenizers.py @@ -0,0 +1,467 @@ +from unittest import result + +from .aether import apply_CuTF_decoding, apply_Jamo_decoding, apply_nybble_decoding, create_CuTF_encoding, pretok_with_targeted_handling, apply_encoding, apply_specialized_decoding, create_shuffled_unicode_mapping, apply_character_decoding, pretok_with_targeted_handling_list +from .consts import * +from .bpe_utils import bytes_to_unicode_original, bytes_to_unicode_special +from transformers.tokenization_utils import PreTrainedTokenizer +import os, json, shutil +from typing import Optional + + +# Approximately modeled after byt5 tokenizer +# https://github.com/huggingface/transformers/blob/v4.57.1/src/transformers/models/byt5/tokenization_byt5.py + +class AetherByteTokenizer(PreTrainedTokenizer): + def __init__(self, lang, encoding, shuffle=False, offset_mapping=None,**kwargs): + self.lang = lang + self.encoding = encoding + self.shuffle = shuffle + if shuffle: + self.unicode_start = lang_ranges[lang][0] + if offset_mapping is not None: + self.offset_mapping = offset_mapping + else: + self.offset_mapping = create_shuffled_unicode_mapping(*lang_ranges[lang]) # list that acts as offset2newoffset + self.offset_unmapping = {v: k for k, v in enumerate(self.offset_mapping)} # newoffset2offset, for decoding + self.target_regex = target_regexes[lang] + if encoding == "utf8": + Aether_info = Aethers["utf8-"+lang] + if lang == "Ko": + # This is to limit Korean to full syllables only for this mode + self.target_regex = target_regexes["Ko-SO"] + else: + Aether_info = Aethers[encoding] + self.aether_type = Aether_info['type'] + # Mapping == aether2symbol + self.mapping = bytes_to_unicode_special(Aether_info['pieces'], OFFSET) if self.aether_type == "bytes" else {a: a for a in Aether_info['pieces']}|bytes_to_unicode_original() + self.index2aether = self._make_index2aether(self.mapping) + self.index2symbol = {index: self.mapping[aether] for index, aether in self.index2aether.items()} + self.symbol2index = {value: key for key, value in self.index2symbol.items()} + self.target_process = lambda x: apply_encoding(x, encoding, self.mapping, OFFSET, self.shuffle, self.offset_mapping if self.shuffle else None, self.unicode_start if self.shuffle else None) #text2symbols + self.nontarget_process = lambda x: apply_encoding(x, "utf8", self.mapping) #text2symbols + + # We want bos and eos tokens for our models + kwargs.setdefault('bos_token', '') + kwargs.setdefault('eos_token', '') + kwargs.setdefault('special_tokens_pattern', 'bos') + super().__init__(lang=lang, encoding=encoding, shuffle=shuffle, **kwargs) + + def _make_index2aether(self, mapping): + index2aether = {index: aether_id for index, aether_id in enumerate(mapping)} + return index2aether + + @property + def vocab_size(self): + return len(self.mapping) + len(self.added_tokens_encoder) + + @classmethod + def from_pretrained(cls, pretrained_model_name_or_path, *args, **kwargs): + mapping_path = os.path.join(pretrained_model_name_or_path, "offset_mapping.json") + if os.path.exists(mapping_path): + with open(mapping_path, "r") as f: + kwargs["offset_mapping"] = json.load(f) + return super().from_pretrained(pretrained_model_name_or_path, *args, **kwargs) + + def get_vocab(self): + # vocab is symbol2tok_id + vocab = {self.convert_ids_to_tokens(i): i for i in self.index2aether} + vocab.update(self.added_tokens_encoder) + return vocab + + def pretokenize(self, input): + return pretok_with_targeted_handling(input, None, self.target_regex, self.target_process, self.nontarget_process) + + def _tokenize(self, text: str) -> list[str]: + """Take as input a string and return a list of strings (tokens) for words/sub-words""" + pretokenized = self.pretokenize(text) + tokens = [] + for chunk in pretokenized: + tokens+=chunk + return tokens + + def _convert_token_to_id(self, token): + # Symbol to id + if token in self.added_tokens_encoder: + return self.added_tokens_encoder[token] + tok_id = self.symbol2index[token] + return tok_id + + def _convert_id_to_token(self, index): + """Converts an index (integer) in a token (str) using the vocab.""" + if index in self.added_tokens_decoder: + return self.added_tokens_decoder[index] + return self.index2symbol[index] + + def _apply_decoding(self, tokens): + if self.aether_type == "strings": + if self.encoding == "Jamo": + result = apply_Jamo_decoding(tokens, self.mapping) + elif self.encoding == "Jamo-varlen": + result = apply_Jamo_decoding(tokens, self.mapping) + elif self.aether_type == "bytes": + if self.encoding == "nybbles": + result = apply_nybble_decoding(tokens, self.mapping, OFFSET) + else: + result = apply_specialized_decoding(tokens, self.encoding, self.mapping, OFFSET, self.shuffle, self.offset_unmapping if self.shuffle else None, self.unicode_start if self.shuffle else None) + return result + + def convert_tokens_to_string(self, tokens): + special_tokens_set = set(self.all_special_tokens) + """Converts a sequence of tokens (symbols) into a single string.""" + final_string = "" + current_iter = [] + for token in tokens: + if token in special_tokens_set: + # Resolve current_iter + if current_iter: + resolved = self._apply_decoding(current_iter) + final_string += resolved + current_iter = [] + final_string += token + else: + current_iter.append(token) + if current_iter: + resolved = self._apply_decoding(current_iter) + final_string += resolved + return final_string + + + # Tokenizer has no vocab file + def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> tuple[str]: + return () + + def save_pretrained(self, save_directory, **kwargs): + saved = super().save_pretrained(save_directory, **kwargs) + # Copy this file and all local dependencies into the directory + src_dir = os.path.dirname(__file__) + for fname in ["aethertokenizers.py", "aether.py", "consts.py", "bpe_utils.py"]: + shutil.copy(os.path.join(src_dir, fname), os.path.join(save_directory, fname)) + + if self.shuffle: + mapping_path = os.path.join(save_directory, "offset_mapping.json") + with open(mapping_path, "w") as f: + json.dump(self.offset_mapping, f) + + config_path = os.path.join(save_directory, "tokenizer_config.json") + with open(config_path, "r") as f: + config = json.load(f) + config["auto_map"] = { + "AutoTokenizer": ["aethertokenizers.AetherByteTokenizer", None] + } + + config["shuffle"] = self.shuffle + + with open(config_path, "w") as f: + json.dump(config, f, indent=2) + + return saved + +class AetherCharTokenizer(PreTrainedTokenizer): + def __init__(self, lang, **kwargs): + self.lang = lang + self.unicode_start = lang_ranges[lang][0] + self.unicode_end = lang_ranges[lang][1] + self.target_regex = target_regexes[lang] + if lang == "Ko": + # This is to limit Korean to full syllables only for this mode + self.target_regex = target_regexes["Ko-SO"] + self.target_process = lambda x: x + self.mapping = bytes_to_unicode_original() + self.symbol2index = {value: key for key, value in self.mapping.items()} + self.nontarget_process = lambda x: apply_encoding(x, "utf8", self.mapping) #text2symbols + + # We want bos and eos tokens for our models + kwargs.setdefault('bos_token', '') + kwargs.setdefault('eos_token', '') + kwargs.setdefault('special_tokens_pattern', 'bos') + super().__init__(lang=lang, **kwargs) + + @property + def vocab_size(self): + return len(self.mapping) + (self.unicode_end - self.unicode_start + 1) + len(self.added_tokens_encoder) + + + def get_vocab(self): + # vocab is symbol2tok_id + vocab = {self.convert_ids_to_tokens(i): i for i in self.mapping} + vocab.update({chr(i): i - self.unicode_start + len(self.mapping) for i in range(self.unicode_start, self.unicode_end+1)}) + vocab.update(self.added_tokens_encoder) + return vocab + + def pretokenize(self, input): + return pretok_with_targeted_handling(input, None, self.target_regex, self.target_process, self.nontarget_process) + + def _tokenize(self, text: str) -> list[str]: + """Take as input a string and return a list of strings (tokens) for words/sub-words""" + pretokenized = self.pretokenize(text) + tokens = [] + for chunk in pretokenized: + tokens+=chunk + return tokens + + def _convert_token_to_id(self, token): + # Symbol to token_id + if token in self.added_tokens_encoder: + return self.added_tokens_encoder[token] + if token in self.symbol2index: + return self.symbol2index[token] + if self.unicode_start <= ord(token) <= self.unicode_end: + return ord(token) - self.unicode_start + len(self.mapping) + raise ValueError(f"Token {token} not in vocab") + + def _convert_id_to_token(self, index): + """Converts an index (integer) in a token (str) using the vocab.""" + if index in self.added_tokens_decoder: + return self.added_tokens_decoder[index] + if index < len(self.mapping): + return self.mapping[index] + return chr(index - len(self.mapping) + self.unicode_start) + + def _apply_decoding(self, tokens): + result = apply_character_decoding(tokens, self.mapping) + return result + + def convert_tokens_to_string(self, tokens): + special_tokens_set = set(self.all_special_tokens) + """Converts a sequence of tokens (symbols) into a single string.""" + final_string = "" + current_iter = [] + for token in tokens: + if token in special_tokens_set: + # Resolve current_iter + if current_iter: + resolved = self._apply_decoding(current_iter) + final_string += resolved + current_iter = [] + final_string += token + else: + current_iter.append(token) + if current_iter: + resolved = self._apply_decoding(current_iter) + final_string += resolved + return final_string + + + # Tokenizer has no vocab file + def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> tuple[str]: + return () + + def save_pretrained(self, save_directory, **kwargs): + saved = super().save_pretrained(save_directory, **kwargs) + # Copy this file and all local dependencies into the directory + src_dir = os.path.dirname(__file__) + for fname in ["aethertokenizers.py", "aether.py", "consts.py", "bpe_utils.py"]: + shutil.copy(os.path.join(src_dir, fname), os.path.join(save_directory, fname)) + + config_path = os.path.join(save_directory, "tokenizer_config.json") + with open(config_path, "r") as f: + config = json.load(f) + config["auto_map"] = { + "AutoTokenizer": ["aethertokenizers.AetherCharTokenizer", None] + } + + with open(config_path, "w") as f: + json.dump(config, f, indent=2) + + return saved + + +# A comparable tokenizer in UTF-8. +# Could just use ByT5 too, but controls for special tokens this way + +class UTF8ByteTokenizer(PreTrainedTokenizer): + def __init__(self, **kwargs): + self.mapping = bytes_to_unicode_special([]) #AKA index2symbol + self.symbol2index = {value: key for key, value in self.mapping.items()} + # We want bos and eos tokens for our models + kwargs.setdefault('bos_token', '') + kwargs.setdefault('eos_token', '') + kwargs.setdefault('special_tokens_pattern', 'bos') + super().__init__(**kwargs) + + @property + def vocab_size(self): + return 256 + len(self.added_tokens_encoder) + + + def get_vocab(self): + # vocab is symbol2tok_id + vocab = {self.convert_ids_to_tokens(i): i for i in range(256)} + vocab.update(self.added_tokens_encoder) + return vocab + + + def _tokenize(self, text: str) -> list[str]: + """Take as input a string and return a list of strings (tokens) for words/sub-words""" + text_bytes = text.encode('utf-8') + tokens = [self.mapping[b] for b in text_bytes] + return tokens + + def _convert_token_to_id(self, token): + # Symbol to id + if token in self.added_tokens_encoder: + return self.added_tokens_encoder[token] + return self.symbol2index[token] + + def _convert_id_to_token(self, index): + """Converts an index (integer) in a token (str) using the vocab.""" + if index in self.added_tokens_decoder: + return self.added_tokens_decoder[index] + return self.mapping[index] + + def convert_tokens_to_string(self, tokens): + """Converts a sequence of tokens (symbols) into a single string.""" + symbol2byte = self.get_vocab() + final_string = "" + curr_bstring = b"" + for token in tokens: + if token in self.all_special_tokens: + # Resolve current bstring + if curr_bstring: + string = curr_bstring.decode("utf-8", errors="replace") + final_string += string + curr_bstring = b"" + final_string += token + else: + curr_bstring += bytes([symbol2byte[token]]) + if curr_bstring: + string = curr_bstring.decode("utf-8", errors="replace") + final_string += string + return final_string + + + # Tokenizer has no vocab file + def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> tuple[str]: + return () + + def save_pretrained(self, save_directory, **kwargs): + saved = super().save_pretrained(save_directory, **kwargs) + + src_dir = os.path.dirname(__file__) + for fname in ["aethertokenizers.py", "aether.py", "consts.py", "bpe_utils.py"]: + shutil.copy(os.path.join(src_dir, fname), os.path.join(save_directory, fname)) + + config_path = os.path.join(save_directory, "tokenizer_config.json") + with open(config_path, "r") as f: + config = json.load(f) + config["auto_map"] = { + "AutoTokenizer": ["aethertokenizers.UTF8ByteTokenizer", None] + } + with open(config_path, "w") as f: + json.dump(config, f, indent=2) + + return saved + + + +class AetherCuTFTokenizer(PreTrainedTokenizer): + def __init__(self, lang, encode_length, indexing_strategy="shared", **kwargs): + self.lang = lang + self.encode_length = encode_length + self.indexing_strategy = indexing_strategy + self.target_regex = target_regexes[lang] + if lang == "Ko": + # This is to limit Korean to full syllables only for this mode + self.target_regex = target_regexes["Ko-SO"] + self.unicode_start = lang_ranges[lang][0] + self.unicode_end = lang_ranges[lang][1] + + raw_mapping, self.num_CuTF_indices = create_CuTF_encoding(self.unicode_start, self.unicode_end, encode_length) + mapping = dict() + if indexing_strategy == "shared": + for codepoint in raw_mapping: + mapping[codepoint] = [256+ a for a in raw_mapping[codepoint]] + elif indexing_strategy == "unique": + for codepoint in raw_mapping: + mapping[codepoint] = [256+ (encode_length -1 - i) * self.num_CuTF_indices + a for i, a in enumerate(raw_mapping[codepoint])] + self.mapping = mapping + self.target_process = lambda x: [rep for a in x for rep in self.mapping[ord(a)]] + self.nontarget_process = lambda x: [a for a in x.encode('utf8')] + + # We want bos and eos tokens for our models + kwargs.setdefault('bos_token', '') + kwargs.setdefault('eos_token', '') + kwargs.setdefault('special_tokens_pattern', 'bos') + super().__init__(lang=lang, encode_length = encode_length, indexing_strategy= indexing_strategy, **kwargs) + + @property + def vocab_size(self): + if self.indexing_strategy == "shared": + return 256 + self.num_CuTF_indices + len(self.added_tokens_encoder) + + elif self.indexing_strategy == "unique": + # This is the simple answer, but in reality the full bytespace isn't utilized. + # return 256 + self.num_CuTF_indices * self.encode_length + len(self.added_tokens_encoder) + # Need to use the highest number as reference + max_index = self.mapping[self.unicode_end][0] + return max_index + len(self.added_tokens_encoder) + 1 + + + def get_vocab(self): + vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size-len(self.added_tokens_encoder))} + vocab.update(self.added_tokens_encoder) + return vocab + + def _tokenize(self, text: str) -> list[str]: + """Take as input a string and return a list of strings (tokens) for words/sub-words""" + return pretok_with_targeted_handling_list(text, None, self.target_regex, self.target_process, self.nontarget_process) + + def _convert_token_to_id(self, token): + # Symbol to id + if token in self.added_tokens_encoder: + return self.added_tokens_encoder[token] + return token + + def _convert_id_to_token(self, index): + """Converts an index (integer) in a token (str) using the vocab.""" + if index in self.added_tokens_decoder: + return self.added_tokens_decoder[index] + return index + + def _apply_decoding(self, tokens): + result = apply_CuTF_decoding(tokens, self.encode_length, self.num_CuTF_indices, self.unicode_start, self.indexing_strategy) + return result + + def convert_tokens_to_string(self, tokens): + special_tokens_set = set(self.all_special_tokens) + """Converts a sequence of tokens (symbols) into a single string.""" + final_string = "" + current_iter = [] + for token in tokens: + if token in special_tokens_set: + # Resolve current_iter + if current_iter: + resolved = self._apply_decoding(current_iter) + final_string += resolved + current_iter = [] + final_string += token + else: + current_iter.append(token) + if current_iter: + resolved = self._apply_decoding(current_iter) + final_string += resolved + return final_string + + + # Tokenizer has no vocab file + def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> tuple[str]: + return () + + def save_pretrained(self, save_directory, **kwargs): + saved = super().save_pretrained(save_directory, **kwargs) + # Copy this file and all local dependencies into the directory + src_dir = os.path.dirname(__file__) + for fname in ["aethertokenizers.py", "aether.py", "consts.py", "bpe_utils.py"]: + shutil.copy(os.path.join(src_dir, fname), os.path.join(save_directory, fname)) + + config_path = os.path.join(save_directory, "tokenizer_config.json") + with open(config_path, "r") as f: + config = json.load(f) + config["auto_map"] = { + "AutoTokenizer": ["aethertokenizers.AetherCuTFTokenizer", None] + } + + with open(config_path, "w") as f: + json.dump(config, f, indent=2) + + return saved \ No newline at end of file diff --git a/all_results.json b/all_results.json new file mode 100644 index 0000000..2e66ea6 --- /dev/null +++ b/all_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 1.0, + "total_flos": 9.745954960876503e+18, + "train_loss": 0.052523610041691704, + "train_runtime": 7246.7996, + "train_samples": 3494359, + "train_samples_per_second": 482.193, + "train_steps_per_second": 7.534 +} \ No newline at end of file diff --git a/bpe_utils.py b/bpe_utils.py new file mode 100644 index 0000000..8289c8c --- /dev/null +++ b/bpe_utils.py @@ -0,0 +1,64 @@ +from .consts import OFFSET + +def bytes_to_unicode_original(): + # Can call this through + # from transformers.models.gpt2.tokenization_gpt2 import bytes_to_unicode + """ + Returns list of utf-8 byte and a mapping to unicode strings. We specifically avoids mapping to whitespace/control + characters the bpe code barfs on. + + The reversible bpe codes work on unicode strings. This means you need a large # of unicode characters in your vocab + if you want to avoid UNKs. When you're at something like a 10B token dataset you end up needing around 5K for + decent coverage. This is a significant percentage of your normal, say, 32K bpe vocab. To avoid that, we want lookup + tables between utf-8 bytes and unicode strings. + """ + bs = ( + list(range(ord("!"), ord("~") + 1)) + list(range(ord("¡"), ord("¬") + 1)) + list(range(ord("®"), ord("ÿ") + 1)) + ) + cs = bs[:] + n = 0 + for b in range(2**8): + if b not in bs: + bs.append(b) + cs.append(2**8 + n) + n += 1 + cs = [chr(n) for n in cs] + return dict(zip(bs, cs)) + +def bytes_to_unicode_special(special_bytes, offset=OFFSET): + bs = ( + list(range(ord("!"), ord("~") + 1)) + list(range(ord("¡"), ord("¬") + 1)) + list(range(ord("®"), ord("ÿ") + 1)) + ) + cs = bs[:] + n = 0 + for b in range(2**8): + if b not in bs: + bs.append(b) + cs.append(2**8 + n) + n += 1 + + for b in special_bytes: + bs.append(offset+b) + cs.append(2**8 + n) + n += 1 + + cs = [chr(n) for n in cs] + return dict(zip(bs, cs)) + +byte2unic = bytes_to_unicode_original() +unic2byte = {value: key for key, value in byte2unic.items()} + +def decode_uniced(scrambled): + unic_nums = [unic2byte[a] for a in scrambled] + char_byte = [chr(a) for a in unic_nums] + full = "".join(char_byte) + return full.encode("latin-1").decode() + + +def find_aether_bytes(encoding, range_start, range_end): + aether_bytes = set() + for i in range(range_start, range_end): + pieces = chr(i).encode(encoding) + for piece in pieces: + aether_bytes.add(piece) + return aether_bytes \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..44098d8 --- /dev/null +++ b/config.json @@ -0,0 +1,32 @@ +{ + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 279, + "dtype": "float32", + "eos_token_id": 280, + "head_dim": 64, + "hidden_act": "silu", + "hidden_size": 768, + "initializer_range": 0.02, + "intermediate_size": 3072, + "max_position_embeddings": 4096, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 12, + "num_hidden_layers": 12, + "num_key_value_heads": 12, + "pad_token_id": null, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_parameters": { + "rope_theta": 500000.0, + "rope_type": "default" + }, + "tie_word_embeddings": false, + "transformers_version": "5.0.0", + "use_cache": false, + "vocab_size": 288 +} diff --git a/consts.py b/consts.py new file mode 100644 index 0000000..5a00c4d --- /dev/null +++ b/consts.py @@ -0,0 +1,57 @@ +OFFSET = 10000 + +# Regexes +llama_regex = r"(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+" +target_regexes = { + # This is the one with leading space, could be used as a pretok later + # "Ko": r"(?P ?[가-힣ㄱ-ㅣ]+)|(?P(?:[^가-힣ㄱ-ㅣ ]| (?![가-힣ㄱ-ㅣ]))+)", + "Ko": r"(?P[가-힣ㄱ-ㅣ]+)|(?P[^가-힣ㄱ-ㅣ]+)", + "Ko-SO": r"(?P[가-힣]+)|(?P[^가-힣]+)", # Syllables Only + "Zh": r"(?P[\u4e00-\u9fa5]+)|(?P[^\u4e00-\u9fa5]+)", + } + +lang_ranges = { + "Ko": (0xAC00, 0xD7A3), + "Zh": (0x4E00, 0x9FA5)} + +# per-encoding pieces +used_bytes ={} +Aethers = { + 'Johab': { + 'type': "bytes", + 'pieces': [65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253] + }, + 'Jamo': { + 'type': "strings", + 'pieces': ['ᄀ', 'ᄁ', 'ᄂ', 'ᄃ', 'ᄄ', 'ᄅ', 'ᄆ', 'ᄇ', 'ᄈ', 'ᄉ', 'ᄊ', 'ᄋ', 'ᄌ', 'ᄍ', 'ᄎ', 'ᄏ', 'ᄐ', 'ᄑ', 'ᄒ', 'ᅡ', 'ᅢ', 'ᅣ', 'ᅤ', 'ᅥ', 'ᅦ', 'ᅧ', 'ᅨ', 'ᅩ', 'ᅪ', 'ᅫ', 'ᅬ', 'ᅭ', 'ᅮ', 'ᅯ', 'ᅰ', 'ᅱ', 'ᅲ', 'ᅳ', 'ᅴ', 'ᅵ', 'ᆨ', 'ᆩ', 'ᆪ', 'ᆫ', 'ᆬ', 'ᆭ', 'ᆮ', 'ᆯ', 'ᆰ', 'ᆱ', 'ᆲ', 'ᆳ', 'ᆴ', 'ᆵ', 'ᆶ', 'ᆷ', 'ᆸ', 'ᆹ', 'ᆺ', 'ᆻ', 'ᆼ', 'ᆽ', 'ᆾ', 'ᆿ', 'ᇀ', 'ᇁ', 'ᇂ', '∅'] + }, + 'Jamo-varlen': { + 'type': "strings", + 'pieces': ['ᄀ', 'ᄁ', 'ᄂ', 'ᄃ', 'ᄄ', 'ᄅ', 'ᄆ', 'ᄇ', 'ᄈ', 'ᄉ', 'ᄊ', 'ᄋ', 'ᄌ', 'ᄍ', 'ᄎ', 'ᄏ', 'ᄐ', 'ᄑ', 'ᄒ', 'ᅡ', 'ᅢ', 'ᅣ', 'ᅤ', 'ᅥ', 'ᅦ', 'ᅧ', 'ᅨ', 'ᅩ', 'ᅪ', 'ᅫ', 'ᅬ', 'ᅭ', 'ᅮ', 'ᅯ', 'ᅰ', 'ᅱ', 'ᅲ', 'ᅳ', 'ᅴ', 'ᅵ', 'ᆨ', 'ᆩ', 'ᆪ', 'ᆫ', 'ᆬ', 'ᆭ', 'ᆮ', 'ᆯ', 'ᆰ', 'ᆱ', 'ᆲ', 'ᆳ', 'ᆴ', 'ᆵ', 'ᆶ', 'ᆷ', 'ᆸ', 'ᆹ', 'ᆺ', 'ᆻ', 'ᆼ', 'ᆽ', 'ᆾ', 'ᆿ', 'ᇀ', 'ᇁ', 'ᇂ'] + }, + 'GBK': { + 'type': "bytes", + 'pieces': [64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254] + }, + 'utf-16-be': { + 'type': "bytes", + 'pieces': range(0, 256) + }, + 'cp949': { + 'type': "bytes", + 'pieces': [65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254] + }, + 'nybbles': { + 'type': "bytes", + 'pieces': range(0, 16) + }, + 'utf8-Ko': { + 'type': "bytes", + 'pieces': [128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 234, 235, 236, 237] + } + , + 'utf8-Zh': { + 'type': "bytes", + 'pieces': [128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 228,229, 230, 231, 232, 233] + } +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..eddac2b --- /dev/null +++ b/generation_config.json @@ -0,0 +1,11 @@ +{ + "_from_model_config": true, + "bos_token_id": 279, + "eos_token_id": [ + 280 + ], + "output_attentions": false, + "output_hidden_states": false, + "transformers_version": "5.0.0", + "use_cache": true +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..42631bc --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c68f15489df01c76fd0dea86abdc7513ad6ae3250769f4e623154bf76107f68 +size 454843368 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..b86ac50 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,35 @@ +{ + "added_tokens_decoder": { + "279": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "280": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "additional_special_tokens": null, + "backend": "custom", + "bos_token": "", + "encode_length": 3, + "eos_token": "", + "indexing_strategy": "shared", + "lang": "Ko", + "model_max_length": 1000000000000000019884624838656, + "tokenizer_class": "AetherCuTFTokenizer", + "auto_map": { + "AutoTokenizer": [ + "aethertokenizers.AetherCuTFTokenizer", + null + ] + } +} \ No newline at end of file diff --git a/train_results.json b/train_results.json new file mode 100644 index 0000000..2e66ea6 --- /dev/null +++ b/train_results.json @@ -0,0 +1,9 @@ +{ + "epoch": 1.0, + "total_flos": 9.745954960876503e+18, + "train_loss": 0.052523610041691704, + "train_runtime": 7246.7996, + "train_samples": 3494359, + "train_samples_per_second": 482.193, + "train_steps_per_second": 7.534 +} \ No newline at end of file diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..fd1b354 --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,1569 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 500, + "global_step": 54600, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.004578796509125542, + "grad_norm": 3.342534303665161, + "learning_rate": 2.28021978021978e-05, + "loss": 3.3527626953125, + "step": 250 + }, + { + "epoch": 0.009157593018251083, + "grad_norm": 4.355895042419434, + "learning_rate": 4.5695970695970695e-05, + "loss": 1.733473876953125, + "step": 500 + }, + { + "epoch": 0.013736389527376625, + "grad_norm": 1.9130524396896362, + "learning_rate": 6.85897435897436e-05, + "loss": 1.234192626953125, + "step": 750 + }, + { + "epoch": 0.018315186036502167, + "grad_norm": 1.457084059715271, + "learning_rate": 9.148351648351648e-05, + "loss": 1.120093017578125, + "step": 1000 + }, + { + "epoch": 0.022893982545627706, + "grad_norm": 1.005660891532898, + "learning_rate": 0.00011437728937728938, + "loss": 1.04007568359375, + "step": 1250 + }, + { + "epoch": 0.02747277905475325, + "grad_norm": 0.6642019748687744, + "learning_rate": 0.00013727106227106228, + "loss": 0.979659912109375, + "step": 1500 + }, + { + "epoch": 0.03205157556387879, + "grad_norm": 0.4342072308063507, + "learning_rate": 0.00016016483516483518, + "loss": 0.92401513671875, + "step": 1750 + }, + { + "epoch": 0.03663037207300433, + "grad_norm": 0.30285128951072693, + "learning_rate": 0.00018305860805860804, + "loss": 0.8480064086914062, + "step": 2000 + }, + { + "epoch": 0.041209168582129876, + "grad_norm": 0.3807497024536133, + "learning_rate": 0.00020595238095238094, + "loss": 0.8091620483398437, + "step": 2250 + }, + { + "epoch": 0.04578796509125541, + "grad_norm": 0.2316020131111145, + "learning_rate": 0.00022884615384615383, + "loss": 0.786201171875, + "step": 2500 + }, + { + "epoch": 0.050366761600380956, + "grad_norm": 0.24112890660762787, + "learning_rate": 0.0002517399267399267, + "loss": 0.7673829956054687, + "step": 2750 + }, + { + "epoch": 0.0549455581095065, + "grad_norm": 0.209824338555336, + "learning_rate": 0.00027463369963369965, + "loss": 0.7533621826171875, + "step": 3000 + }, + { + "epoch": 0.059524354618632036, + "grad_norm": 0.1774982362985611, + "learning_rate": 0.0002975274725274725, + "loss": 0.7406448364257813, + "step": 3250 + }, + { + "epoch": 0.06410315112775758, + "grad_norm": 0.18220750987529755, + "learning_rate": 0.00032042124542124543, + "loss": 0.7278763427734375, + "step": 3500 + }, + { + "epoch": 0.06868194763688312, + "grad_norm": 0.15290245413780212, + "learning_rate": 0.0003433150183150183, + "loss": 0.7186447143554687, + "step": 3750 + }, + { + "epoch": 0.07326074414600867, + "grad_norm": 0.15288487076759338, + "learning_rate": 0.00036620879120879117, + "loss": 0.7105338134765625, + "step": 4000 + }, + { + "epoch": 0.07783954065513421, + "grad_norm": 0.1429632157087326, + "learning_rate": 0.00038910256410256414, + "loss": 0.702983154296875, + "step": 4250 + }, + { + "epoch": 0.08241833716425975, + "grad_norm": 0.13405482470989227, + "learning_rate": 0.000411996336996337, + "loss": 0.6989120483398438, + "step": 4500 + }, + { + "epoch": 0.08699713367338528, + "grad_norm": 0.12790542840957642, + "learning_rate": 0.00043489010989010993, + "loss": 0.6919224243164063, + "step": 4750 + }, + { + "epoch": 0.09157593018251083, + "grad_norm": 0.12135355174541473, + "learning_rate": 0.0004577838827838828, + "loss": 0.6880528564453126, + "step": 5000 + }, + { + "epoch": 0.09615472669163637, + "grad_norm": 0.11120759695768356, + "learning_rate": 0.0004806776556776557, + "loss": 0.6828602294921875, + "step": 5250 + }, + { + "epoch": 0.10073352320076191, + "grad_norm": 0.10983391106128693, + "learning_rate": 0.0004996031746031746, + "loss": 0.675973876953125, + "step": 5500 + }, + { + "epoch": 0.10531231970988746, + "grad_norm": 0.13301989436149597, + "learning_rate": 0.000497059422059422, + "loss": 0.6739141845703125, + "step": 5750 + }, + { + "epoch": 0.109891116219013, + "grad_norm": 0.0955018475651741, + "learning_rate": 0.0004945156695156695, + "loss": 0.667236083984375, + "step": 6000 + }, + { + "epoch": 0.11446991272813854, + "grad_norm": 0.10058317333459854, + "learning_rate": 0.0004919719169719169, + "loss": 0.6618697509765625, + "step": 6250 + }, + { + "epoch": 0.11904870923726407, + "grad_norm": 0.0959748700261116, + "learning_rate": 0.0004894281644281644, + "loss": 0.6591820068359375, + "step": 6500 + }, + { + "epoch": 0.12362750574638962, + "grad_norm": 0.09256613999605179, + "learning_rate": 0.0004868844118844119, + "loss": 0.6521591186523438, + "step": 6750 + }, + { + "epoch": 0.12820630225551516, + "grad_norm": 0.09235742688179016, + "learning_rate": 0.00048434065934065937, + "loss": 0.6495025634765625, + "step": 7000 + }, + { + "epoch": 0.1327850987646407, + "grad_norm": 0.09277229756116867, + "learning_rate": 0.0004817969067969068, + "loss": 0.6460286865234375, + "step": 7250 + }, + { + "epoch": 0.13736389527376625, + "grad_norm": 0.0949406772851944, + "learning_rate": 0.00047925315425315426, + "loss": 0.6419014282226563, + "step": 7500 + }, + { + "epoch": 0.1419426917828918, + "grad_norm": 0.09082936495542526, + "learning_rate": 0.00047670940170940176, + "loss": 0.64010888671875, + "step": 7750 + }, + { + "epoch": 0.14652148829201733, + "grad_norm": 0.0872732549905777, + "learning_rate": 0.0004741656491656492, + "loss": 0.6353873901367187, + "step": 8000 + }, + { + "epoch": 0.15110028480114288, + "grad_norm": 0.08977802842855453, + "learning_rate": 0.00047162189662189666, + "loss": 0.634181640625, + "step": 8250 + }, + { + "epoch": 0.15567908131026842, + "grad_norm": 0.08916153013706207, + "learning_rate": 0.0004690781440781441, + "loss": 0.6323787841796875, + "step": 8500 + }, + { + "epoch": 0.16025787781939396, + "grad_norm": 0.08718685060739517, + "learning_rate": 0.00046653439153439155, + "loss": 0.6297101440429688, + "step": 8750 + }, + { + "epoch": 0.1648366743285195, + "grad_norm": 0.08362448215484619, + "learning_rate": 0.000463990638990639, + "loss": 0.6261585083007812, + "step": 9000 + }, + { + "epoch": 0.16941547083764502, + "grad_norm": 0.08333246409893036, + "learning_rate": 0.0004614468864468865, + "loss": 0.625505615234375, + "step": 9250 + }, + { + "epoch": 0.17399426734677056, + "grad_norm": 0.08667623996734619, + "learning_rate": 0.00045890313390313395, + "loss": 0.62279931640625, + "step": 9500 + }, + { + "epoch": 0.1785730638558961, + "grad_norm": 0.08940846472978592, + "learning_rate": 0.0004563593813593814, + "loss": 0.6183375244140625, + "step": 9750 + }, + { + "epoch": 0.18315186036502165, + "grad_norm": 0.0858025848865509, + "learning_rate": 0.00045381562881562884, + "loss": 0.6180489501953125, + "step": 10000 + }, + { + "epoch": 0.1877306568741472, + "grad_norm": 0.08406202495098114, + "learning_rate": 0.0004512718762718763, + "loss": 0.6160538940429687, + "step": 10250 + }, + { + "epoch": 0.19230945338327274, + "grad_norm": 0.08236856758594513, + "learning_rate": 0.00044872812372812374, + "loss": 0.61333447265625, + "step": 10500 + }, + { + "epoch": 0.19688824989239828, + "grad_norm": 0.07843296974897385, + "learning_rate": 0.00044618437118437124, + "loss": 0.6112908935546875, + "step": 10750 + }, + { + "epoch": 0.20146704640152382, + "grad_norm": 0.08862949162721634, + "learning_rate": 0.0004436406186406187, + "loss": 0.6111968994140625, + "step": 11000 + }, + { + "epoch": 0.20604584291064937, + "grad_norm": 0.07972651720046997, + "learning_rate": 0.00044109686609686613, + "loss": 0.60775244140625, + "step": 11250 + }, + { + "epoch": 0.2106246394197749, + "grad_norm": 0.0773809626698494, + "learning_rate": 0.0004385531135531136, + "loss": 0.6076170654296875, + "step": 11500 + }, + { + "epoch": 0.21520343592890046, + "grad_norm": 0.08130772411823273, + "learning_rate": 0.00043600936100936103, + "loss": 0.60721044921875, + "step": 11750 + }, + { + "epoch": 0.219782232438026, + "grad_norm": 0.08655981719493866, + "learning_rate": 0.0004334656084656085, + "loss": 0.6045301513671875, + "step": 12000 + }, + { + "epoch": 0.22436102894715154, + "grad_norm": 0.07923956960439682, + "learning_rate": 0.000430921855921856, + "loss": 0.603427490234375, + "step": 12250 + }, + { + "epoch": 0.22893982545627709, + "grad_norm": 0.07150240242481232, + "learning_rate": 0.0004283781033781034, + "loss": 0.6024298095703124, + "step": 12500 + }, + { + "epoch": 0.2335186219654026, + "grad_norm": 0.0733460783958435, + "learning_rate": 0.00042583435083435087, + "loss": 0.599318115234375, + "step": 12750 + }, + { + "epoch": 0.23809741847452814, + "grad_norm": 0.08155152201652527, + "learning_rate": 0.0004232905982905983, + "loss": 0.600461181640625, + "step": 13000 + }, + { + "epoch": 0.2426762149836537, + "grad_norm": 0.08487660437822342, + "learning_rate": 0.00042074684574684577, + "loss": 0.599893310546875, + "step": 13250 + }, + { + "epoch": 0.24725501149277923, + "grad_norm": 0.07263469696044922, + "learning_rate": 0.0004182030932030932, + "loss": 0.5981270751953125, + "step": 13500 + }, + { + "epoch": 0.2518338080019048, + "grad_norm": 0.0769294798374176, + "learning_rate": 0.0004156593406593407, + "loss": 0.597044921875, + "step": 13750 + }, + { + "epoch": 0.2564126045110303, + "grad_norm": 0.0794798731803894, + "learning_rate": 0.00041311558811558816, + "loss": 0.5964545288085937, + "step": 14000 + }, + { + "epoch": 0.2609914010201559, + "grad_norm": 0.07667657732963562, + "learning_rate": 0.0004105718355718356, + "loss": 0.5919962158203125, + "step": 14250 + }, + { + "epoch": 0.2655701975292814, + "grad_norm": 0.07880273461341858, + "learning_rate": 0.00040802808302808306, + "loss": 0.5930234375, + "step": 14500 + }, + { + "epoch": 0.2701489940384069, + "grad_norm": 0.06861519813537598, + "learning_rate": 0.0004054843304843305, + "loss": 0.5921943969726563, + "step": 14750 + }, + { + "epoch": 0.2747277905475325, + "grad_norm": 0.0762229785323143, + "learning_rate": 0.0004029405779405779, + "loss": 0.5893192749023437, + "step": 15000 + }, + { + "epoch": 0.279306587056658, + "grad_norm": 0.08483275771141052, + "learning_rate": 0.00040039682539682545, + "loss": 0.589997314453125, + "step": 15250 + }, + { + "epoch": 0.2838853835657836, + "grad_norm": 0.07121487706899643, + "learning_rate": 0.0003978530728530729, + "loss": 0.5913633422851563, + "step": 15500 + }, + { + "epoch": 0.2884641800749091, + "grad_norm": 0.06958875805139542, + "learning_rate": 0.0003953093203093203, + "loss": 0.5887347412109375, + "step": 15750 + }, + { + "epoch": 0.29304297658403466, + "grad_norm": 0.08195552974939346, + "learning_rate": 0.00039276556776556774, + "loss": 0.5877152099609375, + "step": 16000 + }, + { + "epoch": 0.2976217730931602, + "grad_norm": 0.07579973340034485, + "learning_rate": 0.0003902218152218152, + "loss": 0.5861239013671875, + "step": 16250 + }, + { + "epoch": 0.30220056960228575, + "grad_norm": 0.06779183447360992, + "learning_rate": 0.00038767806267806263, + "loss": 0.583551025390625, + "step": 16500 + }, + { + "epoch": 0.30677936611141127, + "grad_norm": 0.06482373923063278, + "learning_rate": 0.00038513431013431014, + "loss": 0.58420751953125, + "step": 16750 + }, + { + "epoch": 0.31135816262053684, + "grad_norm": 0.07873611152172089, + "learning_rate": 0.0003825905575905576, + "loss": 0.5830638427734375, + "step": 17000 + }, + { + "epoch": 0.31593695912966235, + "grad_norm": 0.07843663543462753, + "learning_rate": 0.00038004680504680503, + "loss": 0.5826964111328125, + "step": 17250 + }, + { + "epoch": 0.3205157556387879, + "grad_norm": 0.07587391883134842, + "learning_rate": 0.0003775030525030525, + "loss": 0.5826300048828125, + "step": 17500 + }, + { + "epoch": 0.32509455214791344, + "grad_norm": 0.0711434856057167, + "learning_rate": 0.0003749592999592999, + "loss": 0.58239306640625, + "step": 17750 + }, + { + "epoch": 0.329673348657039, + "grad_norm": 0.07004653662443161, + "learning_rate": 0.0003724155474155474, + "loss": 0.5806968994140626, + "step": 18000 + }, + { + "epoch": 0.3342521451661645, + "grad_norm": 0.07417979836463928, + "learning_rate": 0.0003698717948717949, + "loss": 0.5811688232421875, + "step": 18250 + }, + { + "epoch": 0.33883094167529004, + "grad_norm": 0.06397568434476852, + "learning_rate": 0.0003673280423280423, + "loss": 0.578081787109375, + "step": 18500 + }, + { + "epoch": 0.3434097381844156, + "grad_norm": 0.06324686855077744, + "learning_rate": 0.00036478428978428977, + "loss": 0.577623046875, + "step": 18750 + }, + { + "epoch": 0.34798853469354113, + "grad_norm": 0.06681980192661285, + "learning_rate": 0.0003622405372405372, + "loss": 0.5780208740234375, + "step": 19000 + }, + { + "epoch": 0.3525673312026667, + "grad_norm": 0.07156607508659363, + "learning_rate": 0.00035969678469678466, + "loss": 0.57647021484375, + "step": 19250 + }, + { + "epoch": 0.3571461277117922, + "grad_norm": 0.06628810614347458, + "learning_rate": 0.00035715303215303216, + "loss": 0.5765814208984374, + "step": 19500 + }, + { + "epoch": 0.3617249242209178, + "grad_norm": 0.06292615085840225, + "learning_rate": 0.0003546092796092796, + "loss": 0.574661865234375, + "step": 19750 + }, + { + "epoch": 0.3663037207300433, + "grad_norm": 0.06046818569302559, + "learning_rate": 0.00035206552706552706, + "loss": 0.5743336791992187, + "step": 20000 + }, + { + "epoch": 0.3708825172391689, + "grad_norm": 0.06358954310417175, + "learning_rate": 0.0003495217745217745, + "loss": 0.5736307373046875, + "step": 20250 + }, + { + "epoch": 0.3754613137482944, + "grad_norm": 0.06913734972476959, + "learning_rate": 0.00034697802197802195, + "loss": 0.5728591918945313, + "step": 20500 + }, + { + "epoch": 0.38004011025741996, + "grad_norm": 0.06982014328241348, + "learning_rate": 0.0003444342694342694, + "loss": 0.57314013671875, + "step": 20750 + }, + { + "epoch": 0.3846189067665455, + "grad_norm": 0.06307590007781982, + "learning_rate": 0.0003418905168905169, + "loss": 0.5726715087890625, + "step": 21000 + }, + { + "epoch": 0.38919770327567105, + "grad_norm": 0.0676989033818245, + "learning_rate": 0.00033934676434676435, + "loss": 0.5700736083984375, + "step": 21250 + }, + { + "epoch": 0.39377649978479656, + "grad_norm": 0.08213786780834198, + "learning_rate": 0.0003368030118030118, + "loss": 0.5705738525390625, + "step": 21500 + }, + { + "epoch": 0.3983552962939221, + "grad_norm": 0.06425507366657257, + "learning_rate": 0.00033425925925925924, + "loss": 0.56998388671875, + "step": 21750 + }, + { + "epoch": 0.40293409280304765, + "grad_norm": 0.06765250116586685, + "learning_rate": 0.0003317155067155067, + "loss": 0.5707243041992187, + "step": 22000 + }, + { + "epoch": 0.40751288931217317, + "grad_norm": 0.07066161930561066, + "learning_rate": 0.00032917175417175414, + "loss": 0.5674595947265625, + "step": 22250 + }, + { + "epoch": 0.41209168582129874, + "grad_norm": 0.07193256914615631, + "learning_rate": 0.00032662800162800164, + "loss": 0.5684276123046875, + "step": 22500 + }, + { + "epoch": 0.41667048233042425, + "grad_norm": 0.06980223953723907, + "learning_rate": 0.0003240842490842491, + "loss": 0.5667611694335938, + "step": 22750 + }, + { + "epoch": 0.4212492788395498, + "grad_norm": 0.06676173955202103, + "learning_rate": 0.00032154049654049653, + "loss": 0.5669996337890625, + "step": 23000 + }, + { + "epoch": 0.42582807534867534, + "grad_norm": 0.06899011880159378, + "learning_rate": 0.000318996743996744, + "loss": 0.5668095703125, + "step": 23250 + }, + { + "epoch": 0.4304068718578009, + "grad_norm": 0.06902206689119339, + "learning_rate": 0.00031645299145299143, + "loss": 0.5664789428710938, + "step": 23500 + }, + { + "epoch": 0.4349856683669264, + "grad_norm": 0.06520352512598038, + "learning_rate": 0.0003139092389092389, + "loss": 0.5667901000976563, + "step": 23750 + }, + { + "epoch": 0.439564464876052, + "grad_norm": 0.06156212091445923, + "learning_rate": 0.0003113654863654864, + "loss": 0.5641578369140625, + "step": 24000 + }, + { + "epoch": 0.4441432613851775, + "grad_norm": 0.06374220550060272, + "learning_rate": 0.0003088217338217338, + "loss": 0.5657506103515625, + "step": 24250 + }, + { + "epoch": 0.4487220578943031, + "grad_norm": 0.06591662019491196, + "learning_rate": 0.00030627798127798127, + "loss": 0.56384130859375, + "step": 24500 + }, + { + "epoch": 0.4533008544034286, + "grad_norm": 0.06188199669122696, + "learning_rate": 0.0003037342287342287, + "loss": 0.56164990234375, + "step": 24750 + }, + { + "epoch": 0.45787965091255417, + "grad_norm": 0.06126328930258751, + "learning_rate": 0.00030119047619047616, + "loss": 0.5632687377929687, + "step": 25000 + }, + { + "epoch": 0.4624584474216797, + "grad_norm": 0.06881853938102722, + "learning_rate": 0.0002986467236467236, + "loss": 0.5626099853515625, + "step": 25250 + }, + { + "epoch": 0.4670372439308052, + "grad_norm": 0.06128287687897682, + "learning_rate": 0.0002961029711029711, + "loss": 0.5612110595703125, + "step": 25500 + }, + { + "epoch": 0.4716160404399308, + "grad_norm": 0.06223996356129646, + "learning_rate": 0.00029355921855921856, + "loss": 0.562291015625, + "step": 25750 + }, + { + "epoch": 0.4761948369490563, + "grad_norm": 0.0587678961455822, + "learning_rate": 0.000291015466015466, + "loss": 0.562425048828125, + "step": 26000 + }, + { + "epoch": 0.48077363345818186, + "grad_norm": 0.06476189941167831, + "learning_rate": 0.00028847171347171346, + "loss": 0.559524658203125, + "step": 26250 + }, + { + "epoch": 0.4853524299673074, + "grad_norm": 0.05949617549777031, + "learning_rate": 0.0002859279609279609, + "loss": 0.5606175537109375, + "step": 26500 + }, + { + "epoch": 0.48993122647643295, + "grad_norm": 0.0664505809545517, + "learning_rate": 0.00028338420838420835, + "loss": 0.5592316284179687, + "step": 26750 + }, + { + "epoch": 0.49451002298555846, + "grad_norm": 0.06326252967119217, + "learning_rate": 0.00028084045584045585, + "loss": 0.560334716796875, + "step": 27000 + }, + { + "epoch": 0.49908881949468403, + "grad_norm": 0.065422423183918, + "learning_rate": 0.0002782967032967033, + "loss": 0.558829833984375, + "step": 27250 + }, + { + "epoch": 0.5036676160038096, + "grad_norm": 0.0695870891213417, + "learning_rate": 0.00027575295075295075, + "loss": 0.558175537109375, + "step": 27500 + }, + { + "epoch": 0.5082464125129351, + "grad_norm": 0.06475581973791122, + "learning_rate": 0.0002732091982091982, + "loss": 0.5586744384765625, + "step": 27750 + }, + { + "epoch": 0.5128252090220606, + "grad_norm": 0.06314811110496521, + "learning_rate": 0.00027066544566544564, + "loss": 0.5569791259765625, + "step": 28000 + }, + { + "epoch": 0.5174040055311862, + "grad_norm": 0.06873711943626404, + "learning_rate": 0.0002681216931216931, + "loss": 0.5555595092773438, + "step": 28250 + }, + { + "epoch": 0.5219828020403118, + "grad_norm": 0.06372768431901932, + "learning_rate": 0.0002655779405779406, + "loss": 0.55628759765625, + "step": 28500 + }, + { + "epoch": 0.5265615985494373, + "grad_norm": 0.06410428136587143, + "learning_rate": 0.00026303418803418804, + "loss": 0.5568046875, + "step": 28750 + }, + { + "epoch": 0.5311403950585628, + "grad_norm": 0.0655541867017746, + "learning_rate": 0.0002604904354904355, + "loss": 0.555463623046875, + "step": 29000 + }, + { + "epoch": 0.5357191915676883, + "grad_norm": 0.061745524406433105, + "learning_rate": 0.00025794668294668293, + "loss": 0.5544102783203125, + "step": 29250 + }, + { + "epoch": 0.5402979880768138, + "grad_norm": 0.06390813738107681, + "learning_rate": 0.0002554029304029304, + "loss": 0.5550816650390625, + "step": 29500 + }, + { + "epoch": 0.5448767845859395, + "grad_norm": 0.06350462883710861, + "learning_rate": 0.0002528591778591779, + "loss": 0.5545914306640625, + "step": 29750 + }, + { + "epoch": 0.549455581095065, + "grad_norm": 0.07035643607378006, + "learning_rate": 0.0002503154253154253, + "loss": 0.5539207763671875, + "step": 30000 + }, + { + "epoch": 0.5540343776041905, + "grad_norm": 0.060600634664297104, + "learning_rate": 0.0002477716727716728, + "loss": 0.5543575439453124, + "step": 30250 + }, + { + "epoch": 0.558613174113316, + "grad_norm": 0.07064654678106308, + "learning_rate": 0.0002452279202279202, + "loss": 0.5540496826171875, + "step": 30500 + }, + { + "epoch": 0.5631919706224416, + "grad_norm": 0.06378184258937836, + "learning_rate": 0.00024268416768416767, + "loss": 0.55242919921875, + "step": 30750 + }, + { + "epoch": 0.5677707671315672, + "grad_norm": 0.062876857817173, + "learning_rate": 0.00024014041514041514, + "loss": 0.5535072021484375, + "step": 31000 + }, + { + "epoch": 0.5723495636406927, + "grad_norm": 0.06333809345960617, + "learning_rate": 0.0002375966625966626, + "loss": 0.551296142578125, + "step": 31250 + }, + { + "epoch": 0.5769283601498182, + "grad_norm": 0.06588946282863617, + "learning_rate": 0.00023505291005291004, + "loss": 0.5515674438476562, + "step": 31500 + }, + { + "epoch": 0.5815071566589438, + "grad_norm": 0.05932895466685295, + "learning_rate": 0.0002325091575091575, + "loss": 0.5509488525390625, + "step": 31750 + }, + { + "epoch": 0.5860859531680693, + "grad_norm": 0.06366723030805588, + "learning_rate": 0.00022996540496540496, + "loss": 0.549708984375, + "step": 32000 + }, + { + "epoch": 0.5906647496771948, + "grad_norm": 0.06046701595187187, + "learning_rate": 0.0002274216524216524, + "loss": 0.548451904296875, + "step": 32250 + }, + { + "epoch": 0.5952435461863204, + "grad_norm": 0.060656215995550156, + "learning_rate": 0.00022487789987789988, + "loss": 0.55017724609375, + "step": 32500 + }, + { + "epoch": 0.5998223426954459, + "grad_norm": 0.06226223334670067, + "learning_rate": 0.00022233414733414733, + "loss": 0.5499310913085937, + "step": 32750 + }, + { + "epoch": 0.6044011392045715, + "grad_norm": 0.06266618520021439, + "learning_rate": 0.00021979039479039477, + "loss": 0.5480341796875, + "step": 33000 + }, + { + "epoch": 0.608979935713697, + "grad_norm": 0.06450474262237549, + "learning_rate": 0.00021724664224664225, + "loss": 0.55114453125, + "step": 33250 + }, + { + "epoch": 0.6135587322228225, + "grad_norm": 0.06394919753074646, + "learning_rate": 0.0002147028897028897, + "loss": 0.5472305908203124, + "step": 33500 + }, + { + "epoch": 0.618137528731948, + "grad_norm": 0.06171587482094765, + "learning_rate": 0.00021215913715913714, + "loss": 0.547580322265625, + "step": 33750 + }, + { + "epoch": 0.6227163252410737, + "grad_norm": 0.058975059539079666, + "learning_rate": 0.00020961538461538462, + "loss": 0.5463271484375, + "step": 34000 + }, + { + "epoch": 0.6272951217501992, + "grad_norm": 0.061736807227134705, + "learning_rate": 0.00020707163207163206, + "loss": 0.5462554931640625, + "step": 34250 + }, + { + "epoch": 0.6318739182593247, + "grad_norm": 0.06212234124541283, + "learning_rate": 0.00020452787952787954, + "loss": 0.546873291015625, + "step": 34500 + }, + { + "epoch": 0.6364527147684502, + "grad_norm": 0.06781437247991562, + "learning_rate": 0.00020198412698412699, + "loss": 0.5456260986328125, + "step": 34750 + }, + { + "epoch": 0.6410315112775758, + "grad_norm": 0.05848696827888489, + "learning_rate": 0.00019944037444037443, + "loss": 0.5449661865234375, + "step": 35000 + }, + { + "epoch": 0.6456103077867014, + "grad_norm": 0.06378760188817978, + "learning_rate": 0.0001968966218966219, + "loss": 0.545228515625, + "step": 35250 + }, + { + "epoch": 0.6501891042958269, + "grad_norm": 0.06400810182094574, + "learning_rate": 0.00019435286935286935, + "loss": 0.545167724609375, + "step": 35500 + }, + { + "epoch": 0.6547679008049524, + "grad_norm": 0.06151324138045311, + "learning_rate": 0.0001918091168091168, + "loss": 0.545171875, + "step": 35750 + }, + { + "epoch": 0.659346697314078, + "grad_norm": 0.06354264169931412, + "learning_rate": 0.00018926536426536428, + "loss": 0.545235107421875, + "step": 36000 + }, + { + "epoch": 0.6639254938232035, + "grad_norm": 0.05942839756608009, + "learning_rate": 0.00018672161172161172, + "loss": 0.5439722290039063, + "step": 36250 + }, + { + "epoch": 0.668504290332329, + "grad_norm": 0.0653739720582962, + "learning_rate": 0.00018417785917785917, + "loss": 0.5418724365234375, + "step": 36500 + }, + { + "epoch": 0.6730830868414546, + "grad_norm": 0.062099047005176544, + "learning_rate": 0.00018163410663410664, + "loss": 0.5428839721679688, + "step": 36750 + }, + { + "epoch": 0.6776618833505801, + "grad_norm": 0.06153613328933716, + "learning_rate": 0.0001790903540903541, + "loss": 0.5429735717773437, + "step": 37000 + }, + { + "epoch": 0.6822406798597057, + "grad_norm": 0.06720417737960815, + "learning_rate": 0.00017654660154660154, + "loss": 0.5424810791015625, + "step": 37250 + }, + { + "epoch": 0.6868194763688312, + "grad_norm": 0.06468328833580017, + "learning_rate": 0.000174002849002849, + "loss": 0.5409627685546875, + "step": 37500 + }, + { + "epoch": 0.6913982728779567, + "grad_norm": 0.06758266687393188, + "learning_rate": 0.00017145909645909646, + "loss": 0.5416390380859375, + "step": 37750 + }, + { + "epoch": 0.6959770693870823, + "grad_norm": 0.06019122526049614, + "learning_rate": 0.0001689153439153439, + "loss": 0.5392860717773438, + "step": 38000 + }, + { + "epoch": 0.7005558658962079, + "grad_norm": 0.06067493185400963, + "learning_rate": 0.00016637159137159138, + "loss": 0.540711669921875, + "step": 38250 + }, + { + "epoch": 0.7051346624053334, + "grad_norm": 0.0617203414440155, + "learning_rate": 0.00016382783882783883, + "loss": 0.543611572265625, + "step": 38500 + }, + { + "epoch": 0.7097134589144589, + "grad_norm": 0.06383258104324341, + "learning_rate": 0.00016128408628408628, + "loss": 0.54111669921875, + "step": 38750 + }, + { + "epoch": 0.7142922554235844, + "grad_norm": 0.06264094263315201, + "learning_rate": 0.00015874033374033375, + "loss": 0.5409336547851562, + "step": 39000 + }, + { + "epoch": 0.7188710519327101, + "grad_norm": 0.0582389310002327, + "learning_rate": 0.0001561965811965812, + "loss": 0.5385667114257813, + "step": 39250 + }, + { + "epoch": 0.7234498484418356, + "grad_norm": 0.06511319428682327, + "learning_rate": 0.00015365282865282865, + "loss": 0.5388533325195313, + "step": 39500 + }, + { + "epoch": 0.7280286449509611, + "grad_norm": 0.06048616021871567, + "learning_rate": 0.00015110907610907612, + "loss": 0.539404296875, + "step": 39750 + }, + { + "epoch": 0.7326074414600866, + "grad_norm": 0.06342992931604385, + "learning_rate": 0.00014856532356532357, + "loss": 0.5397015380859375, + "step": 40000 + }, + { + "epoch": 0.7371862379692121, + "grad_norm": 0.0633598044514656, + "learning_rate": 0.00014602157102157101, + "loss": 0.5383480834960938, + "step": 40250 + }, + { + "epoch": 0.7417650344783377, + "grad_norm": 0.0615195594727993, + "learning_rate": 0.0001434778184778185, + "loss": 0.5385945434570313, + "step": 40500 + }, + { + "epoch": 0.7463438309874633, + "grad_norm": 0.05627000704407692, + "learning_rate": 0.00014093406593406594, + "loss": 0.53756591796875, + "step": 40750 + }, + { + "epoch": 0.7509226274965888, + "grad_norm": 0.06461521238088608, + "learning_rate": 0.00013839031339031338, + "loss": 0.5365997314453125, + "step": 41000 + }, + { + "epoch": 0.7555014240057143, + "grad_norm": 0.06126422807574272, + "learning_rate": 0.00013584656084656086, + "loss": 0.5386915893554688, + "step": 41250 + }, + { + "epoch": 0.7600802205148399, + "grad_norm": 0.05774404853582382, + "learning_rate": 0.0001333028083028083, + "loss": 0.5380045776367187, + "step": 41500 + }, + { + "epoch": 0.7646590170239654, + "grad_norm": 0.06210438162088394, + "learning_rate": 0.00013075905575905575, + "loss": 0.5373387451171875, + "step": 41750 + }, + { + "epoch": 0.769237813533091, + "grad_norm": 0.06252729892730713, + "learning_rate": 0.00012821530321530323, + "loss": 0.5376387939453126, + "step": 42000 + }, + { + "epoch": 0.7738166100422165, + "grad_norm": 0.06452998518943787, + "learning_rate": 0.00012567155067155067, + "loss": 0.5358265380859375, + "step": 42250 + }, + { + "epoch": 0.7783954065513421, + "grad_norm": 0.0634666383266449, + "learning_rate": 0.00012312779812779812, + "loss": 0.5373413696289062, + "step": 42500 + }, + { + "epoch": 0.7829742030604676, + "grad_norm": 0.06478540599346161, + "learning_rate": 0.0001205840455840456, + "loss": 0.53590234375, + "step": 42750 + }, + { + "epoch": 0.7875529995695931, + "grad_norm": 0.06542439013719559, + "learning_rate": 0.00011804029304029304, + "loss": 0.5361544189453125, + "step": 43000 + }, + { + "epoch": 0.7921317960787186, + "grad_norm": 0.061942797154188156, + "learning_rate": 0.0001154965404965405, + "loss": 0.5337443237304688, + "step": 43250 + }, + { + "epoch": 0.7967105925878442, + "grad_norm": 0.05985584482550621, + "learning_rate": 0.00011295278795278796, + "loss": 0.5335606689453125, + "step": 43500 + }, + { + "epoch": 0.8012893890969698, + "grad_norm": 0.05933161452412605, + "learning_rate": 0.00011040903540903541, + "loss": 0.5334180908203126, + "step": 43750 + }, + { + "epoch": 0.8058681856060953, + "grad_norm": 0.06226200982928276, + "learning_rate": 0.00010786528286528287, + "loss": 0.5355284423828125, + "step": 44000 + }, + { + "epoch": 0.8104469821152208, + "grad_norm": 0.06400461494922638, + "learning_rate": 0.00010532153032153033, + "loss": 0.534538818359375, + "step": 44250 + }, + { + "epoch": 0.8150257786243463, + "grad_norm": 0.05959906429052353, + "learning_rate": 0.00010277777777777778, + "loss": 0.533162109375, + "step": 44500 + }, + { + "epoch": 0.819604575133472, + "grad_norm": 0.06476127356290817, + "learning_rate": 0.00010023402523402524, + "loss": 0.5328413696289063, + "step": 44750 + }, + { + "epoch": 0.8241833716425975, + "grad_norm": 0.06384240090847015, + "learning_rate": 9.76902726902727e-05, + "loss": 0.5334242553710937, + "step": 45000 + }, + { + "epoch": 0.828762168151723, + "grad_norm": 0.06064584106206894, + "learning_rate": 9.514652014652015e-05, + "loss": 0.5317213745117187, + "step": 45250 + }, + { + "epoch": 0.8333409646608485, + "grad_norm": 0.06094207242131233, + "learning_rate": 9.260276760276761e-05, + "loss": 0.5321604614257812, + "step": 45500 + }, + { + "epoch": 0.8379197611699741, + "grad_norm": 0.06116236746311188, + "learning_rate": 9.005901505901507e-05, + "loss": 0.5310498657226562, + "step": 45750 + }, + { + "epoch": 0.8424985576790996, + "grad_norm": 0.06410134583711624, + "learning_rate": 8.751526251526252e-05, + "loss": 0.5318917846679687, + "step": 46000 + }, + { + "epoch": 0.8470773541882252, + "grad_norm": 0.059746153652668, + "learning_rate": 8.497150997150998e-05, + "loss": 0.5324998168945313, + "step": 46250 + }, + { + "epoch": 0.8516561506973507, + "grad_norm": 0.06164969503879547, + "learning_rate": 8.242775742775744e-05, + "loss": 0.5311779174804687, + "step": 46500 + }, + { + "epoch": 0.8562349472064763, + "grad_norm": 0.06264585256576538, + "learning_rate": 7.988400488400489e-05, + "loss": 0.5288109741210938, + "step": 46750 + }, + { + "epoch": 0.8608137437156018, + "grad_norm": 0.06309522688388824, + "learning_rate": 7.734025234025235e-05, + "loss": 0.5295328369140625, + "step": 47000 + }, + { + "epoch": 0.8653925402247273, + "grad_norm": 0.06573944538831711, + "learning_rate": 7.479649979649981e-05, + "loss": 0.5310571899414063, + "step": 47250 + }, + { + "epoch": 0.8699713367338529, + "grad_norm": 0.06289469450712204, + "learning_rate": 7.225274725274724e-05, + "loss": 0.5286151733398438, + "step": 47500 + }, + { + "epoch": 0.8745501332429784, + "grad_norm": 0.0591423362493515, + "learning_rate": 6.97089947089947e-05, + "loss": 0.5285560913085937, + "step": 47750 + }, + { + "epoch": 0.879128929752104, + "grad_norm": 0.06373240053653717, + "learning_rate": 6.716524216524216e-05, + "loss": 0.52852880859375, + "step": 48000 + }, + { + "epoch": 0.8837077262612295, + "grad_norm": 0.06770361214876175, + "learning_rate": 6.462148962148962e-05, + "loss": 0.5289181518554688, + "step": 48250 + }, + { + "epoch": 0.888286522770355, + "grad_norm": 0.06282202154397964, + "learning_rate": 6.207773707773707e-05, + "loss": 0.5284034423828124, + "step": 48500 + }, + { + "epoch": 0.8928653192794805, + "grad_norm": 0.06374731659889221, + "learning_rate": 5.953398453398454e-05, + "loss": 0.5279761962890624, + "step": 48750 + }, + { + "epoch": 0.8974441157886062, + "grad_norm": 0.060556381940841675, + "learning_rate": 5.699023199023199e-05, + "loss": 0.5265902709960938, + "step": 49000 + }, + { + "epoch": 0.9020229122977317, + "grad_norm": 0.06243932247161865, + "learning_rate": 5.444647944647945e-05, + "loss": 0.5270482149991123, + "step": 49250 + }, + { + "epoch": 0.9066017088068572, + "grad_norm": 0.05966382101178169, + "learning_rate": 5.1902726902726907e-05, + "loss": 0.5288416748046875, + "step": 49500 + }, + { + "epoch": 0.9111805053159827, + "grad_norm": 0.06264135241508484, + "learning_rate": 4.935897435897436e-05, + "loss": 0.5250919189453125, + "step": 49750 + }, + { + "epoch": 0.9157593018251083, + "grad_norm": 0.060594044625759125, + "learning_rate": 4.681522181522182e-05, + "loss": 0.527917724609375, + "step": 50000 + }, + { + "epoch": 0.9203380983342339, + "grad_norm": 0.06120523810386658, + "learning_rate": 4.4271469271469275e-05, + "loss": 0.5259210205078125, + "step": 50250 + }, + { + "epoch": 0.9249168948433594, + "grad_norm": 0.0615825429558754, + "learning_rate": 4.172771672771673e-05, + "loss": 0.5261221923828125, + "step": 50500 + }, + { + "epoch": 0.9294956913524849, + "grad_norm": 0.06489884108304977, + "learning_rate": 3.918396418396418e-05, + "loss": 0.52599853515625, + "step": 50750 + }, + { + "epoch": 0.9340744878616104, + "grad_norm": 0.06422311812639236, + "learning_rate": 3.664021164021164e-05, + "loss": 0.5265205078125, + "step": 51000 + }, + { + "epoch": 0.938653284370736, + "grad_norm": 0.06154436618089676, + "learning_rate": 3.409645909645909e-05, + "loss": 0.52559375, + "step": 51250 + }, + { + "epoch": 0.9432320808798615, + "grad_norm": 0.06325343251228333, + "learning_rate": 3.155270655270655e-05, + "loss": 0.526312255859375, + "step": 51500 + }, + { + "epoch": 0.9478108773889871, + "grad_norm": 0.060577958822250366, + "learning_rate": 2.900895400895401e-05, + "loss": 0.5263541259765625, + "step": 51750 + }, + { + "epoch": 0.9523896738981126, + "grad_norm": 0.0622745044529438, + "learning_rate": 2.6465201465201467e-05, + "loss": 0.5247227783203126, + "step": 52000 + }, + { + "epoch": 0.9569684704072382, + "grad_norm": 0.061482541263103485, + "learning_rate": 2.3921448921448924e-05, + "loss": 0.522807861328125, + "step": 52250 + }, + { + "epoch": 0.9615472669163637, + "grad_norm": 0.0617360919713974, + "learning_rate": 2.1377696377696378e-05, + "loss": 0.524948486328125, + "step": 52500 + }, + { + "epoch": 0.9661260634254892, + "grad_norm": 0.06259670853614807, + "learning_rate": 1.8833943833943832e-05, + "loss": 0.52488330078125, + "step": 52750 + }, + { + "epoch": 0.9707048599346147, + "grad_norm": 0.0630236268043518, + "learning_rate": 1.629019129019129e-05, + "loss": 0.5252532958984375, + "step": 53000 + }, + { + "epoch": 0.9752836564437404, + "grad_norm": 0.0633007138967514, + "learning_rate": 1.3746438746438745e-05, + "loss": 0.5238004150390625, + "step": 53250 + }, + { + "epoch": 0.9798624529528659, + "grad_norm": 0.05867809057235718, + "learning_rate": 1.1202686202686203e-05, + "loss": 0.5233538818359375, + "step": 53500 + }, + { + "epoch": 0.9844412494619914, + "grad_norm": 0.06631673127412796, + "learning_rate": 8.658933658933658e-06, + "loss": 0.5244312744140625, + "step": 53750 + }, + { + "epoch": 0.9890200459711169, + "grad_norm": 0.06228270381689072, + "learning_rate": 6.115181115181116e-06, + "loss": 0.523883056640625, + "step": 54000 + }, + { + "epoch": 0.9935988424802424, + "grad_norm": 0.06517703086137772, + "learning_rate": 3.5714285714285714e-06, + "loss": 0.52413623046875, + "step": 54250 + }, + { + "epoch": 0.9981776389893681, + "grad_norm": 0.06215018779039383, + "learning_rate": 1.0276760276760277e-06, + "loss": 0.5219486694335937, + "step": 54500 + }, + { + "epoch": 1.0, + "step": 54600, + "total_flos": 9.745954960876503e+18, + "train_loss": 0.052523610041691704, + "train_runtime": 7246.7996, + "train_samples_per_second": 482.193, + "train_steps_per_second": 7.534 + } + ], + "logging_steps": 250, + "max_steps": 54600, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 5460, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 9.745954960876503e+18, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..7e9f74e --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f481d5b0726409662acd47a76e5834f7d48be364a3e2ed6d4ba8dfc81efbee17 +size 5265