From f17555f54ac5ba621dcd1cdf3fec0e15398026ce Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Mon, 4 May 2026 21:37:19 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: PARZ2344/web_llama_sft_random Source: Original Platform --- .gitattributes | 36 + README.md | 61 + all_results.json | 8 + chat_template.jinja | 93 ++ config.json | 39 + generation_config.json | 12 + model-00001-of-00002.safetensors | 3 + model-00002-of-00002.safetensors | 3 + model.safetensors.index.json | 261 ++++ special_tokens_map.json | 26 + tokenizer.json | 3 + tokenizer_config.json | 2068 ++++++++++++++++++++++++++++++ train_results.json | 8 + trainer_log.jsonl | 32 + trainer_state.json | 680 ++++++++++ training_args.bin | 3 + training_loss.png | Bin 0 -> 36537 bytes 17 files changed, 3336 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 all_results.json create mode 100644 chat_template.jinja create mode 100644 config.json create mode 100644 generation_config.json create mode 100644 model-00001-of-00002.safetensors create mode 100644 model-00002-of-00002.safetensors create mode 100644 model.safetensors.index.json create mode 100644 special_tokens_map.json create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json create mode 100644 train_results.json create mode 100644 trainer_log.jsonl create mode 100644 trainer_state.json create mode 100644 training_args.bin create mode 100644 training_loss.png diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..2a2f620 --- /dev/null +++ b/README.md @@ -0,0 +1,61 @@ +--- +library_name: transformers +license: other +base_model: meta-llama/Llama-3.2-3B-Instruct +tags: +- llama-factory +- full +- generated_from_trainer +model-index: +- name: apm_sft_random_llama + results: [] +--- + + + +# apm_sft_random_llama + +This model is a fine-tuned version of [meta-llama/Llama-3.2-3B-Instruct](https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct) on the deep_research_25 dataset. + +## Model description + +More information needed + +## Intended uses & limitations + +More information needed + +## Training and evaluation data + +More information needed + +## Training procedure + +### Training hyperparameters + +The following hyperparameters were used during training: +- learning_rate: 1e-05 +- train_batch_size: 1 +- eval_batch_size: 8 +- seed: 42 +- distributed_type: multi-GPU +- num_devices: 8 +- gradient_accumulation_steps: 8 +- total_train_batch_size: 64 +- total_eval_batch_size: 64 +- optimizer: Use adamw_torch with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments +- lr_scheduler_type: cosine +- lr_scheduler_warmup_ratio: 0.1 +- num_epochs: 3.0 + +### Training results + + + +### Framework versions + +- Transformers 4.52.4 +- Pytorch 2.9.0+cu128 +- Datasets 3.2.0 +- Tokenizers 0.21.1 diff --git a/all_results.json b/all_results.json new file mode 100644 index 0000000..06328b4 --- /dev/null +++ b/all_results.json @@ -0,0 +1,8 @@ +{ + "epoch": 3.0, + "total_flos": 181229773520896.0, + "train_loss": 0.266992773216715, + "train_runtime": 3529.0775, + "train_samples_per_second": 16.577, + "train_steps_per_second": 0.259 +} \ No newline at end of file diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..1bad6a0 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,93 @@ +{{- bos_token }} +{%- if custom_tools is defined %} + {%- set tools = custom_tools %} +{%- endif %} +{%- if not tools_in_user_message is defined %} + {%- set tools_in_user_message = true %} +{%- endif %} +{%- if not date_string is defined %} + {%- if strftime_now is defined %} + {%- set date_string = strftime_now("%d %b %Y") %} + {%- else %} + {%- set date_string = "26 Jul 2024" %} + {%- endif %} +{%- endif %} +{%- if not tools is defined %} + {%- set tools = none %} +{%- endif %} + +{#- This block extracts the system message, so we can slot it into the right place. #} +{%- if messages[0]['role'] == 'system' %} + {%- set system_message = messages[0]['content']|trim %} + {%- set messages = messages[1:] %} +{%- else %} + {%- set system_message = "" %} +{%- endif %} + +{#- System message #} +{{- "<|start_header_id|>system<|end_header_id|>\n\n" }} +{%- if tools is not none %} + {{- "Environment: ipython\n" }} +{%- endif %} +{{- "Cutting Knowledge Date: December 2023\n" }} +{{- "Today Date: " + date_string + "\n\n" }} +{%- if tools is not none and not tools_in_user_message %} + {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }} + {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }} + {{- "Do not use variables.\n\n" }} + {%- for t in tools %} + {{- t | tojson(indent=4) }} + {{- "\n\n" }} + {%- endfor %} +{%- endif %} +{{- system_message }} +{{- "<|eot_id|>" }} + +{#- Custom tools are passed in a user message with some extra guidance #} +{%- if tools_in_user_message and not tools is none %} + {#- Extract the first user message so we can plug it in here #} + {%- if messages | length != 0 %} + {%- set first_user_message = messages[0]['content']|trim %} + {%- set messages = messages[1:] %} + {%- else %} + {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }} +{%- endif %} + {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}} + {{- "Given the following functions, please respond with a JSON for a function call " }} + {{- "with its proper arguments that best answers the given prompt.\n\n" }} + {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }} + {{- "Do not use variables.\n\n" }} + {%- for t in tools %} + {{- t | tojson(indent=4) }} + {{- "\n\n" }} + {%- endfor %} + {{- first_user_message + "<|eot_id|>"}} +{%- endif %} + +{%- for message in messages %} + {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %} + {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }} + {%- elif 'tool_calls' in message %} + {%- if not message.tool_calls|length == 1 %} + {{- raise_exception("This model only supports single tool-calls at once!") }} + {%- endif %} + {%- set tool_call = message.tool_calls[0].function %} + {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}} + {{- '{"name": "' + tool_call.name + '", ' }} + {{- '"parameters": ' }} + {{- tool_call.arguments | tojson }} + {{- "}" }} + {{- "<|eot_id|>" }} + {%- elif message.role == "tool" or message.role == "ipython" %} + {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }} + {%- if message.content is mapping or message.content is iterable %} + {{- message.content | tojson }} + {%- else %} + {{- message.content }} + {%- endif %} + {{- "<|eot_id|>" }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }} +{%- endif %} diff --git a/config.json b/config.json new file mode 100644 index 0000000..319a0ad --- /dev/null +++ b/config.json @@ -0,0 +1,39 @@ +{ + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 128000, + "eos_token_id": [ + 128001, + 128008, + 128009 + ], + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 3072, + "initializer_range": 0.02, + "intermediate_size": 8192, + "max_position_embeddings": 131072, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 24, + "num_hidden_layers": 28, + "num_key_value_heads": 8, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": { + "factor": 32.0, + "high_freq_factor": 4.0, + "low_freq_factor": 1.0, + "original_max_position_embeddings": 8192, + "rope_type": "llama3" + }, + "rope_theta": 500000.0, + "tie_word_embeddings": true, + "torch_dtype": "bfloat16", + "transformers_version": "4.52.4", + "use_cache": false, + "vocab_size": 128256 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..86546ab --- /dev/null +++ b/generation_config.json @@ -0,0 +1,12 @@ +{ + "bos_token_id": 128000, + "do_sample": true, + "eos_token_id": [ + 128001, + 128008, + 128009 + ], + "temperature": 0.6, + "top_p": 0.9, + "transformers_version": "4.52.4" +} diff --git a/model-00001-of-00002.safetensors b/model-00001-of-00002.safetensors new file mode 100644 index 0000000..6d4284d --- /dev/null +++ b/model-00001-of-00002.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:65e7e7ca498a603c9864bf034f02abffb706dd1e71ad4855081d2ed63f34c8cb +size 4965799096 diff --git a/model-00002-of-00002.safetensors b/model-00002-of-00002.safetensors new file mode 100644 index 0000000..4cd4d79 --- /dev/null +++ b/model-00002-of-00002.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:504a677695dd37326c1a97e2fc641f79fe210ff73a10ecf1af07af48599f4c09 +size 1459729952 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000..d3a1f0f --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,261 @@ +{ + "metadata": { + "total_size": 6425499648 + }, + "weight_map": { + "model.embed_tokens.weight": "model-00001-of-00002.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.21.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.input_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00002-of-00002.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00002-of-00002.safetensors", + "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors", + "model.norm.weight": "model-00002-of-00002.safetensors" + } +} diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..14daf45 --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,26 @@ +{ + "additional_special_tokens": [ + { + "content": "<|eom_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } + ], + "bos_token": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": "<|eot_id|>" +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..1c1d8d5 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b +size 17209920 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..8e7819a --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,2068 @@ +{ + "added_tokens_decoder": { + "128000": { + "content": "<|begin_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128001": { + "content": "<|end_of_text|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128002": { + "content": "<|reserved_special_token_0|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128003": { + "content": "<|reserved_special_token_1|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128004": { + "content": "<|finetune_right_pad_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128005": { + "content": "<|reserved_special_token_2|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128006": { + "content": "<|start_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128007": { + "content": "<|end_header_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128008": { + "content": "<|eom_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128009": { + "content": "<|eot_id|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128010": { + "content": "<|python_tag|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128011": { + "content": "<|reserved_special_token_3|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128012": { + "content": "<|reserved_special_token_4|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128013": { + "content": "<|reserved_special_token_5|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128014": { + "content": "<|reserved_special_token_6|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128015": { + "content": "<|reserved_special_token_7|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128016": { + "content": "<|reserved_special_token_8|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128017": { + "content": "<|reserved_special_token_9|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128018": { + "content": "<|reserved_special_token_10|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128019": { + "content": "<|reserved_special_token_11|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128020": { + "content": "<|reserved_special_token_12|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128021": { + "content": "<|reserved_special_token_13|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128022": { + "content": "<|reserved_special_token_14|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128023": { + "content": "<|reserved_special_token_15|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128024": { + "content": "<|reserved_special_token_16|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128025": { + "content": "<|reserved_special_token_17|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128026": { + "content": "<|reserved_special_token_18|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128027": { + "content": "<|reserved_special_token_19|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128028": { + "content": "<|reserved_special_token_20|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128029": { + "content": "<|reserved_special_token_21|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128030": { + "content": "<|reserved_special_token_22|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128031": { + "content": "<|reserved_special_token_23|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128032": { + "content": "<|reserved_special_token_24|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128033": { + "content": "<|reserved_special_token_25|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128034": { + "content": "<|reserved_special_token_26|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128035": { + "content": "<|reserved_special_token_27|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128036": { + "content": "<|reserved_special_token_28|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128037": { + "content": "<|reserved_special_token_29|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128038": { + "content": "<|reserved_special_token_30|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128039": { + "content": "<|reserved_special_token_31|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128040": { + "content": "<|reserved_special_token_32|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128041": { + "content": "<|reserved_special_token_33|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128042": { + "content": "<|reserved_special_token_34|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128043": { + "content": "<|reserved_special_token_35|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128044": { + "content": "<|reserved_special_token_36|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128045": { + "content": "<|reserved_special_token_37|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128046": { + "content": "<|reserved_special_token_38|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128047": { + "content": "<|reserved_special_token_39|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128048": { + "content": "<|reserved_special_token_40|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128049": { + "content": "<|reserved_special_token_41|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128050": { + "content": "<|reserved_special_token_42|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128051": { + "content": "<|reserved_special_token_43|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128052": { + "content": "<|reserved_special_token_44|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128053": { + "content": "<|reserved_special_token_45|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128054": { + "content": "<|reserved_special_token_46|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128055": { + "content": "<|reserved_special_token_47|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128056": { + "content": "<|reserved_special_token_48|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128057": { + "content": "<|reserved_special_token_49|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128058": { + "content": "<|reserved_special_token_50|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128059": { + "content": "<|reserved_special_token_51|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128060": { + "content": "<|reserved_special_token_52|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128061": { + "content": "<|reserved_special_token_53|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128062": { + "content": "<|reserved_special_token_54|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128063": { + "content": "<|reserved_special_token_55|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128064": { + "content": "<|reserved_special_token_56|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128065": { + "content": "<|reserved_special_token_57|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128066": { + "content": "<|reserved_special_token_58|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128067": { + "content": "<|reserved_special_token_59|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128068": { + "content": "<|reserved_special_token_60|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128069": { + "content": "<|reserved_special_token_61|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128070": { + "content": "<|reserved_special_token_62|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128071": { + "content": "<|reserved_special_token_63|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128072": { + "content": "<|reserved_special_token_64|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128073": { + "content": "<|reserved_special_token_65|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128074": { + "content": "<|reserved_special_token_66|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128075": { + "content": "<|reserved_special_token_67|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128076": { + "content": "<|reserved_special_token_68|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128077": { + "content": "<|reserved_special_token_69|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128078": { + "content": "<|reserved_special_token_70|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128079": { + "content": "<|reserved_special_token_71|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128080": { + "content": "<|reserved_special_token_72|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128081": { + "content": "<|reserved_special_token_73|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128082": { + "content": "<|reserved_special_token_74|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128083": { + "content": "<|reserved_special_token_75|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128084": { + "content": "<|reserved_special_token_76|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128085": { + "content": "<|reserved_special_token_77|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128086": { + "content": "<|reserved_special_token_78|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128087": { + "content": "<|reserved_special_token_79|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128088": { + "content": "<|reserved_special_token_80|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128089": { + "content": "<|reserved_special_token_81|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128090": { + "content": "<|reserved_special_token_82|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128091": { + "content": "<|reserved_special_token_83|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128092": { + "content": "<|reserved_special_token_84|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128093": { + "content": "<|reserved_special_token_85|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128094": { + "content": "<|reserved_special_token_86|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128095": { + "content": "<|reserved_special_token_87|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128096": { + "content": "<|reserved_special_token_88|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128097": { + "content": "<|reserved_special_token_89|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128098": { + "content": "<|reserved_special_token_90|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128099": { + "content": "<|reserved_special_token_91|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128100": { + "content": "<|reserved_special_token_92|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128101": { + "content": "<|reserved_special_token_93|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128102": { + "content": "<|reserved_special_token_94|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128103": { + "content": "<|reserved_special_token_95|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128104": { + "content": "<|reserved_special_token_96|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128105": { + "content": "<|reserved_special_token_97|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128106": { + "content": "<|reserved_special_token_98|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128107": { + "content": "<|reserved_special_token_99|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128108": { + "content": "<|reserved_special_token_100|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128109": { + "content": "<|reserved_special_token_101|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128110": { + "content": "<|reserved_special_token_102|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128111": { + "content": "<|reserved_special_token_103|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128112": { + "content": "<|reserved_special_token_104|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128113": { + "content": "<|reserved_special_token_105|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128114": { + "content": "<|reserved_special_token_106|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128115": { + "content": "<|reserved_special_token_107|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128116": { + "content": "<|reserved_special_token_108|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128117": { + "content": "<|reserved_special_token_109|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128118": { + "content": "<|reserved_special_token_110|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128119": { + "content": "<|reserved_special_token_111|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128120": { + "content": "<|reserved_special_token_112|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128121": { + "content": "<|reserved_special_token_113|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128122": { + "content": "<|reserved_special_token_114|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128123": { + "content": "<|reserved_special_token_115|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128124": { + "content": "<|reserved_special_token_116|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128125": { + "content": "<|reserved_special_token_117|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128126": { + "content": "<|reserved_special_token_118|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128127": { + "content": "<|reserved_special_token_119|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128128": { + "content": "<|reserved_special_token_120|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128129": { + "content": "<|reserved_special_token_121|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128130": { + "content": "<|reserved_special_token_122|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128131": { + "content": "<|reserved_special_token_123|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128132": { + "content": "<|reserved_special_token_124|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128133": { + "content": "<|reserved_special_token_125|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128134": { + "content": "<|reserved_special_token_126|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128135": { + "content": "<|reserved_special_token_127|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128136": { + "content": "<|reserved_special_token_128|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128137": { + "content": "<|reserved_special_token_129|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128138": { + "content": "<|reserved_special_token_130|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128139": { + "content": "<|reserved_special_token_131|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128140": { + "content": "<|reserved_special_token_132|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128141": { + "content": "<|reserved_special_token_133|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128142": { + "content": "<|reserved_special_token_134|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128143": { + "content": "<|reserved_special_token_135|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128144": { + "content": "<|reserved_special_token_136|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128145": { + "content": "<|reserved_special_token_137|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128146": { + "content": "<|reserved_special_token_138|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128147": { + "content": "<|reserved_special_token_139|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128148": { + "content": "<|reserved_special_token_140|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128149": { + "content": "<|reserved_special_token_141|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128150": { + "content": "<|reserved_special_token_142|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128151": { + "content": "<|reserved_special_token_143|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128152": { + "content": "<|reserved_special_token_144|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128153": { + "content": "<|reserved_special_token_145|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128154": { + "content": "<|reserved_special_token_146|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128155": { + "content": "<|reserved_special_token_147|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128156": { + "content": "<|reserved_special_token_148|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128157": { + "content": "<|reserved_special_token_149|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128158": { + "content": "<|reserved_special_token_150|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128159": { + "content": "<|reserved_special_token_151|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128160": { + "content": "<|reserved_special_token_152|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128161": { + "content": "<|reserved_special_token_153|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128162": { + "content": "<|reserved_special_token_154|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128163": { + "content": "<|reserved_special_token_155|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128164": { + "content": "<|reserved_special_token_156|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128165": { + "content": "<|reserved_special_token_157|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128166": { + "content": "<|reserved_special_token_158|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128167": { + "content": "<|reserved_special_token_159|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128168": { + "content": "<|reserved_special_token_160|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128169": { + "content": "<|reserved_special_token_161|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128170": { + "content": "<|reserved_special_token_162|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128171": { + "content": "<|reserved_special_token_163|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128172": { + "content": "<|reserved_special_token_164|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128173": { + "content": "<|reserved_special_token_165|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128174": { + "content": "<|reserved_special_token_166|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128175": { + "content": "<|reserved_special_token_167|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128176": { + "content": "<|reserved_special_token_168|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128177": { + "content": "<|reserved_special_token_169|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128178": { + "content": "<|reserved_special_token_170|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128179": { + "content": "<|reserved_special_token_171|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128180": { + "content": "<|reserved_special_token_172|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128181": { + "content": "<|reserved_special_token_173|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128182": { + "content": "<|reserved_special_token_174|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128183": { + "content": "<|reserved_special_token_175|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128184": { + "content": "<|reserved_special_token_176|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128185": { + "content": "<|reserved_special_token_177|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128186": { + "content": "<|reserved_special_token_178|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128187": { + "content": "<|reserved_special_token_179|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128188": { + "content": "<|reserved_special_token_180|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128189": { + "content": "<|reserved_special_token_181|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128190": { + "content": "<|reserved_special_token_182|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128191": { + "content": "<|reserved_special_token_183|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128192": { + "content": "<|reserved_special_token_184|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128193": { + "content": "<|reserved_special_token_185|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128194": { + "content": "<|reserved_special_token_186|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128195": { + "content": "<|reserved_special_token_187|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128196": { + "content": "<|reserved_special_token_188|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128197": { + "content": "<|reserved_special_token_189|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128198": { + "content": "<|reserved_special_token_190|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128199": { + "content": "<|reserved_special_token_191|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128200": { + "content": "<|reserved_special_token_192|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128201": { + "content": "<|reserved_special_token_193|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128202": { + "content": "<|reserved_special_token_194|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128203": { + "content": "<|reserved_special_token_195|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128204": { + "content": "<|reserved_special_token_196|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128205": { + "content": "<|reserved_special_token_197|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128206": { + "content": "<|reserved_special_token_198|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128207": { + "content": "<|reserved_special_token_199|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128208": { + "content": "<|reserved_special_token_200|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128209": { + "content": "<|reserved_special_token_201|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128210": { + "content": "<|reserved_special_token_202|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128211": { + "content": "<|reserved_special_token_203|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128212": { + "content": "<|reserved_special_token_204|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128213": { + "content": "<|reserved_special_token_205|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128214": { + "content": "<|reserved_special_token_206|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128215": { + "content": "<|reserved_special_token_207|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128216": { + "content": "<|reserved_special_token_208|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128217": { + "content": "<|reserved_special_token_209|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128218": { + "content": "<|reserved_special_token_210|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128219": { + "content": "<|reserved_special_token_211|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128220": { + "content": "<|reserved_special_token_212|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128221": { + "content": "<|reserved_special_token_213|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128222": { + "content": "<|reserved_special_token_214|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128223": { + "content": "<|reserved_special_token_215|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128224": { + "content": "<|reserved_special_token_216|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128225": { + "content": "<|reserved_special_token_217|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128226": { + "content": "<|reserved_special_token_218|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128227": { + "content": "<|reserved_special_token_219|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128228": { + "content": "<|reserved_special_token_220|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128229": { + "content": "<|reserved_special_token_221|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128230": { + "content": "<|reserved_special_token_222|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128231": { + "content": "<|reserved_special_token_223|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128232": { + "content": "<|reserved_special_token_224|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128233": { + "content": "<|reserved_special_token_225|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128234": { + "content": "<|reserved_special_token_226|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128235": { + "content": "<|reserved_special_token_227|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128236": { + "content": "<|reserved_special_token_228|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128237": { + "content": "<|reserved_special_token_229|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128238": { + "content": "<|reserved_special_token_230|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128239": { + "content": "<|reserved_special_token_231|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128240": { + "content": "<|reserved_special_token_232|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128241": { + "content": "<|reserved_special_token_233|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128242": { + "content": "<|reserved_special_token_234|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128243": { + "content": "<|reserved_special_token_235|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128244": { + "content": "<|reserved_special_token_236|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128245": { + "content": "<|reserved_special_token_237|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128246": { + "content": "<|reserved_special_token_238|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128247": { + "content": "<|reserved_special_token_239|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128248": { + "content": "<|reserved_special_token_240|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128249": { + "content": "<|reserved_special_token_241|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128250": { + "content": "<|reserved_special_token_242|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128251": { + "content": "<|reserved_special_token_243|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128252": { + "content": "<|reserved_special_token_244|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128253": { + "content": "<|reserved_special_token_245|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128254": { + "content": "<|reserved_special_token_246|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "128255": { + "content": "<|reserved_special_token_247|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "additional_special_tokens": [ + "<|eom_id|>" + ], + "bos_token": "<|begin_of_text|>", + "clean_up_tokenization_spaces": true, + "eos_token": "<|eot_id|>", + "extra_special_tokens": {}, + "model_input_names": [ + "input_ids", + "attention_mask" + ], + "model_max_length": 131072, + "pad_token": "<|eot_id|>", + "padding_side": "right", + "split_special_tokens": false, + "tokenizer_class": "PreTrainedTokenizer" +} diff --git a/train_results.json b/train_results.json new file mode 100644 index 0000000..06328b4 --- /dev/null +++ b/train_results.json @@ -0,0 +1,8 @@ +{ + "epoch": 3.0, + "total_flos": 181229773520896.0, + "train_loss": 0.266992773216715, + "train_runtime": 3529.0775, + "train_samples_per_second": 16.577, + "train_steps_per_second": 0.259 +} \ No newline at end of file diff --git a/trainer_log.jsonl b/trainer_log.jsonl new file mode 100644 index 0000000..b873b03 --- /dev/null +++ b/trainer_log.jsonl @@ -0,0 +1,32 @@ +{"current_steps": 610, "total_steps": 915, "loss": 0.8327, "lr": 3.0403920675946826e-06, "epoch": 2.0032813781788352, "percentage": 66.67, "elapsed_time": "0:02:00", "remaining_time": "0:01:00"} +{"current_steps": 620, "total_steps": 915, "loss": 0.7782, "lr": 2.8662692339278387e-06, "epoch": 2.036095159967186, "percentage": 67.76, "elapsed_time": "0:03:54", "remaining_time": "0:01:51"} +{"current_steps": 630, "total_steps": 915, "loss": 0.7875, "lr": 2.6952551585633947e-06, "epoch": 2.0689089417555375, "percentage": 68.85, "elapsed_time": "0:05:30", "remaining_time": "0:02:29"} +{"current_steps": 640, "total_steps": 915, "loss": 0.7708, "lr": 2.52759900200232e-06, "epoch": 2.1017227235438884, "percentage": 69.95, "elapsed_time": "0:07:31", "remaining_time": "0:03:14"} +{"current_steps": 650, "total_steps": 915, "loss": 0.7927, "lr": 2.3635450323954773e-06, "epoch": 2.1345365053322394, "percentage": 71.04, "elapsed_time": "0:09:09", "remaining_time": "0:03:43"} +{"current_steps": 660, "total_steps": 915, "loss": 0.7885, "lr": 2.2033322696549197e-06, "epoch": 2.1673502871205907, "percentage": 72.13, "elapsed_time": "0:10:50", "remaining_time": "0:04:11"} +{"current_steps": 670, "total_steps": 915, "loss": 0.7626, "lr": 2.0471941372116793e-06, "epoch": 2.2001640689089417, "percentage": 73.22, "elapsed_time": "0:12:42", "remaining_time": "0:04:38"} +{"current_steps": 680, "total_steps": 915, "loss": 0.7754, "lr": 1.8953581219273987e-06, "epoch": 2.232977850697293, "percentage": 74.32, "elapsed_time": "0:14:33", "remaining_time": "0:05:01"} +{"current_steps": 690, "total_steps": 915, "loss": 0.7783, "lr": 1.7480454426552773e-06, "epoch": 2.265791632485644, "percentage": 75.41, "elapsed_time": "0:16:17", "remaining_time": "0:05:18"} +{"current_steps": 700, "total_steps": 915, "loss": 0.7705, "lr": 1.6054707279332865e-06, "epoch": 2.298605414273995, "percentage": 76.5, "elapsed_time": "0:18:15", "remaining_time": "0:05:36"} +{"current_steps": 710, "total_steps": 915, "loss": 0.7699, "lr": 1.4678417032791653e-06, "epoch": 2.3314191960623463, "percentage": 77.6, "elapsed_time": "0:19:52", "remaining_time": "0:05:44"} +{"current_steps": 720, "total_steps": 915, "loss": 0.7526, "lr": 1.335358888542862e-06, "epoch": 2.364232977850697, "percentage": 78.69, "elapsed_time": "0:21:39", "remaining_time": "0:05:51"} +{"current_steps": 730, "total_steps": 915, "loss": 0.7918, "lr": 1.20821530575733e-06, "epoch": 2.3970467596390486, "percentage": 79.78, "elapsed_time": "0:23:35", "remaining_time": "0:05:58"} +{"current_steps": 740, "total_steps": 915, "loss": 0.7815, "lr": 1.0865961979133245e-06, "epoch": 2.4298605414273995, "percentage": 80.87, "elapsed_time": "0:25:10", "remaining_time": "0:05:57"} +{"current_steps": 750, "total_steps": 915, "loss": 0.7731, "lr": 9.706787590679685e-07, "epoch": 2.462674323215751, "percentage": 81.97, "elapsed_time": "0:26:39", "remaining_time": "0:05:51"} +{"current_steps": 760, "total_steps": 915, "loss": 0.7666, "lr": 8.606318761802584e-07, "epoch": 2.495488105004102, "percentage": 83.06, "elapsed_time": "0:28:16", "remaining_time": "0:05:45"} +{"current_steps": 770, "total_steps": 915, "loss": 0.7657, "lr": 7.566158830496917e-07, "epoch": 2.5283018867924527, "percentage": 84.15, "elapsed_time": "0:29:58", "remaining_time": "0:05:38"} +{"current_steps": 780, "total_steps": 915, "loss": 0.7798, "lr": 6.587823267164911e-07, "epoch": 2.561115668580804, "percentage": 85.25, "elapsed_time": "0:31:27", "remaining_time": "0:05:26"} +{"current_steps": 790, "total_steps": 915, "loss": 0.7816, "lr": 5.672737466637701e-07, "epoch": 2.593929450369155, "percentage": 86.34, "elapsed_time": "0:33:28", "remaining_time": "0:05:17"} +{"current_steps": 800, "total_steps": 915, "loss": 0.7837, "lr": 4.822234671433552e-07, "epoch": 2.626743232157506, "percentage": 87.43, "elapsed_time": "0:35:41", "remaining_time": "0:05:07"} +{"current_steps": 810, "total_steps": 915, "loss": 0.7747, "lr": 4.03755402927804e-07, "epoch": 2.6595570139458573, "percentage": 88.52, "elapsed_time": "0:39:29", "remaining_time": "0:05:07"} +{"current_steps": 820, "total_steps": 915, "loss": 0.7793, "lr": 3.319838787716634e-07, "epoch": 2.6923707957342082, "percentage": 89.62, "elapsed_time": "0:41:31", "remaining_time": "0:04:48"} +{"current_steps": 830, "total_steps": 915, "loss": 0.7542, "lr": 2.6701346284499e-07, "epoch": 2.7251845775225596, "percentage": 90.71, "elapsed_time": "0:43:29", "remaining_time": "0:04:27"} +{"current_steps": 840, "total_steps": 915, "loss": 0.7844, "lr": 2.0893881438180275e-07, "epoch": 2.7579983593109105, "percentage": 91.8, "elapsed_time": "0:45:26", "remaining_time": "0:04:03"} +{"current_steps": 850, "total_steps": 915, "loss": 0.7643, "lr": 1.578445457654637e-07, "epoch": 2.790812141099262, "percentage": 92.9, "elapsed_time": "0:47:15", "remaining_time": "0:03:36"} +{"current_steps": 860, "total_steps": 915, "loss": 0.7673, "lr": 1.1380509925189853e-07, "epoch": 2.823625922887613, "percentage": 93.99, "elapsed_time": "0:49:01", "remaining_time": "0:03:08"} +{"current_steps": 870, "total_steps": 915, "loss": 0.769, "lr": 7.688463851028227e-08, "epoch": 2.8564397046759638, "percentage": 95.08, "elapsed_time": "0:51:01", "remaining_time": "0:02:38"} +{"current_steps": 880, "total_steps": 915, "loss": 0.7799, "lr": 4.713695513920147e-08, "epoch": 2.889253486464315, "percentage": 96.17, "elapsed_time": "0:52:45", "remaining_time": "0:02:05"} +{"current_steps": 890, "total_steps": 915, "loss": 0.785, "lr": 2.4605390294497043e-08, "epoch": 2.922067268252666, "percentage": 97.27, "elapsed_time": "0:54:45", "remaining_time": "0:01:32"} +{"current_steps": 900, "total_steps": 915, "loss": 0.7753, "lr": 9.322771542978892e-09, "epoch": 2.954881050041017, "percentage": 98.36, "elapsed_time": "0:56:44", "remaining_time": "0:00:56"} +{"current_steps": 910, "total_steps": 915, "loss": 0.7911, "lr": 1.3113650340046413e-09, "epoch": 2.9876948318293683, "percentage": 99.45, "elapsed_time": "0:58:10", "remaining_time": "0:00:19"} +{"current_steps": 914, "total_steps": 915, "epoch": 3.0, "percentage": 99.89, "elapsed_time": "0:58:47", "remaining_time": "0:00:03"} diff --git a/trainer_state.json b/trainer_state.json new file mode 100644 index 0000000..a570947 --- /dev/null +++ b/trainer_state.json @@ -0,0 +1,680 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 3.0, + "eval_steps": 500, + "global_step": 914, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.03281378178835111, + "grad_norm": 7.324607115951123, + "learning_rate": 9.782608695652175e-07, + "loss": 1.6507, + "step": 10 + }, + { + "epoch": 0.06562756357670221, + "grad_norm": 2.6718330452258137, + "learning_rate": 2.065217391304348e-06, + "loss": 1.5293, + "step": 20 + }, + { + "epoch": 0.09844134536505332, + "grad_norm": 2.0449760430297435, + "learning_rate": 3.152173913043479e-06, + "loss": 1.3882, + "step": 30 + }, + { + "epoch": 0.13125512715340443, + "grad_norm": 1.7054134520698259, + "learning_rate": 4.239130434782609e-06, + "loss": 1.3212, + "step": 40 + }, + { + "epoch": 0.16406890894175555, + "grad_norm": 1.5625383091861629, + "learning_rate": 5.3260869565217395e-06, + "loss": 1.2339, + "step": 50 + }, + { + "epoch": 0.19688269073010664, + "grad_norm": 1.5467201317331245, + "learning_rate": 6.41304347826087e-06, + "loss": 1.2209, + "step": 60 + }, + { + "epoch": 0.22969647251845776, + "grad_norm": 1.5441409779630197, + "learning_rate": 7.500000000000001e-06, + "loss": 1.1981, + "step": 70 + }, + { + "epoch": 0.26251025430680885, + "grad_norm": 1.5810387088382716, + "learning_rate": 8.586956521739131e-06, + "loss": 1.1633, + "step": 80 + }, + { + "epoch": 0.29532403609515995, + "grad_norm": 1.4632034044381061, + "learning_rate": 9.673913043478262e-06, + "loss": 1.1815, + "step": 90 + }, + { + "epoch": 0.3281378178835111, + "grad_norm": 1.6769268374195203, + "learning_rate": 9.998215114657564e-06, + "loss": 1.1741, + "step": 100 + }, + { + "epoch": 0.3609515996718622, + "grad_norm": 1.564606037108119, + "learning_rate": 9.98947588668843e-06, + "loss": 1.13, + "step": 110 + }, + { + "epoch": 0.3937653814602133, + "grad_norm": 1.4659283460430306, + "learning_rate": 9.973467196782484e-06, + "loss": 1.1339, + "step": 120 + }, + { + "epoch": 0.4265791632485644, + "grad_norm": 1.3900414131292986, + "learning_rate": 9.950212368945013e-06, + "loss": 1.1501, + "step": 130 + }, + { + "epoch": 0.4593929450369155, + "grad_norm": 1.533573901481892, + "learning_rate": 9.91974528450737e-06, + "loss": 1.1374, + "step": 140 + }, + { + "epoch": 0.4922067268252666, + "grad_norm": 1.4930324222767686, + "learning_rate": 9.882110332763275e-06, + "loss": 1.1316, + "step": 150 + }, + { + "epoch": 0.5250205086136177, + "grad_norm": 1.4649309232116048, + "learning_rate": 9.83736234629543e-06, + "loss": 1.1199, + "step": 160 + }, + { + "epoch": 0.5578342904019689, + "grad_norm": 1.4519140933698538, + "learning_rate": 9.785566521086695e-06, + "loss": 1.1163, + "step": 170 + }, + { + "epoch": 0.5906480721903199, + "grad_norm": 1.4591675768161227, + "learning_rate": 9.726798321532205e-06, + "loss": 1.125, + "step": 180 + }, + { + "epoch": 0.623461853978671, + "grad_norm": 1.3084115670512635, + "learning_rate": 9.661143370490846e-06, + "loss": 1.1385, + "step": 190 + }, + { + "epoch": 0.6562756357670222, + "grad_norm": 1.428698415469623, + "learning_rate": 9.588697324536254e-06, + "loss": 1.0995, + "step": 200 + }, + { + "epoch": 0.6890894175553732, + "grad_norm": 1.3195726786177668, + "learning_rate": 9.509565734589105e-06, + "loss": 1.105, + "step": 210 + }, + { + "epoch": 0.7219031993437244, + "grad_norm": 1.4644538779508807, + "learning_rate": 9.423863892133754e-06, + "loss": 1.0949, + "step": 220 + }, + { + "epoch": 0.7547169811320755, + "grad_norm": 1.4442654685795682, + "learning_rate": 9.33171666124326e-06, + "loss": 1.1097, + "step": 230 + }, + { + "epoch": 0.7875307629204266, + "grad_norm": 1.4628666514015816, + "learning_rate": 9.233258296657547e-06, + "loss": 1.0915, + "step": 240 + }, + { + "epoch": 0.8203445447087777, + "grad_norm": 1.4584300507718668, + "learning_rate": 9.128632248179761e-06, + "loss": 1.0952, + "step": 250 + }, + { + "epoch": 0.8531583264971287, + "grad_norm": 1.3907705989649481, + "learning_rate": 9.017990951675764e-06, + "loss": 1.1072, + "step": 260 + }, + { + "epoch": 0.8859721082854799, + "grad_norm": 1.44611189547518, + "learning_rate": 8.901495606981339e-06, + "loss": 1.0908, + "step": 270 + }, + { + "epoch": 0.918785890073831, + "grad_norm": 1.4283544277374125, + "learning_rate": 8.779315943040629e-06, + "loss": 1.0934, + "step": 280 + }, + { + "epoch": 0.9515996718621821, + "grad_norm": 1.3679340439375278, + "learning_rate": 8.65162997061802e-06, + "loss": 1.0902, + "step": 290 + }, + { + "epoch": 0.9844134536505332, + "grad_norm": 1.4053097778093493, + "learning_rate": 8.518623722943747e-06, + "loss": 1.0826, + "step": 300 + }, + { + "epoch": 1.0164068908941755, + "grad_norm": 1.358051135209502, + "learning_rate": 8.380490984671105e-06, + "loss": 1.0004, + "step": 310 + }, + { + "epoch": 1.0492206726825266, + "grad_norm": 1.4920803109368017, + "learning_rate": 8.23743300954015e-06, + "loss": 0.9529, + "step": 320 + }, + { + "epoch": 1.0820344544708778, + "grad_norm": 1.391421362612517, + "learning_rate": 8.089658227159239e-06, + "loss": 0.9108, + "step": 330 + }, + { + "epoch": 1.114848236259229, + "grad_norm": 1.6241278760522855, + "learning_rate": 7.937381939331628e-06, + "loss": 0.9279, + "step": 340 + }, + { + "epoch": 1.14766201804758, + "grad_norm": 1.4776439598223472, + "learning_rate": 7.780826006369586e-06, + "loss": 0.9332, + "step": 350 + }, + { + "epoch": 1.1804757998359312, + "grad_norm": 1.531082298396227, + "learning_rate": 7.620218523852987e-06, + "loss": 0.9503, + "step": 360 + }, + { + "epoch": 1.2132895816242821, + "grad_norm": 1.4148212125540105, + "learning_rate": 7.4557934903034035e-06, + "loss": 0.9409, + "step": 370 + }, + { + "epoch": 1.2461033634126333, + "grad_norm": 1.569873004540387, + "learning_rate": 7.287790466257854e-06, + "loss": 0.9228, + "step": 380 + }, + { + "epoch": 1.2789171452009844, + "grad_norm": 1.50216408338208, + "learning_rate": 7.116454225238909e-06, + "loss": 0.9354, + "step": 390 + }, + { + "epoch": 1.3117309269893356, + "grad_norm": 1.4646843748186438, + "learning_rate": 6.942034397129702e-06, + "loss": 0.9372, + "step": 400 + }, + { + "epoch": 1.3445447087776867, + "grad_norm": 1.5763941611606018, + "learning_rate": 6.764785104473411e-06, + "loss": 0.9169, + "step": 410 + }, + { + "epoch": 1.3773584905660377, + "grad_norm": 1.4076340592270304, + "learning_rate": 6.584964592227135e-06, + "loss": 0.9235, + "step": 420 + }, + { + "epoch": 1.4101722723543888, + "grad_norm": 1.5649588493962718, + "learning_rate": 6.402834851509564e-06, + "loss": 0.926, + "step": 430 + }, + { + "epoch": 1.44298605414274, + "grad_norm": 1.370995330259522, + "learning_rate": 6.2186612378906545e-06, + "loss": 0.9327, + "step": 440 + }, + { + "epoch": 1.475799835931091, + "grad_norm": 1.4405579132350896, + "learning_rate": 6.0327120847794415e-06, + "loss": 0.9461, + "step": 450 + }, + { + "epoch": 1.5086136177194422, + "grad_norm": 1.5139954304026377, + "learning_rate": 5.845258312473252e-06, + "loss": 0.9479, + "step": 460 + }, + { + "epoch": 1.5414273995077932, + "grad_norm": 1.5381263145609851, + "learning_rate": 5.656573033437932e-06, + "loss": 0.9217, + "step": 470 + }, + { + "epoch": 1.5742411812961445, + "grad_norm": 1.5176651174503644, + "learning_rate": 5.466931154394171e-06, + "loss": 0.9402, + "step": 480 + }, + { + "epoch": 1.6070549630844955, + "grad_norm": 1.64200137668156, + "learning_rate": 5.276608975789683e-06, + "loss": 0.925, + "step": 490 + }, + { + "epoch": 1.6398687448728466, + "grad_norm": 1.4813527495999301, + "learning_rate": 5.085883789240764e-06, + "loss": 0.9268, + "step": 500 + }, + { + "epoch": 1.6726825266611978, + "grad_norm": 1.5116269108263887, + "learning_rate": 4.8950334735297746e-06, + "loss": 0.9095, + "step": 510 + }, + { + "epoch": 1.7054963084495487, + "grad_norm": 1.4490396822436944, + "learning_rate": 4.704336089747135e-06, + "loss": 0.9341, + "step": 520 + }, + { + "epoch": 1.7383100902379, + "grad_norm": 1.5628125198488163, + "learning_rate": 4.514069476167716e-06, + "loss": 0.932, + "step": 530 + }, + { + "epoch": 1.771123872026251, + "grad_norm": 1.4687886059027409, + "learning_rate": 4.324510843451851e-06, + "loss": 0.9311, + "step": 540 + }, + { + "epoch": 1.8039376538146021, + "grad_norm": 1.5227999474981233, + "learning_rate": 4.135936370760759e-06, + "loss": 0.9046, + "step": 550 + }, + { + "epoch": 1.8367514356029533, + "grad_norm": 1.4388610554454382, + "learning_rate": 3.9486208033748315e-06, + "loss": 0.9378, + "step": 560 + }, + { + "epoch": 1.8695652173913042, + "grad_norm": 1.3581055291483148, + "learning_rate": 3.762837052401004e-06, + "loss": 0.9235, + "step": 570 + }, + { + "epoch": 1.9023789991796556, + "grad_norm": 1.5099266943158, + "learning_rate": 3.5788557971524695e-06, + "loss": 0.9444, + "step": 580 + }, + { + "epoch": 1.9351927809680065, + "grad_norm": 1.5169803759126903, + "learning_rate": 3.3969450907799966e-06, + "loss": 0.9279, + "step": 590 + }, + { + "epoch": 1.9680065627563577, + "grad_norm": 1.3979513991441848, + "learning_rate": 3.217369969729476e-06, + "loss": 0.9115, + "step": 600 + }, + { + "epoch": 2.0032813781788352, + "grad_norm": 2.105088244846462, + "learning_rate": 3.0403920675946826e-06, + "loss": 0.8327, + "step": 610 + }, + { + "epoch": 2.036095159967186, + "grad_norm": 1.6690335971877808, + "learning_rate": 2.8662692339278387e-06, + "loss": 0.7782, + "step": 620 + }, + { + "epoch": 2.0689089417555375, + "grad_norm": 1.705302067748598, + "learning_rate": 2.6952551585633947e-06, + "loss": 0.7875, + "step": 630 + }, + { + "epoch": 2.1017227235438884, + "grad_norm": 1.6066988675182772, + "learning_rate": 2.52759900200232e-06, + "loss": 0.7708, + "step": 640 + }, + { + "epoch": 2.1345365053322394, + "grad_norm": 1.5203406830167119, + "learning_rate": 2.3635450323954773e-06, + "loss": 0.7927, + "step": 650 + }, + { + "epoch": 2.1673502871205907, + "grad_norm": 1.626389642459301, + "learning_rate": 2.2033322696549197e-06, + "loss": 0.7885, + "step": 660 + }, + { + "epoch": 2.2001640689089417, + "grad_norm": 1.7664606984797573, + "learning_rate": 2.0471941372116793e-06, + "loss": 0.7626, + "step": 670 + }, + { + "epoch": 2.232977850697293, + "grad_norm": 1.5793686740276833, + "learning_rate": 1.8953581219273987e-06, + "loss": 0.7754, + "step": 680 + }, + { + "epoch": 2.265791632485644, + "grad_norm": 1.6724958298486645, + "learning_rate": 1.7480454426552773e-06, + "loss": 0.7783, + "step": 690 + }, + { + "epoch": 2.298605414273995, + "grad_norm": 1.5819324740235794, + "learning_rate": 1.6054707279332865e-06, + "loss": 0.7705, + "step": 700 + }, + { + "epoch": 2.3314191960623463, + "grad_norm": 1.6323457128335996, + "learning_rate": 1.4678417032791653e-06, + "loss": 0.7699, + "step": 710 + }, + { + "epoch": 2.364232977850697, + "grad_norm": 1.7857816578985155, + "learning_rate": 1.335358888542862e-06, + "loss": 0.7526, + "step": 720 + }, + { + "epoch": 2.3970467596390486, + "grad_norm": 1.5935449756290072, + "learning_rate": 1.20821530575733e-06, + "loss": 0.7918, + "step": 730 + }, + { + "epoch": 2.4298605414273995, + "grad_norm": 1.693473193818772, + "learning_rate": 1.0865961979133245e-06, + "loss": 0.7815, + "step": 740 + }, + { + "epoch": 2.462674323215751, + "grad_norm": 1.6977380314021282, + "learning_rate": 9.706787590679685e-07, + "loss": 0.7731, + "step": 750 + }, + { + "epoch": 2.495488105004102, + "grad_norm": 1.701374033891568, + "learning_rate": 8.606318761802584e-07, + "loss": 0.7666, + "step": 760 + }, + { + "epoch": 2.5283018867924527, + "grad_norm": 1.8239897351347403, + "learning_rate": 7.566158830496917e-07, + "loss": 0.7657, + "step": 770 + }, + { + "epoch": 2.561115668580804, + "grad_norm": 1.7594557186973525, + "learning_rate": 6.587823267164911e-07, + "loss": 0.7798, + "step": 780 + }, + { + "epoch": 2.593929450369155, + "grad_norm": 1.6074400677010736, + "learning_rate": 5.672737466637701e-07, + "loss": 0.7816, + "step": 790 + }, + { + "epoch": 2.626743232157506, + "grad_norm": 1.589497024974972, + "learning_rate": 4.822234671433552e-07, + "loss": 0.7837, + "step": 800 + }, + { + "epoch": 2.6595570139458573, + "grad_norm": 1.5745731469005892, + "learning_rate": 4.03755402927804e-07, + "loss": 0.7747, + "step": 810 + }, + { + "epoch": 2.6923707957342082, + "grad_norm": 1.6156048098137663, + "learning_rate": 3.319838787716634e-07, + "loss": 0.7793, + "step": 820 + }, + { + "epoch": 2.7251845775225596, + "grad_norm": 1.6695478870964964, + "learning_rate": 2.6701346284499e-07, + "loss": 0.7542, + "step": 830 + }, + { + "epoch": 2.7579983593109105, + "grad_norm": 1.5107440160015784, + "learning_rate": 2.0893881438180275e-07, + "loss": 0.7844, + "step": 840 + }, + { + "epoch": 2.790812141099262, + "grad_norm": 1.6477028499246302, + "learning_rate": 1.578445457654637e-07, + "loss": 0.7643, + "step": 850 + }, + { + "epoch": 2.823625922887613, + "grad_norm": 1.5854989560185238, + "learning_rate": 1.1380509925189853e-07, + "loss": 0.7673, + "step": 860 + }, + { + "epoch": 2.8564397046759638, + "grad_norm": 1.6787763051906506, + "learning_rate": 7.688463851028227e-08, + "loss": 0.769, + "step": 870 + }, + { + "epoch": 2.889253486464315, + "grad_norm": 1.581911801204859, + "learning_rate": 4.713695513920147e-08, + "loss": 0.7799, + "step": 880 + }, + { + "epoch": 2.922067268252666, + "grad_norm": 1.697466745242019, + "learning_rate": 2.4605390294497043e-08, + "loss": 0.785, + "step": 890 + }, + { + "epoch": 2.954881050041017, + "grad_norm": 1.6425019567953216, + "learning_rate": 9.322771542978892e-09, + "loss": 0.7753, + "step": 900 + }, + { + "epoch": 2.9876948318293683, + "grad_norm": 1.5783466963735138, + "learning_rate": 1.3113650340046413e-09, + "loss": 0.7911, + "step": 910 + }, + { + "epoch": 3.0, + "step": 914, + "total_flos": 181229773520896.0, + "train_loss": 0.266992773216715, + "train_runtime": 3529.0775, + "train_samples_per_second": 16.577, + "train_steps_per_second": 0.259 + } + ], + "logging_steps": 10, + "max_steps": 915, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 200, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 181229773520896.0, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +} diff --git a/training_args.bin b/training_args.bin new file mode 100644 index 0000000..5ad8f0a --- /dev/null +++ b/training_args.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:90dd3d267d14dbdca9eafe6602aba67e5bbe51012a9cccf660a487fae7628844 +size 8017 diff --git a/training_loss.png b/training_loss.png new file mode 100644 index 0000000000000000000000000000000000000000..f0fdce22f04fb799b7a8e7e37d1f98e1325ebf11 GIT binary patch literal 36537 zcmd?Rg;!Nu)Hi%cMNmW}L_$eL8tJkSlu}T-Q@T490fP|f7D=T+Iu0RS(%mK9UGLo9 z=Xu8WKYWhixL5W$d+)W@Tyy?nE4idJOkTu=pYU{z z&cc5L?ZlPs6f6zx9Cd6AP%=7pR%VuVX2!a=91Lu1jV&#>**H1a_*ibev9q(X6=Y{O z|G!VLS=tz}OXtzOg1g{ZNvhbQP=q?jUyO9IG-DJhkm1>rhp(JsSH>LeUfG{DZ0?7) zNxS6G?~DgC-ugsRpTGAGXQV8f)iZ|1#=ci!E(MYUMz_jNoZ zs;%wxl!)uvwOj6Ao3{m-{6d;5ZN)-L+uXOz2lUB(-+q8Uzit?MlHH<(KkZT&FP=Pk zk{s|7b)ATa$XM?>N*w+@Opfya{u#{s^#4DXv7R&rQk9+^EiV_YM{??sa~n!0^(&}d z{!z{_m~Vh@6lYHw92^`e%~*A^)%w(Vd^ralA4ny{+hWTT9&N%EB`v%+s8_X~CL8yu zSzmXdBhKFAj8m`SPqF2MFUinz%0t(TtT-V@U9PA|L3@*=fnL>;F$-#^NsR99ZjP9C zb*CwxGqxWud-r4CTq!(6vu6ciYICcUSltd1*&}y$c0OYhr|5c*N32|6c-iy(c*6Po zbpHmIev|cn>Fif2ay|?7kiMPS5IJ7et5m{H9K~Y}YDvNoVwGDh_ntg`DzB-zwk{tn zZDGNtm=Pu?diZ-H<^B)Hac4iH1ov<$_R3Jl3HONd^{SZN<$>)%y`a8jLc-vpmZf#f z;ZHvJWLpO_LAIy61H>dGeSbUA#yyN}^G9=sLj^{BHazB!jg9YqCS{%O^gQDxW|Gb9 z6P=rzbK4(tl#UlJZC-NOYGx|k>6W@V%9ZbV?xC2i-mGUm6Fx;JcE3NM^DvU*w2K`%zsT@ab~hYuf~v86P(Mf2c?H2+pHikpL>^b(4lJScZq zb=s^aa}Sb<;SFe{@r;ET|5Xxh8yx)~FfpP2QL+&Eq12 zsHiBE{iPkT(;Z?TAD>v)wNjZ^uV)VTMl7xQDPtXmjZxS*IIr;?uJ|QANBvS};itSJ z&8Se&@4~{sz;FUnku~i&ewEv>O}jV!rOeg^L#eO9!CkXCc_k$!&l0qf8D5~waFKP} zpYRkz#R=Ni>T|um!DIky&u>jjSFO7fO{?ux^Prg>UA5UBH{~al2*K*qw4><0hSLcY5zD`EAy;i<{)KSo~tvisdG3qd4 zzVo-sdva1+S5FUl=fGieUbvC0$nL9m-rjGIf-p{&GIIF$M{S=ydj@malOjbH?=+1k zOf47CsDbtnNsD)3zJ2@lFVlfQqj=}kVP4s^5D6Oh&}@xz-QtmFQc^Q-FHl!6+9{4W zbi@kq$H{iUc$j(4PPRLxqwjx&3IG26`#~MenOEx^EjxSAa&~#(-@k9ZCnY&gzP-Rf zkkn#JN=9b8Ub+7@JluE8B26NQS{PoZugF#~Zq$8$>~~`$F*S7u4w-uOc%`eU+t}jb z;;7S106ilkyZ}r*;d;fkY`_S0^YSj2?R*>aZO4|-yGZ!a)6;9iC&tS8EGssj7P<>L ztTZ_<#5JxA72M#rx^>0R8*aFLxR9VUl+Ed3*;ruEO3GF`d+qDjuU7n&$*HL}b74wf z6B5>Mp7T4dsrP58EeN3@Gqo5*Xy8`=NQ+-`bsMj6E)<;C3li~#bai^P>rW#V55toc zT%<=S=4iI>4rs|i7K05z&%j_k6TtU1E>2^{)@-b-Z=jX<#*OC7e4`Iw>ch;JZDc9e zZ7)VIxc>+k_Nco5w6d_0ttIaVe#s4Wx6OL9@ro$et6#{p+=5F=_)*g{GwvfN8+C+k zkQSt4`H>$^`0UL^XwARFXPbmYZJ4v!?2@3d9x{m7Z0DQk{V6^ZD71D**(=X9qJgYsb9`ayq;*rR!D4O`V+!5Ta9X6T{8v`YQ8rrAf(oxZ3+$z4dpQIriWN_hO|{W`3^sl*PMb(!;a`h z7nJ~9VXNGcUxVdpX=zcebm93-#%_3Wu!$OVSqR|ktJ z)~fcUsdBLeRgy#yBQ5Q_3)D_t-@e^CJM8r2GHfIFIN6l&C8X)7C)1LC_3ApLUp^%b zdy<)k06G?ym69CG2LBtekf5v9jXLAo86wmlt&dkm2|IHQ4GoQ2)m`Q}TQjj`t2jS9 z9(CJpC%DP$uVzt-?hRU^DlqY3VZ-QO@%*2FQ$Z>5pfNxy8Es`@+3myOE%hidz zme$t8v*UGVSc0JA=s=^T-XHIMe5^Y~_OFtXHo$UK(*+4XdiW5dQ}oy@%-q)2R^WK0 zp!8t+b19U+D3}{U(SutMHrqJ-IRmXojCYEix3LzFKFZOoL~0a!)d6h`pJg`hnKg_A zNo&7<|7H%Fux_M+fZBv4y4}i--so4eBqAZHul2zOm3YFeOA_n40@hw^>yWtY5Dv#bomBWC0nxh_XjW~xOJ29R~@ej6J#~?TD z&*UViXG_YOPzzB!-$(kra=tyl?L{V`bG2n=$l!XfKAk1WPazg03md&QsJA+v@e^AWe8)sf=Pi-d%H zvZ9Cj6M8T-4%7_9PFCNx#YEK@%D>ch;fVSszty+k*jV+s(?}85A`EgWLA&&PPxDTn zKY#u(qkEE8Ey;cJ-gMDoag4jJS2726MscSN9}4Q!zBl2)Xh;eYT{^DXst|HRM+YAJ ziJjD3#KYr+G~gB#XVY@G&%|0V?y0=|SLAZS8+nmqIg77MkdRK>?B0$2NF)zUM=CKe!B zFVw{b$laIgIQ(B{DBke09x*R|J!AoSr_aNZZ_F+PMrEdNVDNfqujO9OeOfv?cXVBg ztp!w(*G0#zT4y$++w~j-dMkr@Pa&0O2=^yGzJSFcdc49tp?7$A_|(QGpYzS1TN8Sh zPb;Z9p`u@|KG^Az3o0GEn-(j-dVZFc;Gw#Du4pcD^fw7tTiw3zb)Sb~&pIS;vHK{! zvy;QV_?d>AmYMmh;$#~7zZ)8ks&ZNdUXFAorg~@*BsrY#n;su_lkfKL6}$+P(Wqwq z4L3#-S%!$_&?zoS79G5XKeb4i;#efAc+^HJ3@s03G@x-t2S7mM*uL*gi*zntD}VwI z#K~Bz^9@>?Iy&ZIN4bYtYb$BF^FSsq{JAcPV6nkmohmkRuiRV?i_$q#hqck>quu4V zpR65S_ty*-yHh0Gv(wY>kg{vG0R}N~cMEE)gMwPRQH#GgT3WEaIXkP5jC5v8o-D1i*<7No*L>Pv7#e(nH9U9Fo3 zg)5o4(-DA(E;;Um$8naW(6?a+iK6)%zBPdE8=xQs>5#wH>>13{Ylh*ixe6oinX|bu z;p*Wbs-mJ23xRAlQ5AP`a>6{4nw)%~I)DM}uw<=lS-I4P;ldTl^n7!7Ps>JyLr?Ag zOqISenV34rNvf-Pb)J3(1_ls>*3%!EFnbT;DGw=;f+r+2vLE#ll96b_W0~EmUPTR2 zEeci3$71*??=ZFTyM8F7QP7;v$5`@j{w4;H0mb1e4Nd1w%Zit)SSN3WVY|#7w`hmQ z#EeY3-+QbN;gDfwRH(X4Md`3QEFMHH@)JTRPQkhD0*%`jSaaXYH=h<;jFp`3ji?|y z&OP3F{ziYchLox*MV;^EnY>17p|uNIhK7uQ;U%(amUr`^u)0(hwM36C62LITbj zF5;m=CoVh!J`*Io^}%vfRb8DD>c{Nd+-uiBfNXrKN-dB!O8|#|@b{;OvJN;=t4V09 z1ORwnSs$RojMyBtA~PLG*H8eI4wuF&gpzk7 zjpyLtlUg<0FuF=X@f-jIyyeNn`6hZGqU!%lVf5@vzr#H}Vm`@>Bs9V+G!~&A50Om~3#6*1k`tQ)Y3O}VC_(?z4 z%g@X_WyQtd)1OG~&ro9LH6Q-XH{n*-7R7zIQqbuHeA0O~Sige;r4E4hYMAk`t2#n$?u)F=ll3CqIoH1)Jp&lMsgWkLbwUtQ~wQCO?Lpi ziYnsIx%3RE0x;q)>i%Ll|Xr zw`QA>O1?{>piai)jtjhLjzzuX%kero;VssMYow&U1Iu$TLD`#*e>*#+Jrg<$Pu+~) zR0eFS+#$J+ih1T9UVi}B0qX-N&S=@XsgNpd+}6F6Wkb3G1tUY$K7q1R(qx98jGgoI+?=dnOphxQLZJ6Sr7)hM%@8qo5bgGdsaKhLPB z5C-6?yv##kngJlh4XWvNQc^`@9N{g>)jx??nmG~XBSo)`n+3KivtlnVZ94JK_;W9* zcLo;}Vw^!)n6eW*JX+4l`1Ol&T*RX!ORZR!{At+4!hCxSMzp&|rN`+}SB{n#wa0-D zf{@c+CLu-oaAe}lq1QcHcPMdmRDHfJs%|DotOF46v};A`c~0T$2;1*>Mg5}X&Be4R zex_)VsS(>_@P+*O^TJE;)F@Y(B~m(-xz%6=%1GDR)}I%TeF<*!@`e{rc-$-=S#>?} zCKOF~B&#d2o>9)%#{;EgOGHs!y$ukhI0US_-b|pnZ;fF=7cdBdx^EQ~ltGOOSwgpN z-+l|lgCgS65B)r4{^iVq0|=s@cJ83>e1T;_$J%v21GTRA(7rvA)BE;g-&au|qhn*+ zs^4NEEGDD(Xv}fqutAVoQ|UjnL($0%n{t)?TuIY8W!Q!V>PLDvEKZnbJB^1r!~=+0 zn(GS{5S`EDt3_5c3RzdB(#763%;ObrOaLNT2(6WP_s9wb04IqBhRFpmwG1di4!Y%hCrxY< zNQDjP1OGUpuC5cz?ogiIg#`D#sC5e_M56*G>`%e->G$s^yk>*d897xgr3>*=PEP!Q zl@UU|m=eLIU5%=(tvvuvpi*XM9EN^jx>bxm4FFUf7$Ph9am1qhazU2N`6ej|$zf4OYbl4*hIZBYnX9JL6pp;S z{NLUm#C3jz3l-ZPGpG9#Ok7-FK`kf%fQx(x7_R$qcz8G$z>IMqjxLoQbODqCUxE5F zN=H^FKMMiWiLlDcOYh1+jynWNv1GSjElS9dE$L}M?aU^CnWv(Mv&4q&(e;6%Eliy)O{*nr5^hwD$K$Vnj{) z3m>v5jsbmcg6vC>0lbwm3u(N^k@42{HmHwPJXe?RL2-vv1su=WcWcXLxX={QUvBog zX#8I`07&y#3e8r;cnL!A@1I0rz%mtw^D%9)0>r?IA;ksWK*17 zT`)!@(Y-EhI-Mf)oP^Xe=|iUZEj|51Z|^Hcndsku%?TbzN?rs7_-27o=R9nM#^v4a zLQ^G8&r|y|*r?P6zM}3&8<280|2!t-gy5>sWJf{<3BAI?ViP?8FDt`^84bth@$P#= z2uc-(j4!0DTsQlqM)eaf5@(z&aHS zE|=21?&ueFdxQN-ni)}pAbdEGHG-{v0t}vBR5TvoYTocRLfs*w`XB`yglJ@e5oO8+ zEwRf9dmceskgmnLdwTHj@N{5LAaX=iRIx-zXz1@4%c^D|@`$#v;UwG^Csh0pg5Kwg%ZvuNjB9A(m8h%LEwWm8$4>nyY z0dv~ItPKpx4D<_cw{cehy;+$+qs7N)vlx+(Xbor8YyQg6=i#tgIFJunLCe&qai+$k zaO*d*+`&6XsXYDWYltufFxNy~p6SKT+E~nXhfvN#)<>WNCxa{lX@f67IqJhNI}?mh z+`ISg)maoP;qk&kQ40tN>{gMee_I^POR{uRmg2LTe9$3wenR0{1j_$}h8Z*Z!cXD; zufmX5FsRPd*~paCt#Kl>JWA10M8S%ZtX|!>04mIY7CfKd%0>rOl;ob`2B^_7nw2hj zZiidU_|K%4f4S_MAS-#cSB$m>knm4q0hqH1wFHq2b3yZ_7`PYGpD3V;vbC|XDpi*_ zEv?u6VPo3X@Z#00R|b^kG5~ag$|#?yO$P0Yt6nKYJ@>X-84UjQ>pr5-d$iPw3C&v@#iU%PxWe}x@5Pm^N+n|5HQnM;c zM5O(R=x-7Q5ch^!^QjD*jmNaCeyU)yB|l@;syd03dW>`70=G$T`csj45a)wj7ZRWp z{Ll>P!}JjwCo>lKUqg-+n-v&Ycf%V3Z7h(c;Ik4=PEHw-Q+MIVB#usZSy(_;t9SH% zTl5qo7Xz*jOs_TGfs2KZ1gPZ|6izjKweFyJAIt?z?yn5>Os};fS9$K;xpRw^wcU}* z^7D1n&2RZ5e6+NW5Zf;~%(%6#c-XM97&Ly#o2e+|2U zFm(`GD)^~(o^q~gmbr`n#3u47$(VVYQpm|-K)4Y1>Lnzl=H})y_S2XDU7p)@L8>U} zXP?l}>t?($ssXA3bsVr(`i6%2dW}~g0{acgTt!I{H`2HOf(7s&tPdd36OF+8UxBQ= zd-raz(3FfV0^hH0YW=G!rMND9bi4P_b*G(p>KG3pcxPI3f6Q;l5H`Wr;KuQnsJcmDkJRAflAW8NZVI5!q@w^nDX20ac8aiOA&1=dS{o%v?SRHaF zVQMJi7xD2kda5`%Ib~jj9fXv5seSWND*C5#Kc3r*Md zj9{R>wlPZ0p1?A)9HFNrN>7rJ;;_n9UteFb*>DrNp>2o21up>BF$iixd{9spuX~}- zc5`Aum633dM-s6gwabUkY-Ja{Kp_gd!EH#ne*nOBrf9?hi&^zp6MU!njyNgM5KvAB zQ-1Cv4luvS@&}0?eE^yMn_`Y!bi67|;O~n<13@cK9+kxHRLEza?(5oo(l}$2dh$f| z1MrZSnwlNJAl!mrJ~+(=IbgmfUCo-l+?H?;JUMZ@LM_|`iRhnR1B>l8Izuxa{e~+_ z^7(U+k?Qsb!LV$D?<4;P3(b$lBXX?Vao}k8S&ss~HvaW%`V4IrbMswsk@?z8BV3>`we9LV=D35w!HGH5e$Bq~8^) zgYAU#Hi(M9{aH7`p*8tq`+P@wg}2t2WYgpN3){W)5lkoaAnKYxzsUM8o)NJR``lADK->Gq_`AXs2_eqNWW&+|q; zXzO{d`xc@{f9ZgiH;A2W5>`P*b`eXse;KEml(|>2t50od>km86S28vlRJCWs3M2Z7 z!lL+-;WDraKS{Bbz6WX$PAtp+7HA`vX+H~yIi{zSh|z&KGe{`_u>o;AVMf8jT+$do zj0xaa?SLu~V-BB;ZI<2hP^vOPF50)5gP?9g{FfpG2N>|+#>oL!OA_554Y;w=$v*SD zgVQdMu#SVE!}*Yjm-^1^S)NGAX>WFbi!f1eQF>-3I0_DbKqkZWiGMJ)>$2FFK?ZeF zKu&n`J-XHSi(X~1jPt1lYcJ;Dqt}60EY`3654oeHM{R73P^oOAAA7R$ydYn5zyeGI zbTEXfnyFIY3-b~LdayT;q2u!V#=4N-UNsQ`;;1;gOE1pf#A3JA_tX81S?0+T%=bfP zhF}cA4hwym^fb?~odSX4p!3hIcPD=|mI)?ho&_=JTExhHWV^|{?@Owvlhvo=@#wf# z+%(=d-?E`jn1d?v+VGmxNFr%9>CaTs2VNPz#Y}A$22xLnws{SXlI|mYCv%Xk(vxti zFG^fhx-5k?nRPf4Q^|&0IYq9tuxUq>AIa2OlK0_d4E_GTI+QsSvmUes6Pio{k(GEw zi=b@wL%C=26CB$!uj{Dsqi0z#dRlot3}VzmXF?}vpmNWjUt$dT2=2sI$uSreXD16{ zJs^m$I0(T^&&D~98z1SK{>Z;s()-%IIU+{F5mUP%`gEoFu5o`E@+5owNR%?rb}tMp zC%{f7m7U;Of^}?r0C>DTRVL;Dfb3#dk_54$3SSsT?%*P@XVJUCvAsN3f(9nXe1u|3 zxsWlnV;bIwghb{Cf7AL$I%5SwXgp7X!OS2gCdPUIl0Zy)*%98Ae9R6L;)l#*PVy2S zy^z%_LDvH6jjHFZ*2p z#oYQ$R}m!?=!*mh%7{sNhm}=iNfg|0D4xt(GIx7D((y|w=21@1_p!gyQO$;ja+{9T zFlIqjoK*&v@(~Q;zMrq5gqtL`<5t6KZbKd^a9CA^4Ti7jvd92qZ!m}a-|NmO7cY%y z=J!C~d}b=FNXFq+Z#JEOn}o;hm%d|2KT?k0Cbi=}$Qk$y-3!285ZT`UCSM@J(!m7X zPtWoq6xq27zCcxui0j^c@G=g)}VhRFJeyZKC7nx2{Y6H*`msH~ySJ#=+-!EZ4@OpTPg0xd(d8C)g?|A92YkxiEb|X z0h0-y1F|y6l;r`y)gi|Pvm(oyrhH5mLM|*P^}0)F{Wro03aZ+>V>1{BZ;X3z<%xnH z7?I;LuoY`n5!9vI#~tslFZ|5No(3mIAEa)0W)LWlgZq?XCmUGDd#i|1gYW=haB&bz z`H6%?fzzfgVn3a2_=z24J3@hlbQ2CDKscL53%EWlz>GT~tGPEKh29wek@H&V0^)oF zN|*zmUK@A>ibW+K+iM+t9)EhgkF+Bk?}2o}Hd17+w>DZ@0TmUg*pR}GJVe2*CuLRr zjTE7!z6^wdBTB1#XC?9?Ws8i6N&oY82LEoa0;0w#qWM%%95?I*W>oQ@=LdQRYJlZB z^_q{nEVClWjf;y5Fc{V(l!9ph7OB90CnqOosMEfE^J;361R5VhParr(EvgoO<<+kp z6;_^w-6&H+t7Xq{VlqRcuE%LacA={fLO=8KxuE_(g!Tx~Q$tHecRcjNwM68Ly1#h!ZM+iU*R9+i{$A!wsYd06~d} z&XthRxwF{gE`|oO5+z_e|B~q~<>5GW_K%#2tcTU~J|LPXW~t8IbSnd#4gXiaHb$a1 zgUGSt=7ww?(4HDpebDpT#j3f{_ei}(ho4;a>TzYb zN@mEE(bOD!Ihuf$_S#=u0`h7g^0J_9inWe6{PAfgL&RTn7pQo%JG$~TC0S^816 zzFxXOjY6cZ-X97>Xiqdd0VZ@lY(f=>&ICseNS8>%5zMrjdWcPK`y-vGm}?P!RuV4) zMzB@d)OqQ{_~y-L>KI)4%PnoVmG07BBY)@3l6k(YjHW6tVj;zc97Y8|Kp$)jM6*Fu zNu(DATFPEmYJyNqL9^|WF>dnf2j%jhxVSioB;o8@nhf-qRn&sUZcwip{cqT|Z4&zh z@uc?_7|B69KsjUPaO@Vy#s=KkCtk1P&2RO1?oQF~-!C&~T#4(&C5Xiv*3*Ewz1 z=i)T!y#rHLkMwUKSPVAD(yn@?%dReza%i(UR>dG=QT7@#O9i79a?F>nUu)mt;PcOg zD!3gZ-DWV$7}^CxUZ2+H%E<4(xk?@6=kHJOG>F>0_lUX9JO%8E6xqh$pO~rm{4ZX6 zTn#HR(Y%x@?>G43sl&&5j>!-M9yuUk!PU92wd38ko`tijH2_ID1peII)|RZ4qq*x+ z1^br5udW7ZXe@VN1z~3nT6`vlO>;oABl1F&6v#r+L1Jf-pr~x7iJg7~2iAHa&Jpv= zmoE>jtT?!hI%X$pyg@`9SQ-RKx&WH=eA#mL3~VtYD4~#>-ZzIZ9XFiF01J!aHrg7s z?aYHlDNXm?K4$JODQkc%O1`MJVQ1UTb-1Jrg$e>OY|u( z02VHg1^?M%Aj0jsWXa#YWrV>OF&`=ZYfk&Nl;eCio8VyaXlob$7`3dM&6QGyxeD9U ztFp^cgY-NfKVW~-Yrh`WFH_(ZBm-TM>K`g?IREN7Vmz+Tocb&$iK9JiV~_P22y*+s z8Tr;SQqyPgV$}z01Y@JF!G9+3CiDge7d^ws^}!dtkNanFuir#nYr=sjxgJ|Vam`?@ zp)HjS0k^8R0cdv+=_hVAP!}3^0UGsRR?53F>CbZS^&cYw!$=}(@Y4H!ox9{-9;>v^KI5hS8m$|Zxgnzx{Xz1*@HiX;v;bTvpTzLr({Ka-Ou{_vSdKA#w!j6 zw*S@~2d7$*Hkn$graEz``t+VE#`%6m*;}Lu93!{yVjhhuL0N@`Mm36zKoVoJ{PKmn zcY3Z}fbC%(hpBy!$PJKV?_Ab)mAgFc-|(xxt*Zv}(O3Z*pp|_1#dXOm>peTFUyl}P zRIPouiaKzc8-JPw^!bS@4LrA2C&MXYi5P{Ug*q{Cy|U<>5}+7?d{Y$p=FAIxZ51k} zx35gdNZK}JIZ}f7swA#cY$@0?5lx*ZM@st3J zO@zk+_jvN9-oMWbGkyPEhta-FgCS1=L(&KP<4Y6;!Uzr*hiD}7%Q_=|h@$oO<}Xm9 zMQLTe;>(3OQ%gY&e!@O(!06AxC-;aUKuA%^@=zEH3_U9bhTi=pMVmRUTO|JPms|Ot zVp#^m$COh(!Z+`Ihi~$~V;;{w3dF$9RcVhMKu>qv(#rf3jW81PVEFR%oA*G3K3{3a zjW1lgRo_uIp^CsgRX&)MzMkx-SsNGI5+OTIB= zQ-}#6@rwZ_=fjxlro#CfxAfSwPX8!dd9G@D~CJTV}R^OBIA27u~8EZ7`(!$ zTFbF&?*5fDNz;E1#{2h`A{1uqAJOQJ&BV}f*T_5<&m`eA9$3JXPRzPs8RBvmr6dSX66Z=JZonO=LZzkjFMqV_p10 zJ~cVJwuzEfxQ7UT)v!AaD7Gzb`IVblQJ@l7WYh7CDc53R9B#LONzbIk2 zFAVr{3o16tyb^U~s#^&QF+IxEi+zb9fL|e3=XC!BjA351n)>E|tsS$LrK zUi$zOeA}CgcYY3R+Lzs<6=t7hWr+;dxrRBhTE8N1M+8Le$ zo0NWRC?I(B$m`ar5}__*#jd5sBwG(nJ~qW<8yV4mr~o&JPC#ahHzt>63D@GMwHIpP zvSb61z9HxY=(Tps{27=r$!$nwDbY~Ao)ZP$>R1cGu9lE;;QQ|doZ$uJWhzOvQ#R{_ zP#cFWb~3m~BDf0A4HZex$1gD^Goc>ljDEP9WK~*OQEp`vLXG62Xk?v8Mt^^Ps07D2 zPNrtrP!;GmWa6@pA=F4&i;joueWCkUqTMQ$#EX<<&mTsq>HUizuYce4$M|E$Yu(x$ zhjB0%9>Sntz4_h} z{nUWWsc|qN&nRwcsQ;1K}_&tn67# z{`T!#O9Xl_(~!k)yfWXAB7|(};`GwgD#pP=&`8W&Br@!SVKdYyoS5-E>Pa{NxZ~Yt zlGP^)vK=EASNDEcczFNs*VJmZj#yZuPV!9eSQx`s+H?-HM-7EQFZyslp7#bygqjct z$O4!tX#l?p$JZ16sAz2Y^nU}0V~gJg)A9~wPM zUSNgj1KiJ4FZ~MUD(2*_3nu{<80+BP3atL>`MGLTJOUnXyT;l}o&|Xhzw+HsHfgo0 z2!Wg22ljH*Sl!>FcMp(}aG2-gaDWTUW7?l<(#Je80G(LRK{Zw`G$8@Ibl>q~VfUz1 z&8sC!ng(%ic?_q`3ug`=~9OA>T+vTN7xfJWF$&Ykj@gHWJ#rrBO<-c6Drp2iXFr*wXlu z&3}XFywya90;exwtQ>2Z*D$!|A!;_PB>HraO80T;&U=6)5xtjS z6LFda7H5!$j(=$6skE^S$G6v-t9bl)2#;L?uPW^Go-N7@YJ6B7vnqzxkp(`* zOE{d?y#@Q(CJ**zHcgq=y)oX@d%PJ@3_`ZbxjZA5_Q+JMLh-W+WA$=om~*N)Gj^By zWOuMebh1SmfCDjXya?G!z*7$Uhu!jBja+y#6pQXS0gsHw@Z>-_3=Y(REA7QsqjU#f zYe@Dldf1NhqWf;c!Z)U&$WbREM9JN(OpY~X?;tP+oxWA)6v9KBy6&QIy}~(&u)is& z-&xp^!u@Ynzy8mUle=l;7BQ>O71xdAWIt+|LNEkJ|GUNoU$HEglwG3^YBb&GKUwua z6t*WOsVMSWx~h_V94Gyk4w~~t^hpIz)dS#Bl+Q3Y9=vkEG9o(^L&XmY_#%m`tWsJf zw{Bt%c@?7TD6A(^Q5P|_xuV>%yw=%=#UHAaCfq|HQXM?=L->6n>`8ykk9JZQlm0Bu zi=kJx90g#2HsmebE)Wy@{7IH zcm(U$y<1o3eAao7|9$l#2FJHwzdqi2!AWyFL2!j7UCA5mw82XW9P0yT~ASc}U7#Q}n60kf8`u@Ewb==i!4k#siZ+(W=)<2(m0tKoLA3 z3Je#za+NvU=SPLyMgm%0N-?;nlLX-7FIIThzs1&G7Gg>^b|JFsPdk&Xz;qHmsaGX< zT!73}^k-O}pJo^=W^XmB!&VgWJGDenpo+KIUqBw?83A3X-G+BixZhen-oi;?EylU} zP3}28+zYpwPAh9p`(f0-5HY?9YxM%^fR2ju=XKT7nMk%BQ?h3dqhcW%a)M|l4kRRzhwS_U5W#};)+gy(CcvMag(dGp^#tRDERA9ImQL=pT zZ>uv)b#_$zi342>A?Z;mEqtaL0{xq6-IFhM8hqce^jv<=*tkaROg)da0P`3^7Ec+5 z{wM^I5|qTM}T~224$Q%d=Z4Z0u~s z1D$=n?R5)#-)+_JT)5QYw871K840REn48dc6UITs%{|gzUm+|XnJH>k*<9uQ90ee6 zYfo&5ZhEB|S?F%=nTaXET3q9YdqrBZRx92o znzb4ruMp?5RsRwsAhe*x6c_X ztZ6a{tb!GAX=F_uwOPNPEZDkQ>n@Og(xA3*bG#enBm3XB07{R`AJgYgmp*OIMv$vm zed}}7KtsZV*<*<$Taun03?vY>ZY$1|#?Khsy;R zxr+}lfaZE+ry3KO(6rxEit`%6rePhc6Pq{N{TJk{;c&-40&M=EI0$PV7~3+s+B`qe zkPqQ6aHRD-)zCs|o_KJdYgCNu4L3Ic>?^bW|O&&(jKxR2S2)&R4TX1R>#R7iwfTn8z z(kurJ=4)_JNKA?B-`&c7cMFJKuk#Wg$;ZlP_@mFY={e)FnP%&PxiKF!a?oRBdS3|W z)a=y^H2xq+<q{v&gSPo{U{f;G)Ef$x207%(4wvd~IcAayYDkLK+ge zJB5i+esvFxjkP-g8!%C;py#gpB zB8+qoQY8@f0k;aDYmF#?_AA7$gPxMwB#Uz}Yqr5ILSSvIEEAkYo`Ulc$U&h(QA0TO z2i@NwtO5(|x|s9*kVIfiTC0*UDB@F!uI^(b7`X_%_dSQ|Ask*Pt}%@2%neChuyUB7 z_l$s(j_veUvnMXk?UPm`sw=Rc<0?%qbSG(~ub@kGb==^Ad;pI$6dL3%%e z+zJE1gHV%OoL=t+kQE&|QSN2Vdy|2I&( z-v2M_GTY@T+{?R7|8#KENvD0hT7z_668vz9j^%c$wAY#9hp@4ob@mv?1mF>v(Gysc z&zRzVBNGSb#B07XNFisfF$OD~x#6fwyL=rq-)K>Lo>(Kjz|edfST^>UP~?e+N7cG% zXM8mUAMY)|G9m&DC8KXeNH?tDQLdd!HCRUbs7V?zv$?LEftS>iGq<4?ZgtBQ<+j_W zl&AXdEG^%-lQ;B@V{Aa%g3aEL5#Qma7<6@5R2>>3&jORzU(dp{cm$R^eTzgA$_lb` z_b666@Y;^CX0XV!jay2bOXR*!5)=VW8=j1Af~l3yNF;#k2UCd5tO*|Brk zNhkL%gx}NvzhDAs($*;JwRk#T;k=D_tQjJr&|ro%4M2}TCpd{we3sNGI2JYq9YexP zdxb>lo)1rcF@%T+r0hsu+8<+Elcdgh3$$>)C0Z%Cr`zMKZ&8$d7L>j^;D6E_gljj$sl8~^>>4v(ZaBp68SW_k)&(B+*<};I!w=tzh05RijO0V56O!XGcaJp4ARU9h3YH!F&hkJ`%MAML=pOW{e@{rAIAX2H;tv=R)#l+j;i!R@z`a&$(us+IF#VCDPs)28Fp( zXk8dNy9j3#{dLIU;4^qXwIDjQKwwR-Zw+U%`PGiSJ>hz;Zh=x_0=G$WFyuN0n|2k#^LdhvXh7DW zWsQP8j`T^vL5EuKN2q$*%u6>^?GQr- z3y;-}K(|6k`9XpX!pQuEY!5CD@Fl;)Sx1lTzXzD^1U1m5=rI)kIDPEhh~A`~>tgWr zhRG{^>k($;xLTRgX_=#~#v;U$K=Tu)=LDeb&CC5NQC~e2ZKPKLSs&?(~!20k;>!UySl(IAX92ga} zbGQe(Ragx{tcO{>m{I?`DcibH{WwR%jsqI@z@crKywa|mo)EBdt?_(~|6`}p%3PF6 z%wW7qTZ$#yv~O@+iKUsNmSZV`?YnfLk2I-hgX~FOPdf+|c5(z_Yr~EfK@pvbEA=xz zu)xE!8c(*QTAX!{TvAyl@L#b6Z|^Xi-socJ?+p9+K{BPt>Qs~(Er%H#&f05XARY8x zmDSCxF!I@_%s~rF3uA?dep+!~?9EXFtVox-7trhjg8tOcV$Z3!#%P;c=(dSDNUEqy zUfoT@RF^(2I-7H&6e)V>-VQk?tdx6hFtzS+&6k!6i@6f}7|eNOgEKKKi1WbsRTu_G zd6W2$R)ytLu1l>}rC6p!i&hd>O{itxBb3HPLC4xkij z3amjC#p4$RD$7^CXw2#=r;(R2zNd0Le(k!YjLz}m@C)m4VZ3TcSn-V5SBJy(^kMv& z1Kmi$CyWpH1-4z^uCr0aJ`GB?5>vWE+wJi8C<`@sOYp5p5wA!>pX;_U8C4rJ%>BOe>#@eo`oug)_VIQk&wJrLh@^?pMJV%2*;)?8C?c3E41bA%& zY_#3-|98i5rD@%UXe{M6t7g0(%B(y=_nVk)woZ?{oi6_kCvT{E$93uX7?5JHTp!rI zne3ot+jly&R$v52#h|0XJCsJ%l(W}xYKd6LYWrE7EPSS7L zIo>!1ETiG#3nd)5$>Kam+XG=bh zjH&JY7hA9l>(JmNmlF$U(P!Mk?XkZ4)y=KeJxYg#Q0?OA8zX!_hLZ`48(U9UqbQql z$mz(;xO-S{$7bW{@V}#2pf10F-*NzRX_51`&un^{c$lg%qpTFpHUsai;I+KD&1ZB( zZAn918&=m1mJu`F)Dkr*(GPYFGzseF6Q)01&Xu?cS%Xrgjgvr5e{U-NVZ#j1U@jW4I{w{zIm&GHy1rNP5_inWhQ3AB`uU-sGua0uxtp~vn`l5x z5=N}dw}vfV7aT)@5(Nuho#L$^P-yfnXZIDTc@qD-q-J84&AN`V&k7qKuae@Xhz~GZ zIalD8+ddz(`{orp{Rt;@Rgs&?F^EaDVr>SC`%+7!rj&+v?S_v*+lbSh6LCTQQdeP|lsUKZBHe~9 zP_gq=)I3aw=e>OpNxV}jIpp=uh&&gTJYA`YvTFy;mq%|<*(tEyl>axd8kJrs2G*Gv z|I(h@b5hA=joEp;Pa&H4_*Ir_6^H?OW=0la8%6^66Q^VPj_=&D=LX^uH~r{lO2_<* zwM{NhTgC*ojIvNf$ymotwd%{=gQ;I!V1^`4{Xc8?#S>~FsNiOz%GeKrk~w7G(5%M1 zu_*y)mLg;`n4pFGz$V!nn9xt@1LXp~1fIpilAnosCj*=)9kK2^Gv=o1QE z88gP7jIDZhGV}8x1=Af+LjUg(=7U<*EstlW9<7pIqobsk5R~+{g7_SA%l)9WKMVL$ zx$RGm#&vMYY^c{1?OoDTxQ_R~Tt>?)>;4MBdOgSEd7kIvJbmle};a>3x5Ht#h_>XT- ze_DO5&_ICAfAyo|`s|2~L1M0dS|pnt^XK{jb8IY0&o`@X2tQCKbMB)3Lm9o8zH>G) z+ur#|zW&_7|1xYcR^Bt;dS7nGzN+Djl@HykcO3QXu9k-1;O$84yy?+AvAtsWk8Vpe zPuuP_WD3QLw;-GDg4a(ncfg*SGn$P}TFf>~$W%JYbxzdn-UxS?~K^JBu&; zD0L%_byqv4P(@t$Jrpq<-d61RE-jpN;%ApN^w*c?OtA@%KPDY^R9KM>IcLcz&gD{E zf0);Fu${L|FShyP!`H-fRo=2SeKPveSF^Vdc&U?3Dk&>KoJ~9*7*7kBU!R97W+c1BbK@`!fwaMEenq-S@8?oY`bba1!^tB`L9o%*Ogu$)+u_aMo-fO{B87@>MFWEV%3M|wnRx?(Pv3I`T zRZ8oyX-#A-CF^BE>=hnHlc~G3Z*kmzV{mK2glXlMVpr59rR=s_*zd4q9`uv&TzRG3 zVmaUM*~lT9VYz{lCY)RN-hGd(IOR)I7V4TUjlFf_zY1suvxJ75TT@PVtS7#QkmKR2 zfvrBs9x)wP^nFP%f=>Irv;jPKb7v@NS=bhqnMi99VOI{6yDH zDFOvVdZt>D(G|v$9K|f%ri2gs<9t6lI-b+}%2p$6#~w(z?Mbg`mZK|cNP+o2*MT!F)3QZOr4_a-z&VoI#KGCX;o>>pJ$zKczdc=friD^Q6T1{Gc0dPn8XzDOrr(@JPW&FjncB zGQ~S>+`bK6tDKbqc-|=?CNomOI838Ew}oxlf9VTeqggRg-Ks~nRyRjHil;py*c=UeZ=<`7*P5mPJYU#E|RUk*p32JSxAB9udkuo!t@jIPCYPC7 zDVp0=6{1P~HPnC-*JkJCx{JrO!`UpW{d0dtIO9uW2g(ays@-}9 zC~+I{ESUz=Vm1w3^FSf`N9ju(7RVDfth^J;QucxT1`jPd{W(c1{#8A(n8p&16ab=M zH{Ypu9UD)?@dOln|0NZEkdGqAG>t?=iqWd$k+Iw zWU*w;4Ori$Zj=SAoYe1oG`|JcZiNAmM@AENf^P9LyR7f^@$UU6uDhHEg9)yd-_w?? z=V(aWToKV(gXd^_$|;Yefl;;Upu|M9rUg;u=8hTd|6V?O6r!)Iv!)?`S$e!>(=LdO zfKu$bt739J^)E5AVzl{)SE06IHBN#!bK>3nlG#ZQir*CV_FG8U$@tycVaM7$jXs<& zBVxKm!|(31FdfFk`TuF@A=)0$yhVoA3E&s8Fj>9Z!8No4_ibm-WR1HpFAH~T{S1X@_IEXL^RT{{Urnibyp$Sfu-Y7^lTe=Oy=v@mO~u>*LMMkG$E2l zgH1JIH-|>5xFU6$KY?j5{koPy z8|)wR|G&BzSUrDAgMn#A@5hY@eRLJ0jYkj`kUD$+r+)zj)8)%u?8??lul9)SJ^Tba zKWARVg6`guT*acXzNa?zhbv7)9Da?e^t!np$1pAbe30EXe}I4m_4GvYRw+3dfi?us+#g<}U zua2iq-n)OLo*7c7c8iT)0Dr|CwN$xZOPKGS4rO-ubOYT#_?Sy&ayoZY>SW0TGDnP7Gdt-GWf*TjYeN8worB zEBQ8-D)a$ts90ZHxnaEX5&Dpnl1@G(T4%{NlK+Gt{}WRk>luN??t*bfmCCw`1AuW{&@?q!+W-XX8@Fkz-F! z-~O3NDZ4ZgaR3|Y1Dw%3LR=hXk2<+cj5|-f=U;FQ!or$ZLlW_+*{N_P-_G!4`BZ%> z;Jnmp5x-rjsnda-Q&l~aJa#?d>ih>kKwB$T3O7|zSHfj5K=xMy$=5@)kq_1r_d!^vA zu82fV@Ywx5&5ssZ*ZEUh8SdTx)iqlR7B6X}#7S)R?yDP06wnup-r0G&(9gd3kY6j; z$$J!MBL8ez1J>Kc8IsH9B;BXc=te_3^dU#IcH8cC2 zKr2#C`$sa;DGj(pCG_(|36$i=@-Tu#A3r{NK{+Mt#l%7r4_9i{2uBrT(Y`Gxj9-am zZ|q3uyoqC8`}7f`E2sDC3I%sMW(D%?k~d=zgXV}+sx%>gY;)NS041rv&wcSF;|d*% z@;Oj9RI7s7A>tgEI>e!@(8O6t9kZjy+ zj=tC9qf7XTMmPeITr8NJOj5OcZTY+#;aG65=mfo3AA_Me!=KaSTUrbC52`xw6r=jx z)%*BIs1;5X%WInf?=?X>0F9jN!eeK~a($0SCWdr?pe+H)LLXvrxt7^^S4g1Y^UYag zb4-4le4k1<5swQn9h89=roVS38NTlgV+aw5SNq2`p#Ri85 z9tsqXg!jvbpZ?Eb=H?@zA~D}ay<=(mr8TcWPB-d#kfM7OqZd`(Ipm>%XMBB#;qeQc zY&ow5a@2jyv6C{GxOUXilw982u91TxTbHs>m(od#i`ywzsyr0uG|L4Qn-31#8SiD| z+~Fj$$E>}0BmKSCH>9YtwLwOK5abm3L>!VjRd<7x8mhFdx-Aw+mAQCQysG^m-|eUT zJK8!XYDX4?auj*@ZhX07@KxbIbD44z$S1V!s!s`uCcAE+9CW@yHhXTor8J z$;E#Us|oQ1U9PN^bbI&Tsh&!pCGlE7SVAlKxB!jv3W}-!g(TiNm5tjdk$Y}5ir&CB zv}cC5Kd}ig1E@MIIWI)I_9s6IO)9id(=cci7k%S2p_VlMGyA&#?V3RbiN30Ut{08X zwrRuu<>#PQ$N`+0Z7fo=aQ4N*FP!?z1S0?+Vf`&o;Y(8~ynVyX{42dgJ-=fV0H%^o zd6%e=m+aq-4f!1v_4Z7*iGQ*%R%+RVLwVh!yvrv7ZuSg6#R<{+JU@ToTd^B=6#}y~ z)32Uap~UVqG&I<&B);abVD=3%RTi)&m+bX_nZ^gEs zsTsEPbcBFbZ%x$m?XR<` zu@*E|rBTwiJMwfS(f#waloX0B$tBe@Kix`$sM|~LK6U;$uiH=&NMS^fSl4i!L;TFb zn=5>C90>)GrG}u&@2lhe&dL{MGrsAsrUV%Ds_|utKR@)$HP^J|d-eXFfK5>e@b7{27CZHLK@Rerf7?RUUBi-k^z2a`F@-1 z?l;#W0~c?jUfy+;G>lAi}x*1KdffWu-zZui*a`?Xh*V>=j9A0QIUA%&%R9y?bCRZl}h0R~{e?K;zo+1{HC0o_efDBytr6VvT@R8m-E`oy86U(%iUXyda!1 z9}+;V%Doz1eRm_eHg$xLo{}lLWkPTUPS=gPtwr|oejVyy5`qEB+>7dcB=(OvXlEn) zwx;o@_HKqo8@>einf_4J~5LDZPHZ?+2>2Y@U*o?oCOAV~w{ZC8{clTwS+c zac>ho@l6#8hvd;0d&xFvMM%r7HrwSdGM?y*f#@pPwW1IC-H8|?q2tkM(8yg4~xX7IVzS_VX-K# zQCnKrHn!8LQSkmdiRztnEPid1=Ih^-dT-w`ElD$a)+c^v@SkoK_BkhY%L+A|Llhbw z@f*?1oDJJ<;qA4{!%=bcA?@22rEX^Q;@LV%Kq3b&Q;F?R3ab>LhQ1oZ*P7vQ2IEn@ zzksDIx$ez?+n_$y}{&XhkYqgH3oN)p%h9i=;yvoq_ z{0(+uS`<*GMp`Sr1}SWBF)jE0y=2PieI5x7%cpbvTQ+8*?%57R^^5+O%zhC)X z(RADA)#;8Ky(?pxeqt|$`Js{Li-6-;RjSZVGp5&2RQrUj@c32*HfM*459bb*6$=8j z(oSu!!tLy%eMjnipsn<}mQ6@BP)*-dtk6&hMX&;e>oNDmjp$|!9gl8wTf6Xs{^Rxz zDd9UQ3KDjiIMW>E`hC4C@sgbM^W7z%_0sE)-43ze)GAo;#6L5K;R7ZCPl<`zchri) zQ%p=2Qg(~!^wx!r>V=|>9Gwp@RCd49**&R2b;M9GGW~YpmM{J1z09wiR(oi0>)(>L zxT%M?%KuU$$sn6&hfhBj4c`xgGMwlGcfc~mdKhyL-%IXx=Ld3U#SV?g>fT2hssLQ- z_#ON@dFk&JYv=LxF!ciXdZ}qY38LgK<-+&-QQD(sIN#W1O3P*Cw@!`6@0>DHs10@q z!8)G%{e!;jBlboq7wLV5u0Q#vk0fWk{IoM#4TOkIAZb0@C_MZPY| zrdKtG6jO{LdrPqI`jTw9GMXkfof40laqM0FI9;E95TO zx4SYfZ7@U&smqn&a#zqwow6Z*-=i;1f7=_wSWU;YK=x&6TI8tfh11C$Z0upO@Q!M zaWBEzOWmjp3EbqPT5|txNv#tU+_aUuEoaXtdr|qkO1Aqox>UOBBght^9d=5%9d z;h)IbuA(gpT{WgyJ~YebsZjYfsXZg);OIB~bhgR?YH)tEk!(4>BNd`>{5aHmg$V}? zVdo%rNA!IHK;=Z|(&l^$9}*73z)lAt-!{g^rupZJV3&OJGhQ{haN=Id{W!>-V!~JP zMhB4m*E*w$d zpPth1Gi477Jhqpx(R+SZHGXq?mtTiZr8s#J|BUe)*I|41JsJo|7&9`06I2y0qd!qPg zo88{Ckp4s0_WH-9;q{^T`M5*8AJM}EB;3H0N~Xh<<7X?iLWoJt!{pbFmstYH{;a!g z`CdAE6ib9~w;-D2i3jvtki+-WL7)rI6_M0h-X-`6F}atWzOLlmNzFXsg9SIf-$MW>t5f+& z!iricWdA-GtIdC38obeYpS{+fH6xP45?)U7g zIJ8Z7XK?{!5T2e56wTvFp*>fAe)xW+XsOS9qFyaJOFhESRAn@^Zot_&9T_@KB-(A zOO@RAfBX#IRz6^nlcez@ZfEnE!i2iq=mk#kgMrFhs9w)C$@ea=JBsY%ol{h1&$#(_ zDClA+fD3MzOq$ZfqHiwyadEI;P*3pSw!4%TuOuouqrW-*u1)W}uRjr8ET<#%_ck9d z`79IIwBgu*Tx4<(?)BW$wOHql`h-4C%6C3TG@dyv|B{;ZHhejF$}84zoX+a(-+LpC zRhNV+VNLI>Kgh2)Zj(l3iK4++tA-a?R$XW8RNP$n?^@ch6?T?OUm|kA@SMochNe;b z@TL0`cYeB0H1zzsfGao{!j?=R5@p`2icjwiB zH5o4WbmaBJ0{QoX*lOv_1R>MAXwmN`^aoTT_=`$l6iJ3IPCuM}UB{P4MYGO-A*KHB z7`b)GPg7}AJ@2)GItQOKrXukGOxTmw91ia5b7Uv*d$6@G?HmN$zW#gs+bn};85 z_(B+Z!|(+m5(vpifSvCir_tg_DeSIW%c3St)X03D4mE8vLmWiFhmx zgmWXIj3>-{;Wb2vWf82y8!uwZzJLGY?gb)5XsIn}y-H&G%{zn=bwM=`Z(&KW%W$!J zyCd@y)aYn72Z^oF!gmN<`^4jXGTq(aIH$=AhF|udeY4i)LKyu30L3{zEk{4Csj^~0 zjoNNs6D5$lPba%wHR+?;*^D@D5?B3L_Vz3caENASBH4`a(mC(Fd?A056TVvTmIApE zKAnwto9OBgCc?h0<3|jO95?%H{I(S;t=AF}kT{sSU;Oi~MEL#T!>0YulnTrlvj57H zjnH(EU3xlv87{7+w z&n~$&PR``u;LT9tq+P}neZS;aXVj&}XSe)q#I@=qs9cP4V|INMjDy@utWry%18WV&*V2GPL2FopZ*i>z9+RU zsBNi-x)4Tq$#y*KAKOtvX6Yz_jdoOgA4HkIRFcABoF9WqNILT?1^PV>5_sj(@r$fBYvP3UgGs|18T;2 z=_+>Sfe#xeyc{=q!?o4A*AfKT33x^~={<)C$w(v~m2Iw6W@w{~Wd(Uf91yi#9YqV0xAGi|#(Ba-}uN6?o#r%jq0qio-#1)$JF~s&V?KLW#!% z>p212?qh!)WrtzW&4zRgxU#?KKADWv4LCo4g`uKIfH;T|ieN%NX z{IcpEv5}XT=lgfQ*z0f5YPVk_B_aWKRHof zoT$+ajOsdwdixW&>7e($DxiDAGhv6Oe$V!l?}c$C4S6DDG-x&k4M(_*!48#K#D@7u z=^QP3pNIfA5@F?#hjc=?RBg0}!`?4gsSAL*Gi3T{xIW87v_Ki|#By2;@XS~^9Am7Ngh>n94p4{TvS_D%4mk1v zpkDdK(;m_ndr8L5K@3AAeV$eTwLE&9#-P!GdPU}W!#jx1fd$pBuc|MM+|NFed2emW z9Ml1XRA*aw4oPO_X|zB>9m%p+$@&^a>wXakmIAj^vU69RhRh5*y65y+#e$>hG0U7M$cBaS4REA)!OHZDQL`cTfl6W2_f^FY(%iyilWPyS zi>0Ii{s)f1dS(p&g+w2O)|%Z?%-`G&J!WuHfDrThDX(2ys2V^cWB;pj^&UPDt1OUq z<5=oKpe0mc?wn_gPQDB&tsAod^Q1rYjzV0e!QuwJL?4`vInwLSVhtJK+q5GZ=gygM z8D@wT0JU@gMq zpueC1U$I&(FH3-QP=jn#F8fb#Cm2!}RCQFM+LT8aPKdpK3+o}Yn**IS_cq`HzfD{@ z4;N=)710Y?V0Lj^z~FWi(nswgm?AnVMrJld((Y=+#za^QT#*q4fN+_K0sXZwv1HAB z3vwC~;1!+oJo9IR9s$udq~LQbj?G^nsT4-Lw7yN~&0wE6?owFrA8=bPr8v}AsB9oo zZ6q7S47O%MqQp=#D-io7FOjm0@zMPfr>?B>NYxU5rjbvY*brV zhd|(EqYei&D0QSSklDAwHm8U8dMSB&+b`|%OcU)afV;*IO{{IYUWC}j%D)`YJO!zJ z_LR6~e#}qzZEwg?8^CKr`TODvJ`3{^R+xUNYJKAnud0FVC`0aDP##cOJ%5G672xkY ziDPPqPWGzk#e@~vOF(mt2_Zmj@39RQ*630I-V^ym==jqS)`SsO-fzKDn?QuUWIT!z zWKJch!dfI3B+C{mZXF>>gIf^E_d4ikzS{Ts`N{H{Ev+|_wZBWdHa%VlrrbB^!b9^`8p~lIMFBP+LKBV429Y!KPu)+< zjV^f=myvt+0_BZN>?MZ+1ksLPv*T(6jKa=n2wKpgl^5u2ZP}_&#g|iMcN1uV`NdAg zR@*f2KB*enpI#@g5fy@7j`4Ykmk~G4b*0|9TBf=L<%5dJUXWo z{K)m3+{4|FzwyKT!>=+Db*7zy<@*h|6KFTLyapLJvv^%yeZ4K_AUJgghq z+XX}l^zZvX{%qy|X)h}=V!?`b9pxzxeaUVqe;5MHOJ5IW=Iv>CtfS146{7?hSd|~l zjP}SC9SFo4XB5v+SgVLZ4l)3}U~dfY0E7hi;#6iHr^ta>qUI3hD0-Vwcn0jrUrj(z z;HUE#=Z&(ar6lxrd|xkZ?8_vxoKq3an^2PXSu!>y-~80hM&oFGRJg2S zYG=@P$|pd3m!|e?|Ed|<0X`IKj&xdI@MoJ!Ye7@Nc#3RoYwCc_<|CfJ#=YQtH#K0I zKVi-wrVTyF8(C?HdCw)?QxLE6yBKTlkW%`ZA>)a`Mkq=JUtVa;_6`z8cQi(e4bg={ zQSebvol|#03xw4`fIM8duc!4J(Ab z>f02J)&?ot`jpNw5Kbm$9irXxoA){G%j_@$l?7JaUgq}bu-yRjf7G_@?DNrTsx=8= zyUcaBl`QLKE_)xuC!Bwxy=q*TFspE0ndSO@QP1Mt9t~^<=;P1R*?A(;uqGn@=+Enq zO>HO+6jzB{9=g>w`MkjMoIy~?E)Xyu6Z($VyQV^p5`9GT*@Y>=7k;HEe!GgwBf;>a z=u1D5)Nw_1^kvLFbCtll{SD<7V!f zqKEBA-yRnm*5b9GyP^JKKp#}A5QHhXcB1ZtBna+xQHJU&24}@AF36usr-BX$Bhmd} z0AT_Myz#KJ+YSAJ+lt}W1YTU}6)Xwb{Q~E6i0hyotAQFDgU#l|F|t%k_3QV)Smnp; z>gReeeSD9Zr3T_uvafR;tm{mi{F)8N{&^M-k z;^!t>qmM6ZrB`6R%J7WN@u+{)_lU z9hkeybk@YLuke`xnnLdO?(Q&zhQmmTKB1=;OU6%jTiGuOO=wmP!{Zc$#0J(n4Q92S zKZb{Ojsk$lQV~4p*E1H);1K`0ckHz&|m3G6t4Kqc=IVkg>tHl|M_41 zGb%YvZ%FRohcqg}Ml&T0B3`KctV{{i1>KCy1dc)zC32bo_WrMboV{G))7~gRO+u#G zsvZQLfuA%+oJ(fK`x&{A#Da8n672E4Ug4wUe!R%aiiC5ucl00=M>1dS0QszY=|oRq z1Oh+c?pvBfC_@o`0$X9izv_U-;rYkElg(RKT60F%E^E(ueL~15!t#(2OJn)SmLJaJLWJ9>Tpy)uv2T}p?sbf0sFhVD9dep4r+uGl{YK=zDpm?wxx03QUC zAc4Euu5hJ>U?h#~2`1LPc1@S5bm4Ns8?}V7GK3)hA%WTay zJ%a=k7^YbwcW9lTi%mK4L@WRcT2@vS8JP_LMD_zXd6A>6t-TA$9unEv*+dpKBIN85 z>&D$fcwG}YZV0Z1&p;8f$s9`hnwHczd_)YRKdeO|s+^U4_B+|_X=B@Qf-U;zy_HZ3QOR_ZbGl|GB*fBS)sq080 z(SkUEA!At%??Yr125}~)IS%NQ?)zvIu-F6_b2Ib?3?nl@5P6H1L}1x*xWyS^Vi-;nn%gusz7G?RZ^M7w_&6@O6$QsVBa>FD6c2$B?m zW495}KVNfRYRwFBwx5fByhvZ2xucVhAgXCtk-Q{Qm5A(Ufs<=<7m!ficpbj*H+n|b zA_LS(bICA`CZ1i%vC~xhU2l%Cox-4I=___VN=S4622O82 zRl2Wf9L5625QvwOybjC$b#IB^EVS}HHDs&h#Ti>{dPJNH?^>{Fz4j}nkKg(B+@Tu+ z{_NWCx9`0FWN+4!(F6who%4^=>U?>VC>nx;<@(fLG*O>F+kH0gL!-5>yrR?e?+d>~ zu1dD7%vntRnp*W1c;sYA-66@;sMk2rTY33dFoFRTRuFFe^Vcsr1WYi=P^cLi9wEV2 zcq`Ypny}q6Pt}7uG7bNfeJ%X}7^4coDdsD%YWHihpp(gBJtnZYwNuvJ!$VD5n}`Uq zN{WcsfzfS0-7mSr;u-ZmDX&eW(Rp{$DRc8f%gf7V^LS?&Ev=Y~5pYC&YbNy8NI*dl z&)BzNq!O+wTLe9(qY@LtVf9|uEK%W3U4Y?JVIyLSydgvp$Ro;1B|&t5Sh?pCt*xzX zjyufasUZix|5-M*4XzA|g7C2aIWu$f-5~Ru`s+85rS-U|h&e1QjO6p06j2W1j1yp5D3<$cE`E(#Z%g$h4I7`iL7m==gdk1H)!RL&Le>zYlP7ZV`#! zM$+2BveSli)&V9PYMPq25!-h@^@$cAA79=3_cCf~G^=ZCk`Kd&UiaC>G+Dq8atmoE z9o?kVBdv;kq4RtumJ`F=%8Ogc1JptjCOqr{;U>*5LT~ zN}p!f)&4g(TQAS4YV;pDa9|_44rHyYxGOijkI%Q{~=dJVI3C;^WCykw*1- z{{7`+PfAPmMb<|enBI8*Nt|Ci3oE0}B8LNvyjrBY^?v?TOeV;(=qz!C;qonV^4+I) zv|c_I7!o4u;E!A=o$=ML@6&p}z^Y%RZPlc<_4ynQt4pu@#NXF z)5ud>v+qJw1M{H0sGOVIYa~8@%{lOg7?y(WbC<1l#h@{QX3rj8Bo!U0&SITQ?mXh@Fk?6gnG1kbD+YR#qmR80Na%LLNEKtOwhC285jx_eV}%J_vKZp{CESurtRz zM1Yx^_+wbi;{mdTxc#1bh=dacgqX{ue+~vA64>0_{5By#6VYnyJUoiHXO!OF-hy7M z*R$PzS#-^wez|3h`~J=WUOqTF$jKQ*l#{`@j$NHu#E1C()_3HQs0DfOt?*W5rA*pIZv4|Z=|VcCs7Up#C27Y)l`W+P5h*DLj|mH( ztzB9EY7eRhS&%n1(&4Ve^P#6_JeHD{W^Ge`Djgg9avyGHR9svz0!a>tiS2s$@Zn*^ z6F!j-`Ow=-9o^uMcwdr_LFO4l!+ppv-5gTwi}(-BKGYaQSmtSEt+S3bRV$3CjEDHk0a_n*3E#Z}r*L0{$c;v4oK$a74z)hz*gQen?GWmyjT`vV0?U z7-sEkm14?NmHRr6G#%>n&s#CS5q>?0I2I(zyi2Ol)YKeCs0g$8wZmX8cGA;t!YzW4 z)!@>v?>WQd6 z@6MYoxEWnQCZiCa0!uAtNIr`5>0&z~RFb2#i_FaN5tle93dOlAJ-y zpuJH`qU80(pZl@Csi~^^6kZ)Lsq~{{hF_4iwe<(ALE+)s=jP{;h`e7^lo@fG#505y znsH%IRBUVzX2RK8tfJMZom34CBjEXCiiniDy1FmZe*#DMMVu!E)s0)_>%j193nDPE zMap1tLL6Hl63ejgV|?Mi#Qn+3qem2A1@1rA7IM@rk8*SAuoVS-w!SZkl`jKRz6wDq zGfTfLk>YEDFrCEs!!)e}DYP;a=Q*Y2=Kc`b9`@o7i#(TdQZL{ttvt^Y&bI$ED>8ll l%?^C4|L-jMKb~UW4C8)n{$XRzcbkM?XXRDoUdb5w{y(LQrZ)fp literal 0 HcmV?d00001