初始化项目，由ModelHub XC社区提供模型
Model: iproskurina/SmolLM2-360M-biasinbios-pt-factory-real-base-npacking Source: Original Platform
This commit is contained in:
ModelHub XC
35
.gitattributes
vendored
Normal file
35
.gitattributes
vendored
Normal file
@@ -0,0 +1,35 @@
|
|||||||
|
*.7z filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.arrow filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.bin filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.ftz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.gz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.h5 filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.joblib filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.model filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.npy filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.npz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.ot filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.parquet filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pb filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pickle filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pkl filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pt filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pth filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.rar filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||||
|
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.tar filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.tflite filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.tgz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.wasm filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.xz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.zip filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.zst filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||||
58
README.md
Normal file
58
README.md
Normal file
@@ -0,0 +1,58 @@
|
|||||||
|
---
|
||||||
|
library_name: transformers
|
||||||
|
license: other
|
||||||
|
base_model: HuggingFaceTB/SmolLM2-360M
|
||||||
|
tags:
|
||||||
|
- llama-factory
|
||||||
|
- full
|
||||||
|
- generated_from_trainer
|
||||||
|
model-index:
|
||||||
|
- name: SmolLM2_360M_pretrain_real
|
||||||
|
results: []
|
||||||
|
---
|
||||||
|
|
||||||
|
<!-- This model card has been generated automatically according to the information the Trainer had access to. You
|
||||||
|
should probably proofread and complete it, then remove this comment. -->
|
||||||
|
|
||||||
|
# SmolLM2_360M_pretrain_real
|
||||||
|
|
||||||
|
This model is a fine-tuned version of [HuggingFaceTB/SmolLM2-360M](https://huggingface.co/HuggingFaceTB/SmolLM2-360M) on the bias_in_bios_pt_train_real dataset.
|
||||||
|
|
||||||
|
## Model description
|
||||||
|
|
||||||
|
More information needed
|
||||||
|
|
||||||
|
## Intended uses & limitations
|
||||||
|
|
||||||
|
More information needed
|
||||||
|
|
||||||
|
## Training and evaluation data
|
||||||
|
|
||||||
|
More information needed
|
||||||
|
|
||||||
|
## Training procedure
|
||||||
|
|
||||||
|
### Training hyperparameters
|
||||||
|
|
||||||
|
The following hyperparameters were used during training:
|
||||||
|
- learning_rate: 5e-05
|
||||||
|
- train_batch_size: 2
|
||||||
|
- eval_batch_size: 8
|
||||||
|
- seed: 42
|
||||||
|
- gradient_accumulation_steps: 16
|
||||||
|
- total_train_batch_size: 32
|
||||||
|
- optimizer: Use OptimizerNames.ADAMW_TORCH_FUSED with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
|
||||||
|
- lr_scheduler_type: cosine
|
||||||
|
- lr_scheduler_warmup_ratio: 0.1
|
||||||
|
- num_epochs: 1.0
|
||||||
|
|
||||||
|
### Training results
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
### Framework versions
|
||||||
|
|
||||||
|
- Transformers 4.57.1
|
||||||
|
- Pytorch 2.10.0+cu128
|
||||||
|
- Datasets 4.0.0
|
||||||
|
- Tokenizers 0.22.2
|
||||||
8
all_results.json
Normal file
8
all_results.json
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
{
|
||||||
|
"epoch": 1.0,
|
||||||
|
"total_flos": 5361727389638400.0,
|
||||||
|
"train_loss": 2.783692882907006,
|
||||||
|
"train_runtime": 2875.5796,
|
||||||
|
"train_samples_per_second": 9.651,
|
||||||
|
"train_steps_per_second": 0.302
|
||||||
|
}
|
||||||
4
chat_template.jinja
Normal file
4
chat_template.jinja
Normal file
@@ -0,0 +1,4 @@
|
|||||||
|
{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% endif %}{% if system_message is defined %}{{ 'System: ' + system_message + '<|endoftext|>' + '
|
||||||
|
' }}{% endif %}{% for message in loop_messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ 'Human: ' + content + '<|endoftext|>' + '
|
||||||
|
Assistant:' }}{% elif message['role'] == 'assistant' %}{{ content + '<|endoftext|>' + '
|
||||||
|
' }}{% endif %}{% endfor %}
|
||||||
4
checkpoint-868/chat_template.jinja
Normal file
4
checkpoint-868/chat_template.jinja
Normal file
@@ -0,0 +1,4 @@
|
|||||||
|
{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% endif %}{% if system_message is defined %}{{ 'System: ' + system_message + '<|endoftext|>' + '
|
||||||
|
' }}{% endif %}{% for message in loop_messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ 'Human: ' + content + '<|endoftext|>' + '
|
||||||
|
Assistant:' }}{% elif message['role'] == 'assistant' %}{{ content + '<|endoftext|>' + '
|
||||||
|
' }}{% endif %}{% endfor %}
|
||||||
32
checkpoint-868/config.json
Normal file
32
checkpoint-868/config.json
Normal file
@@ -0,0 +1,32 @@
|
|||||||
|
{
|
||||||
|
"architectures": [
|
||||||
|
"LlamaForCausalLM"
|
||||||
|
],
|
||||||
|
"attention_bias": false,
|
||||||
|
"attention_dropout": 0.0,
|
||||||
|
"bos_token_id": 0,
|
||||||
|
"dtype": "float32",
|
||||||
|
"eos_token_id": 0,
|
||||||
|
"head_dim": 64,
|
||||||
|
"hidden_act": "silu",
|
||||||
|
"hidden_size": 960,
|
||||||
|
"initializer_range": 0.02,
|
||||||
|
"intermediate_size": 2560,
|
||||||
|
"is_llama_config": true,
|
||||||
|
"max_position_embeddings": 8192,
|
||||||
|
"mlp_bias": false,
|
||||||
|
"model_type": "llama",
|
||||||
|
"num_attention_heads": 15,
|
||||||
|
"num_hidden_layers": 32,
|
||||||
|
"num_key_value_heads": 5,
|
||||||
|
"pad_token_id": 0,
|
||||||
|
"pretraining_tp": 1,
|
||||||
|
"rms_norm_eps": 1e-05,
|
||||||
|
"rope_interleaved": false,
|
||||||
|
"rope_scaling": null,
|
||||||
|
"rope_theta": 100000,
|
||||||
|
"tie_word_embeddings": true,
|
||||||
|
"transformers_version": "4.57.1",
|
||||||
|
"use_cache": false,
|
||||||
|
"vocab_size": 49152
|
||||||
|
}
|
||||||
9
checkpoint-868/generation_config.json
Normal file
9
checkpoint-868/generation_config.json
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
{
|
||||||
|
"_from_model_config": true,
|
||||||
|
"bos_token_id": 0,
|
||||||
|
"eos_token_id": [
|
||||||
|
0
|
||||||
|
],
|
||||||
|
"pad_token_id": 0,
|
||||||
|
"transformers_version": "4.57.1"
|
||||||
|
}
|
||||||
48901
checkpoint-868/merges.txt
Normal file
48901
checkpoint-868/merges.txt
Normal file
File diff suppressed because it is too large
Load Diff
3
checkpoint-868/model.safetensors
Normal file
3
checkpoint-868/model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:672f35cabb2488b43aa1f8ee5e7bd8fd32cd265ef3e74a295be5c67658412020
|
||||||
|
size 1447317080
|
||||||
3
checkpoint-868/optimizer.pt
Normal file
3
checkpoint-868/optimizer.pt
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:1a51439c060c956e84d802193f6ceac9380cd3604e494e844979fcc3566bcc73
|
||||||
|
size 2894817739
|
||||||
3
checkpoint-868/rng_state.pth
Normal file
3
checkpoint-868/rng_state.pth
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:250560ab3d528161ab3659b120def6e4a9ab4b457e3399603bbcfa40db3efc90
|
||||||
|
size 14645
|
||||||
3
checkpoint-868/scheduler.pt
Normal file
3
checkpoint-868/scheduler.pt
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:f2c3fadb1edcfe8ead120a2002df424969bb0eb1826d352d5da6bfdb969f89c1
|
||||||
|
size 1465
|
||||||
43
checkpoint-868/special_tokens_map.json
Normal file
43
checkpoint-868/special_tokens_map.json
Normal file
@@ -0,0 +1,43 @@
|
|||||||
|
{
|
||||||
|
"additional_special_tokens": [
|
||||||
|
"<|endoftext|>",
|
||||||
|
"<|im_start|>",
|
||||||
|
"<|im_end|>",
|
||||||
|
"<repo_name>",
|
||||||
|
"<reponame>",
|
||||||
|
"<file_sep>",
|
||||||
|
"<filename>",
|
||||||
|
"<gh_stars>",
|
||||||
|
"<issue_start>",
|
||||||
|
"<issue_comment>",
|
||||||
|
"<issue_closed>",
|
||||||
|
"<jupyter_start>",
|
||||||
|
"<jupyter_text>",
|
||||||
|
"<jupyter_code>",
|
||||||
|
"<jupyter_output>",
|
||||||
|
"<jupyter_script>",
|
||||||
|
"<empty_output>"
|
||||||
|
],
|
||||||
|
"bos_token": {
|
||||||
|
"content": "<|endoftext|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false
|
||||||
|
},
|
||||||
|
"eos_token": {
|
||||||
|
"content": "<|endoftext|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false
|
||||||
|
},
|
||||||
|
"pad_token": "<|endoftext|>",
|
||||||
|
"unk_token": {
|
||||||
|
"content": "<|endoftext|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false
|
||||||
|
}
|
||||||
|
}
|
||||||
244949
checkpoint-868/tokenizer.json
Normal file
244949
checkpoint-868/tokenizer.json
Normal file
File diff suppressed because it is too large
Load Diff
171
checkpoint-868/tokenizer_config.json
Normal file
171
checkpoint-868/tokenizer_config.json
Normal file
@@ -0,0 +1,171 @@
|
|||||||
|
{
|
||||||
|
"add_prefix_space": false,
|
||||||
|
"added_tokens_decoder": {
|
||||||
|
"0": {
|
||||||
|
"content": "<|endoftext|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"1": {
|
||||||
|
"content": "<|im_start|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"2": {
|
||||||
|
"content": "<|im_end|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"3": {
|
||||||
|
"content": "<repo_name>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"4": {
|
||||||
|
"content": "<reponame>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"5": {
|
||||||
|
"content": "<file_sep>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"6": {
|
||||||
|
"content": "<filename>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"7": {
|
||||||
|
"content": "<gh_stars>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"8": {
|
||||||
|
"content": "<issue_start>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"9": {
|
||||||
|
"content": "<issue_comment>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"10": {
|
||||||
|
"content": "<issue_closed>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"11": {
|
||||||
|
"content": "<jupyter_start>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"12": {
|
||||||
|
"content": "<jupyter_text>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"13": {
|
||||||
|
"content": "<jupyter_code>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"14": {
|
||||||
|
"content": "<jupyter_output>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"15": {
|
||||||
|
"content": "<jupyter_script>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"16": {
|
||||||
|
"content": "<empty_output>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"additional_special_tokens": [
|
||||||
|
"<|endoftext|>",
|
||||||
|
"<|im_start|>",
|
||||||
|
"<|im_end|>",
|
||||||
|
"<repo_name>",
|
||||||
|
"<reponame>",
|
||||||
|
"<file_sep>",
|
||||||
|
"<filename>",
|
||||||
|
"<gh_stars>",
|
||||||
|
"<issue_start>",
|
||||||
|
"<issue_comment>",
|
||||||
|
"<issue_closed>",
|
||||||
|
"<jupyter_start>",
|
||||||
|
"<jupyter_text>",
|
||||||
|
"<jupyter_code>",
|
||||||
|
"<jupyter_output>",
|
||||||
|
"<jupyter_script>",
|
||||||
|
"<empty_output>"
|
||||||
|
],
|
||||||
|
"bos_token": "<|endoftext|>",
|
||||||
|
"clean_up_tokenization_spaces": false,
|
||||||
|
"eos_token": "<|endoftext|>",
|
||||||
|
"extra_special_tokens": {},
|
||||||
|
"model_max_length": 8192,
|
||||||
|
"pad_token": "<|endoftext|>",
|
||||||
|
"padding_side": "right",
|
||||||
|
"split_special_tokens": false,
|
||||||
|
"tokenizer_class": "GPT2Tokenizer",
|
||||||
|
"unk_token": "<|endoftext|>",
|
||||||
|
"vocab_size": 49152
|
||||||
|
}
|
||||||
90
checkpoint-868/trainer_state.json
Normal file
90
checkpoint-868/trainer_state.json
Normal file
@@ -0,0 +1,90 @@
|
|||||||
|
{
|
||||||
|
"best_global_step": null,
|
||||||
|
"best_metric": null,
|
||||||
|
"best_model_checkpoint": null,
|
||||||
|
"epoch": 1.0,
|
||||||
|
"eval_steps": 500,
|
||||||
|
"global_step": 868,
|
||||||
|
"is_hyper_param_search": false,
|
||||||
|
"is_local_process_zero": true,
|
||||||
|
"is_world_process_zero": true,
|
||||||
|
"log_history": [
|
||||||
|
{
|
||||||
|
"epoch": 0.11530700490054771,
|
||||||
|
"grad_norm": 2.9323058128356934,
|
||||||
|
"learning_rate": 4.9970880369844344e-05,
|
||||||
|
"loss": 2.995,
|
||||||
|
"step": 100
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.23061400980109542,
|
||||||
|
"grad_norm": 2.415233612060547,
|
||||||
|
"learning_rate": 4.7505488501875907e-05,
|
||||||
|
"loss": 2.809,
|
||||||
|
"step": 200
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.3459210147016431,
|
||||||
|
"grad_norm": 1.7806472778320312,
|
||||||
|
"learning_rate": 4.1447386246547995e-05,
|
||||||
|
"loss": 2.7775,
|
||||||
|
"step": 300
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.46122801960219084,
|
||||||
|
"grad_norm": 1.9444694519042969,
|
||||||
|
"learning_rate": 3.27636713364243e-05,
|
||||||
|
"loss": 2.7791,
|
||||||
|
"step": 400
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.5765350245027385,
|
||||||
|
"grad_norm": 2.093379020690918,
|
||||||
|
"learning_rate": 2.2840586632035014e-05,
|
||||||
|
"loss": 2.7299,
|
||||||
|
"step": 500
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.6918420294032862,
|
||||||
|
"grad_norm": 1.8663088083267212,
|
||||||
|
"learning_rate": 1.3262224364443493e-05,
|
||||||
|
"loss": 2.7359,
|
||||||
|
"step": 600
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.807149034303834,
|
||||||
|
"grad_norm": 2.0475974082946777,
|
||||||
|
"learning_rate": 5.557646282733725e-06,
|
||||||
|
"loss": 2.7394,
|
||||||
|
"step": 700
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.9224560392043817,
|
||||||
|
"grad_norm": 1.9257546663284302,
|
||||||
|
"learning_rate": 9.56788706753814e-07,
|
||||||
|
"loss": 2.7317,
|
||||||
|
"step": 800
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"logging_steps": 100,
|
||||||
|
"max_steps": 868,
|
||||||
|
"num_input_tokens_seen": 0,
|
||||||
|
"num_train_epochs": 1,
|
||||||
|
"save_steps": 500,
|
||||||
|
"stateful_callbacks": {
|
||||||
|
"TrainerControl": {
|
||||||
|
"args": {
|
||||||
|
"should_epoch_stop": false,
|
||||||
|
"should_evaluate": false,
|
||||||
|
"should_log": false,
|
||||||
|
"should_save": true,
|
||||||
|
"should_training_stop": true
|
||||||
|
},
|
||||||
|
"attributes": {}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"total_flos": 5361727389638400.0,
|
||||||
|
"train_batch_size": 2,
|
||||||
|
"trial_name": null,
|
||||||
|
"trial_params": null
|
||||||
|
}
|
||||||
3
checkpoint-868/training_args.bin
Normal file
3
checkpoint-868/training_args.bin
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:8c8211fa9f42421d17ed348efb104b6813733638a757c0698b50a2318e4e417d
|
||||||
|
size 6225
|
||||||
1
checkpoint-868/vocab.json
Normal file
1
checkpoint-868/vocab.json
Normal file
File diff suppressed because one or more lines are too long
32
config.json
Normal file
32
config.json
Normal file
@@ -0,0 +1,32 @@
|
|||||||
|
{
|
||||||
|
"architectures": [
|
||||||
|
"LlamaForCausalLM"
|
||||||
|
],
|
||||||
|
"attention_bias": false,
|
||||||
|
"attention_dropout": 0.0,
|
||||||
|
"bos_token_id": 0,
|
||||||
|
"dtype": "float32",
|
||||||
|
"eos_token_id": 0,
|
||||||
|
"head_dim": 64,
|
||||||
|
"hidden_act": "silu",
|
||||||
|
"hidden_size": 960,
|
||||||
|
"initializer_range": 0.02,
|
||||||
|
"intermediate_size": 2560,
|
||||||
|
"is_llama_config": true,
|
||||||
|
"max_position_embeddings": 8192,
|
||||||
|
"mlp_bias": false,
|
||||||
|
"model_type": "llama",
|
||||||
|
"num_attention_heads": 15,
|
||||||
|
"num_hidden_layers": 32,
|
||||||
|
"num_key_value_heads": 5,
|
||||||
|
"pad_token_id": 0,
|
||||||
|
"pretraining_tp": 1,
|
||||||
|
"rms_norm_eps": 1e-05,
|
||||||
|
"rope_interleaved": false,
|
||||||
|
"rope_scaling": null,
|
||||||
|
"rope_theta": 100000,
|
||||||
|
"tie_word_embeddings": true,
|
||||||
|
"transformers_version": "4.57.1",
|
||||||
|
"use_cache": false,
|
||||||
|
"vocab_size": 49152
|
||||||
|
}
|
||||||
9
generation_config.json
Normal file
9
generation_config.json
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
{
|
||||||
|
"_from_model_config": true,
|
||||||
|
"bos_token_id": 0,
|
||||||
|
"eos_token_id": [
|
||||||
|
0
|
||||||
|
],
|
||||||
|
"pad_token_id": 0,
|
||||||
|
"transformers_version": "4.57.1"
|
||||||
|
}
|
||||||
48901
merges.txt
Normal file
48901
merges.txt
Normal file
File diff suppressed because it is too large
Load Diff
3
model.safetensors
Normal file
3
model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:672f35cabb2488b43aa1f8ee5e7bd8fd32cd265ef3e74a295be5c67658412020
|
||||||
|
size 1447317080
|
||||||
43
special_tokens_map.json
Normal file
43
special_tokens_map.json
Normal file
@@ -0,0 +1,43 @@
|
|||||||
|
{
|
||||||
|
"additional_special_tokens": [
|
||||||
|
"<|endoftext|>",
|
||||||
|
"<|im_start|>",
|
||||||
|
"<|im_end|>",
|
||||||
|
"<repo_name>",
|
||||||
|
"<reponame>",
|
||||||
|
"<file_sep>",
|
||||||
|
"<filename>",
|
||||||
|
"<gh_stars>",
|
||||||
|
"<issue_start>",
|
||||||
|
"<issue_comment>",
|
||||||
|
"<issue_closed>",
|
||||||
|
"<jupyter_start>",
|
||||||
|
"<jupyter_text>",
|
||||||
|
"<jupyter_code>",
|
||||||
|
"<jupyter_output>",
|
||||||
|
"<jupyter_script>",
|
||||||
|
"<empty_output>"
|
||||||
|
],
|
||||||
|
"bos_token": {
|
||||||
|
"content": "<|endoftext|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false
|
||||||
|
},
|
||||||
|
"eos_token": {
|
||||||
|
"content": "<|endoftext|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false
|
||||||
|
},
|
||||||
|
"pad_token": "<|endoftext|>",
|
||||||
|
"unk_token": {
|
||||||
|
"content": "<|endoftext|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false
|
||||||
|
}
|
||||||
|
}
|
||||||
244949
tokenizer.json
Normal file
244949
tokenizer.json
Normal file
File diff suppressed because it is too large
Load Diff
171
tokenizer_config.json
Normal file
171
tokenizer_config.json
Normal file
@@ -0,0 +1,171 @@
|
|||||||
|
{
|
||||||
|
"add_prefix_space": false,
|
||||||
|
"added_tokens_decoder": {
|
||||||
|
"0": {
|
||||||
|
"content": "<|endoftext|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"1": {
|
||||||
|
"content": "<|im_start|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"2": {
|
||||||
|
"content": "<|im_end|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"3": {
|
||||||
|
"content": "<repo_name>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"4": {
|
||||||
|
"content": "<reponame>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"5": {
|
||||||
|
"content": "<file_sep>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"6": {
|
||||||
|
"content": "<filename>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"7": {
|
||||||
|
"content": "<gh_stars>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"8": {
|
||||||
|
"content": "<issue_start>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"9": {
|
||||||
|
"content": "<issue_comment>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"10": {
|
||||||
|
"content": "<issue_closed>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"11": {
|
||||||
|
"content": "<jupyter_start>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"12": {
|
||||||
|
"content": "<jupyter_text>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"13": {
|
||||||
|
"content": "<jupyter_code>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"14": {
|
||||||
|
"content": "<jupyter_output>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"15": {
|
||||||
|
"content": "<jupyter_script>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"16": {
|
||||||
|
"content": "<empty_output>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"additional_special_tokens": [
|
||||||
|
"<|endoftext|>",
|
||||||
|
"<|im_start|>",
|
||||||
|
"<|im_end|>",
|
||||||
|
"<repo_name>",
|
||||||
|
"<reponame>",
|
||||||
|
"<file_sep>",
|
||||||
|
"<filename>",
|
||||||
|
"<gh_stars>",
|
||||||
|
"<issue_start>",
|
||||||
|
"<issue_comment>",
|
||||||
|
"<issue_closed>",
|
||||||
|
"<jupyter_start>",
|
||||||
|
"<jupyter_text>",
|
||||||
|
"<jupyter_code>",
|
||||||
|
"<jupyter_output>",
|
||||||
|
"<jupyter_script>",
|
||||||
|
"<empty_output>"
|
||||||
|
],
|
||||||
|
"bos_token": "<|endoftext|>",
|
||||||
|
"clean_up_tokenization_spaces": false,
|
||||||
|
"eos_token": "<|endoftext|>",
|
||||||
|
"extra_special_tokens": {},
|
||||||
|
"model_max_length": 8192,
|
||||||
|
"pad_token": "<|endoftext|>",
|
||||||
|
"padding_side": "right",
|
||||||
|
"split_special_tokens": false,
|
||||||
|
"tokenizer_class": "GPT2Tokenizer",
|
||||||
|
"unk_token": "<|endoftext|>",
|
||||||
|
"vocab_size": 49152
|
||||||
|
}
|
||||||
8
train_results.json
Normal file
8
train_results.json
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
{
|
||||||
|
"epoch": 1.0,
|
||||||
|
"total_flos": 5361727389638400.0,
|
||||||
|
"train_loss": 2.783692882907006,
|
||||||
|
"train_runtime": 2875.5796,
|
||||||
|
"train_samples_per_second": 9.651,
|
||||||
|
"train_steps_per_second": 0.302
|
||||||
|
}
|
||||||
9
trainer_log.jsonl
Normal file
9
trainer_log.jsonl
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
{"current_steps": 100, "total_steps": 868, "loss": 2.995, "lr": 4.9970880369844344e-05, "epoch": 0.11530700490054771, "percentage": 11.52, "elapsed_time": "0:05:30", "remaining_time": "0:42:21"}
|
||||||
|
{"current_steps": 200, "total_steps": 868, "loss": 2.809, "lr": 4.7505488501875907e-05, "epoch": 0.23061400980109542, "percentage": 23.04, "elapsed_time": "0:11:02", "remaining_time": "0:36:53"}
|
||||||
|
{"current_steps": 300, "total_steps": 868, "loss": 2.7775, "lr": 4.1447386246547995e-05, "epoch": 0.3459210147016431, "percentage": 34.56, "elapsed_time": "0:16:32", "remaining_time": "0:31:19"}
|
||||||
|
{"current_steps": 400, "total_steps": 868, "loss": 2.7791, "lr": 3.27636713364243e-05, "epoch": 0.46122801960219084, "percentage": 46.08, "elapsed_time": "0:22:03", "remaining_time": "0:25:48"}
|
||||||
|
{"current_steps": 500, "total_steps": 868, "loss": 2.7299, "lr": 2.2840586632035014e-05, "epoch": 0.5765350245027385, "percentage": 57.6, "elapsed_time": "0:27:32", "remaining_time": "0:20:16"}
|
||||||
|
{"current_steps": 600, "total_steps": 868, "loss": 2.7359, "lr": 1.3262224364443493e-05, "epoch": 0.6918420294032862, "percentage": 69.12, "elapsed_time": "0:33:02", "remaining_time": "0:14:45"}
|
||||||
|
{"current_steps": 700, "total_steps": 868, "loss": 2.7394, "lr": 5.557646282733725e-06, "epoch": 0.807149034303834, "percentage": 80.65, "elapsed_time": "0:38:32", "remaining_time": "0:09:15"}
|
||||||
|
{"current_steps": 800, "total_steps": 868, "loss": 2.7317, "lr": 9.56788706753814e-07, "epoch": 0.9224560392043817, "percentage": 92.17, "elapsed_time": "0:44:03", "remaining_time": "0:03:44"}
|
||||||
|
{"current_steps": 868, "total_steps": 868, "epoch": 1.0, "percentage": 100.0, "elapsed_time": "0:47:55", "remaining_time": "0:00:00"}
|
||||||
99
trainer_state.json
Normal file
99
trainer_state.json
Normal file
@@ -0,0 +1,99 @@
|
|||||||
|
{
|
||||||
|
"best_global_step": null,
|
||||||
|
"best_metric": null,
|
||||||
|
"best_model_checkpoint": null,
|
||||||
|
"epoch": 1.0,
|
||||||
|
"eval_steps": 500,
|
||||||
|
"global_step": 868,
|
||||||
|
"is_hyper_param_search": false,
|
||||||
|
"is_local_process_zero": true,
|
||||||
|
"is_world_process_zero": true,
|
||||||
|
"log_history": [
|
||||||
|
{
|
||||||
|
"epoch": 0.11530700490054771,
|
||||||
|
"grad_norm": 2.9323058128356934,
|
||||||
|
"learning_rate": 4.9970880369844344e-05,
|
||||||
|
"loss": 2.995,
|
||||||
|
"step": 100
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.23061400980109542,
|
||||||
|
"grad_norm": 2.415233612060547,
|
||||||
|
"learning_rate": 4.7505488501875907e-05,
|
||||||
|
"loss": 2.809,
|
||||||
|
"step": 200
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.3459210147016431,
|
||||||
|
"grad_norm": 1.7806472778320312,
|
||||||
|
"learning_rate": 4.1447386246547995e-05,
|
||||||
|
"loss": 2.7775,
|
||||||
|
"step": 300
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.46122801960219084,
|
||||||
|
"grad_norm": 1.9444694519042969,
|
||||||
|
"learning_rate": 3.27636713364243e-05,
|
||||||
|
"loss": 2.7791,
|
||||||
|
"step": 400
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.5765350245027385,
|
||||||
|
"grad_norm": 2.093379020690918,
|
||||||
|
"learning_rate": 2.2840586632035014e-05,
|
||||||
|
"loss": 2.7299,
|
||||||
|
"step": 500
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.6918420294032862,
|
||||||
|
"grad_norm": 1.8663088083267212,
|
||||||
|
"learning_rate": 1.3262224364443493e-05,
|
||||||
|
"loss": 2.7359,
|
||||||
|
"step": 600
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.807149034303834,
|
||||||
|
"grad_norm": 2.0475974082946777,
|
||||||
|
"learning_rate": 5.557646282733725e-06,
|
||||||
|
"loss": 2.7394,
|
||||||
|
"step": 700
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.9224560392043817,
|
||||||
|
"grad_norm": 1.9257546663284302,
|
||||||
|
"learning_rate": 9.56788706753814e-07,
|
||||||
|
"loss": 2.7317,
|
||||||
|
"step": 800
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.0,
|
||||||
|
"step": 868,
|
||||||
|
"total_flos": 5361727389638400.0,
|
||||||
|
"train_loss": 2.783692882907006,
|
||||||
|
"train_runtime": 2875.5796,
|
||||||
|
"train_samples_per_second": 9.651,
|
||||||
|
"train_steps_per_second": 0.302
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"logging_steps": 100,
|
||||||
|
"max_steps": 868,
|
||||||
|
"num_input_tokens_seen": 0,
|
||||||
|
"num_train_epochs": 1,
|
||||||
|
"save_steps": 500,
|
||||||
|
"stateful_callbacks": {
|
||||||
|
"TrainerControl": {
|
||||||
|
"args": {
|
||||||
|
"should_epoch_stop": false,
|
||||||
|
"should_evaluate": false,
|
||||||
|
"should_log": false,
|
||||||
|
"should_save": true,
|
||||||
|
"should_training_stop": true
|
||||||
|
},
|
||||||
|
"attributes": {}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"total_flos": 5361727389638400.0,
|
||||||
|
"train_batch_size": 2,
|
||||||
|
"trial_name": null,
|
||||||
|
"trial_params": null
|
||||||
|
}
|
||||||
3
training_args.bin
Normal file
3
training_args.bin
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:8c8211fa9f42421d17ed348efb104b6813733638a757c0698b50a2318e4e417d
|
||||||
|
size 6225
|
||||||
BIN
training_loss.png
Normal file
BIN
training_loss.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 33 KiB |
1
vocab.json
Normal file
1
vocab.json
Normal file
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user