初始化项目，由ModelHub XC社区提供模型
Model: Henry236/nilechat-eg-lora-heuristic Source: Original Platform
This commit is contained in:
ModelHub XC
36
.gitattributes
vendored
Normal file
36
.gitattributes
vendored
Normal file
@@ -0,0 +1,36 @@
|
|||||||
|
*.7z filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.arrow filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.bin filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.ftz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.gz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.h5 filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.joblib filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.model filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.npy filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.npz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.ot filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.parquet filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pb filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pickle filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pkl filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pt filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pth filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.rar filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||||
|
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.tar filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.tflite filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.tgz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.wasm filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.xz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.zip filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.zst filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||||
|
tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
||||||
48
adapter_config.json
Normal file
48
adapter_config.json
Normal file
@@ -0,0 +1,48 @@
|
|||||||
|
{
|
||||||
|
"alora_invocation_tokens": null,
|
||||||
|
"alpha_pattern": {},
|
||||||
|
"arrow_config": null,
|
||||||
|
"auto_mapping": null,
|
||||||
|
"base_model_name_or_path": "Henry236/nilechat-eg-stage1-general",
|
||||||
|
"bias": "none",
|
||||||
|
"corda_config": null,
|
||||||
|
"ensure_weight_tying": false,
|
||||||
|
"eva_config": null,
|
||||||
|
"exclude_modules": null,
|
||||||
|
"fan_in_fan_out": false,
|
||||||
|
"inference_mode": true,
|
||||||
|
"init_lora_weights": true,
|
||||||
|
"layer_replication": null,
|
||||||
|
"layers_pattern": null,
|
||||||
|
"layers_to_transform": null,
|
||||||
|
"loftq_config": {},
|
||||||
|
"lora_alpha": 128,
|
||||||
|
"lora_bias": false,
|
||||||
|
"lora_dropout": 0.05,
|
||||||
|
"lora_ga_config": null,
|
||||||
|
"megatron_config": null,
|
||||||
|
"megatron_core": "megatron.core",
|
||||||
|
"modules_to_save": null,
|
||||||
|
"peft_type": "LORA",
|
||||||
|
"peft_version": "0.19.1",
|
||||||
|
"qalora_group_size": 16,
|
||||||
|
"r": 64,
|
||||||
|
"rank_pattern": {},
|
||||||
|
"revision": null,
|
||||||
|
"target_modules": [
|
||||||
|
"q_proj",
|
||||||
|
"k_proj",
|
||||||
|
"down_proj",
|
||||||
|
"gate_proj",
|
||||||
|
"o_proj",
|
||||||
|
"up_proj",
|
||||||
|
"v_proj"
|
||||||
|
],
|
||||||
|
"target_parameters": null,
|
||||||
|
"task_type": "CAUSAL_LM",
|
||||||
|
"trainable_token_indices": null,
|
||||||
|
"use_bdlora": null,
|
||||||
|
"use_dora": false,
|
||||||
|
"use_qalora": false,
|
||||||
|
"use_rslora": false
|
||||||
|
}
|
||||||
3
adapter_model.safetensors
Normal file
3
adapter_model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:84bee3b2b1ae9c217490ad2ac4609d3ff42555dec476edfccd268b7df5490190
|
||||||
|
size 476904192
|
||||||
5
added_tokens.json
Normal file
5
added_tokens.json
Normal file
@@ -0,0 +1,5 @@
|
|||||||
|
{
|
||||||
|
"<image_soft_token>": 262144,
|
||||||
|
"<|end-of-read|>": 262145,
|
||||||
|
"<|end-of-write|>": 262146
|
||||||
|
}
|
||||||
47
chat_template.jinja
Normal file
47
chat_template.jinja
Normal file
@@ -0,0 +1,47 @@
|
|||||||
|
{{ bos_token }}
|
||||||
|
{%- if messages[0]['role'] == 'system' -%}
|
||||||
|
{%- if messages[0]['content'] is string -%}
|
||||||
|
{%- set first_user_prefix = messages[0]['content'] + '
|
||||||
|
|
||||||
|
' -%}
|
||||||
|
{%- else -%}
|
||||||
|
{%- set first_user_prefix = messages[0]['content'][0]['text'] + '
|
||||||
|
|
||||||
|
' -%}
|
||||||
|
{%- endif -%}
|
||||||
|
{%- set loop_messages = messages[1:] -%}
|
||||||
|
{%- else -%}
|
||||||
|
{%- set first_user_prefix = "" -%}
|
||||||
|
{%- set loop_messages = messages -%}
|
||||||
|
{%- endif -%}
|
||||||
|
{%- for message in loop_messages -%}
|
||||||
|
{%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%}
|
||||||
|
{{ raise_exception("Conversation roles must alternate user/assistant/user/assistant/...") }}
|
||||||
|
{%- endif -%}
|
||||||
|
{%- if (message['role'] == 'assistant') -%}
|
||||||
|
{%- set role = "model" -%}
|
||||||
|
{%- else -%}
|
||||||
|
{%- set role = message['role'] -%}
|
||||||
|
{%- endif -%}
|
||||||
|
{{ '<start_of_turn>' + role + '
|
||||||
|
' + (first_user_prefix if loop.first else "") }}
|
||||||
|
{%- if message['content'] is string -%}
|
||||||
|
{{ message['content'] | trim }}
|
||||||
|
{%- elif message['content'] is iterable -%}
|
||||||
|
{%- for item in message['content'] -%}
|
||||||
|
{%- if item['type'] == 'image' -%}
|
||||||
|
{{ '<start_of_image>' }}
|
||||||
|
{%- elif item['type'] == 'text' -%}
|
||||||
|
{{ item['text'] | trim }}
|
||||||
|
{%- endif -%}
|
||||||
|
{%- endfor -%}
|
||||||
|
{%- else -%}
|
||||||
|
{{ raise_exception("Invalid content type") }}
|
||||||
|
{%- endif -%}
|
||||||
|
{{ '<end_of_turn>
|
||||||
|
' }}
|
||||||
|
{%- endfor -%}
|
||||||
|
{%- if add_generation_prompt -%}
|
||||||
|
{{'<start_of_turn>model
|
||||||
|
'}}
|
||||||
|
{%- endif -%}
|
||||||
207
checkpoint-150/README.md
Normal file
207
checkpoint-150/README.md
Normal file
@@ -0,0 +1,207 @@
|
|||||||
|
---
|
||||||
|
base_model: Henry236/nilechat-eg-stage1-general
|
||||||
|
library_name: peft
|
||||||
|
pipeline_tag: text-generation
|
||||||
|
tags:
|
||||||
|
- base_model:adapter:Henry236/nilechat-eg-stage1-general
|
||||||
|
- lora
|
||||||
|
- transformers
|
||||||
|
---
|
||||||
|
|
||||||
|
# Model Card for Model ID
|
||||||
|
|
||||||
|
<!-- Provide a quick summary of what the model is/does. -->
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## Model Details
|
||||||
|
|
||||||
|
### Model Description
|
||||||
|
|
||||||
|
<!-- Provide a longer summary of what this model is. -->
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
- **Developed by:** [More Information Needed]
|
||||||
|
- **Funded by [optional]:** [More Information Needed]
|
||||||
|
- **Shared by [optional]:** [More Information Needed]
|
||||||
|
- **Model type:** [More Information Needed]
|
||||||
|
- **Language(s) (NLP):** [More Information Needed]
|
||||||
|
- **License:** [More Information Needed]
|
||||||
|
- **Finetuned from model [optional]:** [More Information Needed]
|
||||||
|
|
||||||
|
### Model Sources [optional]
|
||||||
|
|
||||||
|
<!-- Provide the basic links for the model. -->
|
||||||
|
|
||||||
|
- **Repository:** [More Information Needed]
|
||||||
|
- **Paper [optional]:** [More Information Needed]
|
||||||
|
- **Demo [optional]:** [More Information Needed]
|
||||||
|
|
||||||
|
## Uses
|
||||||
|
|
||||||
|
<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
|
||||||
|
|
||||||
|
### Direct Use
|
||||||
|
|
||||||
|
<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Downstream Use [optional]
|
||||||
|
|
||||||
|
<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Out-of-Scope Use
|
||||||
|
|
||||||
|
<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Bias, Risks, and Limitations
|
||||||
|
|
||||||
|
<!-- This section is meant to convey both technical and sociotechnical limitations. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Recommendations
|
||||||
|
|
||||||
|
<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
|
||||||
|
|
||||||
|
Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
|
||||||
|
|
||||||
|
## How to Get Started with the Model
|
||||||
|
|
||||||
|
Use the code below to get started with the model.
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Training Details
|
||||||
|
|
||||||
|
### Training Data
|
||||||
|
|
||||||
|
<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Training Procedure
|
||||||
|
|
||||||
|
<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
|
||||||
|
|
||||||
|
#### Preprocessing [optional]
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
|
||||||
|
#### Training Hyperparameters
|
||||||
|
|
||||||
|
- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
|
||||||
|
|
||||||
|
#### Speeds, Sizes, Times [optional]
|
||||||
|
|
||||||
|
<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Evaluation
|
||||||
|
|
||||||
|
<!-- This section describes the evaluation protocols and provides the results. -->
|
||||||
|
|
||||||
|
### Testing Data, Factors & Metrics
|
||||||
|
|
||||||
|
#### Testing Data
|
||||||
|
|
||||||
|
<!-- This should link to a Dataset Card if possible. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Factors
|
||||||
|
|
||||||
|
<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Metrics
|
||||||
|
|
||||||
|
<!-- These are the evaluation metrics being used, ideally with a description of why. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Results
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Summary
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## Model Examination [optional]
|
||||||
|
|
||||||
|
<!-- Relevant interpretability work for the model goes here -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Environmental Impact
|
||||||
|
|
||||||
|
<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
|
||||||
|
|
||||||
|
Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
|
||||||
|
|
||||||
|
- **Hardware Type:** [More Information Needed]
|
||||||
|
- **Hours used:** [More Information Needed]
|
||||||
|
- **Cloud Provider:** [More Information Needed]
|
||||||
|
- **Compute Region:** [More Information Needed]
|
||||||
|
- **Carbon Emitted:** [More Information Needed]
|
||||||
|
|
||||||
|
## Technical Specifications [optional]
|
||||||
|
|
||||||
|
### Model Architecture and Objective
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Compute Infrastructure
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Hardware
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Software
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Citation [optional]
|
||||||
|
|
||||||
|
<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
|
||||||
|
|
||||||
|
**BibTeX:**
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
**APA:**
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Glossary [optional]
|
||||||
|
|
||||||
|
<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## More Information [optional]
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Model Card Authors [optional]
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Model Card Contact
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
### Framework versions
|
||||||
|
|
||||||
|
- PEFT 0.19.1
|
||||||
48
checkpoint-150/adapter_config.json
Normal file
48
checkpoint-150/adapter_config.json
Normal file
@@ -0,0 +1,48 @@
|
|||||||
|
{
|
||||||
|
"alora_invocation_tokens": null,
|
||||||
|
"alpha_pattern": {},
|
||||||
|
"arrow_config": null,
|
||||||
|
"auto_mapping": null,
|
||||||
|
"base_model_name_or_path": "Henry236/nilechat-eg-stage1-general",
|
||||||
|
"bias": "none",
|
||||||
|
"corda_config": null,
|
||||||
|
"ensure_weight_tying": false,
|
||||||
|
"eva_config": null,
|
||||||
|
"exclude_modules": null,
|
||||||
|
"fan_in_fan_out": false,
|
||||||
|
"inference_mode": true,
|
||||||
|
"init_lora_weights": true,
|
||||||
|
"layer_replication": null,
|
||||||
|
"layers_pattern": null,
|
||||||
|
"layers_to_transform": null,
|
||||||
|
"loftq_config": {},
|
||||||
|
"lora_alpha": 128,
|
||||||
|
"lora_bias": false,
|
||||||
|
"lora_dropout": 0.05,
|
||||||
|
"lora_ga_config": null,
|
||||||
|
"megatron_config": null,
|
||||||
|
"megatron_core": "megatron.core",
|
||||||
|
"modules_to_save": null,
|
||||||
|
"peft_type": "LORA",
|
||||||
|
"peft_version": "0.19.1",
|
||||||
|
"qalora_group_size": 16,
|
||||||
|
"r": 64,
|
||||||
|
"rank_pattern": {},
|
||||||
|
"revision": null,
|
||||||
|
"target_modules": [
|
||||||
|
"down_proj",
|
||||||
|
"q_proj",
|
||||||
|
"o_proj",
|
||||||
|
"up_proj",
|
||||||
|
"k_proj",
|
||||||
|
"v_proj",
|
||||||
|
"gate_proj"
|
||||||
|
],
|
||||||
|
"target_parameters": null,
|
||||||
|
"task_type": "CAUSAL_LM",
|
||||||
|
"trainable_token_indices": null,
|
||||||
|
"use_bdlora": null,
|
||||||
|
"use_dora": false,
|
||||||
|
"use_qalora": false,
|
||||||
|
"use_rslora": false
|
||||||
|
}
|
||||||
3
checkpoint-150/adapter_model.safetensors
Normal file
3
checkpoint-150/adapter_model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:af5c39a15b1a01ba0a20cea7a153dbca7b68f86c3a14669315aa7ba9e97132d2
|
||||||
|
size 476904192
|
||||||
3
checkpoint-150/optimizer.pt
Normal file
3
checkpoint-150/optimizer.pt
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:4e92a7b72faa11b5dee3a69f947538ed3af113310f47d7017dcd5c2d1a9169ce
|
||||||
|
size 954089079
|
||||||
3
checkpoint-150/rng_state.pth
Normal file
3
checkpoint-150/rng_state.pth
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:682a758eb9a82db8f14e01aeb89e479e02d83811bb49901a4d8fb38e67117e67
|
||||||
|
size 14645
|
||||||
3
checkpoint-150/scheduler.pt
Normal file
3
checkpoint-150/scheduler.pt
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:34384565bf8354bcaad46ad7e6243a7346e204c541fa864f1ebe8968bce29bf3
|
||||||
|
size 1465
|
||||||
83
checkpoint-150/trainer_state.json
Normal file
83
checkpoint-150/trainer_state.json
Normal file
@@ -0,0 +1,83 @@
|
|||||||
|
{
|
||||||
|
"best_global_step": null,
|
||||||
|
"best_metric": null,
|
||||||
|
"best_model_checkpoint": null,
|
||||||
|
"epoch": 0.3861003861003861,
|
||||||
|
"eval_steps": 500,
|
||||||
|
"global_step": 150,
|
||||||
|
"is_hyper_param_search": false,
|
||||||
|
"is_local_process_zero": true,
|
||||||
|
"is_world_process_zero": true,
|
||||||
|
"log_history": [
|
||||||
|
{
|
||||||
|
"epoch": 0.05148005148005148,
|
||||||
|
"grad_norm": 3.7984347343444824,
|
||||||
|
"learning_rate": 2.435897435897436e-06,
|
||||||
|
"loss": 3.0721,
|
||||||
|
"step": 20
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.10296010296010295,
|
||||||
|
"grad_norm": 2.0532360076904297,
|
||||||
|
"learning_rate": 5e-06,
|
||||||
|
"loss": 2.7105,
|
||||||
|
"step": 40
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.15444015444015444,
|
||||||
|
"grad_norm": 1.889143466949463,
|
||||||
|
"learning_rate": 7.564102564102564e-06,
|
||||||
|
"loss": 2.397,
|
||||||
|
"step": 60
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.2059202059202059,
|
||||||
|
"grad_norm": 2.206676959991455,
|
||||||
|
"learning_rate": 9.999949644960027e-06,
|
||||||
|
"loss": 2.1587,
|
||||||
|
"step": 80
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.2574002574002574,
|
||||||
|
"grad_norm": 2.4184727668762207,
|
||||||
|
"learning_rate": 9.9778098230154e-06,
|
||||||
|
"loss": 1.9613,
|
||||||
|
"step": 100
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.3088803088803089,
|
||||||
|
"grad_norm": 2.1078503131866455,
|
||||||
|
"learning_rate": 9.915591603280632e-06,
|
||||||
|
"loss": 1.9187,
|
||||||
|
"step": 120
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.36036036036036034,
|
||||||
|
"grad_norm": 2.228259801864624,
|
||||||
|
"learning_rate": 9.813795930277306e-06,
|
||||||
|
"loss": 1.863,
|
||||||
|
"step": 140
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"logging_steps": 20,
|
||||||
|
"max_steps": 778,
|
||||||
|
"num_input_tokens_seen": 0,
|
||||||
|
"num_train_epochs": 2,
|
||||||
|
"save_steps": 50,
|
||||||
|
"stateful_callbacks": {
|
||||||
|
"TrainerControl": {
|
||||||
|
"args": {
|
||||||
|
"should_epoch_stop": false,
|
||||||
|
"should_evaluate": false,
|
||||||
|
"should_log": false,
|
||||||
|
"should_save": true,
|
||||||
|
"should_training_stop": false
|
||||||
|
},
|
||||||
|
"attributes": {}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"total_flos": 1.0967871654039552e+16,
|
||||||
|
"train_batch_size": 4,
|
||||||
|
"trial_name": null,
|
||||||
|
"trial_params": null
|
||||||
|
}
|
||||||
3
checkpoint-150/training_args.bin
Normal file
3
checkpoint-150/training_args.bin
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:fe6e3348f136c9b50e5b7ff3939a0aca31453e7fbef30d71133ebde6392f9959
|
||||||
|
size 6033
|
||||||
207
checkpoint-200/README.md
Normal file
207
checkpoint-200/README.md
Normal file
@@ -0,0 +1,207 @@
|
|||||||
|
---
|
||||||
|
base_model: Henry236/nilechat-eg-stage1-general
|
||||||
|
library_name: peft
|
||||||
|
pipeline_tag: text-generation
|
||||||
|
tags:
|
||||||
|
- base_model:adapter:Henry236/nilechat-eg-stage1-general
|
||||||
|
- lora
|
||||||
|
- transformers
|
||||||
|
---
|
||||||
|
|
||||||
|
# Model Card for Model ID
|
||||||
|
|
||||||
|
<!-- Provide a quick summary of what the model is/does. -->
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## Model Details
|
||||||
|
|
||||||
|
### Model Description
|
||||||
|
|
||||||
|
<!-- Provide a longer summary of what this model is. -->
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
- **Developed by:** [More Information Needed]
|
||||||
|
- **Funded by [optional]:** [More Information Needed]
|
||||||
|
- **Shared by [optional]:** [More Information Needed]
|
||||||
|
- **Model type:** [More Information Needed]
|
||||||
|
- **Language(s) (NLP):** [More Information Needed]
|
||||||
|
- **License:** [More Information Needed]
|
||||||
|
- **Finetuned from model [optional]:** [More Information Needed]
|
||||||
|
|
||||||
|
### Model Sources [optional]
|
||||||
|
|
||||||
|
<!-- Provide the basic links for the model. -->
|
||||||
|
|
||||||
|
- **Repository:** [More Information Needed]
|
||||||
|
- **Paper [optional]:** [More Information Needed]
|
||||||
|
- **Demo [optional]:** [More Information Needed]
|
||||||
|
|
||||||
|
## Uses
|
||||||
|
|
||||||
|
<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
|
||||||
|
|
||||||
|
### Direct Use
|
||||||
|
|
||||||
|
<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Downstream Use [optional]
|
||||||
|
|
||||||
|
<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Out-of-Scope Use
|
||||||
|
|
||||||
|
<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Bias, Risks, and Limitations
|
||||||
|
|
||||||
|
<!-- This section is meant to convey both technical and sociotechnical limitations. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Recommendations
|
||||||
|
|
||||||
|
<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
|
||||||
|
|
||||||
|
Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
|
||||||
|
|
||||||
|
## How to Get Started with the Model
|
||||||
|
|
||||||
|
Use the code below to get started with the model.
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Training Details
|
||||||
|
|
||||||
|
### Training Data
|
||||||
|
|
||||||
|
<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Training Procedure
|
||||||
|
|
||||||
|
<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
|
||||||
|
|
||||||
|
#### Preprocessing [optional]
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
|
||||||
|
#### Training Hyperparameters
|
||||||
|
|
||||||
|
- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
|
||||||
|
|
||||||
|
#### Speeds, Sizes, Times [optional]
|
||||||
|
|
||||||
|
<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Evaluation
|
||||||
|
|
||||||
|
<!-- This section describes the evaluation protocols and provides the results. -->
|
||||||
|
|
||||||
|
### Testing Data, Factors & Metrics
|
||||||
|
|
||||||
|
#### Testing Data
|
||||||
|
|
||||||
|
<!-- This should link to a Dataset Card if possible. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Factors
|
||||||
|
|
||||||
|
<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Metrics
|
||||||
|
|
||||||
|
<!-- These are the evaluation metrics being used, ideally with a description of why. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Results
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Summary
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## Model Examination [optional]
|
||||||
|
|
||||||
|
<!-- Relevant interpretability work for the model goes here -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Environmental Impact
|
||||||
|
|
||||||
|
<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
|
||||||
|
|
||||||
|
Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
|
||||||
|
|
||||||
|
- **Hardware Type:** [More Information Needed]
|
||||||
|
- **Hours used:** [More Information Needed]
|
||||||
|
- **Cloud Provider:** [More Information Needed]
|
||||||
|
- **Compute Region:** [More Information Needed]
|
||||||
|
- **Carbon Emitted:** [More Information Needed]
|
||||||
|
|
||||||
|
## Technical Specifications [optional]
|
||||||
|
|
||||||
|
### Model Architecture and Objective
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Compute Infrastructure
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Hardware
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Software
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Citation [optional]
|
||||||
|
|
||||||
|
<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
|
||||||
|
|
||||||
|
**BibTeX:**
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
**APA:**
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Glossary [optional]
|
||||||
|
|
||||||
|
<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## More Information [optional]
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Model Card Authors [optional]
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Model Card Contact
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
### Framework versions
|
||||||
|
|
||||||
|
- PEFT 0.19.1
|
||||||
48
checkpoint-200/adapter_config.json
Normal file
48
checkpoint-200/adapter_config.json
Normal file
@@ -0,0 +1,48 @@
|
|||||||
|
{
|
||||||
|
"alora_invocation_tokens": null,
|
||||||
|
"alpha_pattern": {},
|
||||||
|
"arrow_config": null,
|
||||||
|
"auto_mapping": null,
|
||||||
|
"base_model_name_or_path": "Henry236/nilechat-eg-stage1-general",
|
||||||
|
"bias": "none",
|
||||||
|
"corda_config": null,
|
||||||
|
"ensure_weight_tying": false,
|
||||||
|
"eva_config": null,
|
||||||
|
"exclude_modules": null,
|
||||||
|
"fan_in_fan_out": false,
|
||||||
|
"inference_mode": true,
|
||||||
|
"init_lora_weights": true,
|
||||||
|
"layer_replication": null,
|
||||||
|
"layers_pattern": null,
|
||||||
|
"layers_to_transform": null,
|
||||||
|
"loftq_config": {},
|
||||||
|
"lora_alpha": 128,
|
||||||
|
"lora_bias": false,
|
||||||
|
"lora_dropout": 0.05,
|
||||||
|
"lora_ga_config": null,
|
||||||
|
"megatron_config": null,
|
||||||
|
"megatron_core": "megatron.core",
|
||||||
|
"modules_to_save": null,
|
||||||
|
"peft_type": "LORA",
|
||||||
|
"peft_version": "0.19.1",
|
||||||
|
"qalora_group_size": 16,
|
||||||
|
"r": 64,
|
||||||
|
"rank_pattern": {},
|
||||||
|
"revision": null,
|
||||||
|
"target_modules": [
|
||||||
|
"down_proj",
|
||||||
|
"q_proj",
|
||||||
|
"o_proj",
|
||||||
|
"up_proj",
|
||||||
|
"k_proj",
|
||||||
|
"v_proj",
|
||||||
|
"gate_proj"
|
||||||
|
],
|
||||||
|
"target_parameters": null,
|
||||||
|
"task_type": "CAUSAL_LM",
|
||||||
|
"trainable_token_indices": null,
|
||||||
|
"use_bdlora": null,
|
||||||
|
"use_dora": false,
|
||||||
|
"use_qalora": false,
|
||||||
|
"use_rslora": false
|
||||||
|
}
|
||||||
3
checkpoint-200/adapter_model.safetensors
Normal file
3
checkpoint-200/adapter_model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:348baf18ab8817ea9eda9652c79ccce195f8940ed07575d12bbcba6d8ec300bd
|
||||||
|
size 476904192
|
||||||
3
checkpoint-200/optimizer.pt
Normal file
3
checkpoint-200/optimizer.pt
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:d987b412f5725269d5665aa56cc1c18452195fec047547aa5bdda6a10c45f10f
|
||||||
|
size 954089079
|
||||||
3
checkpoint-200/rng_state.pth
Normal file
3
checkpoint-200/rng_state.pth
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:1aa634a4d4cf7c59706a7886b09fc295c65eeab72ebf63757d6b6f742d6e9fee
|
||||||
|
size 14645
|
||||||
3
checkpoint-200/scheduler.pt
Normal file
3
checkpoint-200/scheduler.pt
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:9ba4d38dd8d18b7ff2091d18fff280eb76ec56fe4dbce88d23189117cc21cceb
|
||||||
|
size 1465
|
||||||
104
checkpoint-200/trainer_state.json
Normal file
104
checkpoint-200/trainer_state.json
Normal file
@@ -0,0 +1,104 @@
|
|||||||
|
{
|
||||||
|
"best_global_step": null,
|
||||||
|
"best_metric": null,
|
||||||
|
"best_model_checkpoint": null,
|
||||||
|
"epoch": 0.5148005148005148,
|
||||||
|
"eval_steps": 500,
|
||||||
|
"global_step": 200,
|
||||||
|
"is_hyper_param_search": false,
|
||||||
|
"is_local_process_zero": true,
|
||||||
|
"is_world_process_zero": true,
|
||||||
|
"log_history": [
|
||||||
|
{
|
||||||
|
"epoch": 0.05148005148005148,
|
||||||
|
"grad_norm": 3.7984347343444824,
|
||||||
|
"learning_rate": 2.435897435897436e-06,
|
||||||
|
"loss": 3.0721,
|
||||||
|
"step": 20
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.10296010296010295,
|
||||||
|
"grad_norm": 2.0532360076904297,
|
||||||
|
"learning_rate": 5e-06,
|
||||||
|
"loss": 2.7105,
|
||||||
|
"step": 40
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.15444015444015444,
|
||||||
|
"grad_norm": 1.889143466949463,
|
||||||
|
"learning_rate": 7.564102564102564e-06,
|
||||||
|
"loss": 2.397,
|
||||||
|
"step": 60
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.2059202059202059,
|
||||||
|
"grad_norm": 2.206676959991455,
|
||||||
|
"learning_rate": 9.999949644960027e-06,
|
||||||
|
"loss": 2.1587,
|
||||||
|
"step": 80
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.2574002574002574,
|
||||||
|
"grad_norm": 2.4184727668762207,
|
||||||
|
"learning_rate": 9.9778098230154e-06,
|
||||||
|
"loss": 1.9613,
|
||||||
|
"step": 100
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.3088803088803089,
|
||||||
|
"grad_norm": 2.1078503131866455,
|
||||||
|
"learning_rate": 9.915591603280632e-06,
|
||||||
|
"loss": 1.9187,
|
||||||
|
"step": 120
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.36036036036036034,
|
||||||
|
"grad_norm": 2.228259801864624,
|
||||||
|
"learning_rate": 9.813795930277306e-06,
|
||||||
|
"loss": 1.863,
|
||||||
|
"step": 140
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.4118404118404118,
|
||||||
|
"grad_norm": 3.990351438522339,
|
||||||
|
"learning_rate": 9.673242402909555e-06,
|
||||||
|
"loss": 1.7983,
|
||||||
|
"step": 160
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.46332046332046334,
|
||||||
|
"grad_norm": 2.6102797985076904,
|
||||||
|
"learning_rate": 9.495062675535614e-06,
|
||||||
|
"loss": 1.7845,
|
||||||
|
"step": 180
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.5148005148005148,
|
||||||
|
"grad_norm": 2.4415857791900635,
|
||||||
|
"learning_rate": 9.280691346552308e-06,
|
||||||
|
"loss": 1.7541,
|
||||||
|
"step": 200
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"logging_steps": 20,
|
||||||
|
"max_steps": 778,
|
||||||
|
"num_input_tokens_seen": 0,
|
||||||
|
"num_train_epochs": 2,
|
||||||
|
"save_steps": 50,
|
||||||
|
"stateful_callbacks": {
|
||||||
|
"TrainerControl": {
|
||||||
|
"args": {
|
||||||
|
"should_epoch_stop": false,
|
||||||
|
"should_evaluate": false,
|
||||||
|
"should_log": false,
|
||||||
|
"should_save": true,
|
||||||
|
"should_training_stop": false
|
||||||
|
},
|
||||||
|
"attributes": {}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"total_flos": 1.4632482245799936e+16,
|
||||||
|
"train_batch_size": 4,
|
||||||
|
"trial_name": null,
|
||||||
|
"trial_params": null
|
||||||
|
}
|
||||||
3
checkpoint-200/training_args.bin
Normal file
3
checkpoint-200/training_args.bin
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:fe6e3348f136c9b50e5b7ff3939a0aca31453e7fbef30d71133ebde6392f9959
|
||||||
|
size 6033
|
||||||
207
checkpoint-250/README.md
Normal file
207
checkpoint-250/README.md
Normal file
@@ -0,0 +1,207 @@
|
|||||||
|
---
|
||||||
|
base_model: Henry236/nilechat-eg-stage1-general
|
||||||
|
library_name: peft
|
||||||
|
pipeline_tag: text-generation
|
||||||
|
tags:
|
||||||
|
- base_model:adapter:Henry236/nilechat-eg-stage1-general
|
||||||
|
- lora
|
||||||
|
- transformers
|
||||||
|
---
|
||||||
|
|
||||||
|
# Model Card for Model ID
|
||||||
|
|
||||||
|
<!-- Provide a quick summary of what the model is/does. -->
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## Model Details
|
||||||
|
|
||||||
|
### Model Description
|
||||||
|
|
||||||
|
<!-- Provide a longer summary of what this model is. -->
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
- **Developed by:** [More Information Needed]
|
||||||
|
- **Funded by [optional]:** [More Information Needed]
|
||||||
|
- **Shared by [optional]:** [More Information Needed]
|
||||||
|
- **Model type:** [More Information Needed]
|
||||||
|
- **Language(s) (NLP):** [More Information Needed]
|
||||||
|
- **License:** [More Information Needed]
|
||||||
|
- **Finetuned from model [optional]:** [More Information Needed]
|
||||||
|
|
||||||
|
### Model Sources [optional]
|
||||||
|
|
||||||
|
<!-- Provide the basic links for the model. -->
|
||||||
|
|
||||||
|
- **Repository:** [More Information Needed]
|
||||||
|
- **Paper [optional]:** [More Information Needed]
|
||||||
|
- **Demo [optional]:** [More Information Needed]
|
||||||
|
|
||||||
|
## Uses
|
||||||
|
|
||||||
|
<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
|
||||||
|
|
||||||
|
### Direct Use
|
||||||
|
|
||||||
|
<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Downstream Use [optional]
|
||||||
|
|
||||||
|
<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Out-of-Scope Use
|
||||||
|
|
||||||
|
<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Bias, Risks, and Limitations
|
||||||
|
|
||||||
|
<!-- This section is meant to convey both technical and sociotechnical limitations. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Recommendations
|
||||||
|
|
||||||
|
<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
|
||||||
|
|
||||||
|
Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
|
||||||
|
|
||||||
|
## How to Get Started with the Model
|
||||||
|
|
||||||
|
Use the code below to get started with the model.
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Training Details
|
||||||
|
|
||||||
|
### Training Data
|
||||||
|
|
||||||
|
<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Training Procedure
|
||||||
|
|
||||||
|
<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
|
||||||
|
|
||||||
|
#### Preprocessing [optional]
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
|
||||||
|
#### Training Hyperparameters
|
||||||
|
|
||||||
|
- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
|
||||||
|
|
||||||
|
#### Speeds, Sizes, Times [optional]
|
||||||
|
|
||||||
|
<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Evaluation
|
||||||
|
|
||||||
|
<!-- This section describes the evaluation protocols and provides the results. -->
|
||||||
|
|
||||||
|
### Testing Data, Factors & Metrics
|
||||||
|
|
||||||
|
#### Testing Data
|
||||||
|
|
||||||
|
<!-- This should link to a Dataset Card if possible. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Factors
|
||||||
|
|
||||||
|
<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Metrics
|
||||||
|
|
||||||
|
<!-- These are the evaluation metrics being used, ideally with a description of why. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Results
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Summary
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## Model Examination [optional]
|
||||||
|
|
||||||
|
<!-- Relevant interpretability work for the model goes here -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Environmental Impact
|
||||||
|
|
||||||
|
<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
|
||||||
|
|
||||||
|
Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
|
||||||
|
|
||||||
|
- **Hardware Type:** [More Information Needed]
|
||||||
|
- **Hours used:** [More Information Needed]
|
||||||
|
- **Cloud Provider:** [More Information Needed]
|
||||||
|
- **Compute Region:** [More Information Needed]
|
||||||
|
- **Carbon Emitted:** [More Information Needed]
|
||||||
|
|
||||||
|
## Technical Specifications [optional]
|
||||||
|
|
||||||
|
### Model Architecture and Objective
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Compute Infrastructure
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Hardware
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Software
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Citation [optional]
|
||||||
|
|
||||||
|
<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
|
||||||
|
|
||||||
|
**BibTeX:**
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
**APA:**
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Glossary [optional]
|
||||||
|
|
||||||
|
<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## More Information [optional]
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Model Card Authors [optional]
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Model Card Contact
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
### Framework versions
|
||||||
|
|
||||||
|
- PEFT 0.19.1
|
||||||
48
checkpoint-250/adapter_config.json
Normal file
48
checkpoint-250/adapter_config.json
Normal file
@@ -0,0 +1,48 @@
|
|||||||
|
{
|
||||||
|
"alora_invocation_tokens": null,
|
||||||
|
"alpha_pattern": {},
|
||||||
|
"arrow_config": null,
|
||||||
|
"auto_mapping": null,
|
||||||
|
"base_model_name_or_path": "Henry236/nilechat-eg-stage1-general",
|
||||||
|
"bias": "none",
|
||||||
|
"corda_config": null,
|
||||||
|
"ensure_weight_tying": false,
|
||||||
|
"eva_config": null,
|
||||||
|
"exclude_modules": null,
|
||||||
|
"fan_in_fan_out": false,
|
||||||
|
"inference_mode": true,
|
||||||
|
"init_lora_weights": true,
|
||||||
|
"layer_replication": null,
|
||||||
|
"layers_pattern": null,
|
||||||
|
"layers_to_transform": null,
|
||||||
|
"loftq_config": {},
|
||||||
|
"lora_alpha": 128,
|
||||||
|
"lora_bias": false,
|
||||||
|
"lora_dropout": 0.05,
|
||||||
|
"lora_ga_config": null,
|
||||||
|
"megatron_config": null,
|
||||||
|
"megatron_core": "megatron.core",
|
||||||
|
"modules_to_save": null,
|
||||||
|
"peft_type": "LORA",
|
||||||
|
"peft_version": "0.19.1",
|
||||||
|
"qalora_group_size": 16,
|
||||||
|
"r": 64,
|
||||||
|
"rank_pattern": {},
|
||||||
|
"revision": null,
|
||||||
|
"target_modules": [
|
||||||
|
"down_proj",
|
||||||
|
"q_proj",
|
||||||
|
"o_proj",
|
||||||
|
"up_proj",
|
||||||
|
"k_proj",
|
||||||
|
"v_proj",
|
||||||
|
"gate_proj"
|
||||||
|
],
|
||||||
|
"target_parameters": null,
|
||||||
|
"task_type": "CAUSAL_LM",
|
||||||
|
"trainable_token_indices": null,
|
||||||
|
"use_bdlora": null,
|
||||||
|
"use_dora": false,
|
||||||
|
"use_qalora": false,
|
||||||
|
"use_rslora": false
|
||||||
|
}
|
||||||
3
checkpoint-250/adapter_model.safetensors
Normal file
3
checkpoint-250/adapter_model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:fc01785c6d7bad98246a02e99228f88bef985172876f559213bb9b93aee23aca
|
||||||
|
size 476904192
|
||||||
3
checkpoint-250/optimizer.pt
Normal file
3
checkpoint-250/optimizer.pt
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:87c56aaf210a0338ace6d4d4ef8eebb1082f8e3ceb492c1e5be418dc602887cf
|
||||||
|
size 954089079
|
||||||
3
checkpoint-250/rng_state.pth
Normal file
3
checkpoint-250/rng_state.pth
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:20d4a077a25b167eaadafb80be2d3de5d6ec836a27f2dfd743717935b378fa03
|
||||||
|
size 14645
|
||||||
3
checkpoint-250/scheduler.pt
Normal file
3
checkpoint-250/scheduler.pt
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:bcf087e5a3624be73138ec9da94a25475bf350ba077f9122500f4a271e8f065b
|
||||||
|
size 1465
|
||||||
118
checkpoint-250/trainer_state.json
Normal file
118
checkpoint-250/trainer_state.json
Normal file
@@ -0,0 +1,118 @@
|
|||||||
|
{
|
||||||
|
"best_global_step": null,
|
||||||
|
"best_metric": null,
|
||||||
|
"best_model_checkpoint": null,
|
||||||
|
"epoch": 0.6435006435006435,
|
||||||
|
"eval_steps": 500,
|
||||||
|
"global_step": 250,
|
||||||
|
"is_hyper_param_search": false,
|
||||||
|
"is_local_process_zero": true,
|
||||||
|
"is_world_process_zero": true,
|
||||||
|
"log_history": [
|
||||||
|
{
|
||||||
|
"epoch": 0.05148005148005148,
|
||||||
|
"grad_norm": 3.7984347343444824,
|
||||||
|
"learning_rate": 2.435897435897436e-06,
|
||||||
|
"loss": 3.0721,
|
||||||
|
"step": 20
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.10296010296010295,
|
||||||
|
"grad_norm": 2.0532360076904297,
|
||||||
|
"learning_rate": 5e-06,
|
||||||
|
"loss": 2.7105,
|
||||||
|
"step": 40
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.15444015444015444,
|
||||||
|
"grad_norm": 1.889143466949463,
|
||||||
|
"learning_rate": 7.564102564102564e-06,
|
||||||
|
"loss": 2.397,
|
||||||
|
"step": 60
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.2059202059202059,
|
||||||
|
"grad_norm": 2.206676959991455,
|
||||||
|
"learning_rate": 9.999949644960027e-06,
|
||||||
|
"loss": 2.1587,
|
||||||
|
"step": 80
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.2574002574002574,
|
||||||
|
"grad_norm": 2.4184727668762207,
|
||||||
|
"learning_rate": 9.9778098230154e-06,
|
||||||
|
"loss": 1.9613,
|
||||||
|
"step": 100
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.3088803088803089,
|
||||||
|
"grad_norm": 2.1078503131866455,
|
||||||
|
"learning_rate": 9.915591603280632e-06,
|
||||||
|
"loss": 1.9187,
|
||||||
|
"step": 120
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.36036036036036034,
|
||||||
|
"grad_norm": 2.228259801864624,
|
||||||
|
"learning_rate": 9.813795930277306e-06,
|
||||||
|
"loss": 1.863,
|
||||||
|
"step": 140
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.4118404118404118,
|
||||||
|
"grad_norm": 3.990351438522339,
|
||||||
|
"learning_rate": 9.673242402909555e-06,
|
||||||
|
"loss": 1.7983,
|
||||||
|
"step": 160
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.46332046332046334,
|
||||||
|
"grad_norm": 2.6102797985076904,
|
||||||
|
"learning_rate": 9.495062675535614e-06,
|
||||||
|
"loss": 1.7845,
|
||||||
|
"step": 180
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.5148005148005148,
|
||||||
|
"grad_norm": 2.4415857791900635,
|
||||||
|
"learning_rate": 9.280691346552308e-06,
|
||||||
|
"loss": 1.7541,
|
||||||
|
"step": 200
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.5662805662805663,
|
||||||
|
"grad_norm": 2.875765800476074,
|
||||||
|
"learning_rate": 9.031854407852317e-06,
|
||||||
|
"loss": 1.7665,
|
||||||
|
"step": 220
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.6177606177606177,
|
||||||
|
"grad_norm": 2.55922269821167,
|
||||||
|
"learning_rate": 8.750555348152299e-06,
|
||||||
|
"loss": 1.7272,
|
||||||
|
"step": 240
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"logging_steps": 20,
|
||||||
|
"max_steps": 778,
|
||||||
|
"num_input_tokens_seen": 0,
|
||||||
|
"num_train_epochs": 2,
|
||||||
|
"save_steps": 50,
|
||||||
|
"stateful_callbacks": {
|
||||||
|
"TrainerControl": {
|
||||||
|
"args": {
|
||||||
|
"should_epoch_stop": false,
|
||||||
|
"should_evaluate": false,
|
||||||
|
"should_log": false,
|
||||||
|
"should_save": true,
|
||||||
|
"should_training_stop": false
|
||||||
|
},
|
||||||
|
"attributes": {}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"total_flos": 1.8251403024175104e+16,
|
||||||
|
"train_batch_size": 4,
|
||||||
|
"trial_name": null,
|
||||||
|
"trial_params": null
|
||||||
|
}
|
||||||
3
checkpoint-250/training_args.bin
Normal file
3
checkpoint-250/training_args.bin
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:fe6e3348f136c9b50e5b7ff3939a0aca31453e7fbef30d71133ebde6392f9959
|
||||||
|
size 6033
|
||||||
207
checkpoint-300/README.md
Normal file
207
checkpoint-300/README.md
Normal file
@@ -0,0 +1,207 @@
|
|||||||
|
---
|
||||||
|
base_model: Henry236/nilechat-eg-stage1-general
|
||||||
|
library_name: peft
|
||||||
|
pipeline_tag: text-generation
|
||||||
|
tags:
|
||||||
|
- base_model:adapter:Henry236/nilechat-eg-stage1-general
|
||||||
|
- lora
|
||||||
|
- transformers
|
||||||
|
---
|
||||||
|
|
||||||
|
# Model Card for Model ID
|
||||||
|
|
||||||
|
<!-- Provide a quick summary of what the model is/does. -->
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## Model Details
|
||||||
|
|
||||||
|
### Model Description
|
||||||
|
|
||||||
|
<!-- Provide a longer summary of what this model is. -->
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
- **Developed by:** [More Information Needed]
|
||||||
|
- **Funded by [optional]:** [More Information Needed]
|
||||||
|
- **Shared by [optional]:** [More Information Needed]
|
||||||
|
- **Model type:** [More Information Needed]
|
||||||
|
- **Language(s) (NLP):** [More Information Needed]
|
||||||
|
- **License:** [More Information Needed]
|
||||||
|
- **Finetuned from model [optional]:** [More Information Needed]
|
||||||
|
|
||||||
|
### Model Sources [optional]
|
||||||
|
|
||||||
|
<!-- Provide the basic links for the model. -->
|
||||||
|
|
||||||
|
- **Repository:** [More Information Needed]
|
||||||
|
- **Paper [optional]:** [More Information Needed]
|
||||||
|
- **Demo [optional]:** [More Information Needed]
|
||||||
|
|
||||||
|
## Uses
|
||||||
|
|
||||||
|
<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
|
||||||
|
|
||||||
|
### Direct Use
|
||||||
|
|
||||||
|
<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Downstream Use [optional]
|
||||||
|
|
||||||
|
<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Out-of-Scope Use
|
||||||
|
|
||||||
|
<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Bias, Risks, and Limitations
|
||||||
|
|
||||||
|
<!-- This section is meant to convey both technical and sociotechnical limitations. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Recommendations
|
||||||
|
|
||||||
|
<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
|
||||||
|
|
||||||
|
Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
|
||||||
|
|
||||||
|
## How to Get Started with the Model
|
||||||
|
|
||||||
|
Use the code below to get started with the model.
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Training Details
|
||||||
|
|
||||||
|
### Training Data
|
||||||
|
|
||||||
|
<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Training Procedure
|
||||||
|
|
||||||
|
<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
|
||||||
|
|
||||||
|
#### Preprocessing [optional]
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
|
||||||
|
#### Training Hyperparameters
|
||||||
|
|
||||||
|
- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
|
||||||
|
|
||||||
|
#### Speeds, Sizes, Times [optional]
|
||||||
|
|
||||||
|
<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Evaluation
|
||||||
|
|
||||||
|
<!-- This section describes the evaluation protocols and provides the results. -->
|
||||||
|
|
||||||
|
### Testing Data, Factors & Metrics
|
||||||
|
|
||||||
|
#### Testing Data
|
||||||
|
|
||||||
|
<!-- This should link to a Dataset Card if possible. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Factors
|
||||||
|
|
||||||
|
<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Metrics
|
||||||
|
|
||||||
|
<!-- These are the evaluation metrics being used, ideally with a description of why. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Results
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Summary
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## Model Examination [optional]
|
||||||
|
|
||||||
|
<!-- Relevant interpretability work for the model goes here -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Environmental Impact
|
||||||
|
|
||||||
|
<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
|
||||||
|
|
||||||
|
Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
|
||||||
|
|
||||||
|
- **Hardware Type:** [More Information Needed]
|
||||||
|
- **Hours used:** [More Information Needed]
|
||||||
|
- **Cloud Provider:** [More Information Needed]
|
||||||
|
- **Compute Region:** [More Information Needed]
|
||||||
|
- **Carbon Emitted:** [More Information Needed]
|
||||||
|
|
||||||
|
## Technical Specifications [optional]
|
||||||
|
|
||||||
|
### Model Architecture and Objective
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Compute Infrastructure
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Hardware
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Software
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Citation [optional]
|
||||||
|
|
||||||
|
<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
|
||||||
|
|
||||||
|
**BibTeX:**
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
**APA:**
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Glossary [optional]
|
||||||
|
|
||||||
|
<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## More Information [optional]
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Model Card Authors [optional]
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Model Card Contact
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
### Framework versions
|
||||||
|
|
||||||
|
- PEFT 0.19.1
|
||||||
48
checkpoint-300/adapter_config.json
Normal file
48
checkpoint-300/adapter_config.json
Normal file
@@ -0,0 +1,48 @@
|
|||||||
|
{
|
||||||
|
"alora_invocation_tokens": null,
|
||||||
|
"alpha_pattern": {},
|
||||||
|
"arrow_config": null,
|
||||||
|
"auto_mapping": null,
|
||||||
|
"base_model_name_or_path": "Henry236/nilechat-eg-stage1-general",
|
||||||
|
"bias": "none",
|
||||||
|
"corda_config": null,
|
||||||
|
"ensure_weight_tying": false,
|
||||||
|
"eva_config": null,
|
||||||
|
"exclude_modules": null,
|
||||||
|
"fan_in_fan_out": false,
|
||||||
|
"inference_mode": true,
|
||||||
|
"init_lora_weights": true,
|
||||||
|
"layer_replication": null,
|
||||||
|
"layers_pattern": null,
|
||||||
|
"layers_to_transform": null,
|
||||||
|
"loftq_config": {},
|
||||||
|
"lora_alpha": 128,
|
||||||
|
"lora_bias": false,
|
||||||
|
"lora_dropout": 0.05,
|
||||||
|
"lora_ga_config": null,
|
||||||
|
"megatron_config": null,
|
||||||
|
"megatron_core": "megatron.core",
|
||||||
|
"modules_to_save": null,
|
||||||
|
"peft_type": "LORA",
|
||||||
|
"peft_version": "0.19.1",
|
||||||
|
"qalora_group_size": 16,
|
||||||
|
"r": 64,
|
||||||
|
"rank_pattern": {},
|
||||||
|
"revision": null,
|
||||||
|
"target_modules": [
|
||||||
|
"down_proj",
|
||||||
|
"q_proj",
|
||||||
|
"o_proj",
|
||||||
|
"up_proj",
|
||||||
|
"k_proj",
|
||||||
|
"v_proj",
|
||||||
|
"gate_proj"
|
||||||
|
],
|
||||||
|
"target_parameters": null,
|
||||||
|
"task_type": "CAUSAL_LM",
|
||||||
|
"trainable_token_indices": null,
|
||||||
|
"use_bdlora": null,
|
||||||
|
"use_dora": false,
|
||||||
|
"use_qalora": false,
|
||||||
|
"use_rslora": false
|
||||||
|
}
|
||||||
3
checkpoint-300/adapter_model.safetensors
Normal file
3
checkpoint-300/adapter_model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:868fe83f42a86fb73a40718c88dfcc39713c3013d2418691072ee9e10b4ff1e1
|
||||||
|
size 476904192
|
||||||
3
checkpoint-300/optimizer.pt
Normal file
3
checkpoint-300/optimizer.pt
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:4d71b64c39658b1d376289907a4f80b77a54fed659c9dad7b762de0ac79fcb3c
|
||||||
|
size 954089079
|
||||||
3
checkpoint-300/rng_state.pth
Normal file
3
checkpoint-300/rng_state.pth
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:c45d29cf54546eb1e088ecb0995d1358b78ef385035e0ff4d5ad300775fe0423
|
||||||
|
size 14645
|
||||||
3
checkpoint-300/scheduler.pt
Normal file
3
checkpoint-300/scheduler.pt
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:63438e51f56b7c528b601581df4f8738963dc9870c800effe61cf6a574a9b0d9
|
||||||
|
size 1465
|
||||||
139
checkpoint-300/trainer_state.json
Normal file
139
checkpoint-300/trainer_state.json
Normal file
@@ -0,0 +1,139 @@
|
|||||||
|
{
|
||||||
|
"best_global_step": null,
|
||||||
|
"best_metric": null,
|
||||||
|
"best_model_checkpoint": null,
|
||||||
|
"epoch": 0.7722007722007722,
|
||||||
|
"eval_steps": 500,
|
||||||
|
"global_step": 300,
|
||||||
|
"is_hyper_param_search": false,
|
||||||
|
"is_local_process_zero": true,
|
||||||
|
"is_world_process_zero": true,
|
||||||
|
"log_history": [
|
||||||
|
{
|
||||||
|
"epoch": 0.05148005148005148,
|
||||||
|
"grad_norm": 3.7984347343444824,
|
||||||
|
"learning_rate": 2.435897435897436e-06,
|
||||||
|
"loss": 3.0721,
|
||||||
|
"step": 20
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.10296010296010295,
|
||||||
|
"grad_norm": 2.0532360076904297,
|
||||||
|
"learning_rate": 5e-06,
|
||||||
|
"loss": 2.7105,
|
||||||
|
"step": 40
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.15444015444015444,
|
||||||
|
"grad_norm": 1.889143466949463,
|
||||||
|
"learning_rate": 7.564102564102564e-06,
|
||||||
|
"loss": 2.397,
|
||||||
|
"step": 60
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.2059202059202059,
|
||||||
|
"grad_norm": 2.206676959991455,
|
||||||
|
"learning_rate": 9.999949644960027e-06,
|
||||||
|
"loss": 2.1587,
|
||||||
|
"step": 80
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.2574002574002574,
|
||||||
|
"grad_norm": 2.4184727668762207,
|
||||||
|
"learning_rate": 9.9778098230154e-06,
|
||||||
|
"loss": 1.9613,
|
||||||
|
"step": 100
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.3088803088803089,
|
||||||
|
"grad_norm": 2.1078503131866455,
|
||||||
|
"learning_rate": 9.915591603280632e-06,
|
||||||
|
"loss": 1.9187,
|
||||||
|
"step": 120
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.36036036036036034,
|
||||||
|
"grad_norm": 2.228259801864624,
|
||||||
|
"learning_rate": 9.813795930277306e-06,
|
||||||
|
"loss": 1.863,
|
||||||
|
"step": 140
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.4118404118404118,
|
||||||
|
"grad_norm": 3.990351438522339,
|
||||||
|
"learning_rate": 9.673242402909555e-06,
|
||||||
|
"loss": 1.7983,
|
||||||
|
"step": 160
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.46332046332046334,
|
||||||
|
"grad_norm": 2.6102797985076904,
|
||||||
|
"learning_rate": 9.495062675535614e-06,
|
||||||
|
"loss": 1.7845,
|
||||||
|
"step": 180
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.5148005148005148,
|
||||||
|
"grad_norm": 2.4415857791900635,
|
||||||
|
"learning_rate": 9.280691346552308e-06,
|
||||||
|
"loss": 1.7541,
|
||||||
|
"step": 200
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.5662805662805663,
|
||||||
|
"grad_norm": 2.875765800476074,
|
||||||
|
"learning_rate": 9.031854407852317e-06,
|
||||||
|
"loss": 1.7665,
|
||||||
|
"step": 220
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.6177606177606177,
|
||||||
|
"grad_norm": 2.55922269821167,
|
||||||
|
"learning_rate": 8.750555348152299e-06,
|
||||||
|
"loss": 1.7272,
|
||||||
|
"step": 240
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.6692406692406693,
|
||||||
|
"grad_norm": 3.4958205223083496,
|
||||||
|
"learning_rate": 8.439059022079789e-06,
|
||||||
|
"loss": 1.7349,
|
||||||
|
"step": 260
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.7207207207207207,
|
||||||
|
"grad_norm": 2.582611560821533,
|
||||||
|
"learning_rate": 8.099873414895453e-06,
|
||||||
|
"loss": 1.7273,
|
||||||
|
"step": 280
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.7722007722007722,
|
||||||
|
"grad_norm": 2.274104595184326,
|
||||||
|
"learning_rate": 7.73572944967043e-06,
|
||||||
|
"loss": 1.695,
|
||||||
|
"step": 300
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"logging_steps": 20,
|
||||||
|
"max_steps": 778,
|
||||||
|
"num_input_tokens_seen": 0,
|
||||||
|
"num_train_epochs": 2,
|
||||||
|
"save_steps": 50,
|
||||||
|
"stateful_callbacks": {
|
||||||
|
"TrainerControl": {
|
||||||
|
"args": {
|
||||||
|
"should_epoch_stop": false,
|
||||||
|
"should_evaluate": false,
|
||||||
|
"should_log": false,
|
||||||
|
"should_save": true,
|
||||||
|
"should_training_stop": false
|
||||||
|
},
|
||||||
|
"attributes": {}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"total_flos": 2.188446208396493e+16,
|
||||||
|
"train_batch_size": 4,
|
||||||
|
"trial_name": null,
|
||||||
|
"trial_params": null
|
||||||
|
}
|
||||||
3
checkpoint-300/training_args.bin
Normal file
3
checkpoint-300/training_args.bin
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:fe6e3348f136c9b50e5b7ff3939a0aca31453e7fbef30d71133ebde6392f9959
|
||||||
|
size 6033
|
||||||
207
checkpoint-350/README.md
Normal file
207
checkpoint-350/README.md
Normal file
@@ -0,0 +1,207 @@
|
|||||||
|
---
|
||||||
|
base_model: Henry236/nilechat-eg-stage1-general
|
||||||
|
library_name: peft
|
||||||
|
pipeline_tag: text-generation
|
||||||
|
tags:
|
||||||
|
- base_model:adapter:Henry236/nilechat-eg-stage1-general
|
||||||
|
- lora
|
||||||
|
- transformers
|
||||||
|
---
|
||||||
|
|
||||||
|
# Model Card for Model ID
|
||||||
|
|
||||||
|
<!-- Provide a quick summary of what the model is/does. -->
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## Model Details
|
||||||
|
|
||||||
|
### Model Description
|
||||||
|
|
||||||
|
<!-- Provide a longer summary of what this model is. -->
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
- **Developed by:** [More Information Needed]
|
||||||
|
- **Funded by [optional]:** [More Information Needed]
|
||||||
|
- **Shared by [optional]:** [More Information Needed]
|
||||||
|
- **Model type:** [More Information Needed]
|
||||||
|
- **Language(s) (NLP):** [More Information Needed]
|
||||||
|
- **License:** [More Information Needed]
|
||||||
|
- **Finetuned from model [optional]:** [More Information Needed]
|
||||||
|
|
||||||
|
### Model Sources [optional]
|
||||||
|
|
||||||
|
<!-- Provide the basic links for the model. -->
|
||||||
|
|
||||||
|
- **Repository:** [More Information Needed]
|
||||||
|
- **Paper [optional]:** [More Information Needed]
|
||||||
|
- **Demo [optional]:** [More Information Needed]
|
||||||
|
|
||||||
|
## Uses
|
||||||
|
|
||||||
|
<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
|
||||||
|
|
||||||
|
### Direct Use
|
||||||
|
|
||||||
|
<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Downstream Use [optional]
|
||||||
|
|
||||||
|
<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Out-of-Scope Use
|
||||||
|
|
||||||
|
<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Bias, Risks, and Limitations
|
||||||
|
|
||||||
|
<!-- This section is meant to convey both technical and sociotechnical limitations. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Recommendations
|
||||||
|
|
||||||
|
<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
|
||||||
|
|
||||||
|
Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
|
||||||
|
|
||||||
|
## How to Get Started with the Model
|
||||||
|
|
||||||
|
Use the code below to get started with the model.
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Training Details
|
||||||
|
|
||||||
|
### Training Data
|
||||||
|
|
||||||
|
<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Training Procedure
|
||||||
|
|
||||||
|
<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
|
||||||
|
|
||||||
|
#### Preprocessing [optional]
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
|
||||||
|
#### Training Hyperparameters
|
||||||
|
|
||||||
|
- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
|
||||||
|
|
||||||
|
#### Speeds, Sizes, Times [optional]
|
||||||
|
|
||||||
|
<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Evaluation
|
||||||
|
|
||||||
|
<!-- This section describes the evaluation protocols and provides the results. -->
|
||||||
|
|
||||||
|
### Testing Data, Factors & Metrics
|
||||||
|
|
||||||
|
#### Testing Data
|
||||||
|
|
||||||
|
<!-- This should link to a Dataset Card if possible. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Factors
|
||||||
|
|
||||||
|
<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Metrics
|
||||||
|
|
||||||
|
<!-- These are the evaluation metrics being used, ideally with a description of why. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Results
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Summary
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## Model Examination [optional]
|
||||||
|
|
||||||
|
<!-- Relevant interpretability work for the model goes here -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Environmental Impact
|
||||||
|
|
||||||
|
<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
|
||||||
|
|
||||||
|
Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
|
||||||
|
|
||||||
|
- **Hardware Type:** [More Information Needed]
|
||||||
|
- **Hours used:** [More Information Needed]
|
||||||
|
- **Cloud Provider:** [More Information Needed]
|
||||||
|
- **Compute Region:** [More Information Needed]
|
||||||
|
- **Carbon Emitted:** [More Information Needed]
|
||||||
|
|
||||||
|
## Technical Specifications [optional]
|
||||||
|
|
||||||
|
### Model Architecture and Objective
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Compute Infrastructure
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Hardware
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Software
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Citation [optional]
|
||||||
|
|
||||||
|
<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
|
||||||
|
|
||||||
|
**BibTeX:**
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
**APA:**
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Glossary [optional]
|
||||||
|
|
||||||
|
<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## More Information [optional]
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Model Card Authors [optional]
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Model Card Contact
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
### Framework versions
|
||||||
|
|
||||||
|
- PEFT 0.19.1
|
||||||
48
checkpoint-350/adapter_config.json
Normal file
48
checkpoint-350/adapter_config.json
Normal file
@@ -0,0 +1,48 @@
|
|||||||
|
{
|
||||||
|
"alora_invocation_tokens": null,
|
||||||
|
"alpha_pattern": {},
|
||||||
|
"arrow_config": null,
|
||||||
|
"auto_mapping": null,
|
||||||
|
"base_model_name_or_path": "Henry236/nilechat-eg-stage1-general",
|
||||||
|
"bias": "none",
|
||||||
|
"corda_config": null,
|
||||||
|
"ensure_weight_tying": false,
|
||||||
|
"eva_config": null,
|
||||||
|
"exclude_modules": null,
|
||||||
|
"fan_in_fan_out": false,
|
||||||
|
"inference_mode": true,
|
||||||
|
"init_lora_weights": true,
|
||||||
|
"layer_replication": null,
|
||||||
|
"layers_pattern": null,
|
||||||
|
"layers_to_transform": null,
|
||||||
|
"loftq_config": {},
|
||||||
|
"lora_alpha": 128,
|
||||||
|
"lora_bias": false,
|
||||||
|
"lora_dropout": 0.05,
|
||||||
|
"lora_ga_config": null,
|
||||||
|
"megatron_config": null,
|
||||||
|
"megatron_core": "megatron.core",
|
||||||
|
"modules_to_save": null,
|
||||||
|
"peft_type": "LORA",
|
||||||
|
"peft_version": "0.19.1",
|
||||||
|
"qalora_group_size": 16,
|
||||||
|
"r": 64,
|
||||||
|
"rank_pattern": {},
|
||||||
|
"revision": null,
|
||||||
|
"target_modules": [
|
||||||
|
"down_proj",
|
||||||
|
"q_proj",
|
||||||
|
"o_proj",
|
||||||
|
"up_proj",
|
||||||
|
"k_proj",
|
||||||
|
"v_proj",
|
||||||
|
"gate_proj"
|
||||||
|
],
|
||||||
|
"target_parameters": null,
|
||||||
|
"task_type": "CAUSAL_LM",
|
||||||
|
"trainable_token_indices": null,
|
||||||
|
"use_bdlora": null,
|
||||||
|
"use_dora": false,
|
||||||
|
"use_qalora": false,
|
||||||
|
"use_rslora": false
|
||||||
|
}
|
||||||
3
checkpoint-350/adapter_model.safetensors
Normal file
3
checkpoint-350/adapter_model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:83f56b3fe7844d789bbccefd0b3ac048828637c3f82cb212bd909b351451ece5
|
||||||
|
size 476904192
|
||||||
3
checkpoint-350/optimizer.pt
Normal file
3
checkpoint-350/optimizer.pt
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:a172b231aff8ea87ba016b62c17a146fee9ac17807a0f7ed59afdddbc7ea8d37
|
||||||
|
size 954089079
|
||||||
3
checkpoint-350/rng_state.pth
Normal file
3
checkpoint-350/rng_state.pth
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:cc724354d89f8b6ea7e24bfe39a321e4ef1a96e80bb0ebd11c76b848f0300847
|
||||||
|
size 14645
|
||||||
3
checkpoint-350/scheduler.pt
Normal file
3
checkpoint-350/scheduler.pt
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:f3bce589507b2d7326e2033307b04716715fcf521d5d83c7a7f67248b91895eb
|
||||||
|
size 1465
|
||||||
153
checkpoint-350/trainer_state.json
Normal file
153
checkpoint-350/trainer_state.json
Normal file
@@ -0,0 +1,153 @@
|
|||||||
|
{
|
||||||
|
"best_global_step": null,
|
||||||
|
"best_metric": null,
|
||||||
|
"best_model_checkpoint": null,
|
||||||
|
"epoch": 0.9009009009009009,
|
||||||
|
"eval_steps": 500,
|
||||||
|
"global_step": 350,
|
||||||
|
"is_hyper_param_search": false,
|
||||||
|
"is_local_process_zero": true,
|
||||||
|
"is_world_process_zero": true,
|
||||||
|
"log_history": [
|
||||||
|
{
|
||||||
|
"epoch": 0.05148005148005148,
|
||||||
|
"grad_norm": 3.7984347343444824,
|
||||||
|
"learning_rate": 2.435897435897436e-06,
|
||||||
|
"loss": 3.0721,
|
||||||
|
"step": 20
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.10296010296010295,
|
||||||
|
"grad_norm": 2.0532360076904297,
|
||||||
|
"learning_rate": 5e-06,
|
||||||
|
"loss": 2.7105,
|
||||||
|
"step": 40
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.15444015444015444,
|
||||||
|
"grad_norm": 1.889143466949463,
|
||||||
|
"learning_rate": 7.564102564102564e-06,
|
||||||
|
"loss": 2.397,
|
||||||
|
"step": 60
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.2059202059202059,
|
||||||
|
"grad_norm": 2.206676959991455,
|
||||||
|
"learning_rate": 9.999949644960027e-06,
|
||||||
|
"loss": 2.1587,
|
||||||
|
"step": 80
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.2574002574002574,
|
||||||
|
"grad_norm": 2.4184727668762207,
|
||||||
|
"learning_rate": 9.9778098230154e-06,
|
||||||
|
"loss": 1.9613,
|
||||||
|
"step": 100
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.3088803088803089,
|
||||||
|
"grad_norm": 2.1078503131866455,
|
||||||
|
"learning_rate": 9.915591603280632e-06,
|
||||||
|
"loss": 1.9187,
|
||||||
|
"step": 120
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.36036036036036034,
|
||||||
|
"grad_norm": 2.228259801864624,
|
||||||
|
"learning_rate": 9.813795930277306e-06,
|
||||||
|
"loss": 1.863,
|
||||||
|
"step": 140
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.4118404118404118,
|
||||||
|
"grad_norm": 3.990351438522339,
|
||||||
|
"learning_rate": 9.673242402909555e-06,
|
||||||
|
"loss": 1.7983,
|
||||||
|
"step": 160
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.46332046332046334,
|
||||||
|
"grad_norm": 2.6102797985076904,
|
||||||
|
"learning_rate": 9.495062675535614e-06,
|
||||||
|
"loss": 1.7845,
|
||||||
|
"step": 180
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.5148005148005148,
|
||||||
|
"grad_norm": 2.4415857791900635,
|
||||||
|
"learning_rate": 9.280691346552308e-06,
|
||||||
|
"loss": 1.7541,
|
||||||
|
"step": 200
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.5662805662805663,
|
||||||
|
"grad_norm": 2.875765800476074,
|
||||||
|
"learning_rate": 9.031854407852317e-06,
|
||||||
|
"loss": 1.7665,
|
||||||
|
"step": 220
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.6177606177606177,
|
||||||
|
"grad_norm": 2.55922269821167,
|
||||||
|
"learning_rate": 8.750555348152299e-06,
|
||||||
|
"loss": 1.7272,
|
||||||
|
"step": 240
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.6692406692406693,
|
||||||
|
"grad_norm": 3.4958205223083496,
|
||||||
|
"learning_rate": 8.439059022079789e-06,
|
||||||
|
"loss": 1.7349,
|
||||||
|
"step": 260
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.7207207207207207,
|
||||||
|
"grad_norm": 2.582611560821533,
|
||||||
|
"learning_rate": 8.099873414895453e-06,
|
||||||
|
"loss": 1.7273,
|
||||||
|
"step": 280
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.7722007722007722,
|
||||||
|
"grad_norm": 2.274104595184326,
|
||||||
|
"learning_rate": 7.73572944967043e-06,
|
||||||
|
"loss": 1.695,
|
||||||
|
"step": 300
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.8236808236808236,
|
||||||
|
"grad_norm": 2.8610410690307617,
|
||||||
|
"learning_rate": 7.3495589994995274e-06,
|
||||||
|
"loss": 1.663,
|
||||||
|
"step": 320
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.8751608751608752,
|
||||||
|
"grad_norm": 2.4936399459838867,
|
||||||
|
"learning_rate": 6.944471281782975e-06,
|
||||||
|
"loss": 1.6462,
|
||||||
|
"step": 340
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"logging_steps": 20,
|
||||||
|
"max_steps": 778,
|
||||||
|
"num_input_tokens_seen": 0,
|
||||||
|
"num_train_epochs": 2,
|
||||||
|
"save_steps": 50,
|
||||||
|
"stateful_callbacks": {
|
||||||
|
"TrainerControl": {
|
||||||
|
"args": {
|
||||||
|
"should_epoch_stop": false,
|
||||||
|
"should_evaluate": false,
|
||||||
|
"should_log": false,
|
||||||
|
"should_save": true,
|
||||||
|
"should_training_stop": false
|
||||||
|
},
|
||||||
|
"attributes": {}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"total_flos": 2.5584058931429376e+16,
|
||||||
|
"train_batch_size": 4,
|
||||||
|
"trial_name": null,
|
||||||
|
"trial_params": null
|
||||||
|
}
|
||||||
3
checkpoint-350/training_args.bin
Normal file
3
checkpoint-350/training_args.bin
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:fe6e3348f136c9b50e5b7ff3939a0aca31453e7fbef30d71133ebde6392f9959
|
||||||
|
size 6033
|
||||||
207
checkpoint-400/README.md
Normal file
207
checkpoint-400/README.md
Normal file
@@ -0,0 +1,207 @@
|
|||||||
|
---
|
||||||
|
base_model: Henry236/nilechat-eg-stage1-general
|
||||||
|
library_name: peft
|
||||||
|
pipeline_tag: text-generation
|
||||||
|
tags:
|
||||||
|
- base_model:adapter:Henry236/nilechat-eg-stage1-general
|
||||||
|
- lora
|
||||||
|
- transformers
|
||||||
|
---
|
||||||
|
|
||||||
|
# Model Card for Model ID
|
||||||
|
|
||||||
|
<!-- Provide a quick summary of what the model is/does. -->
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## Model Details
|
||||||
|
|
||||||
|
### Model Description
|
||||||
|
|
||||||
|
<!-- Provide a longer summary of what this model is. -->
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
- **Developed by:** [More Information Needed]
|
||||||
|
- **Funded by [optional]:** [More Information Needed]
|
||||||
|
- **Shared by [optional]:** [More Information Needed]
|
||||||
|
- **Model type:** [More Information Needed]
|
||||||
|
- **Language(s) (NLP):** [More Information Needed]
|
||||||
|
- **License:** [More Information Needed]
|
||||||
|
- **Finetuned from model [optional]:** [More Information Needed]
|
||||||
|
|
||||||
|
### Model Sources [optional]
|
||||||
|
|
||||||
|
<!-- Provide the basic links for the model. -->
|
||||||
|
|
||||||
|
- **Repository:** [More Information Needed]
|
||||||
|
- **Paper [optional]:** [More Information Needed]
|
||||||
|
- **Demo [optional]:** [More Information Needed]
|
||||||
|
|
||||||
|
## Uses
|
||||||
|
|
||||||
|
<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
|
||||||
|
|
||||||
|
### Direct Use
|
||||||
|
|
||||||
|
<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Downstream Use [optional]
|
||||||
|
|
||||||
|
<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Out-of-Scope Use
|
||||||
|
|
||||||
|
<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Bias, Risks, and Limitations
|
||||||
|
|
||||||
|
<!-- This section is meant to convey both technical and sociotechnical limitations. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Recommendations
|
||||||
|
|
||||||
|
<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
|
||||||
|
|
||||||
|
Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
|
||||||
|
|
||||||
|
## How to Get Started with the Model
|
||||||
|
|
||||||
|
Use the code below to get started with the model.
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Training Details
|
||||||
|
|
||||||
|
### Training Data
|
||||||
|
|
||||||
|
<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Training Procedure
|
||||||
|
|
||||||
|
<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
|
||||||
|
|
||||||
|
#### Preprocessing [optional]
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
|
||||||
|
#### Training Hyperparameters
|
||||||
|
|
||||||
|
- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
|
||||||
|
|
||||||
|
#### Speeds, Sizes, Times [optional]
|
||||||
|
|
||||||
|
<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Evaluation
|
||||||
|
|
||||||
|
<!-- This section describes the evaluation protocols and provides the results. -->
|
||||||
|
|
||||||
|
### Testing Data, Factors & Metrics
|
||||||
|
|
||||||
|
#### Testing Data
|
||||||
|
|
||||||
|
<!-- This should link to a Dataset Card if possible. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Factors
|
||||||
|
|
||||||
|
<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Metrics
|
||||||
|
|
||||||
|
<!-- These are the evaluation metrics being used, ideally with a description of why. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Results
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Summary
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## Model Examination [optional]
|
||||||
|
|
||||||
|
<!-- Relevant interpretability work for the model goes here -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Environmental Impact
|
||||||
|
|
||||||
|
<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
|
||||||
|
|
||||||
|
Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
|
||||||
|
|
||||||
|
- **Hardware Type:** [More Information Needed]
|
||||||
|
- **Hours used:** [More Information Needed]
|
||||||
|
- **Cloud Provider:** [More Information Needed]
|
||||||
|
- **Compute Region:** [More Information Needed]
|
||||||
|
- **Carbon Emitted:** [More Information Needed]
|
||||||
|
|
||||||
|
## Technical Specifications [optional]
|
||||||
|
|
||||||
|
### Model Architecture and Objective
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Compute Infrastructure
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Hardware
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Software
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Citation [optional]
|
||||||
|
|
||||||
|
<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
|
||||||
|
|
||||||
|
**BibTeX:**
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
**APA:**
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Glossary [optional]
|
||||||
|
|
||||||
|
<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## More Information [optional]
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Model Card Authors [optional]
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Model Card Contact
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
### Framework versions
|
||||||
|
|
||||||
|
- PEFT 0.19.1
|
||||||
48
checkpoint-400/adapter_config.json
Normal file
48
checkpoint-400/adapter_config.json
Normal file
@@ -0,0 +1,48 @@
|
|||||||
|
{
|
||||||
|
"alora_invocation_tokens": null,
|
||||||
|
"alpha_pattern": {},
|
||||||
|
"arrow_config": null,
|
||||||
|
"auto_mapping": null,
|
||||||
|
"base_model_name_or_path": "Henry236/nilechat-eg-stage1-general",
|
||||||
|
"bias": "none",
|
||||||
|
"corda_config": null,
|
||||||
|
"ensure_weight_tying": false,
|
||||||
|
"eva_config": null,
|
||||||
|
"exclude_modules": null,
|
||||||
|
"fan_in_fan_out": false,
|
||||||
|
"inference_mode": true,
|
||||||
|
"init_lora_weights": true,
|
||||||
|
"layer_replication": null,
|
||||||
|
"layers_pattern": null,
|
||||||
|
"layers_to_transform": null,
|
||||||
|
"loftq_config": {},
|
||||||
|
"lora_alpha": 128,
|
||||||
|
"lora_bias": false,
|
||||||
|
"lora_dropout": 0.05,
|
||||||
|
"lora_ga_config": null,
|
||||||
|
"megatron_config": null,
|
||||||
|
"megatron_core": "megatron.core",
|
||||||
|
"modules_to_save": null,
|
||||||
|
"peft_type": "LORA",
|
||||||
|
"peft_version": "0.19.1",
|
||||||
|
"qalora_group_size": 16,
|
||||||
|
"r": 64,
|
||||||
|
"rank_pattern": {},
|
||||||
|
"revision": null,
|
||||||
|
"target_modules": [
|
||||||
|
"down_proj",
|
||||||
|
"q_proj",
|
||||||
|
"o_proj",
|
||||||
|
"up_proj",
|
||||||
|
"k_proj",
|
||||||
|
"v_proj",
|
||||||
|
"gate_proj"
|
||||||
|
],
|
||||||
|
"target_parameters": null,
|
||||||
|
"task_type": "CAUSAL_LM",
|
||||||
|
"trainable_token_indices": null,
|
||||||
|
"use_bdlora": null,
|
||||||
|
"use_dora": false,
|
||||||
|
"use_qalora": false,
|
||||||
|
"use_rslora": false
|
||||||
|
}
|
||||||
3
checkpoint-400/adapter_model.safetensors
Normal file
3
checkpoint-400/adapter_model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:65e10c1bf60ff6957c27889a479eb4e7525e164e1cb5c47f4e70134741a4546c
|
||||||
|
size 476904192
|
||||||
3
checkpoint-400/optimizer.pt
Normal file
3
checkpoint-400/optimizer.pt
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:c6706b6a95e6f56b0f1adfadc7e602b6ec123c5eb2f4b4741e6d068c49883fae
|
||||||
|
size 954089079
|
||||||
3
checkpoint-400/rng_state.pth
Normal file
3
checkpoint-400/rng_state.pth
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:2be0af236b70053e1a7b7af1d08e1a798ad5a6fb170bd41eda639548c4671f06
|
||||||
|
size 14645
|
||||||
3
checkpoint-400/scheduler.pt
Normal file
3
checkpoint-400/scheduler.pt
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:af18ab36e3aeb4ffa41dc6188da1b4cb93916ad02eed04f7341eef26e3d9ccdc
|
||||||
|
size 1465
|
||||||
174
checkpoint-400/trainer_state.json
Normal file
174
checkpoint-400/trainer_state.json
Normal file
@@ -0,0 +1,174 @@
|
|||||||
|
{
|
||||||
|
"best_global_step": null,
|
||||||
|
"best_metric": null,
|
||||||
|
"best_model_checkpoint": null,
|
||||||
|
"epoch": 1.0283140283140284,
|
||||||
|
"eval_steps": 500,
|
||||||
|
"global_step": 400,
|
||||||
|
"is_hyper_param_search": false,
|
||||||
|
"is_local_process_zero": true,
|
||||||
|
"is_world_process_zero": true,
|
||||||
|
"log_history": [
|
||||||
|
{
|
||||||
|
"epoch": 0.05148005148005148,
|
||||||
|
"grad_norm": 3.7984347343444824,
|
||||||
|
"learning_rate": 2.435897435897436e-06,
|
||||||
|
"loss": 3.0721,
|
||||||
|
"step": 20
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.10296010296010295,
|
||||||
|
"grad_norm": 2.0532360076904297,
|
||||||
|
"learning_rate": 5e-06,
|
||||||
|
"loss": 2.7105,
|
||||||
|
"step": 40
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.15444015444015444,
|
||||||
|
"grad_norm": 1.889143466949463,
|
||||||
|
"learning_rate": 7.564102564102564e-06,
|
||||||
|
"loss": 2.397,
|
||||||
|
"step": 60
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.2059202059202059,
|
||||||
|
"grad_norm": 2.206676959991455,
|
||||||
|
"learning_rate": 9.999949644960027e-06,
|
||||||
|
"loss": 2.1587,
|
||||||
|
"step": 80
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.2574002574002574,
|
||||||
|
"grad_norm": 2.4184727668762207,
|
||||||
|
"learning_rate": 9.9778098230154e-06,
|
||||||
|
"loss": 1.9613,
|
||||||
|
"step": 100
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.3088803088803089,
|
||||||
|
"grad_norm": 2.1078503131866455,
|
||||||
|
"learning_rate": 9.915591603280632e-06,
|
||||||
|
"loss": 1.9187,
|
||||||
|
"step": 120
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.36036036036036034,
|
||||||
|
"grad_norm": 2.228259801864624,
|
||||||
|
"learning_rate": 9.813795930277306e-06,
|
||||||
|
"loss": 1.863,
|
||||||
|
"step": 140
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.4118404118404118,
|
||||||
|
"grad_norm": 3.990351438522339,
|
||||||
|
"learning_rate": 9.673242402909555e-06,
|
||||||
|
"loss": 1.7983,
|
||||||
|
"step": 160
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.46332046332046334,
|
||||||
|
"grad_norm": 2.6102797985076904,
|
||||||
|
"learning_rate": 9.495062675535614e-06,
|
||||||
|
"loss": 1.7845,
|
||||||
|
"step": 180
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.5148005148005148,
|
||||||
|
"grad_norm": 2.4415857791900635,
|
||||||
|
"learning_rate": 9.280691346552308e-06,
|
||||||
|
"loss": 1.7541,
|
||||||
|
"step": 200
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.5662805662805663,
|
||||||
|
"grad_norm": 2.875765800476074,
|
||||||
|
"learning_rate": 9.031854407852317e-06,
|
||||||
|
"loss": 1.7665,
|
||||||
|
"step": 220
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.6177606177606177,
|
||||||
|
"grad_norm": 2.55922269821167,
|
||||||
|
"learning_rate": 8.750555348152299e-06,
|
||||||
|
"loss": 1.7272,
|
||||||
|
"step": 240
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.6692406692406693,
|
||||||
|
"grad_norm": 3.4958205223083496,
|
||||||
|
"learning_rate": 8.439059022079789e-06,
|
||||||
|
"loss": 1.7349,
|
||||||
|
"step": 260
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.7207207207207207,
|
||||||
|
"grad_norm": 2.582611560821533,
|
||||||
|
"learning_rate": 8.099873414895453e-06,
|
||||||
|
"loss": 1.7273,
|
||||||
|
"step": 280
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.7722007722007722,
|
||||||
|
"grad_norm": 2.274104595184326,
|
||||||
|
"learning_rate": 7.73572944967043e-06,
|
||||||
|
"loss": 1.695,
|
||||||
|
"step": 300
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.8236808236808236,
|
||||||
|
"grad_norm": 2.8610410690307617,
|
||||||
|
"learning_rate": 7.3495589994995274e-06,
|
||||||
|
"loss": 1.663,
|
||||||
|
"step": 320
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.8751608751608752,
|
||||||
|
"grad_norm": 2.4936399459838867,
|
||||||
|
"learning_rate": 6.944471281782975e-06,
|
||||||
|
"loss": 1.6462,
|
||||||
|
"step": 340
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.9266409266409267,
|
||||||
|
"grad_norm": 3.510580062866211,
|
||||||
|
"learning_rate": 6.523727824636103e-06,
|
||||||
|
"loss": 1.6662,
|
||||||
|
"step": 360
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.9781209781209781,
|
||||||
|
"grad_norm": 2.398237943649292,
|
||||||
|
"learning_rate": 6.090716206982714e-06,
|
||||||
|
"loss": 1.6664,
|
||||||
|
"step": 380
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.0283140283140284,
|
||||||
|
"grad_norm": 2.8953514099121094,
|
||||||
|
"learning_rate": 5.648922783761443e-06,
|
||||||
|
"loss": 1.6363,
|
||||||
|
"step": 400
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"logging_steps": 20,
|
||||||
|
"max_steps": 778,
|
||||||
|
"num_input_tokens_seen": 0,
|
||||||
|
"num_train_epochs": 2,
|
||||||
|
"save_steps": 50,
|
||||||
|
"stateful_callbacks": {
|
||||||
|
"TrainerControl": {
|
||||||
|
"args": {
|
||||||
|
"should_epoch_stop": false,
|
||||||
|
"should_evaluate": false,
|
||||||
|
"should_log": false,
|
||||||
|
"should_save": true,
|
||||||
|
"should_training_stop": false
|
||||||
|
},
|
||||||
|
"attributes": {}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"total_flos": 2.919243590671565e+16,
|
||||||
|
"train_batch_size": 4,
|
||||||
|
"trial_name": null,
|
||||||
|
"trial_params": null
|
||||||
|
}
|
||||||
3
checkpoint-400/training_args.bin
Normal file
3
checkpoint-400/training_args.bin
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:fe6e3348f136c9b50e5b7ff3939a0aca31453e7fbef30d71133ebde6392f9959
|
||||||
|
size 6033
|
||||||
207
checkpoint-450/README.md
Normal file
207
checkpoint-450/README.md
Normal file
@@ -0,0 +1,207 @@
|
|||||||
|
---
|
||||||
|
base_model: Henry236/nilechat-eg-stage1-general
|
||||||
|
library_name: peft
|
||||||
|
pipeline_tag: text-generation
|
||||||
|
tags:
|
||||||
|
- base_model:adapter:Henry236/nilechat-eg-stage1-general
|
||||||
|
- lora
|
||||||
|
- transformers
|
||||||
|
---
|
||||||
|
|
||||||
|
# Model Card for Model ID
|
||||||
|
|
||||||
|
<!-- Provide a quick summary of what the model is/does. -->
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## Model Details
|
||||||
|
|
||||||
|
### Model Description
|
||||||
|
|
||||||
|
<!-- Provide a longer summary of what this model is. -->
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
- **Developed by:** [More Information Needed]
|
||||||
|
- **Funded by [optional]:** [More Information Needed]
|
||||||
|
- **Shared by [optional]:** [More Information Needed]
|
||||||
|
- **Model type:** [More Information Needed]
|
||||||
|
- **Language(s) (NLP):** [More Information Needed]
|
||||||
|
- **License:** [More Information Needed]
|
||||||
|
- **Finetuned from model [optional]:** [More Information Needed]
|
||||||
|
|
||||||
|
### Model Sources [optional]
|
||||||
|
|
||||||
|
<!-- Provide the basic links for the model. -->
|
||||||
|
|
||||||
|
- **Repository:** [More Information Needed]
|
||||||
|
- **Paper [optional]:** [More Information Needed]
|
||||||
|
- **Demo [optional]:** [More Information Needed]
|
||||||
|
|
||||||
|
## Uses
|
||||||
|
|
||||||
|
<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
|
||||||
|
|
||||||
|
### Direct Use
|
||||||
|
|
||||||
|
<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Downstream Use [optional]
|
||||||
|
|
||||||
|
<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Out-of-Scope Use
|
||||||
|
|
||||||
|
<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Bias, Risks, and Limitations
|
||||||
|
|
||||||
|
<!-- This section is meant to convey both technical and sociotechnical limitations. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Recommendations
|
||||||
|
|
||||||
|
<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
|
||||||
|
|
||||||
|
Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
|
||||||
|
|
||||||
|
## How to Get Started with the Model
|
||||||
|
|
||||||
|
Use the code below to get started with the model.
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Training Details
|
||||||
|
|
||||||
|
### Training Data
|
||||||
|
|
||||||
|
<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Training Procedure
|
||||||
|
|
||||||
|
<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
|
||||||
|
|
||||||
|
#### Preprocessing [optional]
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
|
||||||
|
#### Training Hyperparameters
|
||||||
|
|
||||||
|
- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
|
||||||
|
|
||||||
|
#### Speeds, Sizes, Times [optional]
|
||||||
|
|
||||||
|
<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Evaluation
|
||||||
|
|
||||||
|
<!-- This section describes the evaluation protocols and provides the results. -->
|
||||||
|
|
||||||
|
### Testing Data, Factors & Metrics
|
||||||
|
|
||||||
|
#### Testing Data
|
||||||
|
|
||||||
|
<!-- This should link to a Dataset Card if possible. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Factors
|
||||||
|
|
||||||
|
<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Metrics
|
||||||
|
|
||||||
|
<!-- These are the evaluation metrics being used, ideally with a description of why. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Results
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Summary
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## Model Examination [optional]
|
||||||
|
|
||||||
|
<!-- Relevant interpretability work for the model goes here -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Environmental Impact
|
||||||
|
|
||||||
|
<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
|
||||||
|
|
||||||
|
Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
|
||||||
|
|
||||||
|
- **Hardware Type:** [More Information Needed]
|
||||||
|
- **Hours used:** [More Information Needed]
|
||||||
|
- **Cloud Provider:** [More Information Needed]
|
||||||
|
- **Compute Region:** [More Information Needed]
|
||||||
|
- **Carbon Emitted:** [More Information Needed]
|
||||||
|
|
||||||
|
## Technical Specifications [optional]
|
||||||
|
|
||||||
|
### Model Architecture and Objective
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Compute Infrastructure
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Hardware
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Software
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Citation [optional]
|
||||||
|
|
||||||
|
<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
|
||||||
|
|
||||||
|
**BibTeX:**
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
**APA:**
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Glossary [optional]
|
||||||
|
|
||||||
|
<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## More Information [optional]
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Model Card Authors [optional]
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Model Card Contact
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
### Framework versions
|
||||||
|
|
||||||
|
- PEFT 0.19.1
|
||||||
48
checkpoint-450/adapter_config.json
Normal file
48
checkpoint-450/adapter_config.json
Normal file
@@ -0,0 +1,48 @@
|
|||||||
|
{
|
||||||
|
"alora_invocation_tokens": null,
|
||||||
|
"alpha_pattern": {},
|
||||||
|
"arrow_config": null,
|
||||||
|
"auto_mapping": null,
|
||||||
|
"base_model_name_or_path": "Henry236/nilechat-eg-stage1-general",
|
||||||
|
"bias": "none",
|
||||||
|
"corda_config": null,
|
||||||
|
"ensure_weight_tying": false,
|
||||||
|
"eva_config": null,
|
||||||
|
"exclude_modules": null,
|
||||||
|
"fan_in_fan_out": false,
|
||||||
|
"inference_mode": true,
|
||||||
|
"init_lora_weights": true,
|
||||||
|
"layer_replication": null,
|
||||||
|
"layers_pattern": null,
|
||||||
|
"layers_to_transform": null,
|
||||||
|
"loftq_config": {},
|
||||||
|
"lora_alpha": 128,
|
||||||
|
"lora_bias": false,
|
||||||
|
"lora_dropout": 0.05,
|
||||||
|
"lora_ga_config": null,
|
||||||
|
"megatron_config": null,
|
||||||
|
"megatron_core": "megatron.core",
|
||||||
|
"modules_to_save": null,
|
||||||
|
"peft_type": "LORA",
|
||||||
|
"peft_version": "0.19.1",
|
||||||
|
"qalora_group_size": 16,
|
||||||
|
"r": 64,
|
||||||
|
"rank_pattern": {},
|
||||||
|
"revision": null,
|
||||||
|
"target_modules": [
|
||||||
|
"down_proj",
|
||||||
|
"q_proj",
|
||||||
|
"o_proj",
|
||||||
|
"up_proj",
|
||||||
|
"k_proj",
|
||||||
|
"v_proj",
|
||||||
|
"gate_proj"
|
||||||
|
],
|
||||||
|
"target_parameters": null,
|
||||||
|
"task_type": "CAUSAL_LM",
|
||||||
|
"trainable_token_indices": null,
|
||||||
|
"use_bdlora": null,
|
||||||
|
"use_dora": false,
|
||||||
|
"use_qalora": false,
|
||||||
|
"use_rslora": false
|
||||||
|
}
|
||||||
3
checkpoint-450/adapter_model.safetensors
Normal file
3
checkpoint-450/adapter_model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:e01c961226216896316af745ecc1eda65a29838567997086117129eb6b79298e
|
||||||
|
size 476904192
|
||||||
3
checkpoint-450/optimizer.pt
Normal file
3
checkpoint-450/optimizer.pt
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:66ad1b95f8f6dd61204dc4ac33189b9ea74e2774b74a1313a7dc2d30927f6898
|
||||||
|
size 954089079
|
||||||
3
checkpoint-450/rng_state.pth
Normal file
3
checkpoint-450/rng_state.pth
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:868623ceecc58498b5b92049b52c3f92ff7086856e8c068fb31c948e215fe773
|
||||||
|
size 14645
|
||||||
3
checkpoint-450/scheduler.pt
Normal file
3
checkpoint-450/scheduler.pt
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:d2a5081e3d91128719ab7ad5b9006096e1f92250c0f7b7e1853d57eef456113c
|
||||||
|
size 1465
|
||||||
188
checkpoint-450/trainer_state.json
Normal file
188
checkpoint-450/trainer_state.json
Normal file
@@ -0,0 +1,188 @@
|
|||||||
|
{
|
||||||
|
"best_global_step": null,
|
||||||
|
"best_metric": null,
|
||||||
|
"best_model_checkpoint": null,
|
||||||
|
"epoch": 1.157014157014157,
|
||||||
|
"eval_steps": 500,
|
||||||
|
"global_step": 450,
|
||||||
|
"is_hyper_param_search": false,
|
||||||
|
"is_local_process_zero": true,
|
||||||
|
"is_world_process_zero": true,
|
||||||
|
"log_history": [
|
||||||
|
{
|
||||||
|
"epoch": 0.05148005148005148,
|
||||||
|
"grad_norm": 3.7984347343444824,
|
||||||
|
"learning_rate": 2.435897435897436e-06,
|
||||||
|
"loss": 3.0721,
|
||||||
|
"step": 20
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.10296010296010295,
|
||||||
|
"grad_norm": 2.0532360076904297,
|
||||||
|
"learning_rate": 5e-06,
|
||||||
|
"loss": 2.7105,
|
||||||
|
"step": 40
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.15444015444015444,
|
||||||
|
"grad_norm": 1.889143466949463,
|
||||||
|
"learning_rate": 7.564102564102564e-06,
|
||||||
|
"loss": 2.397,
|
||||||
|
"step": 60
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.2059202059202059,
|
||||||
|
"grad_norm": 2.206676959991455,
|
||||||
|
"learning_rate": 9.999949644960027e-06,
|
||||||
|
"loss": 2.1587,
|
||||||
|
"step": 80
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.2574002574002574,
|
||||||
|
"grad_norm": 2.4184727668762207,
|
||||||
|
"learning_rate": 9.9778098230154e-06,
|
||||||
|
"loss": 1.9613,
|
||||||
|
"step": 100
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.3088803088803089,
|
||||||
|
"grad_norm": 2.1078503131866455,
|
||||||
|
"learning_rate": 9.915591603280632e-06,
|
||||||
|
"loss": 1.9187,
|
||||||
|
"step": 120
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.36036036036036034,
|
||||||
|
"grad_norm": 2.228259801864624,
|
||||||
|
"learning_rate": 9.813795930277306e-06,
|
||||||
|
"loss": 1.863,
|
||||||
|
"step": 140
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.4118404118404118,
|
||||||
|
"grad_norm": 3.990351438522339,
|
||||||
|
"learning_rate": 9.673242402909555e-06,
|
||||||
|
"loss": 1.7983,
|
||||||
|
"step": 160
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.46332046332046334,
|
||||||
|
"grad_norm": 2.6102797985076904,
|
||||||
|
"learning_rate": 9.495062675535614e-06,
|
||||||
|
"loss": 1.7845,
|
||||||
|
"step": 180
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.5148005148005148,
|
||||||
|
"grad_norm": 2.4415857791900635,
|
||||||
|
"learning_rate": 9.280691346552308e-06,
|
||||||
|
"loss": 1.7541,
|
||||||
|
"step": 200
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.5662805662805663,
|
||||||
|
"grad_norm": 2.875765800476074,
|
||||||
|
"learning_rate": 9.031854407852317e-06,
|
||||||
|
"loss": 1.7665,
|
||||||
|
"step": 220
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.6177606177606177,
|
||||||
|
"grad_norm": 2.55922269821167,
|
||||||
|
"learning_rate": 8.750555348152299e-06,
|
||||||
|
"loss": 1.7272,
|
||||||
|
"step": 240
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.6692406692406693,
|
||||||
|
"grad_norm": 3.4958205223083496,
|
||||||
|
"learning_rate": 8.439059022079789e-06,
|
||||||
|
"loss": 1.7349,
|
||||||
|
"step": 260
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.7207207207207207,
|
||||||
|
"grad_norm": 2.582611560821533,
|
||||||
|
"learning_rate": 8.099873414895453e-06,
|
||||||
|
"loss": 1.7273,
|
||||||
|
"step": 280
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.7722007722007722,
|
||||||
|
"grad_norm": 2.274104595184326,
|
||||||
|
"learning_rate": 7.73572944967043e-06,
|
||||||
|
"loss": 1.695,
|
||||||
|
"step": 300
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.8236808236808236,
|
||||||
|
"grad_norm": 2.8610410690307617,
|
||||||
|
"learning_rate": 7.3495589994995274e-06,
|
||||||
|
"loss": 1.663,
|
||||||
|
"step": 320
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.8751608751608752,
|
||||||
|
"grad_norm": 2.4936399459838867,
|
||||||
|
"learning_rate": 6.944471281782975e-06,
|
||||||
|
"loss": 1.6462,
|
||||||
|
"step": 340
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.9266409266409267,
|
||||||
|
"grad_norm": 3.510580062866211,
|
||||||
|
"learning_rate": 6.523727824636103e-06,
|
||||||
|
"loss": 1.6662,
|
||||||
|
"step": 360
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.9781209781209781,
|
||||||
|
"grad_norm": 2.398237943649292,
|
||||||
|
"learning_rate": 6.090716206982714e-06,
|
||||||
|
"loss": 1.6664,
|
||||||
|
"step": 380
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.0283140283140284,
|
||||||
|
"grad_norm": 2.8953514099121094,
|
||||||
|
"learning_rate": 5.648922783761443e-06,
|
||||||
|
"loss": 1.6363,
|
||||||
|
"step": 400
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.0797940797940797,
|
||||||
|
"grad_norm": 2.4946253299713135,
|
||||||
|
"learning_rate": 5.201904615845743e-06,
|
||||||
|
"loss": 1.5998,
|
||||||
|
"step": 420
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.1312741312741312,
|
||||||
|
"grad_norm": 2.965862512588501,
|
||||||
|
"learning_rate": 4.753260830681247e-06,
|
||||||
|
"loss": 1.5663,
|
||||||
|
"step": 440
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"logging_steps": 20,
|
||||||
|
"max_steps": 778,
|
||||||
|
"num_input_tokens_seen": 0,
|
||||||
|
"num_train_epochs": 2,
|
||||||
|
"save_steps": 50,
|
||||||
|
"stateful_callbacks": {
|
||||||
|
"TrainerControl": {
|
||||||
|
"args": {
|
||||||
|
"should_epoch_stop": false,
|
||||||
|
"should_evaluate": false,
|
||||||
|
"should_log": false,
|
||||||
|
"should_save": true,
|
||||||
|
"should_training_stop": false
|
||||||
|
},
|
||||||
|
"attributes": {}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"total_flos": 3.286040134491341e+16,
|
||||||
|
"train_batch_size": 4,
|
||||||
|
"trial_name": null,
|
||||||
|
"trial_params": null
|
||||||
|
}
|
||||||
3
checkpoint-450/training_args.bin
Normal file
3
checkpoint-450/training_args.bin
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:fe6e3348f136c9b50e5b7ff3939a0aca31453e7fbef30d71133ebde6392f9959
|
||||||
|
size 6033
|
||||||
207
checkpoint-50/README.md
Normal file
207
checkpoint-50/README.md
Normal file
@@ -0,0 +1,207 @@
|
|||||||
|
---
|
||||||
|
base_model: Henry236/nilechat-eg-stage1-general
|
||||||
|
library_name: peft
|
||||||
|
pipeline_tag: text-generation
|
||||||
|
tags:
|
||||||
|
- base_model:adapter:Henry236/nilechat-eg-stage1-general
|
||||||
|
- lora
|
||||||
|
- transformers
|
||||||
|
---
|
||||||
|
|
||||||
|
# Model Card for Model ID
|
||||||
|
|
||||||
|
<!-- Provide a quick summary of what the model is/does. -->
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## Model Details
|
||||||
|
|
||||||
|
### Model Description
|
||||||
|
|
||||||
|
<!-- Provide a longer summary of what this model is. -->
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
- **Developed by:** [More Information Needed]
|
||||||
|
- **Funded by [optional]:** [More Information Needed]
|
||||||
|
- **Shared by [optional]:** [More Information Needed]
|
||||||
|
- **Model type:** [More Information Needed]
|
||||||
|
- **Language(s) (NLP):** [More Information Needed]
|
||||||
|
- **License:** [More Information Needed]
|
||||||
|
- **Finetuned from model [optional]:** [More Information Needed]
|
||||||
|
|
||||||
|
### Model Sources [optional]
|
||||||
|
|
||||||
|
<!-- Provide the basic links for the model. -->
|
||||||
|
|
||||||
|
- **Repository:** [More Information Needed]
|
||||||
|
- **Paper [optional]:** [More Information Needed]
|
||||||
|
- **Demo [optional]:** [More Information Needed]
|
||||||
|
|
||||||
|
## Uses
|
||||||
|
|
||||||
|
<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
|
||||||
|
|
||||||
|
### Direct Use
|
||||||
|
|
||||||
|
<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Downstream Use [optional]
|
||||||
|
|
||||||
|
<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Out-of-Scope Use
|
||||||
|
|
||||||
|
<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Bias, Risks, and Limitations
|
||||||
|
|
||||||
|
<!-- This section is meant to convey both technical and sociotechnical limitations. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Recommendations
|
||||||
|
|
||||||
|
<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
|
||||||
|
|
||||||
|
Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
|
||||||
|
|
||||||
|
## How to Get Started with the Model
|
||||||
|
|
||||||
|
Use the code below to get started with the model.
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Training Details
|
||||||
|
|
||||||
|
### Training Data
|
||||||
|
|
||||||
|
<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Training Procedure
|
||||||
|
|
||||||
|
<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
|
||||||
|
|
||||||
|
#### Preprocessing [optional]
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
|
||||||
|
#### Training Hyperparameters
|
||||||
|
|
||||||
|
- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
|
||||||
|
|
||||||
|
#### Speeds, Sizes, Times [optional]
|
||||||
|
|
||||||
|
<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Evaluation
|
||||||
|
|
||||||
|
<!-- This section describes the evaluation protocols and provides the results. -->
|
||||||
|
|
||||||
|
### Testing Data, Factors & Metrics
|
||||||
|
|
||||||
|
#### Testing Data
|
||||||
|
|
||||||
|
<!-- This should link to a Dataset Card if possible. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Factors
|
||||||
|
|
||||||
|
<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Metrics
|
||||||
|
|
||||||
|
<!-- These are the evaluation metrics being used, ideally with a description of why. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Results
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Summary
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## Model Examination [optional]
|
||||||
|
|
||||||
|
<!-- Relevant interpretability work for the model goes here -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Environmental Impact
|
||||||
|
|
||||||
|
<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
|
||||||
|
|
||||||
|
Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
|
||||||
|
|
||||||
|
- **Hardware Type:** [More Information Needed]
|
||||||
|
- **Hours used:** [More Information Needed]
|
||||||
|
- **Cloud Provider:** [More Information Needed]
|
||||||
|
- **Compute Region:** [More Information Needed]
|
||||||
|
- **Carbon Emitted:** [More Information Needed]
|
||||||
|
|
||||||
|
## Technical Specifications [optional]
|
||||||
|
|
||||||
|
### Model Architecture and Objective
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Compute Infrastructure
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Hardware
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Software
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Citation [optional]
|
||||||
|
|
||||||
|
<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
|
||||||
|
|
||||||
|
**BibTeX:**
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
**APA:**
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Glossary [optional]
|
||||||
|
|
||||||
|
<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## More Information [optional]
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Model Card Authors [optional]
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Model Card Contact
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
### Framework versions
|
||||||
|
|
||||||
|
- PEFT 0.19.1
|
||||||
48
checkpoint-50/adapter_config.json
Normal file
48
checkpoint-50/adapter_config.json
Normal file
@@ -0,0 +1,48 @@
|
|||||||
|
{
|
||||||
|
"alora_invocation_tokens": null,
|
||||||
|
"alpha_pattern": {},
|
||||||
|
"arrow_config": null,
|
||||||
|
"auto_mapping": null,
|
||||||
|
"base_model_name_or_path": "Henry236/nilechat-eg-stage1-general",
|
||||||
|
"bias": "none",
|
||||||
|
"corda_config": null,
|
||||||
|
"ensure_weight_tying": false,
|
||||||
|
"eva_config": null,
|
||||||
|
"exclude_modules": null,
|
||||||
|
"fan_in_fan_out": false,
|
||||||
|
"inference_mode": true,
|
||||||
|
"init_lora_weights": true,
|
||||||
|
"layer_replication": null,
|
||||||
|
"layers_pattern": null,
|
||||||
|
"layers_to_transform": null,
|
||||||
|
"loftq_config": {},
|
||||||
|
"lora_alpha": 128,
|
||||||
|
"lora_bias": false,
|
||||||
|
"lora_dropout": 0.05,
|
||||||
|
"lora_ga_config": null,
|
||||||
|
"megatron_config": null,
|
||||||
|
"megatron_core": "megatron.core",
|
||||||
|
"modules_to_save": null,
|
||||||
|
"peft_type": "LORA",
|
||||||
|
"peft_version": "0.19.1",
|
||||||
|
"qalora_group_size": 16,
|
||||||
|
"r": 64,
|
||||||
|
"rank_pattern": {},
|
||||||
|
"revision": null,
|
||||||
|
"target_modules": [
|
||||||
|
"down_proj",
|
||||||
|
"q_proj",
|
||||||
|
"o_proj",
|
||||||
|
"up_proj",
|
||||||
|
"k_proj",
|
||||||
|
"v_proj",
|
||||||
|
"gate_proj"
|
||||||
|
],
|
||||||
|
"target_parameters": null,
|
||||||
|
"task_type": "CAUSAL_LM",
|
||||||
|
"trainable_token_indices": null,
|
||||||
|
"use_bdlora": null,
|
||||||
|
"use_dora": false,
|
||||||
|
"use_qalora": false,
|
||||||
|
"use_rslora": false
|
||||||
|
}
|
||||||
3
checkpoint-50/adapter_model.safetensors
Normal file
3
checkpoint-50/adapter_model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:74b0c9d9354653c2cfe992f41d5801caee481d8302d0609b02c4212909051ddb
|
||||||
|
size 476904192
|
||||||
3
checkpoint-50/optimizer.pt
Normal file
3
checkpoint-50/optimizer.pt
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:b20976ee1a335da0be34cc6498b0ed2a9ada2befa5028cc990accc8862be8885
|
||||||
|
size 954089079
|
||||||
3
checkpoint-50/rng_state.pth
Normal file
3
checkpoint-50/rng_state.pth
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:0d66e21cca4c126c6c58b3a99af811780be30e9c388b9aeb7c865cfd7c6879ca
|
||||||
|
size 14645
|
||||||
3
checkpoint-50/scheduler.pt
Normal file
3
checkpoint-50/scheduler.pt
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:18673dc27e631402a8ee04a6eb5454757d3d272236875b56ac93c9160f2e5aeb
|
||||||
|
size 1465
|
||||||
48
checkpoint-50/trainer_state.json
Normal file
48
checkpoint-50/trainer_state.json
Normal file
@@ -0,0 +1,48 @@
|
|||||||
|
{
|
||||||
|
"best_global_step": null,
|
||||||
|
"best_metric": null,
|
||||||
|
"best_model_checkpoint": null,
|
||||||
|
"epoch": 0.1287001287001287,
|
||||||
|
"eval_steps": 500,
|
||||||
|
"global_step": 50,
|
||||||
|
"is_hyper_param_search": false,
|
||||||
|
"is_local_process_zero": true,
|
||||||
|
"is_world_process_zero": true,
|
||||||
|
"log_history": [
|
||||||
|
{
|
||||||
|
"epoch": 0.05148005148005148,
|
||||||
|
"grad_norm": 3.7984347343444824,
|
||||||
|
"learning_rate": 2.435897435897436e-06,
|
||||||
|
"loss": 3.0721,
|
||||||
|
"step": 20
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.10296010296010295,
|
||||||
|
"grad_norm": 2.0532360076904297,
|
||||||
|
"learning_rate": 5e-06,
|
||||||
|
"loss": 2.7105,
|
||||||
|
"step": 40
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"logging_steps": 20,
|
||||||
|
"max_steps": 778,
|
||||||
|
"num_input_tokens_seen": 0,
|
||||||
|
"num_train_epochs": 2,
|
||||||
|
"save_steps": 50,
|
||||||
|
"stateful_callbacks": {
|
||||||
|
"TrainerControl": {
|
||||||
|
"args": {
|
||||||
|
"should_epoch_stop": false,
|
||||||
|
"should_evaluate": false,
|
||||||
|
"should_log": false,
|
||||||
|
"should_save": true,
|
||||||
|
"should_training_stop": false
|
||||||
|
},
|
||||||
|
"attributes": {}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"total_flos": 3660696604250112.0,
|
||||||
|
"train_batch_size": 4,
|
||||||
|
"trial_name": null,
|
||||||
|
"trial_params": null
|
||||||
|
}
|
||||||
3
checkpoint-50/training_args.bin
Normal file
3
checkpoint-50/training_args.bin
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:fe6e3348f136c9b50e5b7ff3939a0aca31453e7fbef30d71133ebde6392f9959
|
||||||
|
size 6033
|
||||||
207
checkpoint-500/README.md
Normal file
207
checkpoint-500/README.md
Normal file
@@ -0,0 +1,207 @@
|
|||||||
|
---
|
||||||
|
base_model: Henry236/nilechat-eg-stage1-general
|
||||||
|
library_name: peft
|
||||||
|
pipeline_tag: text-generation
|
||||||
|
tags:
|
||||||
|
- base_model:adapter:Henry236/nilechat-eg-stage1-general
|
||||||
|
- lora
|
||||||
|
- transformers
|
||||||
|
---
|
||||||
|
|
||||||
|
# Model Card for Model ID
|
||||||
|
|
||||||
|
<!-- Provide a quick summary of what the model is/does. -->
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## Model Details
|
||||||
|
|
||||||
|
### Model Description
|
||||||
|
|
||||||
|
<!-- Provide a longer summary of what this model is. -->
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
- **Developed by:** [More Information Needed]
|
||||||
|
- **Funded by [optional]:** [More Information Needed]
|
||||||
|
- **Shared by [optional]:** [More Information Needed]
|
||||||
|
- **Model type:** [More Information Needed]
|
||||||
|
- **Language(s) (NLP):** [More Information Needed]
|
||||||
|
- **License:** [More Information Needed]
|
||||||
|
- **Finetuned from model [optional]:** [More Information Needed]
|
||||||
|
|
||||||
|
### Model Sources [optional]
|
||||||
|
|
||||||
|
<!-- Provide the basic links for the model. -->
|
||||||
|
|
||||||
|
- **Repository:** [More Information Needed]
|
||||||
|
- **Paper [optional]:** [More Information Needed]
|
||||||
|
- **Demo [optional]:** [More Information Needed]
|
||||||
|
|
||||||
|
## Uses
|
||||||
|
|
||||||
|
<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
|
||||||
|
|
||||||
|
### Direct Use
|
||||||
|
|
||||||
|
<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Downstream Use [optional]
|
||||||
|
|
||||||
|
<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Out-of-Scope Use
|
||||||
|
|
||||||
|
<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Bias, Risks, and Limitations
|
||||||
|
|
||||||
|
<!-- This section is meant to convey both technical and sociotechnical limitations. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Recommendations
|
||||||
|
|
||||||
|
<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
|
||||||
|
|
||||||
|
Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
|
||||||
|
|
||||||
|
## How to Get Started with the Model
|
||||||
|
|
||||||
|
Use the code below to get started with the model.
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Training Details
|
||||||
|
|
||||||
|
### Training Data
|
||||||
|
|
||||||
|
<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Training Procedure
|
||||||
|
|
||||||
|
<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
|
||||||
|
|
||||||
|
#### Preprocessing [optional]
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
|
||||||
|
#### Training Hyperparameters
|
||||||
|
|
||||||
|
- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
|
||||||
|
|
||||||
|
#### Speeds, Sizes, Times [optional]
|
||||||
|
|
||||||
|
<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Evaluation
|
||||||
|
|
||||||
|
<!-- This section describes the evaluation protocols and provides the results. -->
|
||||||
|
|
||||||
|
### Testing Data, Factors & Metrics
|
||||||
|
|
||||||
|
#### Testing Data
|
||||||
|
|
||||||
|
<!-- This should link to a Dataset Card if possible. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Factors
|
||||||
|
|
||||||
|
<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Metrics
|
||||||
|
|
||||||
|
<!-- These are the evaluation metrics being used, ideally with a description of why. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Results
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Summary
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## Model Examination [optional]
|
||||||
|
|
||||||
|
<!-- Relevant interpretability work for the model goes here -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Environmental Impact
|
||||||
|
|
||||||
|
<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
|
||||||
|
|
||||||
|
Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
|
||||||
|
|
||||||
|
- **Hardware Type:** [More Information Needed]
|
||||||
|
- **Hours used:** [More Information Needed]
|
||||||
|
- **Cloud Provider:** [More Information Needed]
|
||||||
|
- **Compute Region:** [More Information Needed]
|
||||||
|
- **Carbon Emitted:** [More Information Needed]
|
||||||
|
|
||||||
|
## Technical Specifications [optional]
|
||||||
|
|
||||||
|
### Model Architecture and Objective
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Compute Infrastructure
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Hardware
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Software
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Citation [optional]
|
||||||
|
|
||||||
|
<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
|
||||||
|
|
||||||
|
**BibTeX:**
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
**APA:**
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Glossary [optional]
|
||||||
|
|
||||||
|
<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## More Information [optional]
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Model Card Authors [optional]
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Model Card Contact
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
### Framework versions
|
||||||
|
|
||||||
|
- PEFT 0.19.1
|
||||||
48
checkpoint-500/adapter_config.json
Normal file
48
checkpoint-500/adapter_config.json
Normal file
@@ -0,0 +1,48 @@
|
|||||||
|
{
|
||||||
|
"alora_invocation_tokens": null,
|
||||||
|
"alpha_pattern": {},
|
||||||
|
"arrow_config": null,
|
||||||
|
"auto_mapping": null,
|
||||||
|
"base_model_name_or_path": "Henry236/nilechat-eg-stage1-general",
|
||||||
|
"bias": "none",
|
||||||
|
"corda_config": null,
|
||||||
|
"ensure_weight_tying": false,
|
||||||
|
"eva_config": null,
|
||||||
|
"exclude_modules": null,
|
||||||
|
"fan_in_fan_out": false,
|
||||||
|
"inference_mode": true,
|
||||||
|
"init_lora_weights": true,
|
||||||
|
"layer_replication": null,
|
||||||
|
"layers_pattern": null,
|
||||||
|
"layers_to_transform": null,
|
||||||
|
"loftq_config": {},
|
||||||
|
"lora_alpha": 128,
|
||||||
|
"lora_bias": false,
|
||||||
|
"lora_dropout": 0.05,
|
||||||
|
"lora_ga_config": null,
|
||||||
|
"megatron_config": null,
|
||||||
|
"megatron_core": "megatron.core",
|
||||||
|
"modules_to_save": null,
|
||||||
|
"peft_type": "LORA",
|
||||||
|
"peft_version": "0.19.1",
|
||||||
|
"qalora_group_size": 16,
|
||||||
|
"r": 64,
|
||||||
|
"rank_pattern": {},
|
||||||
|
"revision": null,
|
||||||
|
"target_modules": [
|
||||||
|
"down_proj",
|
||||||
|
"q_proj",
|
||||||
|
"o_proj",
|
||||||
|
"up_proj",
|
||||||
|
"k_proj",
|
||||||
|
"v_proj",
|
||||||
|
"gate_proj"
|
||||||
|
],
|
||||||
|
"target_parameters": null,
|
||||||
|
"task_type": "CAUSAL_LM",
|
||||||
|
"trainable_token_indices": null,
|
||||||
|
"use_bdlora": null,
|
||||||
|
"use_dora": false,
|
||||||
|
"use_qalora": false,
|
||||||
|
"use_rslora": false
|
||||||
|
}
|
||||||
3
checkpoint-500/adapter_model.safetensors
Normal file
3
checkpoint-500/adapter_model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:c41cb90d3a7f422d2f606ff677a099bea4534bf724af41f34978a8f3ea14b4e3
|
||||||
|
size 476904192
|
||||||
3
checkpoint-500/optimizer.pt
Normal file
3
checkpoint-500/optimizer.pt
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:94205238de41970fbd1ef26db6347f779134e7d6f8be0c200be2b9431847efe4
|
||||||
|
size 954089079
|
||||||
3
checkpoint-500/rng_state.pth
Normal file
3
checkpoint-500/rng_state.pth
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:7b9ded2eee892a2e1fe9026fd053c69cd50f2576cbaf14f86070ee47c6e334b4
|
||||||
|
size 14645
|
||||||
3
checkpoint-500/scheduler.pt
Normal file
3
checkpoint-500/scheduler.pt
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:20dce4004ca038fd153b39e71a416441470c66304e70ea0e3522c7f94b0039ee
|
||||||
|
size 1465
|
||||||
209
checkpoint-500/trainer_state.json
Normal file
209
checkpoint-500/trainer_state.json
Normal file
@@ -0,0 +1,209 @@
|
|||||||
|
{
|
||||||
|
"best_global_step": null,
|
||||||
|
"best_metric": null,
|
||||||
|
"best_model_checkpoint": null,
|
||||||
|
"epoch": 1.2857142857142856,
|
||||||
|
"eval_steps": 500,
|
||||||
|
"global_step": 500,
|
||||||
|
"is_hyper_param_search": false,
|
||||||
|
"is_local_process_zero": true,
|
||||||
|
"is_world_process_zero": true,
|
||||||
|
"log_history": [
|
||||||
|
{
|
||||||
|
"epoch": 0.05148005148005148,
|
||||||
|
"grad_norm": 3.7984347343444824,
|
||||||
|
"learning_rate": 2.435897435897436e-06,
|
||||||
|
"loss": 3.0721,
|
||||||
|
"step": 20
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.10296010296010295,
|
||||||
|
"grad_norm": 2.0532360076904297,
|
||||||
|
"learning_rate": 5e-06,
|
||||||
|
"loss": 2.7105,
|
||||||
|
"step": 40
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.15444015444015444,
|
||||||
|
"grad_norm": 1.889143466949463,
|
||||||
|
"learning_rate": 7.564102564102564e-06,
|
||||||
|
"loss": 2.397,
|
||||||
|
"step": 60
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.2059202059202059,
|
||||||
|
"grad_norm": 2.206676959991455,
|
||||||
|
"learning_rate": 9.999949644960027e-06,
|
||||||
|
"loss": 2.1587,
|
||||||
|
"step": 80
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.2574002574002574,
|
||||||
|
"grad_norm": 2.4184727668762207,
|
||||||
|
"learning_rate": 9.9778098230154e-06,
|
||||||
|
"loss": 1.9613,
|
||||||
|
"step": 100
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.3088803088803089,
|
||||||
|
"grad_norm": 2.1078503131866455,
|
||||||
|
"learning_rate": 9.915591603280632e-06,
|
||||||
|
"loss": 1.9187,
|
||||||
|
"step": 120
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.36036036036036034,
|
||||||
|
"grad_norm": 2.228259801864624,
|
||||||
|
"learning_rate": 9.813795930277306e-06,
|
||||||
|
"loss": 1.863,
|
||||||
|
"step": 140
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.4118404118404118,
|
||||||
|
"grad_norm": 3.990351438522339,
|
||||||
|
"learning_rate": 9.673242402909555e-06,
|
||||||
|
"loss": 1.7983,
|
||||||
|
"step": 160
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.46332046332046334,
|
||||||
|
"grad_norm": 2.6102797985076904,
|
||||||
|
"learning_rate": 9.495062675535614e-06,
|
||||||
|
"loss": 1.7845,
|
||||||
|
"step": 180
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.5148005148005148,
|
||||||
|
"grad_norm": 2.4415857791900635,
|
||||||
|
"learning_rate": 9.280691346552308e-06,
|
||||||
|
"loss": 1.7541,
|
||||||
|
"step": 200
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.5662805662805663,
|
||||||
|
"grad_norm": 2.875765800476074,
|
||||||
|
"learning_rate": 9.031854407852317e-06,
|
||||||
|
"loss": 1.7665,
|
||||||
|
"step": 220
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.6177606177606177,
|
||||||
|
"grad_norm": 2.55922269821167,
|
||||||
|
"learning_rate": 8.750555348152299e-06,
|
||||||
|
"loss": 1.7272,
|
||||||
|
"step": 240
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.6692406692406693,
|
||||||
|
"grad_norm": 3.4958205223083496,
|
||||||
|
"learning_rate": 8.439059022079789e-06,
|
||||||
|
"loss": 1.7349,
|
||||||
|
"step": 260
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.7207207207207207,
|
||||||
|
"grad_norm": 2.582611560821533,
|
||||||
|
"learning_rate": 8.099873414895453e-06,
|
||||||
|
"loss": 1.7273,
|
||||||
|
"step": 280
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.7722007722007722,
|
||||||
|
"grad_norm": 2.274104595184326,
|
||||||
|
"learning_rate": 7.73572944967043e-06,
|
||||||
|
"loss": 1.695,
|
||||||
|
"step": 300
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.8236808236808236,
|
||||||
|
"grad_norm": 2.8610410690307617,
|
||||||
|
"learning_rate": 7.3495589994995274e-06,
|
||||||
|
"loss": 1.663,
|
||||||
|
"step": 320
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.8751608751608752,
|
||||||
|
"grad_norm": 2.4936399459838867,
|
||||||
|
"learning_rate": 6.944471281782975e-06,
|
||||||
|
"loss": 1.6462,
|
||||||
|
"step": 340
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.9266409266409267,
|
||||||
|
"grad_norm": 3.510580062866211,
|
||||||
|
"learning_rate": 6.523727824636103e-06,
|
||||||
|
"loss": 1.6662,
|
||||||
|
"step": 360
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.9781209781209781,
|
||||||
|
"grad_norm": 2.398237943649292,
|
||||||
|
"learning_rate": 6.090716206982714e-06,
|
||||||
|
"loss": 1.6664,
|
||||||
|
"step": 380
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.0283140283140284,
|
||||||
|
"grad_norm": 2.8953514099121094,
|
||||||
|
"learning_rate": 5.648922783761443e-06,
|
||||||
|
"loss": 1.6363,
|
||||||
|
"step": 400
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.0797940797940797,
|
||||||
|
"grad_norm": 2.4946253299713135,
|
||||||
|
"learning_rate": 5.201904615845743e-06,
|
||||||
|
"loss": 1.5998,
|
||||||
|
"step": 420
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.1312741312741312,
|
||||||
|
"grad_norm": 2.965862512588501,
|
||||||
|
"learning_rate": 4.753260830681247e-06,
|
||||||
|
"loss": 1.5663,
|
||||||
|
"step": 440
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.1827541827541828,
|
||||||
|
"grad_norm": 2.843411922454834,
|
||||||
|
"learning_rate": 4.306603644227821e-06,
|
||||||
|
"loss": 1.578,
|
||||||
|
"step": 460
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.2342342342342343,
|
||||||
|
"grad_norm": 2.5591073036193848,
|
||||||
|
"learning_rate": 3.8655292775206185e-06,
|
||||||
|
"loss": 1.5821,
|
||||||
|
"step": 480
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.2857142857142856,
|
||||||
|
"grad_norm": 3.3472440242767334,
|
||||||
|
"learning_rate": 3.4335890020128382e-06,
|
||||||
|
"loss": 1.5743,
|
||||||
|
"step": 500
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"logging_steps": 20,
|
||||||
|
"max_steps": 778,
|
||||||
|
"num_input_tokens_seen": 0,
|
||||||
|
"num_train_epochs": 2,
|
||||||
|
"save_steps": 50,
|
||||||
|
"stateful_callbacks": {
|
||||||
|
"TrainerControl": {
|
||||||
|
"args": {
|
||||||
|
"should_epoch_stop": false,
|
||||||
|
"should_evaluate": false,
|
||||||
|
"should_log": false,
|
||||||
|
"should_save": true,
|
||||||
|
"should_training_stop": false
|
||||||
|
},
|
||||||
|
"attributes": {}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"total_flos": 3.655544518649856e+16,
|
||||||
|
"train_batch_size": 4,
|
||||||
|
"trial_name": null,
|
||||||
|
"trial_params": null
|
||||||
|
}
|
||||||
3
checkpoint-500/training_args.bin
Normal file
3
checkpoint-500/training_args.bin
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:fe6e3348f136c9b50e5b7ff3939a0aca31453e7fbef30d71133ebde6392f9959
|
||||||
|
size 6033
|
||||||
207
checkpoint-550/README.md
Normal file
207
checkpoint-550/README.md
Normal file
@@ -0,0 +1,207 @@
|
|||||||
|
---
|
||||||
|
base_model: Henry236/nilechat-eg-stage1-general
|
||||||
|
library_name: peft
|
||||||
|
pipeline_tag: text-generation
|
||||||
|
tags:
|
||||||
|
- base_model:adapter:Henry236/nilechat-eg-stage1-general
|
||||||
|
- lora
|
||||||
|
- transformers
|
||||||
|
---
|
||||||
|
|
||||||
|
# Model Card for Model ID
|
||||||
|
|
||||||
|
<!-- Provide a quick summary of what the model is/does. -->
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## Model Details
|
||||||
|
|
||||||
|
### Model Description
|
||||||
|
|
||||||
|
<!-- Provide a longer summary of what this model is. -->
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
- **Developed by:** [More Information Needed]
|
||||||
|
- **Funded by [optional]:** [More Information Needed]
|
||||||
|
- **Shared by [optional]:** [More Information Needed]
|
||||||
|
- **Model type:** [More Information Needed]
|
||||||
|
- **Language(s) (NLP):** [More Information Needed]
|
||||||
|
- **License:** [More Information Needed]
|
||||||
|
- **Finetuned from model [optional]:** [More Information Needed]
|
||||||
|
|
||||||
|
### Model Sources [optional]
|
||||||
|
|
||||||
|
<!-- Provide the basic links for the model. -->
|
||||||
|
|
||||||
|
- **Repository:** [More Information Needed]
|
||||||
|
- **Paper [optional]:** [More Information Needed]
|
||||||
|
- **Demo [optional]:** [More Information Needed]
|
||||||
|
|
||||||
|
## Uses
|
||||||
|
|
||||||
|
<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
|
||||||
|
|
||||||
|
### Direct Use
|
||||||
|
|
||||||
|
<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Downstream Use [optional]
|
||||||
|
|
||||||
|
<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Out-of-Scope Use
|
||||||
|
|
||||||
|
<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Bias, Risks, and Limitations
|
||||||
|
|
||||||
|
<!-- This section is meant to convey both technical and sociotechnical limitations. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Recommendations
|
||||||
|
|
||||||
|
<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
|
||||||
|
|
||||||
|
Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
|
||||||
|
|
||||||
|
## How to Get Started with the Model
|
||||||
|
|
||||||
|
Use the code below to get started with the model.
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Training Details
|
||||||
|
|
||||||
|
### Training Data
|
||||||
|
|
||||||
|
<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Training Procedure
|
||||||
|
|
||||||
|
<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
|
||||||
|
|
||||||
|
#### Preprocessing [optional]
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
|
||||||
|
#### Training Hyperparameters
|
||||||
|
|
||||||
|
- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
|
||||||
|
|
||||||
|
#### Speeds, Sizes, Times [optional]
|
||||||
|
|
||||||
|
<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Evaluation
|
||||||
|
|
||||||
|
<!-- This section describes the evaluation protocols and provides the results. -->
|
||||||
|
|
||||||
|
### Testing Data, Factors & Metrics
|
||||||
|
|
||||||
|
#### Testing Data
|
||||||
|
|
||||||
|
<!-- This should link to a Dataset Card if possible. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Factors
|
||||||
|
|
||||||
|
<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Metrics
|
||||||
|
|
||||||
|
<!-- These are the evaluation metrics being used, ideally with a description of why. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Results
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Summary
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## Model Examination [optional]
|
||||||
|
|
||||||
|
<!-- Relevant interpretability work for the model goes here -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Environmental Impact
|
||||||
|
|
||||||
|
<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
|
||||||
|
|
||||||
|
Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
|
||||||
|
|
||||||
|
- **Hardware Type:** [More Information Needed]
|
||||||
|
- **Hours used:** [More Information Needed]
|
||||||
|
- **Cloud Provider:** [More Information Needed]
|
||||||
|
- **Compute Region:** [More Information Needed]
|
||||||
|
- **Carbon Emitted:** [More Information Needed]
|
||||||
|
|
||||||
|
## Technical Specifications [optional]
|
||||||
|
|
||||||
|
### Model Architecture and Objective
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Compute Infrastructure
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Hardware
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Software
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Citation [optional]
|
||||||
|
|
||||||
|
<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
|
||||||
|
|
||||||
|
**BibTeX:**
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
**APA:**
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Glossary [optional]
|
||||||
|
|
||||||
|
<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## More Information [optional]
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Model Card Authors [optional]
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Model Card Contact
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
### Framework versions
|
||||||
|
|
||||||
|
- PEFT 0.19.1
|
||||||
48
checkpoint-550/adapter_config.json
Normal file
48
checkpoint-550/adapter_config.json
Normal file
@@ -0,0 +1,48 @@
|
|||||||
|
{
|
||||||
|
"alora_invocation_tokens": null,
|
||||||
|
"alpha_pattern": {},
|
||||||
|
"arrow_config": null,
|
||||||
|
"auto_mapping": null,
|
||||||
|
"base_model_name_or_path": "Henry236/nilechat-eg-stage1-general",
|
||||||
|
"bias": "none",
|
||||||
|
"corda_config": null,
|
||||||
|
"ensure_weight_tying": false,
|
||||||
|
"eva_config": null,
|
||||||
|
"exclude_modules": null,
|
||||||
|
"fan_in_fan_out": false,
|
||||||
|
"inference_mode": true,
|
||||||
|
"init_lora_weights": true,
|
||||||
|
"layer_replication": null,
|
||||||
|
"layers_pattern": null,
|
||||||
|
"layers_to_transform": null,
|
||||||
|
"loftq_config": {},
|
||||||
|
"lora_alpha": 128,
|
||||||
|
"lora_bias": false,
|
||||||
|
"lora_dropout": 0.05,
|
||||||
|
"lora_ga_config": null,
|
||||||
|
"megatron_config": null,
|
||||||
|
"megatron_core": "megatron.core",
|
||||||
|
"modules_to_save": null,
|
||||||
|
"peft_type": "LORA",
|
||||||
|
"peft_version": "0.19.1",
|
||||||
|
"qalora_group_size": 16,
|
||||||
|
"r": 64,
|
||||||
|
"rank_pattern": {},
|
||||||
|
"revision": null,
|
||||||
|
"target_modules": [
|
||||||
|
"down_proj",
|
||||||
|
"q_proj",
|
||||||
|
"o_proj",
|
||||||
|
"up_proj",
|
||||||
|
"k_proj",
|
||||||
|
"v_proj",
|
||||||
|
"gate_proj"
|
||||||
|
],
|
||||||
|
"target_parameters": null,
|
||||||
|
"task_type": "CAUSAL_LM",
|
||||||
|
"trainable_token_indices": null,
|
||||||
|
"use_bdlora": null,
|
||||||
|
"use_dora": false,
|
||||||
|
"use_qalora": false,
|
||||||
|
"use_rslora": false
|
||||||
|
}
|
||||||
3
checkpoint-550/adapter_model.safetensors
Normal file
3
checkpoint-550/adapter_model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:018060076d7b548698d510cd610e10ceba80e0f87079954ec8885e1b91447e5c
|
||||||
|
size 476904192
|
||||||
3
checkpoint-550/optimizer.pt
Normal file
3
checkpoint-550/optimizer.pt
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:a00477e95b22e5b511aa1ccf0cfd6ba2301876f17ae71a1d3ad5fd05564de8d1
|
||||||
|
size 954089079
|
||||||
3
checkpoint-550/rng_state.pth
Normal file
3
checkpoint-550/rng_state.pth
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:4dfd58a782bafcdf9c93e47b2c9d6949db84a18439462fe81cd7f58ff04008e1
|
||||||
|
size 14645
|
||||||
3
checkpoint-550/scheduler.pt
Normal file
3
checkpoint-550/scheduler.pt
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:51c13b3842bf71dce76eaa8d99680afe5f0f6fb43dd514fcfa0d067e90e795c4
|
||||||
|
size 1465
|
||||||
223
checkpoint-550/trainer_state.json
Normal file
223
checkpoint-550/trainer_state.json
Normal file
@@ -0,0 +1,223 @@
|
|||||||
|
{
|
||||||
|
"best_global_step": null,
|
||||||
|
"best_metric": null,
|
||||||
|
"best_model_checkpoint": null,
|
||||||
|
"epoch": 1.4144144144144144,
|
||||||
|
"eval_steps": 500,
|
||||||
|
"global_step": 550,
|
||||||
|
"is_hyper_param_search": false,
|
||||||
|
"is_local_process_zero": true,
|
||||||
|
"is_world_process_zero": true,
|
||||||
|
"log_history": [
|
||||||
|
{
|
||||||
|
"epoch": 0.05148005148005148,
|
||||||
|
"grad_norm": 3.7984347343444824,
|
||||||
|
"learning_rate": 2.435897435897436e-06,
|
||||||
|
"loss": 3.0721,
|
||||||
|
"step": 20
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.10296010296010295,
|
||||||
|
"grad_norm": 2.0532360076904297,
|
||||||
|
"learning_rate": 5e-06,
|
||||||
|
"loss": 2.7105,
|
||||||
|
"step": 40
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.15444015444015444,
|
||||||
|
"grad_norm": 1.889143466949463,
|
||||||
|
"learning_rate": 7.564102564102564e-06,
|
||||||
|
"loss": 2.397,
|
||||||
|
"step": 60
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.2059202059202059,
|
||||||
|
"grad_norm": 2.206676959991455,
|
||||||
|
"learning_rate": 9.999949644960027e-06,
|
||||||
|
"loss": 2.1587,
|
||||||
|
"step": 80
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.2574002574002574,
|
||||||
|
"grad_norm": 2.4184727668762207,
|
||||||
|
"learning_rate": 9.9778098230154e-06,
|
||||||
|
"loss": 1.9613,
|
||||||
|
"step": 100
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.3088803088803089,
|
||||||
|
"grad_norm": 2.1078503131866455,
|
||||||
|
"learning_rate": 9.915591603280632e-06,
|
||||||
|
"loss": 1.9187,
|
||||||
|
"step": 120
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.36036036036036034,
|
||||||
|
"grad_norm": 2.228259801864624,
|
||||||
|
"learning_rate": 9.813795930277306e-06,
|
||||||
|
"loss": 1.863,
|
||||||
|
"step": 140
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.4118404118404118,
|
||||||
|
"grad_norm": 3.990351438522339,
|
||||||
|
"learning_rate": 9.673242402909555e-06,
|
||||||
|
"loss": 1.7983,
|
||||||
|
"step": 160
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.46332046332046334,
|
||||||
|
"grad_norm": 2.6102797985076904,
|
||||||
|
"learning_rate": 9.495062675535614e-06,
|
||||||
|
"loss": 1.7845,
|
||||||
|
"step": 180
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.5148005148005148,
|
||||||
|
"grad_norm": 2.4415857791900635,
|
||||||
|
"learning_rate": 9.280691346552308e-06,
|
||||||
|
"loss": 1.7541,
|
||||||
|
"step": 200
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.5662805662805663,
|
||||||
|
"grad_norm": 2.875765800476074,
|
||||||
|
"learning_rate": 9.031854407852317e-06,
|
||||||
|
"loss": 1.7665,
|
||||||
|
"step": 220
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.6177606177606177,
|
||||||
|
"grad_norm": 2.55922269821167,
|
||||||
|
"learning_rate": 8.750555348152299e-06,
|
||||||
|
"loss": 1.7272,
|
||||||
|
"step": 240
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.6692406692406693,
|
||||||
|
"grad_norm": 3.4958205223083496,
|
||||||
|
"learning_rate": 8.439059022079789e-06,
|
||||||
|
"loss": 1.7349,
|
||||||
|
"step": 260
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.7207207207207207,
|
||||||
|
"grad_norm": 2.582611560821533,
|
||||||
|
"learning_rate": 8.099873414895453e-06,
|
||||||
|
"loss": 1.7273,
|
||||||
|
"step": 280
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.7722007722007722,
|
||||||
|
"grad_norm": 2.274104595184326,
|
||||||
|
"learning_rate": 7.73572944967043e-06,
|
||||||
|
"loss": 1.695,
|
||||||
|
"step": 300
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.8236808236808236,
|
||||||
|
"grad_norm": 2.8610410690307617,
|
||||||
|
"learning_rate": 7.3495589994995274e-06,
|
||||||
|
"loss": 1.663,
|
||||||
|
"step": 320
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.8751608751608752,
|
||||||
|
"grad_norm": 2.4936399459838867,
|
||||||
|
"learning_rate": 6.944471281782975e-06,
|
||||||
|
"loss": 1.6462,
|
||||||
|
"step": 340
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.9266409266409267,
|
||||||
|
"grad_norm": 3.510580062866211,
|
||||||
|
"learning_rate": 6.523727824636103e-06,
|
||||||
|
"loss": 1.6662,
|
||||||
|
"step": 360
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.9781209781209781,
|
||||||
|
"grad_norm": 2.398237943649292,
|
||||||
|
"learning_rate": 6.090716206982714e-06,
|
||||||
|
"loss": 1.6664,
|
||||||
|
"step": 380
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.0283140283140284,
|
||||||
|
"grad_norm": 2.8953514099121094,
|
||||||
|
"learning_rate": 5.648922783761443e-06,
|
||||||
|
"loss": 1.6363,
|
||||||
|
"step": 400
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.0797940797940797,
|
||||||
|
"grad_norm": 2.4946253299713135,
|
||||||
|
"learning_rate": 5.201904615845743e-06,
|
||||||
|
"loss": 1.5998,
|
||||||
|
"step": 420
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.1312741312741312,
|
||||||
|
"grad_norm": 2.965862512588501,
|
||||||
|
"learning_rate": 4.753260830681247e-06,
|
||||||
|
"loss": 1.5663,
|
||||||
|
"step": 440
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.1827541827541828,
|
||||||
|
"grad_norm": 2.843411922454834,
|
||||||
|
"learning_rate": 4.306603644227821e-06,
|
||||||
|
"loss": 1.578,
|
||||||
|
"step": 460
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.2342342342342343,
|
||||||
|
"grad_norm": 2.5591073036193848,
|
||||||
|
"learning_rate": 3.8655292775206185e-06,
|
||||||
|
"loss": 1.5821,
|
||||||
|
"step": 480
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.2857142857142856,
|
||||||
|
"grad_norm": 3.3472440242767334,
|
||||||
|
"learning_rate": 3.4335890020128382e-06,
|
||||||
|
"loss": 1.5743,
|
||||||
|
"step": 500
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.3371943371943371,
|
||||||
|
"grad_norm": 2.687406063079834,
|
||||||
|
"learning_rate": 3.0142605468260976e-06,
|
||||||
|
"loss": 1.5407,
|
||||||
|
"step": 520
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.3886743886743886,
|
||||||
|
"grad_norm": 2.7775464057922363,
|
||||||
|
"learning_rate": 2.610920098120424e-06,
|
||||||
|
"loss": 1.5429,
|
||||||
|
"step": 540
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"logging_steps": 20,
|
||||||
|
"max_steps": 778,
|
||||||
|
"num_input_tokens_seen": 0,
|
||||||
|
"num_train_epochs": 2,
|
||||||
|
"save_steps": 50,
|
||||||
|
"stateful_callbacks": {
|
||||||
|
"TrainerControl": {
|
||||||
|
"args": {
|
||||||
|
"should_epoch_stop": false,
|
||||||
|
"should_evaluate": false,
|
||||||
|
"should_log": false,
|
||||||
|
"should_save": true,
|
||||||
|
"should_training_stop": false
|
||||||
|
},
|
||||||
|
"attributes": {}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"total_flos": 4.019577308023603e+16,
|
||||||
|
"train_batch_size": 4,
|
||||||
|
"trial_name": null,
|
||||||
|
"trial_params": null
|
||||||
|
}
|
||||||
3
checkpoint-550/training_args.bin
Normal file
3
checkpoint-550/training_args.bin
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:fe6e3348f136c9b50e5b7ff3939a0aca31453e7fbef30d71133ebde6392f9959
|
||||||
|
size 6033
|
||||||
207
checkpoint-600/README.md
Normal file
207
checkpoint-600/README.md
Normal file
@@ -0,0 +1,207 @@
|
|||||||
|
---
|
||||||
|
base_model: Henry236/nilechat-eg-stage1-general
|
||||||
|
library_name: peft
|
||||||
|
pipeline_tag: text-generation
|
||||||
|
tags:
|
||||||
|
- base_model:adapter:Henry236/nilechat-eg-stage1-general
|
||||||
|
- lora
|
||||||
|
- transformers
|
||||||
|
---
|
||||||
|
|
||||||
|
# Model Card for Model ID
|
||||||
|
|
||||||
|
<!-- Provide a quick summary of what the model is/does. -->
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## Model Details
|
||||||
|
|
||||||
|
### Model Description
|
||||||
|
|
||||||
|
<!-- Provide a longer summary of what this model is. -->
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
- **Developed by:** [More Information Needed]
|
||||||
|
- **Funded by [optional]:** [More Information Needed]
|
||||||
|
- **Shared by [optional]:** [More Information Needed]
|
||||||
|
- **Model type:** [More Information Needed]
|
||||||
|
- **Language(s) (NLP):** [More Information Needed]
|
||||||
|
- **License:** [More Information Needed]
|
||||||
|
- **Finetuned from model [optional]:** [More Information Needed]
|
||||||
|
|
||||||
|
### Model Sources [optional]
|
||||||
|
|
||||||
|
<!-- Provide the basic links for the model. -->
|
||||||
|
|
||||||
|
- **Repository:** [More Information Needed]
|
||||||
|
- **Paper [optional]:** [More Information Needed]
|
||||||
|
- **Demo [optional]:** [More Information Needed]
|
||||||
|
|
||||||
|
## Uses
|
||||||
|
|
||||||
|
<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
|
||||||
|
|
||||||
|
### Direct Use
|
||||||
|
|
||||||
|
<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Downstream Use [optional]
|
||||||
|
|
||||||
|
<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Out-of-Scope Use
|
||||||
|
|
||||||
|
<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Bias, Risks, and Limitations
|
||||||
|
|
||||||
|
<!-- This section is meant to convey both technical and sociotechnical limitations. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Recommendations
|
||||||
|
|
||||||
|
<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
|
||||||
|
|
||||||
|
Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
|
||||||
|
|
||||||
|
## How to Get Started with the Model
|
||||||
|
|
||||||
|
Use the code below to get started with the model.
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Training Details
|
||||||
|
|
||||||
|
### Training Data
|
||||||
|
|
||||||
|
<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Training Procedure
|
||||||
|
|
||||||
|
<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
|
||||||
|
|
||||||
|
#### Preprocessing [optional]
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
|
||||||
|
#### Training Hyperparameters
|
||||||
|
|
||||||
|
- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
|
||||||
|
|
||||||
|
#### Speeds, Sizes, Times [optional]
|
||||||
|
|
||||||
|
<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Evaluation
|
||||||
|
|
||||||
|
<!-- This section describes the evaluation protocols and provides the results. -->
|
||||||
|
|
||||||
|
### Testing Data, Factors & Metrics
|
||||||
|
|
||||||
|
#### Testing Data
|
||||||
|
|
||||||
|
<!-- This should link to a Dataset Card if possible. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Factors
|
||||||
|
|
||||||
|
<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Metrics
|
||||||
|
|
||||||
|
<!-- These are the evaluation metrics being used, ideally with a description of why. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Results
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Summary
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## Model Examination [optional]
|
||||||
|
|
||||||
|
<!-- Relevant interpretability work for the model goes here -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Environmental Impact
|
||||||
|
|
||||||
|
<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
|
||||||
|
|
||||||
|
Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
|
||||||
|
|
||||||
|
- **Hardware Type:** [More Information Needed]
|
||||||
|
- **Hours used:** [More Information Needed]
|
||||||
|
- **Cloud Provider:** [More Information Needed]
|
||||||
|
- **Compute Region:** [More Information Needed]
|
||||||
|
- **Carbon Emitted:** [More Information Needed]
|
||||||
|
|
||||||
|
## Technical Specifications [optional]
|
||||||
|
|
||||||
|
### Model Architecture and Objective
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Compute Infrastructure
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Hardware
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Software
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Citation [optional]
|
||||||
|
|
||||||
|
<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
|
||||||
|
|
||||||
|
**BibTeX:**
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
**APA:**
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Glossary [optional]
|
||||||
|
|
||||||
|
<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## More Information [optional]
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Model Card Authors [optional]
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Model Card Contact
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
### Framework versions
|
||||||
|
|
||||||
|
- PEFT 0.19.1
|
||||||
48
checkpoint-600/adapter_config.json
Normal file
48
checkpoint-600/adapter_config.json
Normal file
@@ -0,0 +1,48 @@
|
|||||||
|
{
|
||||||
|
"alora_invocation_tokens": null,
|
||||||
|
"alpha_pattern": {},
|
||||||
|
"arrow_config": null,
|
||||||
|
"auto_mapping": null,
|
||||||
|
"base_model_name_or_path": "Henry236/nilechat-eg-stage1-general",
|
||||||
|
"bias": "none",
|
||||||
|
"corda_config": null,
|
||||||
|
"ensure_weight_tying": false,
|
||||||
|
"eva_config": null,
|
||||||
|
"exclude_modules": null,
|
||||||
|
"fan_in_fan_out": false,
|
||||||
|
"inference_mode": true,
|
||||||
|
"init_lora_weights": true,
|
||||||
|
"layer_replication": null,
|
||||||
|
"layers_pattern": null,
|
||||||
|
"layers_to_transform": null,
|
||||||
|
"loftq_config": {},
|
||||||
|
"lora_alpha": 128,
|
||||||
|
"lora_bias": false,
|
||||||
|
"lora_dropout": 0.05,
|
||||||
|
"lora_ga_config": null,
|
||||||
|
"megatron_config": null,
|
||||||
|
"megatron_core": "megatron.core",
|
||||||
|
"modules_to_save": null,
|
||||||
|
"peft_type": "LORA",
|
||||||
|
"peft_version": "0.19.1",
|
||||||
|
"qalora_group_size": 16,
|
||||||
|
"r": 64,
|
||||||
|
"rank_pattern": {},
|
||||||
|
"revision": null,
|
||||||
|
"target_modules": [
|
||||||
|
"down_proj",
|
||||||
|
"q_proj",
|
||||||
|
"o_proj",
|
||||||
|
"up_proj",
|
||||||
|
"k_proj",
|
||||||
|
"v_proj",
|
||||||
|
"gate_proj"
|
||||||
|
],
|
||||||
|
"target_parameters": null,
|
||||||
|
"task_type": "CAUSAL_LM",
|
||||||
|
"trainable_token_indices": null,
|
||||||
|
"use_bdlora": null,
|
||||||
|
"use_dora": false,
|
||||||
|
"use_qalora": false,
|
||||||
|
"use_rslora": false
|
||||||
|
}
|
||||||
3
checkpoint-600/adapter_model.safetensors
Normal file
3
checkpoint-600/adapter_model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:a064f59e6dd34dc41dda4de6b89ef6e88615d745e2bcfbce522890fb12b6d83c
|
||||||
|
size 476904192
|
||||||
3
checkpoint-600/optimizer.pt
Normal file
3
checkpoint-600/optimizer.pt
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:de122c90debcaaecb61706a7196cdd723434a7aa4e871434e99d76f908cf61fd
|
||||||
|
size 954089079
|
||||||
3
checkpoint-600/rng_state.pth
Normal file
3
checkpoint-600/rng_state.pth
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:f7480c1be9cea5adb9a6904bd3082a4a08ebec7dad09ee933f1209395eb07c92
|
||||||
|
size 14645
|
||||||
3
checkpoint-600/scheduler.pt
Normal file
3
checkpoint-600/scheduler.pt
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:ca506b040d2b04d5fe5b09bf367664da37f08a5af9ab3e9eb9dddd3c178e3f5d
|
||||||
|
size 1465
|
||||||
244
checkpoint-600/trainer_state.json
Normal file
244
checkpoint-600/trainer_state.json
Normal file
@@ -0,0 +1,244 @@
|
|||||||
|
{
|
||||||
|
"best_global_step": null,
|
||||||
|
"best_metric": null,
|
||||||
|
"best_model_checkpoint": null,
|
||||||
|
"epoch": 1.5431145431145432,
|
||||||
|
"eval_steps": 500,
|
||||||
|
"global_step": 600,
|
||||||
|
"is_hyper_param_search": false,
|
||||||
|
"is_local_process_zero": true,
|
||||||
|
"is_world_process_zero": true,
|
||||||
|
"log_history": [
|
||||||
|
{
|
||||||
|
"epoch": 0.05148005148005148,
|
||||||
|
"grad_norm": 3.7984347343444824,
|
||||||
|
"learning_rate": 2.435897435897436e-06,
|
||||||
|
"loss": 3.0721,
|
||||||
|
"step": 20
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.10296010296010295,
|
||||||
|
"grad_norm": 2.0532360076904297,
|
||||||
|
"learning_rate": 5e-06,
|
||||||
|
"loss": 2.7105,
|
||||||
|
"step": 40
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.15444015444015444,
|
||||||
|
"grad_norm": 1.889143466949463,
|
||||||
|
"learning_rate": 7.564102564102564e-06,
|
||||||
|
"loss": 2.397,
|
||||||
|
"step": 60
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.2059202059202059,
|
||||||
|
"grad_norm": 2.206676959991455,
|
||||||
|
"learning_rate": 9.999949644960027e-06,
|
||||||
|
"loss": 2.1587,
|
||||||
|
"step": 80
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.2574002574002574,
|
||||||
|
"grad_norm": 2.4184727668762207,
|
||||||
|
"learning_rate": 9.9778098230154e-06,
|
||||||
|
"loss": 1.9613,
|
||||||
|
"step": 100
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.3088803088803089,
|
||||||
|
"grad_norm": 2.1078503131866455,
|
||||||
|
"learning_rate": 9.915591603280632e-06,
|
||||||
|
"loss": 1.9187,
|
||||||
|
"step": 120
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.36036036036036034,
|
||||||
|
"grad_norm": 2.228259801864624,
|
||||||
|
"learning_rate": 9.813795930277306e-06,
|
||||||
|
"loss": 1.863,
|
||||||
|
"step": 140
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.4118404118404118,
|
||||||
|
"grad_norm": 3.990351438522339,
|
||||||
|
"learning_rate": 9.673242402909555e-06,
|
||||||
|
"loss": 1.7983,
|
||||||
|
"step": 160
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.46332046332046334,
|
||||||
|
"grad_norm": 2.6102797985076904,
|
||||||
|
"learning_rate": 9.495062675535614e-06,
|
||||||
|
"loss": 1.7845,
|
||||||
|
"step": 180
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.5148005148005148,
|
||||||
|
"grad_norm": 2.4415857791900635,
|
||||||
|
"learning_rate": 9.280691346552308e-06,
|
||||||
|
"loss": 1.7541,
|
||||||
|
"step": 200
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.5662805662805663,
|
||||||
|
"grad_norm": 2.875765800476074,
|
||||||
|
"learning_rate": 9.031854407852317e-06,
|
||||||
|
"loss": 1.7665,
|
||||||
|
"step": 220
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.6177606177606177,
|
||||||
|
"grad_norm": 2.55922269821167,
|
||||||
|
"learning_rate": 8.750555348152299e-06,
|
||||||
|
"loss": 1.7272,
|
||||||
|
"step": 240
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.6692406692406693,
|
||||||
|
"grad_norm": 3.4958205223083496,
|
||||||
|
"learning_rate": 8.439059022079789e-06,
|
||||||
|
"loss": 1.7349,
|
||||||
|
"step": 260
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.7207207207207207,
|
||||||
|
"grad_norm": 2.582611560821533,
|
||||||
|
"learning_rate": 8.099873414895453e-06,
|
||||||
|
"loss": 1.7273,
|
||||||
|
"step": 280
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.7722007722007722,
|
||||||
|
"grad_norm": 2.274104595184326,
|
||||||
|
"learning_rate": 7.73572944967043e-06,
|
||||||
|
"loss": 1.695,
|
||||||
|
"step": 300
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.8236808236808236,
|
||||||
|
"grad_norm": 2.8610410690307617,
|
||||||
|
"learning_rate": 7.3495589994995274e-06,
|
||||||
|
"loss": 1.663,
|
||||||
|
"step": 320
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.8751608751608752,
|
||||||
|
"grad_norm": 2.4936399459838867,
|
||||||
|
"learning_rate": 6.944471281782975e-06,
|
||||||
|
"loss": 1.6462,
|
||||||
|
"step": 340
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.9266409266409267,
|
||||||
|
"grad_norm": 3.510580062866211,
|
||||||
|
"learning_rate": 6.523727824636103e-06,
|
||||||
|
"loss": 1.6662,
|
||||||
|
"step": 360
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.9781209781209781,
|
||||||
|
"grad_norm": 2.398237943649292,
|
||||||
|
"learning_rate": 6.090716206982714e-06,
|
||||||
|
"loss": 1.6664,
|
||||||
|
"step": 380
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.0283140283140284,
|
||||||
|
"grad_norm": 2.8953514099121094,
|
||||||
|
"learning_rate": 5.648922783761443e-06,
|
||||||
|
"loss": 1.6363,
|
||||||
|
"step": 400
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.0797940797940797,
|
||||||
|
"grad_norm": 2.4946253299713135,
|
||||||
|
"learning_rate": 5.201904615845743e-06,
|
||||||
|
"loss": 1.5998,
|
||||||
|
"step": 420
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.1312741312741312,
|
||||||
|
"grad_norm": 2.965862512588501,
|
||||||
|
"learning_rate": 4.753260830681247e-06,
|
||||||
|
"loss": 1.5663,
|
||||||
|
"step": 440
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.1827541827541828,
|
||||||
|
"grad_norm": 2.843411922454834,
|
||||||
|
"learning_rate": 4.306603644227821e-06,
|
||||||
|
"loss": 1.578,
|
||||||
|
"step": 460
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.2342342342342343,
|
||||||
|
"grad_norm": 2.5591073036193848,
|
||||||
|
"learning_rate": 3.8655292775206185e-06,
|
||||||
|
"loss": 1.5821,
|
||||||
|
"step": 480
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.2857142857142856,
|
||||||
|
"grad_norm": 3.3472440242767334,
|
||||||
|
"learning_rate": 3.4335890020128382e-06,
|
||||||
|
"loss": 1.5743,
|
||||||
|
"step": 500
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.3371943371943371,
|
||||||
|
"grad_norm": 2.687406063079834,
|
||||||
|
"learning_rate": 3.0142605468260976e-06,
|
||||||
|
"loss": 1.5407,
|
||||||
|
"step": 520
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.3886743886743886,
|
||||||
|
"grad_norm": 2.7775464057922363,
|
||||||
|
"learning_rate": 2.610920098120424e-06,
|
||||||
|
"loss": 1.5429,
|
||||||
|
"step": 540
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.4401544401544402,
|
||||||
|
"grad_norm": 2.49112606048584,
|
||||||
|
"learning_rate": 2.2268151160284508e-06,
|
||||||
|
"loss": 1.5408,
|
||||||
|
"step": 560
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.4916344916344917,
|
||||||
|
"grad_norm": 2.938004493713379,
|
||||||
|
"learning_rate": 1.8650381880159108e-06,
|
||||||
|
"loss": 1.5605,
|
||||||
|
"step": 580
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.5431145431145432,
|
||||||
|
"grad_norm": 2.910888910293579,
|
||||||
|
"learning_rate": 1.5285021291857705e-06,
|
||||||
|
"loss": 1.53,
|
||||||
|
"step": 600
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"logging_steps": 20,
|
||||||
|
"max_steps": 778,
|
||||||
|
"num_input_tokens_seen": 0,
|
||||||
|
"num_train_epochs": 2,
|
||||||
|
"save_steps": 50,
|
||||||
|
"stateful_callbacks": {
|
||||||
|
"TrainerControl": {
|
||||||
|
"args": {
|
||||||
|
"should_epoch_stop": false,
|
||||||
|
"should_evaluate": false,
|
||||||
|
"should_log": false,
|
||||||
|
"should_save": true,
|
||||||
|
"should_training_stop": false
|
||||||
|
},
|
||||||
|
"attributes": {}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"total_flos": 4.381796882775245e+16,
|
||||||
|
"train_batch_size": 4,
|
||||||
|
"trial_name": null,
|
||||||
|
"trial_params": null
|
||||||
|
}
|
||||||
3
checkpoint-600/training_args.bin
Normal file
3
checkpoint-600/training_args.bin
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:fe6e3348f136c9b50e5b7ff3939a0aca31453e7fbef30d71133ebde6392f9959
|
||||||
|
size 6033
|
||||||
207
checkpoint-650/README.md
Normal file
207
checkpoint-650/README.md
Normal file
@@ -0,0 +1,207 @@
|
|||||||
|
---
|
||||||
|
base_model: Henry236/nilechat-eg-stage1-general
|
||||||
|
library_name: peft
|
||||||
|
pipeline_tag: text-generation
|
||||||
|
tags:
|
||||||
|
- base_model:adapter:Henry236/nilechat-eg-stage1-general
|
||||||
|
- lora
|
||||||
|
- transformers
|
||||||
|
---
|
||||||
|
|
||||||
|
# Model Card for Model ID
|
||||||
|
|
||||||
|
<!-- Provide a quick summary of what the model is/does. -->
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## Model Details
|
||||||
|
|
||||||
|
### Model Description
|
||||||
|
|
||||||
|
<!-- Provide a longer summary of what this model is. -->
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
- **Developed by:** [More Information Needed]
|
||||||
|
- **Funded by [optional]:** [More Information Needed]
|
||||||
|
- **Shared by [optional]:** [More Information Needed]
|
||||||
|
- **Model type:** [More Information Needed]
|
||||||
|
- **Language(s) (NLP):** [More Information Needed]
|
||||||
|
- **License:** [More Information Needed]
|
||||||
|
- **Finetuned from model [optional]:** [More Information Needed]
|
||||||
|
|
||||||
|
### Model Sources [optional]
|
||||||
|
|
||||||
|
<!-- Provide the basic links for the model. -->
|
||||||
|
|
||||||
|
- **Repository:** [More Information Needed]
|
||||||
|
- **Paper [optional]:** [More Information Needed]
|
||||||
|
- **Demo [optional]:** [More Information Needed]
|
||||||
|
|
||||||
|
## Uses
|
||||||
|
|
||||||
|
<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
|
||||||
|
|
||||||
|
### Direct Use
|
||||||
|
|
||||||
|
<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Downstream Use [optional]
|
||||||
|
|
||||||
|
<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Out-of-Scope Use
|
||||||
|
|
||||||
|
<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Bias, Risks, and Limitations
|
||||||
|
|
||||||
|
<!-- This section is meant to convey both technical and sociotechnical limitations. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Recommendations
|
||||||
|
|
||||||
|
<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
|
||||||
|
|
||||||
|
Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
|
||||||
|
|
||||||
|
## How to Get Started with the Model
|
||||||
|
|
||||||
|
Use the code below to get started with the model.
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Training Details
|
||||||
|
|
||||||
|
### Training Data
|
||||||
|
|
||||||
|
<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Training Procedure
|
||||||
|
|
||||||
|
<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
|
||||||
|
|
||||||
|
#### Preprocessing [optional]
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
|
||||||
|
#### Training Hyperparameters
|
||||||
|
|
||||||
|
- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
|
||||||
|
|
||||||
|
#### Speeds, Sizes, Times [optional]
|
||||||
|
|
||||||
|
<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Evaluation
|
||||||
|
|
||||||
|
<!-- This section describes the evaluation protocols and provides the results. -->
|
||||||
|
|
||||||
|
### Testing Data, Factors & Metrics
|
||||||
|
|
||||||
|
#### Testing Data
|
||||||
|
|
||||||
|
<!-- This should link to a Dataset Card if possible. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Factors
|
||||||
|
|
||||||
|
<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Metrics
|
||||||
|
|
||||||
|
<!-- These are the evaluation metrics being used, ideally with a description of why. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Results
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Summary
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## Model Examination [optional]
|
||||||
|
|
||||||
|
<!-- Relevant interpretability work for the model goes here -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Environmental Impact
|
||||||
|
|
||||||
|
<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
|
||||||
|
|
||||||
|
Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
|
||||||
|
|
||||||
|
- **Hardware Type:** [More Information Needed]
|
||||||
|
- **Hours used:** [More Information Needed]
|
||||||
|
- **Cloud Provider:** [More Information Needed]
|
||||||
|
- **Compute Region:** [More Information Needed]
|
||||||
|
- **Carbon Emitted:** [More Information Needed]
|
||||||
|
|
||||||
|
## Technical Specifications [optional]
|
||||||
|
|
||||||
|
### Model Architecture and Objective
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
### Compute Infrastructure
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Hardware
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
#### Software
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Citation [optional]
|
||||||
|
|
||||||
|
<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
|
||||||
|
|
||||||
|
**BibTeX:**
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
**APA:**
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Glossary [optional]
|
||||||
|
|
||||||
|
<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## More Information [optional]
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Model Card Authors [optional]
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
|
||||||
|
## Model Card Contact
|
||||||
|
|
||||||
|
[More Information Needed]
|
||||||
|
### Framework versions
|
||||||
|
|
||||||
|
- PEFT 0.19.1
|
||||||
48
checkpoint-650/adapter_config.json
Normal file
48
checkpoint-650/adapter_config.json
Normal file
@@ -0,0 +1,48 @@
|
|||||||
|
{
|
||||||
|
"alora_invocation_tokens": null,
|
||||||
|
"alpha_pattern": {},
|
||||||
|
"arrow_config": null,
|
||||||
|
"auto_mapping": null,
|
||||||
|
"base_model_name_or_path": "Henry236/nilechat-eg-stage1-general",
|
||||||
|
"bias": "none",
|
||||||
|
"corda_config": null,
|
||||||
|
"ensure_weight_tying": false,
|
||||||
|
"eva_config": null,
|
||||||
|
"exclude_modules": null,
|
||||||
|
"fan_in_fan_out": false,
|
||||||
|
"inference_mode": true,
|
||||||
|
"init_lora_weights": true,
|
||||||
|
"layer_replication": null,
|
||||||
|
"layers_pattern": null,
|
||||||
|
"layers_to_transform": null,
|
||||||
|
"loftq_config": {},
|
||||||
|
"lora_alpha": 128,
|
||||||
|
"lora_bias": false,
|
||||||
|
"lora_dropout": 0.05,
|
||||||
|
"lora_ga_config": null,
|
||||||
|
"megatron_config": null,
|
||||||
|
"megatron_core": "megatron.core",
|
||||||
|
"modules_to_save": null,
|
||||||
|
"peft_type": "LORA",
|
||||||
|
"peft_version": "0.19.1",
|
||||||
|
"qalora_group_size": 16,
|
||||||
|
"r": 64,
|
||||||
|
"rank_pattern": {},
|
||||||
|
"revision": null,
|
||||||
|
"target_modules": [
|
||||||
|
"down_proj",
|
||||||
|
"q_proj",
|
||||||
|
"o_proj",
|
||||||
|
"up_proj",
|
||||||
|
"k_proj",
|
||||||
|
"v_proj",
|
||||||
|
"gate_proj"
|
||||||
|
],
|
||||||
|
"target_parameters": null,
|
||||||
|
"task_type": "CAUSAL_LM",
|
||||||
|
"trainable_token_indices": null,
|
||||||
|
"use_bdlora": null,
|
||||||
|
"use_dora": false,
|
||||||
|
"use_qalora": false,
|
||||||
|
"use_rslora": false
|
||||||
|
}
|
||||||
3
checkpoint-650/adapter_model.safetensors
Normal file
3
checkpoint-650/adapter_model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:efd87accb44c9319617dc93965240822548a110521e494edaf21ba557b000bf3
|
||||||
|
size 476904192
|
||||||
3
checkpoint-650/optimizer.pt
Normal file
3
checkpoint-650/optimizer.pt
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:0b4b9468d6cde0d442eeda76d15f0be9d9bfacc068095e2e66abd0222f382ec2
|
||||||
|
size 954089079
|
||||||
3
checkpoint-650/rng_state.pth
Normal file
3
checkpoint-650/rng_state.pth
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:13f76e57951611272f9ae21c4fc6808c57effc1ef5891964cae13760cec87a29
|
||||||
|
size 14645
|
||||||
3
checkpoint-650/scheduler.pt
Normal file
3
checkpoint-650/scheduler.pt
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:0b777d6482972cfe4da20d01a82e04bfbfb467d5e8b005db9dc27dc6b7936ea6
|
||||||
|
size 1465
|
||||||
258
checkpoint-650/trainer_state.json
Normal file
258
checkpoint-650/trainer_state.json
Normal file
@@ -0,0 +1,258 @@
|
|||||||
|
{
|
||||||
|
"best_global_step": null,
|
||||||
|
"best_metric": null,
|
||||||
|
"best_model_checkpoint": null,
|
||||||
|
"epoch": 1.6718146718146718,
|
||||||
|
"eval_steps": 500,
|
||||||
|
"global_step": 650,
|
||||||
|
"is_hyper_param_search": false,
|
||||||
|
"is_local_process_zero": true,
|
||||||
|
"is_world_process_zero": true,
|
||||||
|
"log_history": [
|
||||||
|
{
|
||||||
|
"epoch": 0.05148005148005148,
|
||||||
|
"grad_norm": 3.7984347343444824,
|
||||||
|
"learning_rate": 2.435897435897436e-06,
|
||||||
|
"loss": 3.0721,
|
||||||
|
"step": 20
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.10296010296010295,
|
||||||
|
"grad_norm": 2.0532360076904297,
|
||||||
|
"learning_rate": 5e-06,
|
||||||
|
"loss": 2.7105,
|
||||||
|
"step": 40
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.15444015444015444,
|
||||||
|
"grad_norm": 1.889143466949463,
|
||||||
|
"learning_rate": 7.564102564102564e-06,
|
||||||
|
"loss": 2.397,
|
||||||
|
"step": 60
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.2059202059202059,
|
||||||
|
"grad_norm": 2.206676959991455,
|
||||||
|
"learning_rate": 9.999949644960027e-06,
|
||||||
|
"loss": 2.1587,
|
||||||
|
"step": 80
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.2574002574002574,
|
||||||
|
"grad_norm": 2.4184727668762207,
|
||||||
|
"learning_rate": 9.9778098230154e-06,
|
||||||
|
"loss": 1.9613,
|
||||||
|
"step": 100
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.3088803088803089,
|
||||||
|
"grad_norm": 2.1078503131866455,
|
||||||
|
"learning_rate": 9.915591603280632e-06,
|
||||||
|
"loss": 1.9187,
|
||||||
|
"step": 120
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.36036036036036034,
|
||||||
|
"grad_norm": 2.228259801864624,
|
||||||
|
"learning_rate": 9.813795930277306e-06,
|
||||||
|
"loss": 1.863,
|
||||||
|
"step": 140
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.4118404118404118,
|
||||||
|
"grad_norm": 3.990351438522339,
|
||||||
|
"learning_rate": 9.673242402909555e-06,
|
||||||
|
"loss": 1.7983,
|
||||||
|
"step": 160
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.46332046332046334,
|
||||||
|
"grad_norm": 2.6102797985076904,
|
||||||
|
"learning_rate": 9.495062675535614e-06,
|
||||||
|
"loss": 1.7845,
|
||||||
|
"step": 180
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.5148005148005148,
|
||||||
|
"grad_norm": 2.4415857791900635,
|
||||||
|
"learning_rate": 9.280691346552308e-06,
|
||||||
|
"loss": 1.7541,
|
||||||
|
"step": 200
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.5662805662805663,
|
||||||
|
"grad_norm": 2.875765800476074,
|
||||||
|
"learning_rate": 9.031854407852317e-06,
|
||||||
|
"loss": 1.7665,
|
||||||
|
"step": 220
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.6177606177606177,
|
||||||
|
"grad_norm": 2.55922269821167,
|
||||||
|
"learning_rate": 8.750555348152299e-06,
|
||||||
|
"loss": 1.7272,
|
||||||
|
"step": 240
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.6692406692406693,
|
||||||
|
"grad_norm": 3.4958205223083496,
|
||||||
|
"learning_rate": 8.439059022079789e-06,
|
||||||
|
"loss": 1.7349,
|
||||||
|
"step": 260
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.7207207207207207,
|
||||||
|
"grad_norm": 2.582611560821533,
|
||||||
|
"learning_rate": 8.099873414895453e-06,
|
||||||
|
"loss": 1.7273,
|
||||||
|
"step": 280
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.7722007722007722,
|
||||||
|
"grad_norm": 2.274104595184326,
|
||||||
|
"learning_rate": 7.73572944967043e-06,
|
||||||
|
"loss": 1.695,
|
||||||
|
"step": 300
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.8236808236808236,
|
||||||
|
"grad_norm": 2.8610410690307617,
|
||||||
|
"learning_rate": 7.3495589994995274e-06,
|
||||||
|
"loss": 1.663,
|
||||||
|
"step": 320
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.8751608751608752,
|
||||||
|
"grad_norm": 2.4936399459838867,
|
||||||
|
"learning_rate": 6.944471281782975e-06,
|
||||||
|
"loss": 1.6462,
|
||||||
|
"step": 340
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.9266409266409267,
|
||||||
|
"grad_norm": 3.510580062866211,
|
||||||
|
"learning_rate": 6.523727824636103e-06,
|
||||||
|
"loss": 1.6662,
|
||||||
|
"step": 360
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 0.9781209781209781,
|
||||||
|
"grad_norm": 2.398237943649292,
|
||||||
|
"learning_rate": 6.090716206982714e-06,
|
||||||
|
"loss": 1.6664,
|
||||||
|
"step": 380
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.0283140283140284,
|
||||||
|
"grad_norm": 2.8953514099121094,
|
||||||
|
"learning_rate": 5.648922783761443e-06,
|
||||||
|
"loss": 1.6363,
|
||||||
|
"step": 400
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.0797940797940797,
|
||||||
|
"grad_norm": 2.4946253299713135,
|
||||||
|
"learning_rate": 5.201904615845743e-06,
|
||||||
|
"loss": 1.5998,
|
||||||
|
"step": 420
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.1312741312741312,
|
||||||
|
"grad_norm": 2.965862512588501,
|
||||||
|
"learning_rate": 4.753260830681247e-06,
|
||||||
|
"loss": 1.5663,
|
||||||
|
"step": 440
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.1827541827541828,
|
||||||
|
"grad_norm": 2.843411922454834,
|
||||||
|
"learning_rate": 4.306603644227821e-06,
|
||||||
|
"loss": 1.578,
|
||||||
|
"step": 460
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.2342342342342343,
|
||||||
|
"grad_norm": 2.5591073036193848,
|
||||||
|
"learning_rate": 3.8655292775206185e-06,
|
||||||
|
"loss": 1.5821,
|
||||||
|
"step": 480
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.2857142857142856,
|
||||||
|
"grad_norm": 3.3472440242767334,
|
||||||
|
"learning_rate": 3.4335890020128382e-06,
|
||||||
|
"loss": 1.5743,
|
||||||
|
"step": 500
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.3371943371943371,
|
||||||
|
"grad_norm": 2.687406063079834,
|
||||||
|
"learning_rate": 3.0142605468260976e-06,
|
||||||
|
"loss": 1.5407,
|
||||||
|
"step": 520
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.3886743886743886,
|
||||||
|
"grad_norm": 2.7775464057922363,
|
||||||
|
"learning_rate": 2.610920098120424e-06,
|
||||||
|
"loss": 1.5429,
|
||||||
|
"step": 540
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.4401544401544402,
|
||||||
|
"grad_norm": 2.49112606048584,
|
||||||
|
"learning_rate": 2.2268151160284508e-06,
|
||||||
|
"loss": 1.5408,
|
||||||
|
"step": 560
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.4916344916344917,
|
||||||
|
"grad_norm": 2.938004493713379,
|
||||||
|
"learning_rate": 1.8650381880159108e-06,
|
||||||
|
"loss": 1.5605,
|
||||||
|
"step": 580
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.5431145431145432,
|
||||||
|
"grad_norm": 2.910888910293579,
|
||||||
|
"learning_rate": 1.5285021291857705e-06,
|
||||||
|
"loss": 1.53,
|
||||||
|
"step": 600
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.5945945945945947,
|
||||||
|
"grad_norm": 2.5734620094299316,
|
||||||
|
"learning_rate": 1.2199165300037358e-06,
|
||||||
|
"loss": 1.5425,
|
||||||
|
"step": 620
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"epoch": 1.646074646074646,
|
||||||
|
"grad_norm": 2.9094061851501465,
|
||||||
|
"learning_rate": 9.417659402690254e-07,
|
||||||
|
"loss": 1.5401,
|
||||||
|
"step": 640
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"logging_steps": 20,
|
||||||
|
"max_steps": 778,
|
||||||
|
"num_input_tokens_seen": 0,
|
||||||
|
"num_train_epochs": 2,
|
||||||
|
"save_steps": 50,
|
||||||
|
"stateful_callbacks": {
|
||||||
|
"TrainerControl": {
|
||||||
|
"args": {
|
||||||
|
"should_epoch_stop": false,
|
||||||
|
"should_evaluate": false,
|
||||||
|
"should_log": false,
|
||||||
|
"should_save": true,
|
||||||
|
"should_training_stop": false
|
||||||
|
},
|
||||||
|
"attributes": {}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"total_flos": 4.743984506608435e+16,
|
||||||
|
"train_batch_size": 4,
|
||||||
|
"trial_name": null,
|
||||||
|
"trial_params": null
|
||||||
|
}
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user