初始化项目，由ModelHub XC社区提供模型
Model: uukuguy/speechless-tora-code-7b-v1.0 Source: Original Platform
This commit is contained in:
ModelHub XC
38
.gitattributes
vendored
Normal file
38
.gitattributes
vendored
Normal file
@@ -0,0 +1,38 @@
|
||||
*.7z filter=lfs diff=lfs merge=lfs -text
|
||||
*.arrow filter=lfs diff=lfs merge=lfs -text
|
||||
*.bin filter=lfs diff=lfs merge=lfs -text
|
||||
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
||||
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
||||
*.ftz filter=lfs diff=lfs merge=lfs -text
|
||||
*.gz filter=lfs diff=lfs merge=lfs -text
|
||||
*.h5 filter=lfs diff=lfs merge=lfs -text
|
||||
*.joblib filter=lfs diff=lfs merge=lfs -text
|
||||
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
||||
*.model filter=lfs diff=lfs merge=lfs -text
|
||||
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
||||
*.npy filter=lfs diff=lfs merge=lfs -text
|
||||
*.npz filter=lfs diff=lfs merge=lfs -text
|
||||
*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||
*.ot filter=lfs diff=lfs merge=lfs -text
|
||||
*.parquet filter=lfs diff=lfs merge=lfs -text
|
||||
*.pb filter=lfs diff=lfs merge=lfs -text
|
||||
*.pickle filter=lfs diff=lfs merge=lfs -text
|
||||
*.pkl filter=lfs diff=lfs merge=lfs -text
|
||||
*.pt filter=lfs diff=lfs merge=lfs -text
|
||||
*.pth filter=lfs diff=lfs merge=lfs -text
|
||||
*.rar filter=lfs diff=lfs merge=lfs -text
|
||||
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar filter=lfs diff=lfs merge=lfs -text
|
||||
*.tflite filter=lfs diff=lfs merge=lfs -text
|
||||
*.tgz filter=lfs diff=lfs merge=lfs -text
|
||||
*.wasm filter=lfs diff=lfs merge=lfs -text
|
||||
*.xz filter=lfs diff=lfs merge=lfs -text
|
||||
*.zip filter=lfs diff=lfs merge=lfs -text
|
||||
*.zst filter=lfs diff=lfs merge=lfs -text
|
||||
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||
pytorch_model-00001-of-00002.bin filter=lfs diff=lfs merge=lfs -text
|
||||
pytorch_model-00002-of-00002.bin filter=lfs diff=lfs merge=lfs -text
|
||||
tokenizer.model filter=lfs diff=lfs merge=lfs -text
|
||||
145
README.md
Normal file
145
README.md
Normal file
@@ -0,0 +1,145 @@
|
||||
---
|
||||
language:
|
||||
- en
|
||||
library_name: transformers
|
||||
pipeline_tag: text-generation
|
||||
datasets:
|
||||
- jondurbin/airoboros-2.2
|
||||
- Open-Orca/OpenOrca
|
||||
- garage-bAInd/Open-Platypus
|
||||
- WizardLM/WizardLM_evol_instruct_V2_196k
|
||||
- TokenBender/python_eval_instruct_51k
|
||||
tags:
|
||||
- llama-2
|
||||
- code
|
||||
license: llama2
|
||||
model-index:
|
||||
- name: SpeechlessCoder
|
||||
results:
|
||||
- task:
|
||||
type: text-generation
|
||||
dataset:
|
||||
type: openai_humaneval
|
||||
name: HumanEval
|
||||
metrics:
|
||||
- name: pass@1
|
||||
type: pass@1
|
||||
value: 51.829
|
||||
verified: false
|
||||
---
|
||||
|
||||
<p><h1> speechless-tora-code-7b-v1.0 </h1></p>
|
||||
|
||||
* [AWQ model(s) for GPU inference.](https://huggingface.co/TheBloke/speechless-tora-code-7B-v1.0-AWQ)
|
||||
* [GPTQ models for GPU inference, with multiple quantisation parameter options.](https://huggingface.co/TheBloke/speechless-tora-code-7B-v1.0-GPTQ)
|
||||
* [2, 3, 4, 5, 6 and 8-bit GGUF models for CPU+GPU inference](https://huggingface.co/TheBloke/speechless-tora-code-7B-v1.0-GGUF)
|
||||
|
||||
Code: https://github.com/uukuguy/speechless
|
||||
|
||||
Use the following dataset to fine-tune llm_agents/tora-code-7b-v1.0 in order to improve the model's reasoning and planning abilities.
|
||||
|
||||
Total 201,981 samples.
|
||||
- jondurbin/airoboros-2.2: Filter categories related to coding, reasoning and planning. 23,462 samples.
|
||||
- Open-Orca/OpenOrca: Filter the 'cot' category in 1M GPT4 dataset. 74,440 samples.
|
||||
- garage-bAInd/Open-Platypus: 100%, 24,926 samples.
|
||||
- WizardLM/WizardLM_evol_instruct_V2_196k: Coding coversation part. 30,185 samples
|
||||
- TokenBender/python_eval_instruct_51k: “python” in output .40,309 samples
|
||||
- Spider: 8,659 samples
|
||||
|
||||
## How to Prompt the Model
|
||||
This model accepts the Alpaca instruction format.
|
||||
|
||||
For example:
|
||||
```
|
||||
You are an intelligent programming assistant.
|
||||
|
||||
### Instruction:
|
||||
Implement a linked list in C++
|
||||
|
||||
### Response:
|
||||
```
|
||||
|
||||
## HumanEval
|
||||
|
||||
| Metric | Value |
|
||||
| --- | --- |
|
||||
| humaneval-python | 51.829 |
|
||||
|
||||
[Big Code Models Leaderboard](https://huggingface.co/spaces/bigcode/bigcode-models-leaderboard)
|
||||
|
||||
CodeLlama-34B-Python: 53.29
|
||||
|
||||
CodeLlama-34B-Instruct: 50.79
|
||||
|
||||
CodeLlama-13B-Instruct: 50.6
|
||||
|
||||
CodeLlama-34B: 45.11
|
||||
|
||||
CodeLlama-13B-Python: 42.89
|
||||
|
||||
CodeLlama-13B: 35.07
|
||||
|
||||
|
||||
## LM-Evaluation-Harness
|
||||
|
||||
[Open LLM Leaderboard](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard)
|
||||
| Metric | Value |
|
||||
| --- | --- |
|
||||
| ARC | 42.66 |
|
||||
| HellaSwag | 65.16 |
|
||||
| MMLU | 38.56 |
|
||||
| TruthfulQA | 42.06 |
|
||||
| Average | 47.11 |
|
||||
|
||||
|
||||
## Parameters
|
||||
|
||||
| | |
|
||||
|------ | ------ |
|
||||
| lr | 2e-4 |
|
||||
| lr_scheduler_type | cosine |
|
||||
| weight_decay | 0.0 |
|
||||
| optim | paged_adamw_8bit |
|
||||
| flash_attention | True |
|
||||
| rerope | False |
|
||||
| max_new_tokens | 4096 |
|
||||
| num_train_epochs | 2 |
|
||||
| bits | 4 |
|
||||
| lora_r | 64 |
|
||||
| lora_alpha | 16 |
|
||||
| lora_dropout | 0.05 |
|
||||
| double_quant | True |
|
||||
| quant_type | nf4 |
|
||||
| dataset_format | airoboros |
|
||||
| mini_batch_size | 2 |
|
||||
| grandient_accumulation_steps | 32 |
|
||||
| bf16 | True |
|
||||
|
||||
A800-80G x 2
|
||||
|
||||
| | |
|
||||
|------ | ------ |
|
||||
| epoch | 2.0 |
|
||||
| etrain_loss | 0.5891 |
|
||||
| etrain_runtime | 19:24:49.43 |
|
||||
| etrain_samples_per_second | 5.664 |
|
||||
| etrain_steps_per_second | 0.044 |
|
||||
| eeval_loss | 0.5872 |
|
||||
| eeval_runtime | 0:00:15.59 |
|
||||
| eeval_samples_per_second | 12.822 |
|
||||
| eeval_steps_per_second | 6.411 |
|
||||
|
||||
|
||||
# [Open LLM Leaderboard Evaluation Results](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard)
|
||||
Detailed results can be found [here](https://huggingface.co/datasets/open-llm-leaderboard/details_uukuguy__speechless-tora-code-7b-v1.0)
|
||||
|
||||
| Metric | Value |
|
||||
|-----------------------|---------------------------|
|
||||
| Avg. | 40.1 |
|
||||
| ARC (25-shot) | 42.66 |
|
||||
| HellaSwag (10-shot) | 65.16 |
|
||||
| MMLU (5-shot) | 38.56 |
|
||||
| TruthfulQA (0-shot) | 42.06 |
|
||||
| Winogrande (5-shot) | 62.9 |
|
||||
| GSM8K (5-shot) | 0.91 |
|
||||
| DROP (3-shot) | 28.48 |
|
||||
6
added_tokens.json
Normal file
6
added_tokens.json
Normal file
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"</s>": 2,
|
||||
"<pad>": 32000,
|
||||
"<s>": 1,
|
||||
"<unk>": 0
|
||||
}
|
||||
28
config.json
Normal file
28
config.json
Normal file
@@ -0,0 +1,28 @@
|
||||
{
|
||||
"_name_or_path": "/opt/local/llm_models/huggingface.co/llm_agents/tora-code-7b-v1.0",
|
||||
"architectures": [
|
||||
"LlamaForCausalLM"
|
||||
],
|
||||
"attention_bias": false,
|
||||
"bos_token_id": 1,
|
||||
"eos_token_id": 2,
|
||||
"hidden_act": "silu",
|
||||
"hidden_size": 4096,
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 11008,
|
||||
"max_position_embeddings": 16384,
|
||||
"model_type": "llama",
|
||||
"num_attention_heads": 32,
|
||||
"num_hidden_layers": 32,
|
||||
"num_key_value_heads": 32,
|
||||
"pad_token_id": 0,
|
||||
"pretraining_tp": 1,
|
||||
"rms_norm_eps": 1e-05,
|
||||
"rope_scaling": null,
|
||||
"rope_theta": 1000000,
|
||||
"tie_word_embeddings": false,
|
||||
"torch_dtype": "float16",
|
||||
"transformers_version": "4.34.0",
|
||||
"use_cache": true,
|
||||
"vocab_size": 32001
|
||||
}
|
||||
6
generation_config.json
Normal file
6
generation_config.json
Normal file
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"_from_model_config": true,
|
||||
"bos_token_id": 1,
|
||||
"eos_token_id": 2,
|
||||
"transformers_version": "4.34.0"
|
||||
}
|
||||
3
pytorch_model-00001-of-00002.bin
Normal file
3
pytorch_model-00001-of-00002.bin
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:34de0eeb73499dff4312178870a24cc62d15caab73c13dcade8b0472e54dbe68
|
||||
size 9976628314
|
||||
3
pytorch_model-00002-of-00002.bin
Normal file
3
pytorch_model-00002-of-00002.bin
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:0426b2042db9f55a1bcac302cc6e7b491eaeb1c9250e16749a1ec5529a48cd3b
|
||||
size 3500318979
|
||||
298
pytorch_model.bin.index.json
Normal file
298
pytorch_model.bin.index.json
Normal file
@@ -0,0 +1,298 @@
|
||||
{
|
||||
"metadata": {
|
||||
"total_size": 13476847616
|
||||
},
|
||||
"weight_map": {
|
||||
"lm_head.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.embed_tokens.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.0.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.0.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.0.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.0.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.0.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.0.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.0.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.0.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.0.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.1.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.1.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.1.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.1.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.1.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.1.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.1.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.1.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.1.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.10.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.10.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.10.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.10.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.10.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.10.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.10.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.10.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.10.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.11.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.11.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.11.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.11.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.11.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.11.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.11.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.11.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.11.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.12.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.12.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.12.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.12.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.12.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.12.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.12.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.12.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.12.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.13.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.13.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.13.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.13.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.13.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.13.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.13.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.13.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.13.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.14.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.14.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.14.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.14.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.14.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.14.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.14.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.14.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.14.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.15.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.15.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.15.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.15.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.15.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.15.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.15.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.15.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.15.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.16.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.16.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.16.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.16.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.16.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.16.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.16.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.16.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.16.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.17.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.17.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.17.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.17.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.17.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.17.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.17.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.17.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.17.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.18.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.18.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.18.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.18.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.18.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.18.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.18.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.18.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.18.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.19.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.19.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.19.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.19.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.19.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.19.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.19.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.19.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.19.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.2.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.2.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.2.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.2.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.2.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.2.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.2.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.2.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.2.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.20.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.20.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.20.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.20.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.20.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.20.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.20.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.20.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.20.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.21.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.21.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.21.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.21.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.21.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.21.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.21.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.21.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.21.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.22.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.22.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.22.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.22.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.22.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.22.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.22.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.22.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.22.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.23.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.23.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.23.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.23.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.23.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.23.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.23.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.23.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.23.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.24.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.24.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.24.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.24.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.24.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.24.self_attn.k_proj.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.24.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.24.self_attn.q_proj.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.24.self_attn.v_proj.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.25.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.25.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.25.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.25.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.25.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.25.self_attn.k_proj.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.25.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.25.self_attn.q_proj.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.25.self_attn.v_proj.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.26.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.26.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.26.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.26.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.26.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.26.self_attn.k_proj.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.26.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.26.self_attn.q_proj.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.26.self_attn.v_proj.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.27.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.27.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.27.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.27.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.27.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.27.self_attn.k_proj.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.27.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.27.self_attn.q_proj.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.27.self_attn.v_proj.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.28.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.28.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.28.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.28.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.28.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.28.self_attn.k_proj.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.28.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.28.self_attn.q_proj.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.28.self_attn.v_proj.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.29.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.29.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.29.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.29.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.29.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.29.self_attn.k_proj.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.29.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.29.self_attn.q_proj.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.29.self_attn.v_proj.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.3.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.3.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.3.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.3.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.3.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.3.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.3.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.3.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.3.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.30.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.30.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.30.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.30.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.30.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.30.self_attn.k_proj.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.30.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.30.self_attn.q_proj.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.30.self_attn.v_proj.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.31.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.31.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.31.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.31.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.31.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.31.self_attn.k_proj.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.31.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.31.self_attn.q_proj.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.31.self_attn.v_proj.weight": "pytorch_model-00002-of-00002.bin",
|
||||
"model.layers.4.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.4.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.4.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.4.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.4.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.4.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.4.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.4.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.4.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.5.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.5.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.5.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.5.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.5.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.5.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.5.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.5.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.5.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.6.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.6.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.6.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.6.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.6.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.6.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.6.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.6.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.6.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.7.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.7.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.7.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.7.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.7.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.7.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.7.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.7.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.7.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.8.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.8.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.8.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.8.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.8.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.8.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.8.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.8.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.8.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.9.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.9.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.9.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.9.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.9.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.9.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.9.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.9.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.layers.9.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
|
||||
"model.norm.weight": "pytorch_model-00002-of-00002.bin"
|
||||
}
|
||||
}
|
||||
6
special_tokens_map.json
Normal file
6
special_tokens_map.json
Normal file
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"bos_token": "<s>",
|
||||
"eos_token": "</s>",
|
||||
"pad_token": "<pad>",
|
||||
"unk_token": "<unk>"
|
||||
}
|
||||
13
tasks/speechless-tora-code-7b-v1.0/merge_peft_adapters.sh
Executable file
13
tasks/speechless-tora-code-7b-v1.0/merge_peft_adapters.sh
Executable file
@@ -0,0 +1,13 @@
|
||||
#!/usr/bin/env bash
|
||||
# From speechless-code-mistral-7b-v1.0
|
||||
SCRIPT_PATH=$(cd $(dirname ${BASH_SOURCE[0]}); pwd)
|
||||
PARENT_PATH=$(cd "${SCRIPT_PATH}/.." ; pwd)
|
||||
|
||||
source ${SCRIPT_PATH}/task.env
|
||||
|
||||
CHECKPOINT_DIR=${TASK_CHECKPOINT_DIR}/checkpoint-3092/adapter_model
|
||||
|
||||
python ${SCRIPT_PATH}/../../scripts/merge_peft_adapters.py \
|
||||
--base_model_name_or_path ${BASE_MODEL_PATH} \
|
||||
--peft_model_path ${CHECKPOINT_DIR} \
|
||||
--merged_model_name_or_path ${TEST_MODEL_PATH} \
|
||||
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"epoch": 2.0,
|
||||
"eval_loss": 0.5871890783309937,
|
||||
"eval_runtime": 15.5979,
|
||||
"eval_samples_per_second": 12.822,
|
||||
"eval_steps_per_second": 6.411,
|
||||
"train_loss": 0.5890884618112218,
|
||||
"train_runtime": 69889.4305,
|
||||
"train_samples_per_second": 5.664,
|
||||
"train_steps_per_second": 0.044
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
---
|
||||
library_name: peft
|
||||
---
|
||||
## Training procedure
|
||||
|
||||
|
||||
The following `bitsandbytes` quantization config was used during training:
|
||||
- quant_method: bitsandbytes
|
||||
- load_in_8bit: False
|
||||
- load_in_4bit: True
|
||||
- llm_int8_threshold: 6.0
|
||||
- llm_int8_skip_modules: None
|
||||
- llm_int8_enable_fp32_cpu_offload: False
|
||||
- llm_int8_has_fp16_weight: False
|
||||
- bnb_4bit_quant_type: nf4
|
||||
- bnb_4bit_use_double_quant: True
|
||||
- bnb_4bit_compute_dtype: bfloat16
|
||||
### Framework versions
|
||||
|
||||
|
||||
- PEFT 0.5.0
|
||||
@@ -0,0 +1,26 @@
|
||||
{
|
||||
"auto_mapping": null,
|
||||
"base_model_name_or_path": "/opt/local/llm_models/huggingface.co/llm_agents/tora-code-7b-v1.0",
|
||||
"bias": "none",
|
||||
"fan_in_fan_out": false,
|
||||
"inference_mode": true,
|
||||
"init_lora_weights": true,
|
||||
"layers_pattern": null,
|
||||
"layers_to_transform": null,
|
||||
"lora_alpha": 16.0,
|
||||
"lora_dropout": 0.05,
|
||||
"modules_to_save": null,
|
||||
"peft_type": "LORA",
|
||||
"r": 64,
|
||||
"revision": null,
|
||||
"target_modules": [
|
||||
"down_proj",
|
||||
"up_proj",
|
||||
"q_proj",
|
||||
"o_proj",
|
||||
"gate_proj",
|
||||
"v_proj",
|
||||
"k_proj"
|
||||
],
|
||||
"task_type": "CAUSAL_LM"
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:fe53a63399829bb762760aac51f487a74255439a8ba937191ccecf3a72040093
|
||||
size 639792909
|
||||
@@ -0,0 +1,34 @@
|
||||
---
|
||||
library_name: peft
|
||||
---
|
||||
## Training procedure
|
||||
|
||||
|
||||
The following `bitsandbytes` quantization config was used during training:
|
||||
- quant_method: bitsandbytes
|
||||
- load_in_8bit: False
|
||||
- load_in_4bit: True
|
||||
- llm_int8_threshold: 6.0
|
||||
- llm_int8_skip_modules: None
|
||||
- llm_int8_enable_fp32_cpu_offload: False
|
||||
- llm_int8_has_fp16_weight: False
|
||||
- bnb_4bit_quant_type: nf4
|
||||
- bnb_4bit_use_double_quant: True
|
||||
- bnb_4bit_compute_dtype: bfloat16
|
||||
|
||||
The following `bitsandbytes` quantization config was used during training:
|
||||
- quant_method: bitsandbytes
|
||||
- load_in_8bit: False
|
||||
- load_in_4bit: True
|
||||
- llm_int8_threshold: 6.0
|
||||
- llm_int8_skip_modules: None
|
||||
- llm_int8_enable_fp32_cpu_offload: False
|
||||
- llm_int8_has_fp16_weight: False
|
||||
- bnb_4bit_quant_type: nf4
|
||||
- bnb_4bit_use_double_quant: True
|
||||
- bnb_4bit_compute_dtype: bfloat16
|
||||
### Framework versions
|
||||
|
||||
- PEFT 0.5.0
|
||||
|
||||
- PEFT 0.5.0
|
||||
@@ -0,0 +1,26 @@
|
||||
{
|
||||
"auto_mapping": null,
|
||||
"base_model_name_or_path": "/opt/local/llm_models/huggingface.co/llm_agents/tora-code-7b-v1.0",
|
||||
"bias": "none",
|
||||
"fan_in_fan_out": false,
|
||||
"inference_mode": true,
|
||||
"init_lora_weights": true,
|
||||
"layers_pattern": null,
|
||||
"layers_to_transform": null,
|
||||
"lora_alpha": 16.0,
|
||||
"lora_dropout": 0.05,
|
||||
"modules_to_save": null,
|
||||
"peft_type": "LORA",
|
||||
"r": 64,
|
||||
"revision": null,
|
||||
"target_modules": [
|
||||
"down_proj",
|
||||
"up_proj",
|
||||
"q_proj",
|
||||
"o_proj",
|
||||
"gate_proj",
|
||||
"v_proj",
|
||||
"k_proj"
|
||||
],
|
||||
"task_type": "CAUSAL_LM"
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:fe53a63399829bb762760aac51f487a74255439a8ba937191ccecf3a72040093
|
||||
size 639792909
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"</s>": 2,
|
||||
"<pad>": 32000,
|
||||
"<s>": 1,
|
||||
"<unk>": 0
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:ded5b21ac4123d95f741097b6ba2f10e14abf2fea6dc83011548234db36631c8
|
||||
size 320970527
|
||||
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:b465ff783be6837c8cdf3aa9aa36366d7695d1fdb6b4888ec0c8a7238379af77
|
||||
size 15735
|
||||
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:f79c2a3250c7c5ce53d25ba72962b46362a9f39b3282d1958c6bee70bd711b2e
|
||||
size 15735
|
||||
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:403f7d4ddb29af77fdfd9e883d0c8935063872d5e73dea699394bc703f947ea8
|
||||
size 627
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"bos_token": "<s>",
|
||||
"eos_token": "</s>",
|
||||
"pad_token": "<pad>",
|
||||
"unk_token": "<unk>"
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
|
||||
size 499723
|
||||
@@ -0,0 +1,52 @@
|
||||
{
|
||||
"add_bos_token": true,
|
||||
"add_eos_token": false,
|
||||
"added_tokens_decoder": {
|
||||
"0": {
|
||||
"content": "<unk>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"1": {
|
||||
"content": "<s>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"2": {
|
||||
"content": "</s>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32000": {
|
||||
"content": "<pad>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
}
|
||||
},
|
||||
"additional_special_tokens": [],
|
||||
"bos_token": "<s>",
|
||||
"clean_up_tokenization_spaces": false,
|
||||
"eos_token": "</s>",
|
||||
"legacy": true,
|
||||
"model_max_length": 1000000000000000019884624838656,
|
||||
"pad_token": "<pad>",
|
||||
"padding_side": "left",
|
||||
"sp_model_kwargs": {},
|
||||
"spaces_between_special_tokens": false,
|
||||
"tokenizer_class": "LlamaTokenizer",
|
||||
"tokenizer_file": null,
|
||||
"unk_token": "<unk>",
|
||||
"use_default_system_prompt": true
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:88c746833b5a7f3b81a2dbfe4fd0d18678e9740460c1a97584c75375f13a6c11
|
||||
size 4667
|
||||
@@ -0,0 +1,34 @@
|
||||
---
|
||||
library_name: peft
|
||||
---
|
||||
## Training procedure
|
||||
|
||||
|
||||
The following `bitsandbytes` quantization config was used during training:
|
||||
- quant_method: bitsandbytes
|
||||
- load_in_8bit: False
|
||||
- load_in_4bit: True
|
||||
- llm_int8_threshold: 6.0
|
||||
- llm_int8_skip_modules: None
|
||||
- llm_int8_enable_fp32_cpu_offload: False
|
||||
- llm_int8_has_fp16_weight: False
|
||||
- bnb_4bit_quant_type: nf4
|
||||
- bnb_4bit_use_double_quant: True
|
||||
- bnb_4bit_compute_dtype: bfloat16
|
||||
|
||||
The following `bitsandbytes` quantization config was used during training:
|
||||
- quant_method: bitsandbytes
|
||||
- load_in_8bit: False
|
||||
- load_in_4bit: True
|
||||
- llm_int8_threshold: 6.0
|
||||
- llm_int8_skip_modules: None
|
||||
- llm_int8_enable_fp32_cpu_offload: False
|
||||
- llm_int8_has_fp16_weight: False
|
||||
- bnb_4bit_quant_type: nf4
|
||||
- bnb_4bit_use_double_quant: True
|
||||
- bnb_4bit_compute_dtype: bfloat16
|
||||
### Framework versions
|
||||
|
||||
- PEFT 0.5.0
|
||||
|
||||
- PEFT 0.5.0
|
||||
@@ -0,0 +1,26 @@
|
||||
{
|
||||
"auto_mapping": null,
|
||||
"base_model_name_or_path": "/opt/local/llm_models/huggingface.co/llm_agents/tora-code-7b-v1.0",
|
||||
"bias": "none",
|
||||
"fan_in_fan_out": false,
|
||||
"inference_mode": true,
|
||||
"init_lora_weights": true,
|
||||
"layers_pattern": null,
|
||||
"layers_to_transform": null,
|
||||
"lora_alpha": 16.0,
|
||||
"lora_dropout": 0.05,
|
||||
"modules_to_save": null,
|
||||
"peft_type": "LORA",
|
||||
"r": 64,
|
||||
"revision": null,
|
||||
"target_modules": [
|
||||
"q_proj",
|
||||
"gate_proj",
|
||||
"down_proj",
|
||||
"v_proj",
|
||||
"k_proj",
|
||||
"o_proj",
|
||||
"up_proj"
|
||||
],
|
||||
"task_type": "CAUSAL_LM"
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:064bb620173b89e327154ada8f72dbd1eeea1d16d1ce351c2240385268dc3dac
|
||||
size 639792909
|
||||
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"epoch": 2.0,
|
||||
"eval_loss": 0.5871890783309937,
|
||||
"eval_runtime": 15.5979,
|
||||
"eval_samples_per_second": 12.822,
|
||||
"eval_steps_per_second": 6.411
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
2023-10-09 06:19:22.839 | INFO | __main__:train:918 - args=Namespace(model_name_or_path='/opt/local/llm_models/huggingface.co/llm_agents/tora-code-7b-v1.0', trust_remote_code=False, force_remove_overlength_samples=False, eval_dataset_size=0.005, max_train_samples=None, max_eval_samples=200, model_max_len=4096, dataset='/opt/local/datasets/Speechless/speechless-thoughts-200k.jsonl', dataset_format='airoboros', output_dir='./outputs/speechless-tora-code-7b-v1.0', overwrite_output_dir=False, do_train=True, do_eval=True, do_predict=False, evaluation_strategy=<IntervalStrategy.STEPS: 'steps'>, prediction_loss_only=False, per_device_train_batch_size=8, per_device_eval_batch_size=1, per_gpu_train_batch_size=None, per_gpu_eval_batch_size=None, gradient_accumulation_steps=8, eval_accumulation_steps=None, eval_delay=0, learning_rate=0.0002, weight_decay=0.0, adam_beta1=0.9, adam_beta2=0.999, adam_epsilon=1e-08, max_grad_norm=0.3, num_train_epochs=2, max_steps=-1, lr_scheduler_type=<SchedulerType.COSINE: 'cosine'>, warmup_ratio=0.005, warmup_steps=20, log_level='passive', log_level_replica='warning', log_on_each_node=True, logging_dir='./outputs/speechless-tora-code-7b-v1.0/runs/Oct09_06-19-22_I156f1f3f410070163e', logging_strategy=<IntervalStrategy.STEPS: 'steps'>, logging_first_step=False, logging_steps=1, logging_nan_inf_filter=True, save_strategy=<IntervalStrategy.STEPS: 'steps'>, save_steps=100, save_total_limit=1, save_safetensors=False, save_on_each_node=False, no_cuda=False, use_cpu=False, use_mps_device=False, seed=10042, data_seed=10042, jit_mode_eval=False, use_ipex=False, bf16=True, fp16=False, fp16_opt_level='O1', half_precision_backend='auto', bf16_full_eval=False, fp16_full_eval=False, tf32=None, local_rank=0, ddp_backend=None, tpu_num_cores=None, tpu_metrics_debug=False, debug=[], dataloader_drop_last=False, eval_steps=100, dataloader_num_workers=3, past_index=-1, run_name='20231009-061916', disable_tqdm=False, remove_unused_columns=False, label_names=None, load_best_model_at_end=False, metric_for_best_model=None, greater_is_better=None, ignore_data_skip=False, sharded_ddp=[], fsdp=[], fsdp_min_num_params=0, fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_grad_ckpt': False}, fsdp_transformer_layer_cls_to_wrap=None, deepspeed=None, label_smoothing_factor=0.0, optim=<OptimizerNames.PAGED_ADAMW_8BIT: 'paged_adamw_8bit'>, optim_args=None, adafactor=False, group_by_length=False, length_column_name='length', report_to=['tensorboard'], ddp_find_unused_parameters=False, ddp_bucket_cap_mb=None, ddp_broadcast_buffers=None, dataloader_pin_memory=True, skip_memory_metrics=True, use_legacy_prediction_loop=False, push_to_hub=False, resume_from_checkpoint=None, hub_model_id=None, hub_strategy=<HubStrategy.EVERY_SAVE: 'every_save'>, hub_token=None, hub_private_repo=False, hub_always_push=False, gradient_checkpointing=True, include_inputs_for_metrics=False, fp16_backend='auto', push_to_hub_model_id=None, push_to_hub_organization=None, push_to_hub_token=None, mp_parameters='', auto_find_batch_size=False, full_determinism=False, torchdynamo=None, ray_scope='last', ddp_timeout=1800, torch_compile=False, torch_compile_backend=None, torch_compile_mode=None, dispatch_batches=None, include_tokens_per_second=False, sortish_sampler=False, predict_with_generate=False, generation_max_length=None, generation_num_beams=None, generation_config=None, task_name='speechless-tora-code-7b-v1.0', flash_attention=True, rerope=False, rerope_window=None, wandb=None, sample_packing=False, cache_dir=None, full_finetune=False, adam8bit=False, double_quant=True, quant_type='nf4', bits=4, lora_r=64, lora_alpha=16.0, lora_dropout=0.05, max_memory_MB=70000, mpt=False, max_shard_size='5GB', repeat_steps=0, distributed_state=Distributed environment: MULTI_GPU Backend: nccl
|
||||
Num processes: 2
|
||||
Process index: 0
|
||||
Local process index: 0
|
||||
Device: cuda:0
|
||||
, _n_gpu=1, __cached__setup_devices=device(type='cuda', index=0), deepspeed_plugin=None)
|
||||
2023-10-09 06:19:22.840 | INFO | __main__:get_accelerate_model:303 - loading base model /opt/local/llm_models/huggingface.co/llm_agents/tora-code-7b-v1.0...
|
||||
2023-10-09 06:19:22.848 | INFO | __main__:train:918 - args=Namespace(model_name_or_path='/opt/local/llm_models/huggingface.co/llm_agents/tora-code-7b-v1.0', trust_remote_code=False, force_remove_overlength_samples=False, eval_dataset_size=0.005, max_train_samples=None, max_eval_samples=200, model_max_len=4096, dataset='/opt/local/datasets/Speechless/speechless-thoughts-200k.jsonl', dataset_format='airoboros', output_dir='./outputs/speechless-tora-code-7b-v1.0', overwrite_output_dir=False, do_train=True, do_eval=True, do_predict=False, evaluation_strategy=<IntervalStrategy.STEPS: 'steps'>, prediction_loss_only=False, per_device_train_batch_size=8, per_device_eval_batch_size=1, per_gpu_train_batch_size=None, per_gpu_eval_batch_size=None, gradient_accumulation_steps=8, eval_accumulation_steps=None, eval_delay=0, learning_rate=0.0002, weight_decay=0.0, adam_beta1=0.9, adam_beta2=0.999, adam_epsilon=1e-08, max_grad_norm=0.3, num_train_epochs=2, max_steps=-1, lr_scheduler_type=<SchedulerType.COSINE: 'cosine'>, warmup_ratio=0.005, warmup_steps=20, log_level='passive', log_level_replica='warning', log_on_each_node=True, logging_dir='./outputs/speechless-tora-code-7b-v1.0/runs/Oct09_06-19-22_I156f1f3f410070163e', logging_strategy=<IntervalStrategy.STEPS: 'steps'>, logging_first_step=False, logging_steps=1, logging_nan_inf_filter=True, save_strategy=<IntervalStrategy.STEPS: 'steps'>, save_steps=100, save_total_limit=1, save_safetensors=False, save_on_each_node=False, no_cuda=False, use_cpu=False, use_mps_device=False, seed=10042, data_seed=10042, jit_mode_eval=False, use_ipex=False, bf16=True, fp16=False, fp16_opt_level='O1', half_precision_backend='auto', bf16_full_eval=False, fp16_full_eval=False, tf32=None, local_rank=1, ddp_backend=None, tpu_num_cores=None, tpu_metrics_debug=False, debug=[], dataloader_drop_last=False, eval_steps=100, dataloader_num_workers=3, past_index=-1, run_name='20231009-061916', disable_tqdm=False, remove_unused_columns=False, label_names=None, load_best_model_at_end=False, metric_for_best_model=None, greater_is_better=None, ignore_data_skip=False, sharded_ddp=[], fsdp=[], fsdp_min_num_params=0, fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_grad_ckpt': False}, fsdp_transformer_layer_cls_to_wrap=None, deepspeed=None, label_smoothing_factor=0.0, optim=<OptimizerNames.PAGED_ADAMW_8BIT: 'paged_adamw_8bit'>, optim_args=None, adafactor=False, group_by_length=False, length_column_name='length', report_to=['tensorboard'], ddp_find_unused_parameters=False, ddp_bucket_cap_mb=None, ddp_broadcast_buffers=None, dataloader_pin_memory=True, skip_memory_metrics=True, use_legacy_prediction_loop=False, push_to_hub=False, resume_from_checkpoint=None, hub_model_id=None, hub_strategy=<HubStrategy.EVERY_SAVE: 'every_save'>, hub_token=None, hub_private_repo=False, hub_always_push=False, gradient_checkpointing=True, include_inputs_for_metrics=False, fp16_backend='auto', push_to_hub_model_id=None, push_to_hub_organization=None, push_to_hub_token=None, mp_parameters='', auto_find_batch_size=False, full_determinism=False, torchdynamo=None, ray_scope='last', ddp_timeout=1800, torch_compile=False, torch_compile_backend=None, torch_compile_mode=None, dispatch_batches=None, include_tokens_per_second=False, sortish_sampler=False, predict_with_generate=False, generation_max_length=None, generation_num_beams=None, generation_config=None, task_name='speechless-tora-code-7b-v1.0', flash_attention=True, rerope=False, rerope_window=None, wandb=None, sample_packing=False, cache_dir=None, full_finetune=False, adam8bit=False, double_quant=True, quant_type='nf4', bits=4, lora_r=64, lora_alpha=16.0, lora_dropout=0.05, max_memory_MB=70000, mpt=False, max_shard_size='5GB', repeat_steps=0, distributed_state=Distributed environment: MULTI_GPU Backend: nccl
|
||||
Num processes: 2
|
||||
Process index: 1
|
||||
Local process index: 1
|
||||
Device: cuda:1
|
||||
, _n_gpu=1, __cached__setup_devices=device(type='cuda', index=1), deepspeed_plugin=None)
|
||||
2023-10-09 06:19:22.849 | INFO | __main__:get_accelerate_model:303 - loading base model /opt/local/llm_models/huggingface.co/llm_agents/tora-code-7b-v1.0...
|
||||
2023-10-09 06:19:51.853 | INFO | __main__:get_accelerate_model:350 - adding LoRA modules...
|
||||
2023-10-09 06:19:53.297 | INFO | __main__:get_accelerate_model:350 - adding LoRA modules...
|
||||
2023-10-09 06:21:03.048 | INFO | __main__:train:942 - loaded model
|
||||
2023-10-09 06:21:03.150 | INFO | __main__:format_dataset:639 - ---------- Formatting dataset for Airoboros. ----------
|
||||
2023-10-09 06:21:03.160 | INFO | __main__:make_data_module:734 - ---------- Splitting dataset into train/eval ----------
|
||||
2023-10-09 06:21:03.205 | INFO | __main__:train:985 - *** Train ***
|
||||
2023-10-09 06:21:06.112 | INFO | __main__:train:942 - loaded model
|
||||
2023-10-09 06:21:06.212 | INFO | __main__:format_dataset:639 - ---------- Formatting dataset for Airoboros. ----------
|
||||
2023-10-09 06:21:06.222 | INFO | __main__:make_data_module:734 - ---------- Splitting dataset into train/eval ----------
|
||||
2023-10-09 06:21:06.265 | INFO | __main__:train:985 - *** Train ***
|
||||
@@ -0,0 +1,88 @@
|
||||
2023-10-09 12:23:12.104 | INFO | __main__:train:918 - args=Namespace(model_name_or_path='/opt/local/llm_models/huggingface.co/llm_agents/tora-code-7b-v1.0', trust_remote_code=False, force_remove_overlength_samples=False, eval_dataset_size=0.005, max_train_samples=None, max_eval_samples=200, model_max_len=4096, dataset='/opt/local/datasets/Speechless/speechless-thoughts-200k.jsonl', dataset_format='airoboros', output_dir='./outputs/speechless-tora-code-7b-v1.0', overwrite_output_dir=False, do_train=True, do_eval=True, do_predict=False, evaluation_strategy=<IntervalStrategy.STEPS: 'steps'>, prediction_loss_only=False, per_device_train_batch_size=4, per_device_eval_batch_size=1, per_gpu_train_batch_size=None, per_gpu_eval_batch_size=None, gradient_accumulation_steps=16, eval_accumulation_steps=None, eval_delay=0, learning_rate=0.0002, weight_decay=0.0, adam_beta1=0.9, adam_beta2=0.999, adam_epsilon=1e-08, max_grad_norm=0.3, num_train_epochs=2, max_steps=-1, lr_scheduler_type=<SchedulerType.COSINE: 'cosine'>, warmup_ratio=0.005, warmup_steps=20, log_level='passive', log_level_replica='warning', log_on_each_node=True, logging_dir='./outputs/speechless-tora-code-7b-v1.0/runs/Oct09_12-23-12_I156f1f3f410070163e', logging_strategy=<IntervalStrategy.STEPS: 'steps'>, logging_first_step=False, logging_steps=1, logging_nan_inf_filter=True, save_strategy=<IntervalStrategy.STEPS: 'steps'>, save_steps=100, save_total_limit=1, save_safetensors=False, save_on_each_node=False, no_cuda=False, use_cpu=False, use_mps_device=False, seed=10042, data_seed=10042, jit_mode_eval=False, use_ipex=False, bf16=True, fp16=False, fp16_opt_level='O1', half_precision_backend='auto', bf16_full_eval=False, fp16_full_eval=False, tf32=None, local_rank=1, ddp_backend=None, tpu_num_cores=None, tpu_metrics_debug=False, debug=[], dataloader_drop_last=False, eval_steps=100, dataloader_num_workers=3, past_index=-1, run_name='20231009-122305', disable_tqdm=False, remove_unused_columns=False, label_names=None, load_best_model_at_end=False, metric_for_best_model=None, greater_is_better=None, ignore_data_skip=False, sharded_ddp=[], fsdp=[], fsdp_min_num_params=0, fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_grad_ckpt': False}, fsdp_transformer_layer_cls_to_wrap=None, deepspeed=None, label_smoothing_factor=0.0, optim=<OptimizerNames.PAGED_ADAMW_8BIT: 'paged_adamw_8bit'>, optim_args=None, adafactor=False, group_by_length=False, length_column_name='length', report_to=['tensorboard'], ddp_find_unused_parameters=False, ddp_bucket_cap_mb=None, ddp_broadcast_buffers=None, dataloader_pin_memory=True, skip_memory_metrics=True, use_legacy_prediction_loop=False, push_to_hub=False, resume_from_checkpoint=None, hub_model_id=None, hub_strategy=<HubStrategy.EVERY_SAVE: 'every_save'>, hub_token=None, hub_private_repo=False, hub_always_push=False, gradient_checkpointing=True, include_inputs_for_metrics=False, fp16_backend='auto', push_to_hub_model_id=None, push_to_hub_organization=None, push_to_hub_token=None, mp_parameters='', auto_find_batch_size=False, full_determinism=False, torchdynamo=None, ray_scope='last', ddp_timeout=1800, torch_compile=False, torch_compile_backend=None, torch_compile_mode=None, dispatch_batches=None, include_tokens_per_second=False, sortish_sampler=False, predict_with_generate=False, generation_max_length=None, generation_num_beams=None, generation_config=None, task_name='speechless-tora-code-7b-v1.0', flash_attention=True, rerope=False, rerope_window=None, wandb=None, sample_packing=False, cache_dir=None, full_finetune=False, adam8bit=False, double_quant=True, quant_type='nf4', bits=4, lora_r=64, lora_alpha=16.0, lora_dropout=0.05, max_memory_MB=32000, mpt=False, max_shard_size='5GB', repeat_steps=0, distributed_state=Distributed environment: MULTI_GPU Backend: nccl
|
||||
Num processes: 2
|
||||
Process index: 1
|
||||
Local process index: 1
|
||||
Device: cuda:1
|
||||
, _n_gpu=1, __cached__setup_devices=device(type='cuda', index=1), deepspeed_plugin=None)
|
||||
2023-10-09 12:23:12.105 | INFO | __main__:get_accelerate_model:303 - loading base model /opt/local/llm_models/huggingface.co/llm_agents/tora-code-7b-v1.0...
|
||||
2023-10-09 12:23:12.105 | INFO | __main__:train:918 - args=Namespace(model_name_or_path='/opt/local/llm_models/huggingface.co/llm_agents/tora-code-7b-v1.0', trust_remote_code=False, force_remove_overlength_samples=False, eval_dataset_size=0.005, max_train_samples=None, max_eval_samples=200, model_max_len=4096, dataset='/opt/local/datasets/Speechless/speechless-thoughts-200k.jsonl', dataset_format='airoboros', output_dir='./outputs/speechless-tora-code-7b-v1.0', overwrite_output_dir=False, do_train=True, do_eval=True, do_predict=False, evaluation_strategy=<IntervalStrategy.STEPS: 'steps'>, prediction_loss_only=False, per_device_train_batch_size=4, per_device_eval_batch_size=1, per_gpu_train_batch_size=None, per_gpu_eval_batch_size=None, gradient_accumulation_steps=16, eval_accumulation_steps=None, eval_delay=0, learning_rate=0.0002, weight_decay=0.0, adam_beta1=0.9, adam_beta2=0.999, adam_epsilon=1e-08, max_grad_norm=0.3, num_train_epochs=2, max_steps=-1, lr_scheduler_type=<SchedulerType.COSINE: 'cosine'>, warmup_ratio=0.005, warmup_steps=20, log_level='passive', log_level_replica='warning', log_on_each_node=True, logging_dir='./outputs/speechless-tora-code-7b-v1.0/runs/Oct09_12-23-12_I156f1f3f410070163e', logging_strategy=<IntervalStrategy.STEPS: 'steps'>, logging_first_step=False, logging_steps=1, logging_nan_inf_filter=True, save_strategy=<IntervalStrategy.STEPS: 'steps'>, save_steps=100, save_total_limit=1, save_safetensors=False, save_on_each_node=False, no_cuda=False, use_cpu=False, use_mps_device=False, seed=10042, data_seed=10042, jit_mode_eval=False, use_ipex=False, bf16=True, fp16=False, fp16_opt_level='O1', half_precision_backend='auto', bf16_full_eval=False, fp16_full_eval=False, tf32=None, local_rank=0, ddp_backend=None, tpu_num_cores=None, tpu_metrics_debug=False, debug=[], dataloader_drop_last=False, eval_steps=100, dataloader_num_workers=3, past_index=-1, run_name='20231009-122305', disable_tqdm=False, remove_unused_columns=False, label_names=None, load_best_model_at_end=False, metric_for_best_model=None, greater_is_better=None, ignore_data_skip=False, sharded_ddp=[], fsdp=[], fsdp_min_num_params=0, fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_grad_ckpt': False}, fsdp_transformer_layer_cls_to_wrap=None, deepspeed=None, label_smoothing_factor=0.0, optim=<OptimizerNames.PAGED_ADAMW_8BIT: 'paged_adamw_8bit'>, optim_args=None, adafactor=False, group_by_length=False, length_column_name='length', report_to=['tensorboard'], ddp_find_unused_parameters=False, ddp_bucket_cap_mb=None, ddp_broadcast_buffers=None, dataloader_pin_memory=True, skip_memory_metrics=True, use_legacy_prediction_loop=False, push_to_hub=False, resume_from_checkpoint=None, hub_model_id=None, hub_strategy=<HubStrategy.EVERY_SAVE: 'every_save'>, hub_token=None, hub_private_repo=False, hub_always_push=False, gradient_checkpointing=True, include_inputs_for_metrics=False, fp16_backend='auto', push_to_hub_model_id=None, push_to_hub_organization=None, push_to_hub_token=None, mp_parameters='', auto_find_batch_size=False, full_determinism=False, torchdynamo=None, ray_scope='last', ddp_timeout=1800, torch_compile=False, torch_compile_backend=None, torch_compile_mode=None, dispatch_batches=None, include_tokens_per_second=False, sortish_sampler=False, predict_with_generate=False, generation_max_length=None, generation_num_beams=None, generation_config=None, task_name='speechless-tora-code-7b-v1.0', flash_attention=True, rerope=False, rerope_window=None, wandb=None, sample_packing=False, cache_dir=None, full_finetune=False, adam8bit=False, double_quant=True, quant_type='nf4', bits=4, lora_r=64, lora_alpha=16.0, lora_dropout=0.05, max_memory_MB=32000, mpt=False, max_shard_size='5GB', repeat_steps=0, distributed_state=Distributed environment: MULTI_GPU Backend: nccl
|
||||
Num processes: 2
|
||||
Process index: 0
|
||||
Local process index: 0
|
||||
Device: cuda:0
|
||||
, _n_gpu=1, __cached__setup_devices=device(type='cuda', index=0), deepspeed_plugin=None)
|
||||
2023-10-09 12:23:12.105 | INFO | __main__:get_accelerate_model:303 - loading base model /opt/local/llm_models/huggingface.co/llm_agents/tora-code-7b-v1.0...
|
||||
2023-10-09 12:23:40.070 | INFO | __main__:get_accelerate_model:350 - adding LoRA modules...
|
||||
2023-10-09 12:23:40.822 | INFO | __main__:get_accelerate_model:350 - adding LoRA modules...
|
||||
2023-10-09 12:24:52.367 | INFO | __main__:train:942 - loaded model
|
||||
2023-10-09 12:24:52.467 | INFO | __main__:format_dataset:639 - ---------- Formatting dataset for Airoboros. ----------
|
||||
2023-10-09 12:24:52.478 | INFO | __main__:make_data_module:734 - ---------- Splitting dataset into train/eval ----------
|
||||
2023-10-09 12:24:52.523 | INFO | __main__:train:985 - *** Train ***
|
||||
2023-10-09 12:24:53.833 | INFO | __main__:train:942 - loaded model
|
||||
2023-10-09 12:24:53.932 | INFO | __main__:format_dataset:639 - ---------- Formatting dataset for Airoboros. ----------
|
||||
2023-10-09 12:24:53.942 | INFO | __main__:make_data_module:734 - ---------- Splitting dataset into train/eval ----------
|
||||
2023-10-09 12:24:53.987 | INFO | __main__:train:985 - *** Train ***
|
||||
2023-10-09 13:02:59.195 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
|
||||
2023-10-09 13:03:00.511 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
|
||||
2023-10-09 13:40:57.489 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
|
||||
2023-10-09 13:40:59.007 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
|
||||
2023-10-09 14:18:56.508 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
|
||||
2023-10-09 14:18:57.522 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
|
||||
2023-10-09 14:56:40.535 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
|
||||
2023-10-09 14:56:41.789 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
|
||||
2023-10-09 15:34:06.992 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
|
||||
2023-10-09 15:34:08.286 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
|
||||
2023-10-09 16:12:02.503 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
|
||||
2023-10-09 16:12:03.732 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
|
||||
2023-10-09 16:49:21.174 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
|
||||
2023-10-09 16:49:22.504 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
|
||||
2023-10-09 17:26:54.676 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
|
||||
2023-10-09 17:26:55.668 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
|
||||
2023-10-09 18:04:46.740 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
|
||||
2023-10-09 18:04:47.949 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
|
||||
2023-10-09 18:42:04.644 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
|
||||
2023-10-09 18:42:05.914 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
|
||||
2023-10-09 19:19:37.187 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
|
||||
2023-10-09 19:19:38.502 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
|
||||
2023-10-09 19:56:57.213 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
|
||||
2023-10-09 19:56:58.406 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
|
||||
2023-10-09 20:34:41.712 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
|
||||
2023-10-09 20:34:43.045 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
|
||||
2023-10-09 21:12:11.382 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
|
||||
2023-10-09 21:12:12.669 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
|
||||
2023-10-09 21:49:45.432 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
|
||||
2023-10-09 21:49:46.721 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
|
||||
2023-10-09 22:27:13.362 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
|
||||
2023-10-09 22:27:14.775 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
|
||||
2023-10-09 23:04:40.350 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
|
||||
2023-10-09 23:04:41.734 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
|
||||
2023-10-09 23:43:00.849 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
|
||||
2023-10-09 23:43:01.897 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
|
||||
2023-10-10 00:20:56.083 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
|
||||
2023-10-10 00:20:57.315 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
|
||||
2023-10-10 00:58:45.672 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
|
||||
2023-10-10 00:58:46.917 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
|
||||
2023-10-10 01:36:16.111 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
|
||||
2023-10-10 01:36:17.107 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
|
||||
2023-10-10 02:14:04.076 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
|
||||
2023-10-10 02:14:05.116 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
|
||||
2023-10-10 02:51:35.732 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
|
||||
2023-10-10 02:51:37.116 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
|
||||
2023-10-10 03:29:10.549 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
|
||||
2023-10-10 03:29:11.807 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
|
||||
2023-10-10 04:07:10.597 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
|
||||
2023-10-10 04:07:12.121 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
|
||||
2023-10-10 04:45:13.080 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
|
||||
2023-10-10 04:45:14.388 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
|
||||
2023-10-10 05:22:29.236 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
|
||||
2023-10-10 05:22:30.313 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
|
||||
2023-10-10 06:00:01.444 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
|
||||
2023-10-10 06:00:02.697 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
|
||||
2023-10-10 06:37:56.658 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
|
||||
2023-10-10 06:37:57.703 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
|
||||
2023-10-10 07:15:19.303 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
|
||||
2023-10-10 07:15:20.460 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
|
||||
2023-10-10 07:49:44.958 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
|
||||
2023-10-10 07:49:44.959 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
|
||||
2023-10-10 07:49:45.497 | INFO | __main__:train:996 - *** Evaluate ***
|
||||
2023-10-10 07:49:45.564 | INFO | __main__:train:996 - *** Evaluate ***
|
||||
@@ -0,0 +1 @@
|
||||
{"run_name": "20231009-122305", "train_runtime": 69889.4305, "train_samples_per_second": 5.664, "train_steps_per_second": 0.044, "train_loss": 0.5890884618112218, "epoch": 2.0, "eval_loss": 0.5871890783309937, "eval_runtime": 15.5979, "eval_samples_per_second": 12.822, "eval_steps_per_second": 6.411}
|
||||
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:b2462cbdd451f1620c45fdc00446bef4939780801d8fc5937b775dd515dee8c8
|
||||
size 7869
|
||||
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:2ed40796c7e6086dc6fed30704e34ee1f93696be1510cda6f81cd0b162e7b03c
|
||||
size 498794
|
||||
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:be1c5672e1dd6fd08655f555c268b13123ff736c6281a9bbef2fc2dd916bb156
|
||||
size 359
|
||||
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"epoch": 2.0,
|
||||
"train_loss": 0.5890884618112218,
|
||||
"train_runtime": 69889.4305,
|
||||
"train_samples_per_second": 5.664,
|
||||
"train_steps_per_second": 0.044
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
11
tasks/speechless-tora-code-7b-v1.0/run_api_server.sh
Executable file
11
tasks/speechless-tora-code-7b-v1.0/run_api_server.sh
Executable file
@@ -0,0 +1,11 @@
|
||||
#!/usr/bin/env bash
|
||||
# From speechless-code-mistral-7b-v1.0
|
||||
SCRIPT_PATH=$(cd $(dirname ${BASH_SOURCE[0]}); pwd)
|
||||
PARENT_PATH=$(cd "${SCRIPT_PATH}/.." ; pwd)
|
||||
|
||||
source ${SCRIPT_PATH}/task.env
|
||||
|
||||
PYTHONPATH=${SCRIPT_PATH}/../../.. \
|
||||
python ../../api/server.py \
|
||||
--model_name_or_path=${TEST_MODEL_PATH} \
|
||||
--model_family vllm
|
||||
59
tasks/speechless-tora-code-7b-v1.0/run_finetune.sh
Executable file
59
tasks/speechless-tora-code-7b-v1.0/run_finetune.sh
Executable file
@@ -0,0 +1,59 @@
|
||||
#!/bin/bash
|
||||
# From speechless-code-mistral-7b-v1.0
|
||||
SCRIPT_PATH=$(cd $(dirname ${BASH_SOURCE[0]}); pwd)
|
||||
PARENT_PATH=$(cd "${SCRIPT_PATH}/.." ; pwd)
|
||||
|
||||
source ${SCRIPT_PATH}/task.env
|
||||
|
||||
PYTHONPATH=${PWD}/../.. \
|
||||
torchrun --nnodes=1 --nproc_per_node=${NUM_GPUS} \
|
||||
../../finetune.py \
|
||||
${DEEPSPEED_STAGE2} \
|
||||
--task_name ${TASK_NAME} \
|
||||
--run_name $(date +%Y%m%d-%H%M%S) \
|
||||
--model_name_or_path ${BASE_MODEL_PATH} \
|
||||
--output_dir ${OUTPUT_DIR} \
|
||||
--num_train_epochs ${NUM_TRAIN_EPOCHS} \
|
||||
--data_seed 10042 \
|
||||
--save_strategy steps \
|
||||
--save_total_limit 1 \
|
||||
--evaluation_strategy steps \
|
||||
--eval_dataset_size ${EVAL_DATASET_SIZE} \
|
||||
--save_steps 100 \
|
||||
--eval_steps 100 \
|
||||
--warmup_steps 20 \
|
||||
--max_eval_samples 200 \
|
||||
--dataloader_num_workers 3 \
|
||||
--logging_strategy steps \
|
||||
--logging_steps 1 \
|
||||
--report_to tensorboard \
|
||||
--remove_unused_columns False \
|
||||
--do_train \
|
||||
--max_memory_MB ${MAX_MEMORY_MB} \
|
||||
--bits 4 \
|
||||
--lora_r ${LORA_R} \
|
||||
--lora_alpha 16 \
|
||||
--lora_dropout 0.05 \
|
||||
--lora_modules all \
|
||||
--double_quant \
|
||||
--quant_type nf4 \
|
||||
--bf16 \
|
||||
--dataset ${DATASET} \
|
||||
--dataset_format ${DATASET_FORMAT} \
|
||||
--max_new_tokens 4096 \
|
||||
--model_max_len 4096 \
|
||||
--per_device_train_batch_size ${PER_DEVICE_TRAIN_BATCH_SIZE} \
|
||||
--gradient_accumulation_steps ${GRADIENT_ACCUMULATION_STEPS} \
|
||||
--per_device_eval_batch_size 1 \
|
||||
--learning_rate ${LEARNING_RATE} \
|
||||
--lr_scheduler_type cosine \
|
||||
--weight_decay 0.0 \
|
||||
--seed 10042 \
|
||||
--optim paged_adamw_8bit \
|
||||
--gradient_checkpointing True \
|
||||
--group_by_length ${GROUP_BY_LENGTH} \
|
||||
--ddp_find_unused_parameters False \
|
||||
--force_remove_overlength_samples False \
|
||||
--flash_attention True \
|
||||
--rerope False \
|
||||
--repeat_steps 0
|
||||
47
tasks/speechless-tora-code-7b-v1.0/task.env
Normal file
47
tasks/speechless-tora-code-7b-v1.0/task.env
Normal file
@@ -0,0 +1,47 @@
|
||||
# From speechless-code-mistral-7b-v1.0
|
||||
# -------------------- Model --------------------
|
||||
export MODELS_ROOT_DIR=/opt/local/llm_models/huggingface.co
|
||||
# FIXME
|
||||
export BASE_MODEL_PATH=${MODELS_ROOT_DIR}/llm_agents/tora-code-7b-v1.0
|
||||
# FIXME
|
||||
export TEST_MODEL_PATH=${MODELS_ROOT_DIR}/speechlessai/speechless-tora-code-7b-v1.0
|
||||
# export TEST_MODEL_PATH=${MODELS_ROOT_DIR}/speechlessai/speechless-nl2sql-mistral-7b-v0.1
|
||||
# export TEST_MODEL_PATH=${MODELS_ROOT_DIR}/speechlessai/speechless-mistral-7b-v0.1
|
||||
|
||||
# -------------------- Dataset --------------------
|
||||
# FIXME
|
||||
# export DATASET=/opt/local/datasets/jondurbin/airoboros-2.2/instructions-clean.jsonl
|
||||
# export DATASET=/opt/local/datasets/Speechless/airoboros-orca-platypus-instructions.jsonl
|
||||
# export DATASET=/opt/local/datasets/Speechless/speechless-spider.jsonl
|
||||
export DATASET=/opt/local/datasets/Speechless/speechless-thoughts-200k.jsonl
|
||||
|
||||
# -------------------- Environment --------------------
|
||||
export OUTPUT_DIR=./outputs
|
||||
# export TORCH_DISTRIBUTED_DEBUG=DETAIL
|
||||
export RAY_memory_monitor_refresh_ms=0
|
||||
|
||||
# -------------------- Task --------------------
|
||||
# FIXME
|
||||
export TASK_NAME=$(basename ${TEST_MODEL_PATH})
|
||||
export TASK_CHECKPOINT_DIR=${OUTPUT_DIR}/${TASK_NAME}
|
||||
export WANDB_PROJECT=${TASK_NAME}
|
||||
|
||||
# -------------------- Task --------------------
|
||||
export DATASET_FORMAT=airoboros
|
||||
export NUM_TRAIN_EPOCHS=2
|
||||
export EVAL_DATASET_SIZE=0.005
|
||||
export LEARNING_RATE=2e-4
|
||||
export LORA_R=64
|
||||
export GROUP_BY_LENGTH=False
|
||||
|
||||
#export DEEPSPEED_STAGE2="--deepspeed deepspeed-stage2.json"
|
||||
|
||||
export NUM_GPUS=2
|
||||
# 2 gpus: 40GB: 2 x 32, 80GB: 4 x 16
|
||||
# 4 gpus: 40GB: 2 x 16, 80GB: 4 x 8
|
||||
export PER_DEVICE_TRAIN_BATCH_SIZE=4
|
||||
export GRADIENT_ACCUMULATION_STEPS=16
|
||||
|
||||
# No more than 85% VRAM.
|
||||
# A100(40GB) 34000, A40(48GB) 40000, A100(80GB) 70000
|
||||
export MAX_MEMORY_MB=32000
|
||||
49
tasks/speechless-tora-code-7b-v1.0/task_a100_40gx2.env
Normal file
49
tasks/speechless-tora-code-7b-v1.0/task_a100_40gx2.env
Normal file
@@ -0,0 +1,49 @@
|
||||
# From speechless-code-mistral-orca-7b-v1.0
|
||||
# -------------------- Model --------------------
|
||||
export MODELS_ROOT_DIR=/opt/local/llm_models/huggingface.co
|
||||
# FIXME
|
||||
export BASE_MODEL_PATH=${MODELS_ROOT_DIR}/Open-Orca/Mistral-7B-OpenOrca
|
||||
# export BASE_MODEL_PATH=${MODELS_ROOT_DIR}/mistralai/Mistral-7B-v0.1
|
||||
# FIXME
|
||||
export TEST_MODEL_PATH=${MODELS_ROOT_DIR}/speechlessai/speechless-code-mistral-orca-7b-v1.0
|
||||
# export TEST_MODEL_PATH=${MODELS_ROOT_DIR}/speechlessai/speechless-code-mistral-7b-v1.0
|
||||
# export TEST_MODEL_PATH=${MODELS_ROOT_DIR}/speechlessai/speechless-nl2sql-mistral-7b-v0.1
|
||||
# export TEST_MODEL_PATH=${MODELS_ROOT_DIR}/speechlessai/speechless-mistral-7b-v0.1
|
||||
|
||||
# -------------------- Dataset --------------------
|
||||
# FIXME
|
||||
# export DATASET=/opt/local/datasets/jondurbin/airoboros-2.2/instructions-clean.jsonl
|
||||
# export DATASET=/opt/local/datasets/Speechless/airoboros-orca-platypus-instructions.jsonl
|
||||
# export DATASET=/opt/local/datasets/Speechless/speechless-spider.jsonl
|
||||
export DATASET=/opt/local/datasets/Speechless/speechless-thoughts-200k.jsonl
|
||||
|
||||
# -------------------- Environment --------------------
|
||||
export OUTPUT_DIR=./outputs
|
||||
export TORCH_DISTRIBUTED_DEBUG=DETAIL
|
||||
export RAY_memory_monitor_refresh_ms=0
|
||||
|
||||
# -------------------- Task --------------------
|
||||
# FIXME
|
||||
export TASK_NAME=$(basename ${TEST_MODEL_PATH})
|
||||
export TASK_CHECKPOINT_DIR=${OUTPUT_DIR}/${TASK_NAME}
|
||||
export WANDB_PROJECT=${TASK_NAME}
|
||||
|
||||
# -------------------- Task --------------------
|
||||
export DATASET_FORMAT=airoboros
|
||||
export NUM_TRAIN_EPOCHS=2
|
||||
export EVAL_DATASET_SIZE=0.005
|
||||
export LEARNING_RATE=2e-4
|
||||
export LORA_R=64
|
||||
export GROUP_BY_LENGTH=False
|
||||
|
||||
#export DEEPSPEED_STAGE2="--deepspeed deepspeed-stage2.json"
|
||||
|
||||
export NUM_GPUS=2
|
||||
# 2 gpus: 40GB: 2 x 32, 80GB: 4 x 16
|
||||
# 4 gpus: 40GB: 2 x 16, 80GB: 4 x 8
|
||||
export PER_DEVICE_TRAIN_BATCH_SIZE=2
|
||||
export GRADIENT_ACCUMULATION_STEPS=32
|
||||
|
||||
# No more than 85% VRAM.
|
||||
# A100(40GB) 34000, A40(48GB) 40000, A100(80GB) 70000
|
||||
export MAX_MEMORY_MB=34000
|
||||
49
tasks/speechless-tora-code-7b-v1.0/task_a100_40gx4.env
Normal file
49
tasks/speechless-tora-code-7b-v1.0/task_a100_40gx4.env
Normal file
@@ -0,0 +1,49 @@
|
||||
# From speechless-code-mistral-orca-7b-v1.0
|
||||
# -------------------- Model --------------------
|
||||
export MODELS_ROOT_DIR=/opt/local/llm_models/huggingface.co
|
||||
# FIXME
|
||||
export BASE_MODEL_PATH=${MODELS_ROOT_DIR}/Open-Orca/Mistral-7B-OpenOrca
|
||||
# export BASE_MODEL_PATH=${MODELS_ROOT_DIR}/mistralai/Mistral-7B-v0.1
|
||||
# FIXME
|
||||
export TEST_MODEL_PATH=${MODELS_ROOT_DIR}/speechlessai/speechless-code-mistral-orca-7b-v1.0
|
||||
# export TEST_MODEL_PATH=${MODELS_ROOT_DIR}/speechlessai/speechless-code-mistral-7b-v1.0
|
||||
# export TEST_MODEL_PATH=${MODELS_ROOT_DIR}/speechlessai/speechless-nl2sql-mistral-7b-v0.1
|
||||
# export TEST_MODEL_PATH=${MODELS_ROOT_DIR}/speechlessai/speechless-mistral-7b-v0.1
|
||||
|
||||
# -------------------- Dataset --------------------
|
||||
# FIXME
|
||||
# export DATASET=/opt/local/datasets/jondurbin/airoboros-2.2/instructions-clean.jsonl
|
||||
# export DATASET=/opt/local/datasets/Speechless/airoboros-orca-platypus-instructions.jsonl
|
||||
# export DATASET=/opt/local/datasets/Speechless/speechless-spider.jsonl
|
||||
export DATASET=/opt/local/datasets/Speechless/speechless-thoughts-200k.jsonl
|
||||
|
||||
# -------------------- Environment --------------------
|
||||
export OUTPUT_DIR=./outputs
|
||||
export TORCH_DISTRIBUTED_DEBUG=DETAIL
|
||||
export RAY_memory_monitor_refresh_ms=0
|
||||
|
||||
# -------------------- Task --------------------
|
||||
# FIXME
|
||||
export TASK_NAME=$(basename ${TEST_MODEL_PATH})
|
||||
export TASK_CHECKPOINT_DIR=${OUTPUT_DIR}/${TASK_NAME}
|
||||
export WANDB_PROJECT=${TASK_NAME}
|
||||
|
||||
# -------------------- Task --------------------
|
||||
export DATASET_FORMAT=airoboros
|
||||
export NUM_TRAIN_EPOCHS=2
|
||||
export EVAL_DATASET_SIZE=0.005
|
||||
export LEARNING_RATE=2e-4
|
||||
export LORA_R=64
|
||||
export GROUP_BY_LENGTH=False
|
||||
|
||||
#export DEEPSPEED_STAGE2="--deepspeed deepspeed-stage2.json"
|
||||
|
||||
export NUM_GPUS=4
|
||||
# 2 gpus: 40GB: 2 x 32, 80GB: 4 x 16
|
||||
# 4 gpus: 40GB: 2 x 16, 80GB: 4 x 8
|
||||
export PER_DEVICE_TRAIN_BATCH_SIZE=2
|
||||
export GRADIENT_ACCUMULATION_STEPS=16
|
||||
|
||||
# No more than 85% VRAM.
|
||||
# A100(40GB) 34000, A40(48GB) 40000, A100(80GB) 70000
|
||||
export MAX_MEMORY_MB=34000
|
||||
47
tasks/speechless-tora-code-7b-v1.0/task_a100_80gx2.env
Normal file
47
tasks/speechless-tora-code-7b-v1.0/task_a100_80gx2.env
Normal file
@@ -0,0 +1,47 @@
|
||||
# From speechless-code-mistral-7b-v1.0
|
||||
# -------------------- Model --------------------
|
||||
export MODELS_ROOT_DIR=/opt/local/llm_models/huggingface.co
|
||||
# FIXME
|
||||
export BASE_MODEL_PATH=${MODELS_ROOT_DIR}/mistralai/Mistral-7B-v0.1
|
||||
# FIXME
|
||||
export TEST_MODEL_PATH=${MODELS_ROOT_DIR}/speechlessai/speechless-code-mistral-7b-v1.0
|
||||
# export TEST_MODEL_PATH=${MODELS_ROOT_DIR}/speechlessai/speechless-nl2sql-mistral-7b-v0.1
|
||||
# export TEST_MODEL_PATH=${MODELS_ROOT_DIR}/speechlessai/speechless-mistral-7b-v0.1
|
||||
|
||||
# -------------------- Dataset --------------------
|
||||
# FIXME
|
||||
# export DATASET=/opt/local/datasets/jondurbin/airoboros-2.2/instructions-clean.jsonl
|
||||
# export DATASET=/opt/local/datasets/Speechless/airoboros-orca-platypus-instructions.jsonl
|
||||
# export DATASET=/opt/local/datasets/Speechless/speechless-spider.jsonl
|
||||
export DATASET=/opt/local/datasets/Speechless/speechless-thoughts-200k.jsonl
|
||||
|
||||
# -------------------- Environment --------------------
|
||||
export OUTPUT_DIR=./outputs
|
||||
export TORCH_DISTRIBUTED_DEBUG=DETAIL
|
||||
export RAY_memory_monitor_refresh_ms=0
|
||||
|
||||
# -------------------- Task --------------------
|
||||
# FIXME
|
||||
export TASK_NAME=$(basename ${TEST_MODEL_PATH})
|
||||
export TASK_CHECKPOINT_DIR=${OUTPUT_DIR}/${TASK_NAME}
|
||||
export WANDB_PROJECT=${TASK_NAME}
|
||||
|
||||
# -------------------- Task --------------------
|
||||
export DATASET_FORMAT=airoboros
|
||||
export NUM_TRAIN_EPOCHS=2
|
||||
export EVAL_DATASET_SIZE=0.005
|
||||
export LEARNING_RATE=2e-4
|
||||
export LORA_R=64
|
||||
export GROUP_BY_LENGTH=False
|
||||
|
||||
#export DEEPSPEED_STAGE2="--deepspeed deepspeed-stage2.json"
|
||||
|
||||
export NUM_GPUS=2
|
||||
# 2 gpus: 40GB: 2 x 32, 80GB: 4 x 16
|
||||
# 4 gpus: 40GB: 2 x 16, 80GB: 4 x 8
|
||||
export PER_DEVICE_TRAIN_BATCH_SIZE=2
|
||||
export GRADIENT_ACCUMULATION_STEPS=32
|
||||
|
||||
# No more than 85% VRAM.
|
||||
# A100(40GB) 34000, A40(48GB) 40000, A100(80GB) 70000
|
||||
export MAX_MEMORY_MB=70000
|
||||
47
tasks/speechless-tora-code-7b-v1.0/task_a40_48gx2.env
Normal file
47
tasks/speechless-tora-code-7b-v1.0/task_a40_48gx2.env
Normal file
@@ -0,0 +1,47 @@
|
||||
# From speechless-code-mistral-7b-v1.0
|
||||
# -------------------- Model --------------------
|
||||
export MODELS_ROOT_DIR=/opt/local/llm_models/huggingface.co
|
||||
# FIXME
|
||||
export BASE_MODEL_PATH=${MODELS_ROOT_DIR}/mistralai/Mistral-7B-v0.1
|
||||
# FIXME
|
||||
export TEST_MODEL_PATH=${MODELS_ROOT_DIR}/speechlessai/speechless-code-mistral-7b-v1.0
|
||||
# export TEST_MODEL_PATH=${MODELS_ROOT_DIR}/speechlessai/speechless-nl2sql-mistral-7b-v0.1
|
||||
# export TEST_MODEL_PATH=${MODELS_ROOT_DIR}/speechlessai/speechless-mistral-7b-v0.1
|
||||
|
||||
# -------------------- Dataset --------------------
|
||||
# FIXME
|
||||
# export DATASET=/opt/local/datasets/jondurbin/airoboros-2.2/instructions-clean.jsonl
|
||||
# export DATASET=/opt/local/datasets/Speechless/airoboros-orca-platypus-instructions.jsonl
|
||||
# export DATASET=/opt/local/datasets/Speechless/speechless-spider.jsonl
|
||||
export DATASET=/opt/local/datasets/Speechless/speechless-thoughts-200k.jsonl
|
||||
|
||||
# -------------------- Environment --------------------
|
||||
export OUTPUT_DIR=./outputs
|
||||
export TORCH_DISTRIBUTED_DEBUG=DETAIL
|
||||
export RAY_memory_monitor_refresh_ms=0
|
||||
|
||||
# -------------------- Task --------------------
|
||||
# FIXME
|
||||
export TASK_NAME=$(basename ${TEST_MODEL_PATH})
|
||||
export TASK_CHECKPOINT_DIR=${OUTPUT_DIR}/${TASK_NAME}
|
||||
export WANDB_PROJECT=${TASK_NAME}
|
||||
|
||||
# -------------------- Task --------------------
|
||||
export DATASET_FORMAT=airoboros
|
||||
export NUM_TRAIN_EPOCHS=2
|
||||
export EVAL_DATASET_SIZE=0.005
|
||||
export LEARNING_RATE=2e-4
|
||||
export LORA_R=64
|
||||
export GROUP_BY_LENGTH=False
|
||||
|
||||
#export DEEPSPEED_STAGE2="--deepspeed deepspeed-stage2.json"
|
||||
|
||||
export NUM_GPUS=2
|
||||
# 2 gpus: 40GB: 2 x 32, 80GB: 4 x 16
|
||||
# 4 gpus: 40GB: 2 x 16, 80GB: 4 x 8
|
||||
export PER_DEVICE_TRAIN_BATCH_SIZE=2
|
||||
export GRADIENT_ACCUMULATION_STEPS=32
|
||||
|
||||
# No more than 85% VRAM.
|
||||
# A100(40GB) 34000, A40(48GB) 40000, A100(80GB) 70000
|
||||
export MAX_MEMORY_MB=40000
|
||||
47
tasks/speechless-tora-code-7b-v1.0/task_a40_48gx4.env
Normal file
47
tasks/speechless-tora-code-7b-v1.0/task_a40_48gx4.env
Normal file
@@ -0,0 +1,47 @@
|
||||
# From speechless-code-mistral-7b-v1.0
|
||||
# -------------------- Model --------------------
|
||||
export MODELS_ROOT_DIR=/opt/local/llm_models/huggingface.co
|
||||
# FIXME
|
||||
export BASE_MODEL_PATH=${MODELS_ROOT_DIR}/mistralai/Mistral-7B-v0.1
|
||||
# FIXME
|
||||
export TEST_MODEL_PATH=${MODELS_ROOT_DIR}/speechlessai/speechless-code-mistral-7b-v1.0
|
||||
# export TEST_MODEL_PATH=${MODELS_ROOT_DIR}/speechlessai/speechless-nl2sql-mistral-7b-v0.1
|
||||
# export TEST_MODEL_PATH=${MODELS_ROOT_DIR}/speechlessai/speechless-mistral-7b-v0.1
|
||||
|
||||
# -------------------- Dataset --------------------
|
||||
# FIXME
|
||||
# export DATASET=/opt/local/datasets/jondurbin/airoboros-2.2/instructions-clean.jsonl
|
||||
# export DATASET=/opt/local/datasets/Speechless/airoboros-orca-platypus-instructions.jsonl
|
||||
# export DATASET=/opt/local/datasets/Speechless/speechless-spider.jsonl
|
||||
export DATASET=/opt/local/datasets/Speechless/speechless-thoughts-200k.jsonl
|
||||
|
||||
# -------------------- Environment --------------------
|
||||
export OUTPUT_DIR=./outputs
|
||||
export TORCH_DISTRIBUTED_DEBUG=DETAIL
|
||||
export RAY_memory_monitor_refresh_ms=0
|
||||
|
||||
# -------------------- Task --------------------
|
||||
# FIXME
|
||||
export TASK_NAME=$(basename ${TEST_MODEL_PATH})
|
||||
export TASK_CHECKPOINT_DIR=${OUTPUT_DIR}/${TASK_NAME}
|
||||
export WANDB_PROJECT=${TASK_NAME}
|
||||
|
||||
# -------------------- Task --------------------
|
||||
export DATASET_FORMAT=airoboros
|
||||
export NUM_TRAIN_EPOCHS=2
|
||||
export EVAL_DATASET_SIZE=0.005
|
||||
export LEARNING_RATE=2e-4
|
||||
export LORA_R=64
|
||||
export GROUP_BY_LENGTH=False
|
||||
|
||||
#export DEEPSPEED_STAGE2="--deepspeed deepspeed-stage2.json"
|
||||
|
||||
export NUM_GPUS=4
|
||||
# 2 gpus: 40GB: 2 x 32, 80GB: 4 x 16
|
||||
# 4 gpus: 40GB: 2 x 16, 80GB: 4 x 8
|
||||
export PER_DEVICE_TRAIN_BATCH_SIZE=2
|
||||
export GRADIENT_ACCUMULATION_STEPS=16
|
||||
|
||||
# No more than 85% VRAM.
|
||||
# A100(40GB) 34000, A40(48GB) 40000, A100(80GB) 70000
|
||||
export MAX_MEMORY_MB=40000
|
||||
93400
tokenizer.json
Normal file
93400
tokenizer.json
Normal file
File diff suppressed because it is too large
Load Diff
3
tokenizer.model
Normal file
3
tokenizer.model
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
|
||||
size 499723
|
||||
50
tokenizer_config.json
Normal file
50
tokenizer_config.json
Normal file
@@ -0,0 +1,50 @@
|
||||
{
|
||||
"add_bos_token": true,
|
||||
"add_eos_token": false,
|
||||
"added_tokens_decoder": {
|
||||
"0": {
|
||||
"content": "<unk>",
|
||||
"lstrip": false,
|
||||
"normalized": true,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"1": {
|
||||
"content": "<s>",
|
||||
"lstrip": false,
|
||||
"normalized": true,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"2": {
|
||||
"content": "</s>",
|
||||
"lstrip": false,
|
||||
"normalized": true,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"32000": {
|
||||
"content": "<pad>",
|
||||
"lstrip": false,
|
||||
"normalized": true,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
}
|
||||
},
|
||||
"additional_special_tokens": [],
|
||||
"bos_token": "<s>",
|
||||
"clean_up_tokenization_spaces": false,
|
||||
"eos_token": "</s>",
|
||||
"legacy": true,
|
||||
"model_max_length": 1000000000000000019884624838656,
|
||||
"pad_token": "<pad>",
|
||||
"sp_model_kwargs": {},
|
||||
"spaces_between_special_tokens": false,
|
||||
"tokenizer_class": "LlamaTokenizer",
|
||||
"unk_token": "<unk>",
|
||||
"use_default_system_prompt": true
|
||||
}
|
||||
Reference in New Issue
Block a user