uukuguy/speechless-tora-code-7b-v1.0
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

初始化项目，由ModelHub XC社区提供模型

Browse Source 
Model: uukuguy/speechless-tora-code-7b-v1.0
Source: Original Platform
This commit is contained in:
ModelHub XC
2026-05-31 22:36:09 +08:00
commit ce79999c49
51 changed files with 131839 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
tasks/speechless-tora-code-7b-v1.0/merge_peft_adapters.sh Executable file
View File

@@ -0,0 +1,13 @@
#!/usr/bin/env bash
# From speechless-code-mistral-7b-v1.0
SCRIPT_PATH=$(cd $(dirname ${BASH_SOURCE[0]}); pwd)
PARENT_PATH=$(cd "${SCRIPT_PATH}/.." ; pwd)
source ${SCRIPT_PATH}/task.env
CHECKPOINT_DIR=${TASK_CHECKPOINT_DIR}/checkpoint-3092/adapter_model
python ${SCRIPT_PATH}/../../scripts/merge_peft_adapters.py \
--base_model_name_or_path ${BASE_MODEL_PATH} \
--peft_model_path ${CHECKPOINT_DIR} \
--merged_model_name_or_path ${TEST_MODEL_PATH} \

11
tasks/speechless-tora-code-7b-v1.0/outputs/speechless-tora-code-7b-v1.0/all_results.json Normal file
View File

@@ -0,0 +1,11 @@
{
"epoch": 2.0,
"eval_loss": 0.5871890783309937,
"eval_runtime": 15.5979,
"eval_samples_per_second": 12.822,
"eval_steps_per_second": 6.411,
"train_loss": 0.5890884618112218,
"train_runtime": 69889.4305,
"train_samples_per_second": 5.664,
"train_steps_per_second": 0.044
}

21
tasks/speechless-tora-code-7b-v1.0/outputs/speechless-tora-code-7b-v1.0/checkpoint-3000/README.md Normal file
View File

@@ -0,0 +1,21 @@
---
library_name: peft
---
## Training procedure
The following `bitsandbytes` quantization config was used during training:
- quant_method: bitsandbytes
- load_in_8bit: False
- load_in_4bit: True
- llm_int8_threshold: 6.0
- llm_int8_skip_modules: None
- llm_int8_enable_fp32_cpu_offload: False
- llm_int8_has_fp16_weight: False
- bnb_4bit_quant_type: nf4
- bnb_4bit_use_double_quant: True
- bnb_4bit_compute_dtype: bfloat16
### Framework versions
- PEFT 0.5.0

26
tasks/speechless-tora-code-7b-v1.0/outputs/speechless-tora-code-7b-v1.0/checkpoint-3000/adapter_config.json Normal file
View File

@@ -0,0 +1,26 @@
{
"auto_mapping": null,
"base_model_name_or_path": "/opt/local/llm_models/huggingface.co/llm_agents/tora-code-7b-v1.0",
"bias": "none",
"fan_in_fan_out": false,
"inference_mode": true,
"init_lora_weights": true,
"layers_pattern": null,
"layers_to_transform": null,
"lora_alpha": 16.0,
"lora_dropout": 0.05,
"modules_to_save": null,
"peft_type": "LORA",
"r": 64,
"revision": null,
"target_modules": [
"down_proj",
"up_proj",
"q_proj",
"o_proj",
"gate_proj",
"v_proj",
"k_proj"
],
"task_type": "CAUSAL_LM"
}

3
tasks/speechless-tora-code-7b-v1.0/outputs/speechless-tora-code-7b-v1.0/checkpoint-3000/adapter_model.bin Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:fe53a63399829bb762760aac51f487a74255439a8ba937191ccecf3a72040093
size 639792909

34
tasks/speechless-tora-code-7b-v1.0/outputs/speechless-tora-code-7b-v1.0/checkpoint-3000/adapter_model/README.md Normal file
View File

@@ -0,0 +1,34 @@
---
library_name: peft
---
## Training procedure
The following `bitsandbytes` quantization config was used during training:
- quant_method: bitsandbytes
- load_in_8bit: False
- load_in_4bit: True
- llm_int8_threshold: 6.0
- llm_int8_skip_modules: None
- llm_int8_enable_fp32_cpu_offload: False
- llm_int8_has_fp16_weight: False
- bnb_4bit_quant_type: nf4
- bnb_4bit_use_double_quant: True
- bnb_4bit_compute_dtype: bfloat16
The following `bitsandbytes` quantization config was used during training:
- quant_method: bitsandbytes
- load_in_8bit: False
- load_in_4bit: True
- llm_int8_threshold: 6.0
- llm_int8_skip_modules: None
- llm_int8_enable_fp32_cpu_offload: False
- llm_int8_has_fp16_weight: False
- bnb_4bit_quant_type: nf4
- bnb_4bit_use_double_quant: True
- bnb_4bit_compute_dtype: bfloat16
### Framework versions
- PEFT 0.5.0
- PEFT 0.5.0

26
tasks/speechless-tora-code-7b-v1.0/outputs/speechless-tora-code-7b-v1.0/checkpoint-3000/adapter_model/adapter_config.json Normal file
View File

@@ -0,0 +1,26 @@
{
"auto_mapping": null,
"base_model_name_or_path": "/opt/local/llm_models/huggingface.co/llm_agents/tora-code-7b-v1.0",
"bias": "none",
"fan_in_fan_out": false,
"inference_mode": true,
"init_lora_weights": true,
"layers_pattern": null,
"layers_to_transform": null,
"lora_alpha": 16.0,
"lora_dropout": 0.05,
"modules_to_save": null,
"peft_type": "LORA",
"r": 64,
"revision": null,
"target_modules": [
"down_proj",
"up_proj",
"q_proj",
"o_proj",
"gate_proj",
"v_proj",
"k_proj"
],
"task_type": "CAUSAL_LM"
}

3
tasks/speechless-tora-code-7b-v1.0/outputs/speechless-tora-code-7b-v1.0/checkpoint-3000/adapter_model/adapter_model.bin Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:fe53a63399829bb762760aac51f487a74255439a8ba937191ccecf3a72040093
size 639792909

6
tasks/speechless-tora-code-7b-v1.0/outputs/speechless-tora-code-7b-v1.0/checkpoint-3000/added_tokens.json Normal file
View File

@@ -0,0 +1,6 @@
{
"</s>": 2,
"<pad>": 32000,
"<s>": 1,
"<unk>": 0
}

3
tasks/speechless-tora-code-7b-v1.0/outputs/speechless-tora-code-7b-v1.0/checkpoint-3000/optimizer.pt Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:ded5b21ac4123d95f741097b6ba2f10e14abf2fea6dc83011548234db36631c8
size 320970527

3
tasks/speechless-tora-code-7b-v1.0/outputs/speechless-tora-code-7b-v1.0/checkpoint-3000/rng_state_0.pth Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:b465ff783be6837c8cdf3aa9aa36366d7695d1fdb6b4888ec0c8a7238379af77
size 15735

3
tasks/speechless-tora-code-7b-v1.0/outputs/speechless-tora-code-7b-v1.0/checkpoint-3000/rng_state_1.pth Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:f79c2a3250c7c5ce53d25ba72962b46362a9f39b3282d1958c6bee70bd711b2e
size 15735

3
tasks/speechless-tora-code-7b-v1.0/outputs/speechless-tora-code-7b-v1.0/checkpoint-3000/scheduler.pt Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:403f7d4ddb29af77fdfd9e883d0c8935063872d5e73dea699394bc703f947ea8
size 627

6
tasks/speechless-tora-code-7b-v1.0/outputs/speechless-tora-code-7b-v1.0/checkpoint-3000/special_tokens_map.json Normal file
View File

@@ -0,0 +1,6 @@
{
"bos_token": "<s>",
"eos_token": "</s>",
"pad_token": "<pad>",
"unk_token": "<unk>"
}

3
tasks/speechless-tora-code-7b-v1.0/outputs/speechless-tora-code-7b-v1.0/checkpoint-3000/tokenizer.model Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
size 499723

52
tasks/speechless-tora-code-7b-v1.0/outputs/speechless-tora-code-7b-v1.0/checkpoint-3000/tokenizer_config.json Normal file
View File

@@ -0,0 +1,52 @@
{
"add_bos_token": true,
"add_eos_token": false,
"added_tokens_decoder": {
"0": {
"content": "<unk>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"1": {
"content": "<s>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"2": {
"content": "</s>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"32000": {
"content": "<pad>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
}
},
"additional_special_tokens": [],
"bos_token": "<s>",
"clean_up_tokenization_spaces": false,
"eos_token": "</s>",
"legacy": true,
"model_max_length": 1000000000000000019884624838656,
"pad_token": "<pad>",
"padding_side": "left",
"sp_model_kwargs": {},
"spaces_between_special_tokens": false,
"tokenizer_class": "LlamaTokenizer",
"tokenizer_file": null,
"unk_token": "<unk>",
"use_default_system_prompt": true
}

18259
tasks/speechless-tora-code-7b-v1.0/outputs/speechless-tora-code-7b-v1.0/checkpoint-3000/trainer_state.json Normal file
View File

File diff suppressed because it is too large Load Diff

3
tasks/speechless-tora-code-7b-v1.0/outputs/speechless-tora-code-7b-v1.0/checkpoint-3000/training_args.bin Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:88c746833b5a7f3b81a2dbfe4fd0d18678e9740460c1a97584c75375f13a6c11
size 4667

34
tasks/speechless-tora-code-7b-v1.0/outputs/speechless-tora-code-7b-v1.0/checkpoint-3092/adapter_model/README.md Normal file
View File

@@ -0,0 +1,34 @@
---
library_name: peft
---
## Training procedure
The following `bitsandbytes` quantization config was used during training:
- quant_method: bitsandbytes
- load_in_8bit: False
- load_in_4bit: True
- llm_int8_threshold: 6.0
- llm_int8_skip_modules: None
- llm_int8_enable_fp32_cpu_offload: False
- llm_int8_has_fp16_weight: False
- bnb_4bit_quant_type: nf4
- bnb_4bit_use_double_quant: True
- bnb_4bit_compute_dtype: bfloat16
The following `bitsandbytes` quantization config was used during training:
- quant_method: bitsandbytes
- load_in_8bit: False
- load_in_4bit: True
- llm_int8_threshold: 6.0
- llm_int8_skip_modules: None
- llm_int8_enable_fp32_cpu_offload: False
- llm_int8_has_fp16_weight: False
- bnb_4bit_quant_type: nf4
- bnb_4bit_use_double_quant: True
- bnb_4bit_compute_dtype: bfloat16
### Framework versions
- PEFT 0.5.0
- PEFT 0.5.0

26
tasks/speechless-tora-code-7b-v1.0/outputs/speechless-tora-code-7b-v1.0/checkpoint-3092/adapter_model/adapter_config.json Normal file
View File

@@ -0,0 +1,26 @@
{
"auto_mapping": null,
"base_model_name_or_path": "/opt/local/llm_models/huggingface.co/llm_agents/tora-code-7b-v1.0",
"bias": "none",
"fan_in_fan_out": false,
"inference_mode": true,
"init_lora_weights": true,
"layers_pattern": null,
"layers_to_transform": null,
"lora_alpha": 16.0,
"lora_dropout": 0.05,
"modules_to_save": null,
"peft_type": "LORA",
"r": 64,
"revision": null,
"target_modules": [
"q_proj",
"gate_proj",
"down_proj",
"v_proj",
"k_proj",
"o_proj",
"up_proj"
],
"task_type": "CAUSAL_LM"
}

3
tasks/speechless-tora-code-7b-v1.0/outputs/speechless-tora-code-7b-v1.0/checkpoint-3092/adapter_model/adapter_model.bin Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:064bb620173b89e327154ada8f72dbd1eeea1d16d1ce351c2240385268dc3dac
size 639792909

0
tasks/speechless-tora-code-7b-v1.0/outputs/speechless-tora-code-7b-v1.0/completed Normal file
View File

7
tasks/speechless-tora-code-7b-v1.0/outputs/speechless-tora-code-7b-v1.0/eval_results.json Normal file
View File

@@ -0,0 +1,7 @@
{
"epoch": 2.0,
"eval_loss": 0.5871890783309937,
"eval_runtime": 15.5979,
"eval_samples_per_second": 12.822,
"eval_steps_per_second": 6.411
}

24
tasks/speechless-tora-code-7b-v1.0/outputs/speechless-tora-code-7b-v1.0/logs/finetune_20231009_061922.log Normal file
View File

@@ -0,0 +1,24 @@
2023-10-09 06:19:22.839 | INFO | __main__:train:918 - args=Namespace(model_name_or_path='/opt/local/llm_models/huggingface.co/llm_agents/tora-code-7b-v1.0', trust_remote_code=False, force_remove_overlength_samples=False, eval_dataset_size=0.005, max_train_samples=None, max_eval_samples=200, model_max_len=4096, dataset='/opt/local/datasets/Speechless/speechless-thoughts-200k.jsonl', dataset_format='airoboros', output_dir='./outputs/speechless-tora-code-7b-v1.0', overwrite_output_dir=False, do_train=True, do_eval=True, do_predict=False, evaluation_strategy=<IntervalStrategy.STEPS: 'steps'>, prediction_loss_only=False, per_device_train_batch_size=8, per_device_eval_batch_size=1, per_gpu_train_batch_size=None, per_gpu_eval_batch_size=None, gradient_accumulation_steps=8, eval_accumulation_steps=None, eval_delay=0, learning_rate=0.0002, weight_decay=0.0, adam_beta1=0.9, adam_beta2=0.999, adam_epsilon=1e-08, max_grad_norm=0.3, num_train_epochs=2, max_steps=-1, lr_scheduler_type=<SchedulerType.COSINE: 'cosine'>, warmup_ratio=0.005, warmup_steps=20, log_level='passive', log_level_replica='warning', log_on_each_node=True, logging_dir='./outputs/speechless-tora-code-7b-v1.0/runs/Oct09_06-19-22_I156f1f3f410070163e', logging_strategy=<IntervalStrategy.STEPS: 'steps'>, logging_first_step=False, logging_steps=1, logging_nan_inf_filter=True, save_strategy=<IntervalStrategy.STEPS: 'steps'>, save_steps=100, save_total_limit=1, save_safetensors=False, save_on_each_node=False, no_cuda=False, use_cpu=False, use_mps_device=False, seed=10042, data_seed=10042, jit_mode_eval=False, use_ipex=False, bf16=True, fp16=False, fp16_opt_level='O1', half_precision_backend='auto', bf16_full_eval=False, fp16_full_eval=False, tf32=None, local_rank=0, ddp_backend=None, tpu_num_cores=None, tpu_metrics_debug=False, debug=[], dataloader_drop_last=False, eval_steps=100, dataloader_num_workers=3, past_index=-1, run_name='20231009-061916', disable_tqdm=False, remove_unused_columns=False, label_names=None, load_best_model_at_end=False, metric_for_best_model=None, greater_is_better=None, ignore_data_skip=False, sharded_ddp=[], fsdp=[], fsdp_min_num_params=0, fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_grad_ckpt': False}, fsdp_transformer_layer_cls_to_wrap=None, deepspeed=None, label_smoothing_factor=0.0, optim=<OptimizerNames.PAGED_ADAMW_8BIT: 'paged_adamw_8bit'>, optim_args=None, adafactor=False, group_by_length=False, length_column_name='length', report_to=['tensorboard'], ddp_find_unused_parameters=False, ddp_bucket_cap_mb=None, ddp_broadcast_buffers=None, dataloader_pin_memory=True, skip_memory_metrics=True, use_legacy_prediction_loop=False, push_to_hub=False, resume_from_checkpoint=None, hub_model_id=None, hub_strategy=<HubStrategy.EVERY_SAVE: 'every_save'>, hub_token=None, hub_private_repo=False, hub_always_push=False, gradient_checkpointing=True, include_inputs_for_metrics=False, fp16_backend='auto', push_to_hub_model_id=None, push_to_hub_organization=None, push_to_hub_token=None, mp_parameters='', auto_find_batch_size=False, full_determinism=False, torchdynamo=None, ray_scope='last', ddp_timeout=1800, torch_compile=False, torch_compile_backend=None, torch_compile_mode=None, dispatch_batches=None, include_tokens_per_second=False, sortish_sampler=False, predict_with_generate=False, generation_max_length=None, generation_num_beams=None, generation_config=None, task_name='speechless-tora-code-7b-v1.0', flash_attention=True, rerope=False, rerope_window=None, wandb=None, sample_packing=False, cache_dir=None, full_finetune=False, adam8bit=False, double_quant=True, quant_type='nf4', bits=4, lora_r=64, lora_alpha=16.0, lora_dropout=0.05, max_memory_MB=70000, mpt=False, max_shard_size='5GB', repeat_steps=0, distributed_state=Distributed environment: MULTI_GPU Backend: nccl
Num processes: 2
Process index: 0
Local process index: 0
Device: cuda:0
, _n_gpu=1, __cached__setup_devices=device(type='cuda', index=0), deepspeed_plugin=None)
2023-10-09 06:19:22.840 | INFO | __main__:get_accelerate_model:303 - loading base model /opt/local/llm_models/huggingface.co/llm_agents/tora-code-7b-v1.0...
2023-10-09 06:19:22.848 | INFO | __main__:train:918 - args=Namespace(model_name_or_path='/opt/local/llm_models/huggingface.co/llm_agents/tora-code-7b-v1.0', trust_remote_code=False, force_remove_overlength_samples=False, eval_dataset_size=0.005, max_train_samples=None, max_eval_samples=200, model_max_len=4096, dataset='/opt/local/datasets/Speechless/speechless-thoughts-200k.jsonl', dataset_format='airoboros', output_dir='./outputs/speechless-tora-code-7b-v1.0', overwrite_output_dir=False, do_train=True, do_eval=True, do_predict=False, evaluation_strategy=<IntervalStrategy.STEPS: 'steps'>, prediction_loss_only=False, per_device_train_batch_size=8, per_device_eval_batch_size=1, per_gpu_train_batch_size=None, per_gpu_eval_batch_size=None, gradient_accumulation_steps=8, eval_accumulation_steps=None, eval_delay=0, learning_rate=0.0002, weight_decay=0.0, adam_beta1=0.9, adam_beta2=0.999, adam_epsilon=1e-08, max_grad_norm=0.3, num_train_epochs=2, max_steps=-1, lr_scheduler_type=<SchedulerType.COSINE: 'cosine'>, warmup_ratio=0.005, warmup_steps=20, log_level='passive', log_level_replica='warning', log_on_each_node=True, logging_dir='./outputs/speechless-tora-code-7b-v1.0/runs/Oct09_06-19-22_I156f1f3f410070163e', logging_strategy=<IntervalStrategy.STEPS: 'steps'>, logging_first_step=False, logging_steps=1, logging_nan_inf_filter=True, save_strategy=<IntervalStrategy.STEPS: 'steps'>, save_steps=100, save_total_limit=1, save_safetensors=False, save_on_each_node=False, no_cuda=False, use_cpu=False, use_mps_device=False, seed=10042, data_seed=10042, jit_mode_eval=False, use_ipex=False, bf16=True, fp16=False, fp16_opt_level='O1', half_precision_backend='auto', bf16_full_eval=False, fp16_full_eval=False, tf32=None, local_rank=1, ddp_backend=None, tpu_num_cores=None, tpu_metrics_debug=False, debug=[], dataloader_drop_last=False, eval_steps=100, dataloader_num_workers=3, past_index=-1, run_name='20231009-061916', disable_tqdm=False, remove_unused_columns=False, label_names=None, load_best_model_at_end=False, metric_for_best_model=None, greater_is_better=None, ignore_data_skip=False, sharded_ddp=[], fsdp=[], fsdp_min_num_params=0, fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_grad_ckpt': False}, fsdp_transformer_layer_cls_to_wrap=None, deepspeed=None, label_smoothing_factor=0.0, optim=<OptimizerNames.PAGED_ADAMW_8BIT: 'paged_adamw_8bit'>, optim_args=None, adafactor=False, group_by_length=False, length_column_name='length', report_to=['tensorboard'], ddp_find_unused_parameters=False, ddp_bucket_cap_mb=None, ddp_broadcast_buffers=None, dataloader_pin_memory=True, skip_memory_metrics=True, use_legacy_prediction_loop=False, push_to_hub=False, resume_from_checkpoint=None, hub_model_id=None, hub_strategy=<HubStrategy.EVERY_SAVE: 'every_save'>, hub_token=None, hub_private_repo=False, hub_always_push=False, gradient_checkpointing=True, include_inputs_for_metrics=False, fp16_backend='auto', push_to_hub_model_id=None, push_to_hub_organization=None, push_to_hub_token=None, mp_parameters='', auto_find_batch_size=False, full_determinism=False, torchdynamo=None, ray_scope='last', ddp_timeout=1800, torch_compile=False, torch_compile_backend=None, torch_compile_mode=None, dispatch_batches=None, include_tokens_per_second=False, sortish_sampler=False, predict_with_generate=False, generation_max_length=None, generation_num_beams=None, generation_config=None, task_name='speechless-tora-code-7b-v1.0', flash_attention=True, rerope=False, rerope_window=None, wandb=None, sample_packing=False, cache_dir=None, full_finetune=False, adam8bit=False, double_quant=True, quant_type='nf4', bits=4, lora_r=64, lora_alpha=16.0, lora_dropout=0.05, max_memory_MB=70000, mpt=False, max_shard_size='5GB', repeat_steps=0, distributed_state=Distributed environment: MULTI_GPU Backend: nccl
Num processes: 2
Process index: 1
Local process index: 1
Device: cuda:1
, _n_gpu=1, __cached__setup_devices=device(type='cuda', index=1), deepspeed_plugin=None)
2023-10-09 06:19:22.849 | INFO | __main__:get_accelerate_model:303 - loading base model /opt/local/llm_models/huggingface.co/llm_agents/tora-code-7b-v1.0...
2023-10-09 06:19:51.853 | INFO | __main__:get_accelerate_model:350 - adding LoRA modules...
2023-10-09 06:19:53.297 | INFO | __main__:get_accelerate_model:350 - adding LoRA modules...
2023-10-09 06:21:03.048 | INFO | __main__:train:942 - loaded model
2023-10-09 06:21:03.150 | INFO | __main__:format_dataset:639 - ---------- Formatting dataset for Airoboros. ----------
2023-10-09 06:21:03.160 | INFO | __main__:make_data_module:734 - ---------- Splitting dataset into train/eval ----------
2023-10-09 06:21:03.205 | INFO | __main__:train:985 - *** Train ***
2023-10-09 06:21:06.112 | INFO | __main__:train:942 - loaded model
2023-10-09 06:21:06.212 | INFO | __main__:format_dataset:639 - ---------- Formatting dataset for Airoboros. ----------
2023-10-09 06:21:06.222 | INFO | __main__:make_data_module:734 - ---------- Splitting dataset into train/eval ----------
2023-10-09 06:21:06.265 | INFO | __main__:train:985 - *** Train ***

88
tasks/speechless-tora-code-7b-v1.0/outputs/speechless-tora-code-7b-v1.0/logs/finetune_20231009_122312.log Normal file
View File

@@ -0,0 +1,88 @@
2023-10-09 12:23:12.104 | INFO | __main__:train:918 - args=Namespace(model_name_or_path='/opt/local/llm_models/huggingface.co/llm_agents/tora-code-7b-v1.0', trust_remote_code=False, force_remove_overlength_samples=False, eval_dataset_size=0.005, max_train_samples=None, max_eval_samples=200, model_max_len=4096, dataset='/opt/local/datasets/Speechless/speechless-thoughts-200k.jsonl', dataset_format='airoboros', output_dir='./outputs/speechless-tora-code-7b-v1.0', overwrite_output_dir=False, do_train=True, do_eval=True, do_predict=False, evaluation_strategy=<IntervalStrategy.STEPS: 'steps'>, prediction_loss_only=False, per_device_train_batch_size=4, per_device_eval_batch_size=1, per_gpu_train_batch_size=None, per_gpu_eval_batch_size=None, gradient_accumulation_steps=16, eval_accumulation_steps=None, eval_delay=0, learning_rate=0.0002, weight_decay=0.0, adam_beta1=0.9, adam_beta2=0.999, adam_epsilon=1e-08, max_grad_norm=0.3, num_train_epochs=2, max_steps=-1, lr_scheduler_type=<SchedulerType.COSINE: 'cosine'>, warmup_ratio=0.005, warmup_steps=20, log_level='passive', log_level_replica='warning', log_on_each_node=True, logging_dir='./outputs/speechless-tora-code-7b-v1.0/runs/Oct09_12-23-12_I156f1f3f410070163e', logging_strategy=<IntervalStrategy.STEPS: 'steps'>, logging_first_step=False, logging_steps=1, logging_nan_inf_filter=True, save_strategy=<IntervalStrategy.STEPS: 'steps'>, save_steps=100, save_total_limit=1, save_safetensors=False, save_on_each_node=False, no_cuda=False, use_cpu=False, use_mps_device=False, seed=10042, data_seed=10042, jit_mode_eval=False, use_ipex=False, bf16=True, fp16=False, fp16_opt_level='O1', half_precision_backend='auto', bf16_full_eval=False, fp16_full_eval=False, tf32=None, local_rank=1, ddp_backend=None, tpu_num_cores=None, tpu_metrics_debug=False, debug=[], dataloader_drop_last=False, eval_steps=100, dataloader_num_workers=3, past_index=-1, run_name='20231009-122305', disable_tqdm=False, remove_unused_columns=False, label_names=None, load_best_model_at_end=False, metric_for_best_model=None, greater_is_better=None, ignore_data_skip=False, sharded_ddp=[], fsdp=[], fsdp_min_num_params=0, fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_grad_ckpt': False}, fsdp_transformer_layer_cls_to_wrap=None, deepspeed=None, label_smoothing_factor=0.0, optim=<OptimizerNames.PAGED_ADAMW_8BIT: 'paged_adamw_8bit'>, optim_args=None, adafactor=False, group_by_length=False, length_column_name='length', report_to=['tensorboard'], ddp_find_unused_parameters=False, ddp_bucket_cap_mb=None, ddp_broadcast_buffers=None, dataloader_pin_memory=True, skip_memory_metrics=True, use_legacy_prediction_loop=False, push_to_hub=False, resume_from_checkpoint=None, hub_model_id=None, hub_strategy=<HubStrategy.EVERY_SAVE: 'every_save'>, hub_token=None, hub_private_repo=False, hub_always_push=False, gradient_checkpointing=True, include_inputs_for_metrics=False, fp16_backend='auto', push_to_hub_model_id=None, push_to_hub_organization=None, push_to_hub_token=None, mp_parameters='', auto_find_batch_size=False, full_determinism=False, torchdynamo=None, ray_scope='last', ddp_timeout=1800, torch_compile=False, torch_compile_backend=None, torch_compile_mode=None, dispatch_batches=None, include_tokens_per_second=False, sortish_sampler=False, predict_with_generate=False, generation_max_length=None, generation_num_beams=None, generation_config=None, task_name='speechless-tora-code-7b-v1.0', flash_attention=True, rerope=False, rerope_window=None, wandb=None, sample_packing=False, cache_dir=None, full_finetune=False, adam8bit=False, double_quant=True, quant_type='nf4', bits=4, lora_r=64, lora_alpha=16.0, lora_dropout=0.05, max_memory_MB=32000, mpt=False, max_shard_size='5GB', repeat_steps=0, distributed_state=Distributed environment: MULTI_GPU Backend: nccl
Num processes: 2
Process index: 1
Local process index: 1
Device: cuda:1
, _n_gpu=1, __cached__setup_devices=device(type='cuda', index=1), deepspeed_plugin=None)
2023-10-09 12:23:12.105 | INFO | __main__:get_accelerate_model:303 - loading base model /opt/local/llm_models/huggingface.co/llm_agents/tora-code-7b-v1.0...
2023-10-09 12:23:12.105 | INFO | __main__:train:918 - args=Namespace(model_name_or_path='/opt/local/llm_models/huggingface.co/llm_agents/tora-code-7b-v1.0', trust_remote_code=False, force_remove_overlength_samples=False, eval_dataset_size=0.005, max_train_samples=None, max_eval_samples=200, model_max_len=4096, dataset='/opt/local/datasets/Speechless/speechless-thoughts-200k.jsonl', dataset_format='airoboros', output_dir='./outputs/speechless-tora-code-7b-v1.0', overwrite_output_dir=False, do_train=True, do_eval=True, do_predict=False, evaluation_strategy=<IntervalStrategy.STEPS: 'steps'>, prediction_loss_only=False, per_device_train_batch_size=4, per_device_eval_batch_size=1, per_gpu_train_batch_size=None, per_gpu_eval_batch_size=None, gradient_accumulation_steps=16, eval_accumulation_steps=None, eval_delay=0, learning_rate=0.0002, weight_decay=0.0, adam_beta1=0.9, adam_beta2=0.999, adam_epsilon=1e-08, max_grad_norm=0.3, num_train_epochs=2, max_steps=-1, lr_scheduler_type=<SchedulerType.COSINE: 'cosine'>, warmup_ratio=0.005, warmup_steps=20, log_level='passive', log_level_replica='warning', log_on_each_node=True, logging_dir='./outputs/speechless-tora-code-7b-v1.0/runs/Oct09_12-23-12_I156f1f3f410070163e', logging_strategy=<IntervalStrategy.STEPS: 'steps'>, logging_first_step=False, logging_steps=1, logging_nan_inf_filter=True, save_strategy=<IntervalStrategy.STEPS: 'steps'>, save_steps=100, save_total_limit=1, save_safetensors=False, save_on_each_node=False, no_cuda=False, use_cpu=False, use_mps_device=False, seed=10042, data_seed=10042, jit_mode_eval=False, use_ipex=False, bf16=True, fp16=False, fp16_opt_level='O1', half_precision_backend='auto', bf16_full_eval=False, fp16_full_eval=False, tf32=None, local_rank=0, ddp_backend=None, tpu_num_cores=None, tpu_metrics_debug=False, debug=[], dataloader_drop_last=False, eval_steps=100, dataloader_num_workers=3, past_index=-1, run_name='20231009-122305', disable_tqdm=False, remove_unused_columns=False, label_names=None, load_best_model_at_end=False, metric_for_best_model=None, greater_is_better=None, ignore_data_skip=False, sharded_ddp=[], fsdp=[], fsdp_min_num_params=0, fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_grad_ckpt': False}, fsdp_transformer_layer_cls_to_wrap=None, deepspeed=None, label_smoothing_factor=0.0, optim=<OptimizerNames.PAGED_ADAMW_8BIT: 'paged_adamw_8bit'>, optim_args=None, adafactor=False, group_by_length=False, length_column_name='length', report_to=['tensorboard'], ddp_find_unused_parameters=False, ddp_bucket_cap_mb=None, ddp_broadcast_buffers=None, dataloader_pin_memory=True, skip_memory_metrics=True, use_legacy_prediction_loop=False, push_to_hub=False, resume_from_checkpoint=None, hub_model_id=None, hub_strategy=<HubStrategy.EVERY_SAVE: 'every_save'>, hub_token=None, hub_private_repo=False, hub_always_push=False, gradient_checkpointing=True, include_inputs_for_metrics=False, fp16_backend='auto', push_to_hub_model_id=None, push_to_hub_organization=None, push_to_hub_token=None, mp_parameters='', auto_find_batch_size=False, full_determinism=False, torchdynamo=None, ray_scope='last', ddp_timeout=1800, torch_compile=False, torch_compile_backend=None, torch_compile_mode=None, dispatch_batches=None, include_tokens_per_second=False, sortish_sampler=False, predict_with_generate=False, generation_max_length=None, generation_num_beams=None, generation_config=None, task_name='speechless-tora-code-7b-v1.0', flash_attention=True, rerope=False, rerope_window=None, wandb=None, sample_packing=False, cache_dir=None, full_finetune=False, adam8bit=False, double_quant=True, quant_type='nf4', bits=4, lora_r=64, lora_alpha=16.0, lora_dropout=0.05, max_memory_MB=32000, mpt=False, max_shard_size='5GB', repeat_steps=0, distributed_state=Distributed environment: MULTI_GPU Backend: nccl
Num processes: 2
Process index: 0
Local process index: 0
Device: cuda:0
, _n_gpu=1, __cached__setup_devices=device(type='cuda', index=0), deepspeed_plugin=None)
2023-10-09 12:23:12.105 | INFO | __main__:get_accelerate_model:303 - loading base model /opt/local/llm_models/huggingface.co/llm_agents/tora-code-7b-v1.0...
2023-10-09 12:23:40.070 | INFO | __main__:get_accelerate_model:350 - adding LoRA modules...
2023-10-09 12:23:40.822 | INFO | __main__:get_accelerate_model:350 - adding LoRA modules...
2023-10-09 12:24:52.367 | INFO | __main__:train:942 - loaded model
2023-10-09 12:24:52.467 | INFO | __main__:format_dataset:639 - ---------- Formatting dataset for Airoboros. ----------
2023-10-09 12:24:52.478 | INFO | __main__:make_data_module:734 - ---------- Splitting dataset into train/eval ----------
2023-10-09 12:24:52.523 | INFO | __main__:train:985 - *** Train ***
2023-10-09 12:24:53.833 | INFO | __main__:train:942 - loaded model
2023-10-09 12:24:53.932 | INFO | __main__:format_dataset:639 - ---------- Formatting dataset for Airoboros. ----------
2023-10-09 12:24:53.942 | INFO | __main__:make_data_module:734 - ---------- Splitting dataset into train/eval ----------
2023-10-09 12:24:53.987 | INFO | __main__:train:985 - *** Train ***
2023-10-09 13:02:59.195 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
2023-10-09 13:03:00.511 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
2023-10-09 13:40:57.489 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
2023-10-09 13:40:59.007 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
2023-10-09 14:18:56.508 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
2023-10-09 14:18:57.522 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
2023-10-09 14:56:40.535 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
2023-10-09 14:56:41.789 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
2023-10-09 15:34:06.992 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
2023-10-09 15:34:08.286 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
2023-10-09 16:12:02.503 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
2023-10-09 16:12:03.732 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
2023-10-09 16:49:21.174 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
2023-10-09 16:49:22.504 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
2023-10-09 17:26:54.676 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
2023-10-09 17:26:55.668 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
2023-10-09 18:04:46.740 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
2023-10-09 18:04:47.949 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
2023-10-09 18:42:04.644 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
2023-10-09 18:42:05.914 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
2023-10-09 19:19:37.187 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
2023-10-09 19:19:38.502 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
2023-10-09 19:56:57.213 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
2023-10-09 19:56:58.406 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
2023-10-09 20:34:41.712 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
2023-10-09 20:34:43.045 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
2023-10-09 21:12:11.382 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
2023-10-09 21:12:12.669 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
2023-10-09 21:49:45.432 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
2023-10-09 21:49:46.721 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
2023-10-09 22:27:13.362 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
2023-10-09 22:27:14.775 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
2023-10-09 23:04:40.350 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
2023-10-09 23:04:41.734 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
2023-10-09 23:43:00.849 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
2023-10-09 23:43:01.897 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
2023-10-10 00:20:56.083 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
2023-10-10 00:20:57.315 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
2023-10-10 00:58:45.672 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
2023-10-10 00:58:46.917 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
2023-10-10 01:36:16.111 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
2023-10-10 01:36:17.107 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
2023-10-10 02:14:04.076 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
2023-10-10 02:14:05.116 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
2023-10-10 02:51:35.732 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
2023-10-10 02:51:37.116 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
2023-10-10 03:29:10.549 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
2023-10-10 03:29:11.807 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
2023-10-10 04:07:10.597 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
2023-10-10 04:07:12.121 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
2023-10-10 04:45:13.080 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
2023-10-10 04:45:14.388 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
2023-10-10 05:22:29.236 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
2023-10-10 05:22:30.313 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
2023-10-10 06:00:01.444 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
2023-10-10 06:00:02.697 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
2023-10-10 06:37:56.658 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
2023-10-10 06:37:57.703 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
2023-10-10 07:15:19.303 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
2023-10-10 07:15:20.460 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
2023-10-10 07:49:44.958 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
2023-10-10 07:49:44.959 | INFO | __main__:save_model:262 - Saving PEFT checkpoint...
2023-10-10 07:49:45.497 | INFO | __main__:train:996 - *** Evaluate ***
2023-10-10 07:49:45.564 | INFO | __main__:train:996 - *** Evaluate ***

1
tasks/speechless-tora-code-7b-v1.0/outputs/speechless-tora-code-7b-v1.0/metrics.json Normal file
View File

@@ -0,0 +1 @@
{"run_name": "20231009-122305", "train_runtime": 69889.4305, "train_samples_per_second": 5.664, "train_steps_per_second": 0.044, "train_loss": 0.5890884618112218, "epoch": 2.0, "eval_loss": 0.5871890783309937, "eval_runtime": 15.5979, "eval_samples_per_second": 12.822, "eval_steps_per_second": 6.411}

3
tasks/speechless-tora-code-7b-v1.0/outputs/speechless-tora-code-7b-v1.0/runs/Oct09_06-19-22_I156f1f3f410070163e/events.out.tfevents.1696832467.I156f1f3f410070163e.7536.0 Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:b2462cbdd451f1620c45fdc00446bef4939780801d8fc5937b775dd515dee8c8
size 7869

3
tasks/speechless-tora-code-7b-v1.0/outputs/speechless-tora-code-7b-v1.0/runs/Oct09_12-23-12_I156f1f3f410070163e/events.out.tfevents.1696854295.I156f1f3f410070163e.1701.0 Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:2ed40796c7e6086dc6fed30704e34ee1f93696be1510cda6f81cd0b162e7b03c
size 498794

3
tasks/speechless-tora-code-7b-v1.0/outputs/speechless-tora-code-7b-v1.0/runs/Oct09_12-23-12_I156f1f3f410070163e/events.out.tfevents.1696924201.I156f1f3f410070163e.1701.1 Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:be1c5672e1dd6fd08655f555c268b13123ff736c6281a9bbef2fc2dd916bb156
size 359

7
tasks/speechless-tora-code-7b-v1.0/outputs/speechless-tora-code-7b-v1.0/train_results.json Normal file
View File

@@ -0,0 +1,7 @@
{
"epoch": 2.0,
"train_loss": 0.5890884618112218,
"train_runtime": 69889.4305,
"train_samples_per_second": 5.664,
"train_steps_per_second": 0.044
}

18820
tasks/speechless-tora-code-7b-v1.0/outputs/speechless-tora-code-7b-v1.0/trainer_state.json Normal file
View File

File diff suppressed because it is too large Load Diff

11
tasks/speechless-tora-code-7b-v1.0/run_api_server.sh Executable file
View File

@@ -0,0 +1,11 @@
#!/usr/bin/env bash
# From speechless-code-mistral-7b-v1.0
SCRIPT_PATH=$(cd $(dirname ${BASH_SOURCE[0]}); pwd)
PARENT_PATH=$(cd "${SCRIPT_PATH}/.." ; pwd)
source ${SCRIPT_PATH}/task.env
PYTHONPATH=${SCRIPT_PATH}/../../.. \
python ../../api/server.py \
--model_name_or_path=${TEST_MODEL_PATH} \
--model_family vllm

59
tasks/speechless-tora-code-7b-v1.0/run_finetune.sh Executable file
View File

@@ -0,0 +1,59 @@
#!/bin/bash
# From speechless-code-mistral-7b-v1.0
SCRIPT_PATH=$(cd $(dirname ${BASH_SOURCE[0]}); pwd)
PARENT_PATH=$(cd "${SCRIPT_PATH}/.." ; pwd)
source ${SCRIPT_PATH}/task.env
PYTHONPATH=${PWD}/../.. \
torchrun --nnodes=1 --nproc_per_node=${NUM_GPUS} \
../../finetune.py \
${DEEPSPEED_STAGE2} \
--task_name ${TASK_NAME} \
--run_name $(date +%Y%m%d-%H%M%S) \
--model_name_or_path ${BASE_MODEL_PATH} \
--output_dir ${OUTPUT_DIR} \
--num_train_epochs ${NUM_TRAIN_EPOCHS} \
--data_seed 10042 \
--save_strategy steps \
--save_total_limit 1 \
--evaluation_strategy steps \
--eval_dataset_size ${EVAL_DATASET_SIZE} \
--save_steps 100 \
--eval_steps 100 \
--warmup_steps 20 \
--max_eval_samples 200 \
--dataloader_num_workers 3 \
--logging_strategy steps \
--logging_steps 1 \
--report_to tensorboard \
--remove_unused_columns False \
--do_train \
--max_memory_MB ${MAX_MEMORY_MB} \
--bits 4 \
--lora_r ${LORA_R} \
--lora_alpha 16 \
--lora_dropout 0.05 \
--lora_modules all \
--double_quant \
--quant_type nf4 \
--bf16 \
--dataset ${DATASET} \
--dataset_format ${DATASET_FORMAT} \
--max_new_tokens 4096 \
--model_max_len 4096 \
--per_device_train_batch_size ${PER_DEVICE_TRAIN_BATCH_SIZE} \
--gradient_accumulation_steps ${GRADIENT_ACCUMULATION_STEPS} \
--per_device_eval_batch_size 1 \
--learning_rate ${LEARNING_RATE} \
--lr_scheduler_type cosine \
--weight_decay 0.0 \
--seed 10042 \
--optim paged_adamw_8bit \
--gradient_checkpointing True \
--group_by_length ${GROUP_BY_LENGTH} \
--ddp_find_unused_parameters False \
--force_remove_overlength_samples False \
--flash_attention True \
--rerope False \
--repeat_steps 0

47
tasks/speechless-tora-code-7b-v1.0/task.env Normal file
View File

@@ -0,0 +1,47 @@
# From speechless-code-mistral-7b-v1.0
# -------------------- Model --------------------
export MODELS_ROOT_DIR=/opt/local/llm_models/huggingface.co
# FIXME
export BASE_MODEL_PATH=${MODELS_ROOT_DIR}/llm_agents/tora-code-7b-v1.0
# FIXME
export TEST_MODEL_PATH=${MODELS_ROOT_DIR}/speechlessai/speechless-tora-code-7b-v1.0
# export TEST_MODEL_PATH=${MODELS_ROOT_DIR}/speechlessai/speechless-nl2sql-mistral-7b-v0.1
# export TEST_MODEL_PATH=${MODELS_ROOT_DIR}/speechlessai/speechless-mistral-7b-v0.1
# -------------------- Dataset --------------------
# FIXME
# export DATASET=/opt/local/datasets/jondurbin/airoboros-2.2/instructions-clean.jsonl
# export DATASET=/opt/local/datasets/Speechless/airoboros-orca-platypus-instructions.jsonl
# export DATASET=/opt/local/datasets/Speechless/speechless-spider.jsonl
export DATASET=/opt/local/datasets/Speechless/speechless-thoughts-200k.jsonl
# -------------------- Environment --------------------
export OUTPUT_DIR=./outputs
# export TORCH_DISTRIBUTED_DEBUG=DETAIL
export RAY_memory_monitor_refresh_ms=0
# -------------------- Task --------------------
# FIXME
export TASK_NAME=$(basename ${TEST_MODEL_PATH})
export TASK_CHECKPOINT_DIR=${OUTPUT_DIR}/${TASK_NAME}
export WANDB_PROJECT=${TASK_NAME}
# -------------------- Task --------------------
export DATASET_FORMAT=airoboros
export NUM_TRAIN_EPOCHS=2
export EVAL_DATASET_SIZE=0.005
export LEARNING_RATE=2e-4
export LORA_R=64
export GROUP_BY_LENGTH=False
#export DEEPSPEED_STAGE2="--deepspeed deepspeed-stage2.json"
export NUM_GPUS=2
# 2 gpus: 40GB: 2 x 32, 80GB: 4 x 16
# 4 gpus: 40GB: 2 x 16, 80GB: 4 x 8
export PER_DEVICE_TRAIN_BATCH_SIZE=4
export GRADIENT_ACCUMULATION_STEPS=16
# No more than 85% VRAM.
# A100(40GB) 34000, A40(48GB) 40000, A100(80GB) 70000
export MAX_MEMORY_MB=32000

49
tasks/speechless-tora-code-7b-v1.0/task_a100_40gx2.env Normal file
View File

@@ -0,0 +1,49 @@
# From speechless-code-mistral-orca-7b-v1.0
# -------------------- Model --------------------
export MODELS_ROOT_DIR=/opt/local/llm_models/huggingface.co
# FIXME
export BASE_MODEL_PATH=${MODELS_ROOT_DIR}/Open-Orca/Mistral-7B-OpenOrca
# export BASE_MODEL_PATH=${MODELS_ROOT_DIR}/mistralai/Mistral-7B-v0.1
# FIXME
export TEST_MODEL_PATH=${MODELS_ROOT_DIR}/speechlessai/speechless-code-mistral-orca-7b-v1.0
# export TEST_MODEL_PATH=${MODELS_ROOT_DIR}/speechlessai/speechless-code-mistral-7b-v1.0
# export TEST_MODEL_PATH=${MODELS_ROOT_DIR}/speechlessai/speechless-nl2sql-mistral-7b-v0.1
# export TEST_MODEL_PATH=${MODELS_ROOT_DIR}/speechlessai/speechless-mistral-7b-v0.1
# -------------------- Dataset --------------------
# FIXME
# export DATASET=/opt/local/datasets/jondurbin/airoboros-2.2/instructions-clean.jsonl
# export DATASET=/opt/local/datasets/Speechless/airoboros-orca-platypus-instructions.jsonl
# export DATASET=/opt/local/datasets/Speechless/speechless-spider.jsonl
export DATASET=/opt/local/datasets/Speechless/speechless-thoughts-200k.jsonl
# -------------------- Environment --------------------
export OUTPUT_DIR=./outputs
export TORCH_DISTRIBUTED_DEBUG=DETAIL
export RAY_memory_monitor_refresh_ms=0
# -------------------- Task --------------------
# FIXME
export TASK_NAME=$(basename ${TEST_MODEL_PATH})
export TASK_CHECKPOINT_DIR=${OUTPUT_DIR}/${TASK_NAME}
export WANDB_PROJECT=${TASK_NAME}
# -------------------- Task --------------------
export DATASET_FORMAT=airoboros
export NUM_TRAIN_EPOCHS=2
export EVAL_DATASET_SIZE=0.005
export LEARNING_RATE=2e-4
export LORA_R=64
export GROUP_BY_LENGTH=False
#export DEEPSPEED_STAGE2="--deepspeed deepspeed-stage2.json"
export NUM_GPUS=2
# 2 gpus: 40GB: 2 x 32, 80GB: 4 x 16
# 4 gpus: 40GB: 2 x 16, 80GB: 4 x 8
export PER_DEVICE_TRAIN_BATCH_SIZE=2
export GRADIENT_ACCUMULATION_STEPS=32
# No more than 85% VRAM.
# A100(40GB) 34000, A40(48GB) 40000, A100(80GB) 70000
export MAX_MEMORY_MB=34000

49
tasks/speechless-tora-code-7b-v1.0/task_a100_40gx4.env Normal file
View File

@@ -0,0 +1,49 @@
# From speechless-code-mistral-orca-7b-v1.0
# -------------------- Model --------------------
export MODELS_ROOT_DIR=/opt/local/llm_models/huggingface.co
# FIXME
export BASE_MODEL_PATH=${MODELS_ROOT_DIR}/Open-Orca/Mistral-7B-OpenOrca
# export BASE_MODEL_PATH=${MODELS_ROOT_DIR}/mistralai/Mistral-7B-v0.1
# FIXME
export TEST_MODEL_PATH=${MODELS_ROOT_DIR}/speechlessai/speechless-code-mistral-orca-7b-v1.0
# export TEST_MODEL_PATH=${MODELS_ROOT_DIR}/speechlessai/speechless-code-mistral-7b-v1.0
# export TEST_MODEL_PATH=${MODELS_ROOT_DIR}/speechlessai/speechless-nl2sql-mistral-7b-v0.1
# export TEST_MODEL_PATH=${MODELS_ROOT_DIR}/speechlessai/speechless-mistral-7b-v0.1
# -------------------- Dataset --------------------
# FIXME
# export DATASET=/opt/local/datasets/jondurbin/airoboros-2.2/instructions-clean.jsonl
# export DATASET=/opt/local/datasets/Speechless/airoboros-orca-platypus-instructions.jsonl
# export DATASET=/opt/local/datasets/Speechless/speechless-spider.jsonl
export DATASET=/opt/local/datasets/Speechless/speechless-thoughts-200k.jsonl
# -------------------- Environment --------------------
export OUTPUT_DIR=./outputs
export TORCH_DISTRIBUTED_DEBUG=DETAIL
export RAY_memory_monitor_refresh_ms=0
# -------------------- Task --------------------
# FIXME
export TASK_NAME=$(basename ${TEST_MODEL_PATH})
export TASK_CHECKPOINT_DIR=${OUTPUT_DIR}/${TASK_NAME}
export WANDB_PROJECT=${TASK_NAME}
# -------------------- Task --------------------
export DATASET_FORMAT=airoboros
export NUM_TRAIN_EPOCHS=2
export EVAL_DATASET_SIZE=0.005
export LEARNING_RATE=2e-4
export LORA_R=64
export GROUP_BY_LENGTH=False
#export DEEPSPEED_STAGE2="--deepspeed deepspeed-stage2.json"
export NUM_GPUS=4
# 2 gpus: 40GB: 2 x 32, 80GB: 4 x 16
# 4 gpus: 40GB: 2 x 16, 80GB: 4 x 8
export PER_DEVICE_TRAIN_BATCH_SIZE=2
export GRADIENT_ACCUMULATION_STEPS=16
# No more than 85% VRAM.
# A100(40GB) 34000, A40(48GB) 40000, A100(80GB) 70000
export MAX_MEMORY_MB=34000

47
tasks/speechless-tora-code-7b-v1.0/task_a100_80gx2.env Normal file
View File

@@ -0,0 +1,47 @@
# From speechless-code-mistral-7b-v1.0
# -------------------- Model --------------------
export MODELS_ROOT_DIR=/opt/local/llm_models/huggingface.co
# FIXME
export BASE_MODEL_PATH=${MODELS_ROOT_DIR}/mistralai/Mistral-7B-v0.1
# FIXME
export TEST_MODEL_PATH=${MODELS_ROOT_DIR}/speechlessai/speechless-code-mistral-7b-v1.0
# export TEST_MODEL_PATH=${MODELS_ROOT_DIR}/speechlessai/speechless-nl2sql-mistral-7b-v0.1
# export TEST_MODEL_PATH=${MODELS_ROOT_DIR}/speechlessai/speechless-mistral-7b-v0.1
# -------------------- Dataset --------------------
# FIXME
# export DATASET=/opt/local/datasets/jondurbin/airoboros-2.2/instructions-clean.jsonl
# export DATASET=/opt/local/datasets/Speechless/airoboros-orca-platypus-instructions.jsonl
# export DATASET=/opt/local/datasets/Speechless/speechless-spider.jsonl
export DATASET=/opt/local/datasets/Speechless/speechless-thoughts-200k.jsonl
# -------------------- Environment --------------------
export OUTPUT_DIR=./outputs
export TORCH_DISTRIBUTED_DEBUG=DETAIL
export RAY_memory_monitor_refresh_ms=0
# -------------------- Task --------------------
# FIXME
export TASK_NAME=$(basename ${TEST_MODEL_PATH})
export TASK_CHECKPOINT_DIR=${OUTPUT_DIR}/${TASK_NAME}
export WANDB_PROJECT=${TASK_NAME}
# -------------------- Task --------------------
export DATASET_FORMAT=airoboros
export NUM_TRAIN_EPOCHS=2
export EVAL_DATASET_SIZE=0.005
export LEARNING_RATE=2e-4
export LORA_R=64
export GROUP_BY_LENGTH=False
#export DEEPSPEED_STAGE2="--deepspeed deepspeed-stage2.json"
export NUM_GPUS=2
# 2 gpus: 40GB: 2 x 32, 80GB: 4 x 16
# 4 gpus: 40GB: 2 x 16, 80GB: 4 x 8
export PER_DEVICE_TRAIN_BATCH_SIZE=2
export GRADIENT_ACCUMULATION_STEPS=32
# No more than 85% VRAM.
# A100(40GB) 34000, A40(48GB) 40000, A100(80GB) 70000
export MAX_MEMORY_MB=70000

47
tasks/speechless-tora-code-7b-v1.0/task_a40_48gx2.env Normal file
View File

@@ -0,0 +1,47 @@
# From speechless-code-mistral-7b-v1.0
# -------------------- Model --------------------
export MODELS_ROOT_DIR=/opt/local/llm_models/huggingface.co
# FIXME
export BASE_MODEL_PATH=${MODELS_ROOT_DIR}/mistralai/Mistral-7B-v0.1
# FIXME
export TEST_MODEL_PATH=${MODELS_ROOT_DIR}/speechlessai/speechless-code-mistral-7b-v1.0
# export TEST_MODEL_PATH=${MODELS_ROOT_DIR}/speechlessai/speechless-nl2sql-mistral-7b-v0.1
# export TEST_MODEL_PATH=${MODELS_ROOT_DIR}/speechlessai/speechless-mistral-7b-v0.1
# -------------------- Dataset --------------------
# FIXME
# export DATASET=/opt/local/datasets/jondurbin/airoboros-2.2/instructions-clean.jsonl
# export DATASET=/opt/local/datasets/Speechless/airoboros-orca-platypus-instructions.jsonl
# export DATASET=/opt/local/datasets/Speechless/speechless-spider.jsonl
export DATASET=/opt/local/datasets/Speechless/speechless-thoughts-200k.jsonl
# -------------------- Environment --------------------
export OUTPUT_DIR=./outputs
export TORCH_DISTRIBUTED_DEBUG=DETAIL
export RAY_memory_monitor_refresh_ms=0
# -------------------- Task --------------------
# FIXME
export TASK_NAME=$(basename ${TEST_MODEL_PATH})
export TASK_CHECKPOINT_DIR=${OUTPUT_DIR}/${TASK_NAME}
export WANDB_PROJECT=${TASK_NAME}
# -------------------- Task --------------------
export DATASET_FORMAT=airoboros
export NUM_TRAIN_EPOCHS=2
export EVAL_DATASET_SIZE=0.005
export LEARNING_RATE=2e-4
export LORA_R=64
export GROUP_BY_LENGTH=False
#export DEEPSPEED_STAGE2="--deepspeed deepspeed-stage2.json"
export NUM_GPUS=2
# 2 gpus: 40GB: 2 x 32, 80GB: 4 x 16
# 4 gpus: 40GB: 2 x 16, 80GB: 4 x 8
export PER_DEVICE_TRAIN_BATCH_SIZE=2
export GRADIENT_ACCUMULATION_STEPS=32
# No more than 85% VRAM.
# A100(40GB) 34000, A40(48GB) 40000, A100(80GB) 70000
export MAX_MEMORY_MB=40000

47
tasks/speechless-tora-code-7b-v1.0/task_a40_48gx4.env Normal file
View File

@@ -0,0 +1,47 @@
# From speechless-code-mistral-7b-v1.0
# -------------------- Model --------------------
export MODELS_ROOT_DIR=/opt/local/llm_models/huggingface.co
# FIXME
export BASE_MODEL_PATH=${MODELS_ROOT_DIR}/mistralai/Mistral-7B-v0.1
# FIXME
export TEST_MODEL_PATH=${MODELS_ROOT_DIR}/speechlessai/speechless-code-mistral-7b-v1.0
# export TEST_MODEL_PATH=${MODELS_ROOT_DIR}/speechlessai/speechless-nl2sql-mistral-7b-v0.1
# export TEST_MODEL_PATH=${MODELS_ROOT_DIR}/speechlessai/speechless-mistral-7b-v0.1
# -------------------- Dataset --------------------
# FIXME
# export DATASET=/opt/local/datasets/jondurbin/airoboros-2.2/instructions-clean.jsonl
# export DATASET=/opt/local/datasets/Speechless/airoboros-orca-platypus-instructions.jsonl
# export DATASET=/opt/local/datasets/Speechless/speechless-spider.jsonl
export DATASET=/opt/local/datasets/Speechless/speechless-thoughts-200k.jsonl
# -------------------- Environment --------------------
export OUTPUT_DIR=./outputs
export TORCH_DISTRIBUTED_DEBUG=DETAIL
export RAY_memory_monitor_refresh_ms=0
# -------------------- Task --------------------
# FIXME
export TASK_NAME=$(basename ${TEST_MODEL_PATH})
export TASK_CHECKPOINT_DIR=${OUTPUT_DIR}/${TASK_NAME}
export WANDB_PROJECT=${TASK_NAME}
# -------------------- Task --------------------
export DATASET_FORMAT=airoboros
export NUM_TRAIN_EPOCHS=2
export EVAL_DATASET_SIZE=0.005
export LEARNING_RATE=2e-4
export LORA_R=64
export GROUP_BY_LENGTH=False
#export DEEPSPEED_STAGE2="--deepspeed deepspeed-stage2.json"
export NUM_GPUS=4
# 2 gpus: 40GB: 2 x 32, 80GB: 4 x 16
# 4 gpus: 40GB: 2 x 16, 80GB: 4 x 8
export PER_DEVICE_TRAIN_BATCH_SIZE=2
export GRADIENT_ACCUMULATION_STEPS=16
# No more than 85% VRAM.
# A100(40GB) 34000, A40(48GB) 40000, A100(80GB) 70000
export MAX_MEMORY_MB=40000