habanoz/TinyLlama-1.1B-intermediate-step-715k-1.5T-lr-5-1epch-airoboros3.1-1k-instruct-V1
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

Update README.md

Browse Source
This commit is contained in:
Huseyin ABANOZ
2023-11-22 16:42:36 +00:00
committed by huggingface-web
parent 6a4a7ba9da
commit 2b961bacab
1 changed files with 6 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
README.md
View File

@@ -25,13 +25,13 @@ accelerate launch $BASE_DIR/qlora/train.py \
--num_train_epochs 1 \ --num_train_epochs 1 \
--logging_steps 1 \ --logging_steps 1 \
--save_strategy steps \ --save_strategy steps \
--save_steps 120 \ --save_steps 75 \
--save_total_limit 2 \ --save_total_limit 2 \
--data_seed 11422 \ --data_seed 11422 \
--evaluation_strategy steps \ --evaluation_strategy steps \
--per_device_eval_batch_size 4 \ --per_device_eval_batch_size 4 \
--eval_dataset_size 0.01 \ --eval_dataset_size 0.01 \
--eval_steps 120 \ --eval_steps 75 \
--max_new_tokens 1024 \ --max_new_tokens 1024 \
--dataloader_num_workers 3 \ --dataloader_num_workers 3 \
--logging_strategy steps \ --logging_strategy steps \
@@ -47,16 +47,15 @@ accelerate launch $BASE_DIR/qlora/train.py \
--dataset habanoz/airoboros-3.1-no-mathjson-max-1k \ --dataset habanoz/airoboros-3.1-no-mathjson-max-1k \
--dataset_format airoboros_chat \ --dataset_format airoboros_chat \
--model_max_len 1024 \ --model_max_len 1024 \
--per_device_train_batch_size 1 \ --per_device_train_batch_size 4 \
--gradient_accumulation_steps 16 \ --gradient_accumulation_steps 4 \
--learning_rate 1e-5 \ --learning_rate 1e-5 \
--adam_beta2 0.999 \ --adam_beta2 0.999 \
--max_grad_norm 0.3 \ --max_grad_norm 0.3 \
--lora_dropout 0.0 \ --lora_dropout 0.0 \
--weight_decay 0.0 \ --weight_decay 0.0 \
--seed 11422 \ --seed 11422 \
--gradient_checkpointing False \ --gradient_checkpointing \
--use_flash_attention_2 \ --use_flash_attention_2 \
--ddp_find_unused_parameters False \ --ddp_find_unused_parameters False
--trust_remote_code True
``` ```