71 lines
1.9 KiB
Markdown
71 lines
1.9 KiB
Markdown
|
# Promt Format
|
|||
|
|
```
|
|||
|
|
alpaca_prompt = """Sen bir doktorsun. Soruları buna göre cevapla.
|
|||
|
|
### <|reserved_special_token_0|>:
|
|||
|
|
{}
|
|||
|
|
|
|||
|
|
### <|reserved_special_token_1|>:
|
|||
|
|
{}"""
|
|||
|
|
```
|
|||
|
|
|
|||
|
|
# Training args
|
|||
|
|
```
|
|||
|
|
batch_size = 128
|
|||
|
|
gradient_accumulation_steps = 32
|
|||
|
|
num_train_epochs = 2
|
|||
|
|
per_device_batch_size = int(batch_size / gradient_accumulation_steps)
|
|||
|
|
training_args = TrainingArguments(
|
|||
|
|
per_device_train_batch_size = per_device_batch_size,
|
|||
|
|
per_device_eval_batch_size = per_device_batch_size,
|
|||
|
|
gradient_accumulation_steps = gradient_accumulation_steps,
|
|||
|
|
save_total_limit = 1,
|
|||
|
|
warmup_steps = int(2000 / batch_size),
|
|||
|
|
num_train_epochs = num_train_epochs,
|
|||
|
|
learning_rate = 1e-4,
|
|||
|
|
fp16 = not is_bfloat16_supported(),
|
|||
|
|
bf16 = is_bfloat16_supported(),
|
|||
|
|
optim = "adamw_8bit",
|
|||
|
|
weight_decay = 0.01,
|
|||
|
|
lr_scheduler_type = "linear",
|
|||
|
|
seed = 3407,
|
|||
|
|
output_dir = output_dir,
|
|||
|
|
save_strategy = "steps",
|
|||
|
|
eval_strategy = "steps",
|
|||
|
|
logging_strategy = "steps",
|
|||
|
|
save_steps = int(5000 / batch_size * num_train_epochs),
|
|||
|
|
eval_steps = int(28900 / batch_size * num_train_epochs),
|
|||
|
|
logging_steps = int(28900 / batch_size * num_train_epochs),
|
|||
|
|
)
|
|||
|
|
```
|
|||
|
|
|
|||
|
|
# Trainer args
|
|||
|
|
|
|||
|
|
```
|
|||
|
|
max_seq_length = 8192
|
|||
|
|
trainer = SFTTrainer(
|
|||
|
|
model = model,
|
|||
|
|
tokenizer = tokenizer,
|
|||
|
|
train_dataset = train_dataset,
|
|||
|
|
eval_dataset = eval_dataset,
|
|||
|
|
dataset_text_field = "text",
|
|||
|
|
max_seq_length = max_seq_length,
|
|||
|
|
dataset_num_proc = 1,
|
|||
|
|
packing = False, # Can make training 5x faster for short sequences.
|
|||
|
|
args = training_args
|
|||
|
|
)
|
|||
|
|
```
|
|||
|
|
|
|||
|
|
# From pretrained args
|
|||
|
|
|
|||
|
|
```
|
|||
|
|
from unsloth import FastLanguageModel
|
|||
|
|
dtype = None
|
|||
|
|
load_in_4bit = False
|
|||
|
|
|
|||
|
|
model, tokenizer = FastLanguageModel.from_pretrained(
|
|||
|
|
model_name = output_dir,
|
|||
|
|
max_seq_length = max_seq_length,
|
|||
|
|
dtype = dtype,
|
|||
|
|
load_in_4bit = load_in_4bit,
|
|||
|
|
)
|
|||
|
|
```
|