86 lines
1.9 KiB
YAML
86 lines
1.9 KiB
YAML
|
pretrain:
|
||
|
|
shuffle_buffer: 10000
|
||
|
|
seed: 1337
|
||
|
|
allow_synthetic_fallback: false
|
||
|
|
max_doc_tokens: 16384
|
||
|
|
max_chars_per_doc: 120000
|
||
|
|
sources:
|
||
|
|
- name: fineweb_edu
|
||
|
|
path: HuggingFaceFW/fineweb-edu
|
||
|
|
config: sample-10BT
|
||
|
|
split: train
|
||
|
|
text_field: text
|
||
|
|
weight: 0.0
|
||
|
|
streaming: true
|
||
|
|
fail_on_error: false
|
||
|
|
- name: cosmopedia
|
||
|
|
path: HuggingFaceTB/cosmopedia
|
||
|
|
config: web_samples_v2
|
||
|
|
split: train
|
||
|
|
text_field: text
|
||
|
|
weight: 0.0
|
||
|
|
streaming: true
|
||
|
|
fail_on_error: false
|
||
|
|
- name: wikipedia
|
||
|
|
path: wikimedia/wikipedia
|
||
|
|
config: 20231101.en
|
||
|
|
split: train
|
||
|
|
text_field: text
|
||
|
|
weight: 0.0
|
||
|
|
streaming: true
|
||
|
|
fail_on_error: false
|
||
|
|
- name: tinystories
|
||
|
|
path: roneneldan/TinyStories
|
||
|
|
split: train
|
||
|
|
text_field: text
|
||
|
|
weight: 0.75
|
||
|
|
streaming: true
|
||
|
|
fail_on_error: false
|
||
|
|
- name: permissive_code
|
||
|
|
path: nuprl/stack-dedup-python-testgen-starcoder-filter-v2
|
||
|
|
split: train
|
||
|
|
text_field: content
|
||
|
|
weight: 0.25
|
||
|
|
streaming: true
|
||
|
|
fail_on_error: false
|
||
|
|
|
||
|
|
validation:
|
||
|
|
sources:
|
||
|
|
- name: wikitext103
|
||
|
|
path: Salesforce/wikitext
|
||
|
|
config: wikitext-103-raw-v1
|
||
|
|
split: validation
|
||
|
|
text_field: text
|
||
|
|
weight: 1.0
|
||
|
|
streaming: true
|
||
|
|
fail_on_error: false
|
||
|
|
|
||
|
|
sft:
|
||
|
|
shuffle_buffer: 5000
|
||
|
|
seed: 2025
|
||
|
|
assistant_only_loss: true
|
||
|
|
allow_synthetic_fallback: true
|
||
|
|
sources:
|
||
|
|
- name: openhermes
|
||
|
|
path: teknium/OpenHermes-2.5
|
||
|
|
split: train
|
||
|
|
weight: 0.45
|
||
|
|
streaming: false
|
||
|
|
fail_on_error: false
|
||
|
|
- name: ultrachat
|
||
|
|
path: HuggingFaceH4/ultrachat_200k
|
||
|
|
split: train_sft
|
||
|
|
weight: 0.35
|
||
|
|
streaming: false
|
||
|
|
fail_on_error: false
|
||
|
|
- name: alpaca
|
||
|
|
path: tatsu-lab/alpaca
|
||
|
|
split: train
|
||
|
|
weight: 0.15
|
||
|
|
streaming: false
|
||
|
|
fail_on_error: false
|
||
|
|
- name: safety_small
|
||
|
|
synthetic: safety_refusals
|
||
|
|
weight: 0.05
|
||
|
|
fail_on_error: false
|