pretrain:
  shuffle_buffer: 10000
  seed: 1337
  allow_synthetic_fallback: false
  max_doc_tokens: 16384
  max_chars_per_doc: 120000
  sources:
    - name: fineweb_edu
      path: HuggingFaceFW/fineweb-edu
      config: sample-10BT
      split: train
      text_field: text
      weight: 0.0
      streaming: true
      fail_on_error: false
    - name: cosmopedia
      path: HuggingFaceTB/cosmopedia
      config: web_samples_v2
      split: train
      text_field: text
      weight: 0.0
      streaming: true
      fail_on_error: false
    - name: wikipedia
      path: wikimedia/wikipedia
      config: 20231101.en
      split: train
      text_field: text
      weight: 0.0
      streaming: true
      fail_on_error: false
    - name: tinystories
      path: roneneldan/TinyStories
      split: train
      text_field: text
      weight: 0.75
      streaming: true
      fail_on_error: false
    - name: permissive_code
      path: nuprl/stack-dedup-python-testgen-starcoder-filter-v2
      split: train
      text_field: content
      weight: 0.25
      streaming: true
      fail_on_error: false

validation:
  sources:
    - name: wikitext103
      path: Salesforce/wikitext
      config: wikitext-103-raw-v1
      split: validation
      text_field: text
      weight: 1.0
      streaming: true
      fail_on_error: false

sft:
  shuffle_buffer: 5000
  seed: 2025
  assistant_only_loss: true
  allow_synthetic_fallback: true
  sources:
    - name: openhermes
      path: teknium/OpenHermes-2.5
      split: train
      weight: 0.45
      streaming: false
      fail_on_error: false
    - name: ultrachat
      path: HuggingFaceH4/ultrachat_200k
      split: train_sft
      weight: 0.35
      streaming: false
      fail_on_error: false
    - name: alpaca
      path: tatsu-lab/alpaca
      split: train
      weight: 0.15
      streaming: false
      fail_on_error: false
    - name: safety_small
      synthetic: safety_refusals
      weight: 0.05
      fail_on_error: false