merge_method: dare_ties
dtype: bfloat16
out_dtype: bfloat16

base_model: UmbrellaInc/W.Project-1B

models:
  - model: UmbrellaInc/Prototype-Virus-1B
    parameters:
      weight: 0.40
      density: 0.60
  - model: UmbrellaInc/PG67A-W-Serum-1B
    parameters:
      weight: 0.35
      density: 0.55


parameters:
  t: 0.60
  lambda: -0.70
  normalize: false
  rescale: true
  rescale_factor: 1.30
  memory_efficient: true
  low_cpu_mem_usage: true

layer_range:
  - value: [4, 22]

tie_word_embeddings: false
tie_output_embeddings: false