33 lines
539 B
YAML
33 lines
539 B
YAML
|
|
merge_method: dare_ties
|
|
dtype: bfloat16
|
|
out_dtype: bfloat16
|
|
|
|
base_model: UmbrellaInc/W.Project-1B
|
|
|
|
models:
|
|
- model: UmbrellaInc/Prototype-Virus-1B
|
|
parameters:
|
|
weight: 0.40
|
|
density: 0.60
|
|
- model: UmbrellaInc/PG67A-W-Serum-1B
|
|
parameters:
|
|
weight: 0.35
|
|
density: 0.55
|
|
|
|
|
|
parameters:
|
|
t: 0.60
|
|
lambda: -0.70
|
|
normalize: false
|
|
rescale: true
|
|
rescale_factor: 1.30
|
|
memory_efficient: true
|
|
low_cpu_mem_usage: true
|
|
|
|
layer_range:
|
|
- value: [4, 22]
|
|
|
|
tie_word_embeddings: false
|
|
tie_output_embeddings: false
|