51 lines
1.5 KiB
YAML
51 lines
1.5 KiB
YAML
|
# BEGIN GENERAL GGUF METADATA
|
||
|
|
id: Llama-3.2-3B-Instruct # Model ID unique between models (author / quantization)
|
||
|
|
model: Llama-3.2-3B-Instruct # Model ID which is used for request construct - should be unique between models (author / quantization)
|
||
|
|
name: Llama-3.2-3B-Instruct # metadata.general.name
|
||
|
|
version: 2 # metadata.version
|
||
|
|
|
||
|
|
# END GENERAL GGUF METADATA
|
||
|
|
|
||
|
|
# BEGIN INFERENCE PARAMETERS
|
||
|
|
# BEGIN REQUIRED
|
||
|
|
stop: # tokenizer.ggml.eos_token_id
|
||
|
|
- <|eot_id|>
|
||
|
|
# END REQUIRED
|
||
|
|
|
||
|
|
# BEGIN OPTIONAL
|
||
|
|
stream: true # Default true?
|
||
|
|
top_p: 0.9 # Ranges: 0 to 1
|
||
|
|
temperature: 0.7 # Ranges: 0 to 1
|
||
|
|
frequency_penalty: 0 # Ranges: 0 to 1
|
||
|
|
presence_penalty: 0 # Ranges: 0 to 1
|
||
|
|
max_tokens: 4096 # Should be default to context length
|
||
|
|
seed: -1
|
||
|
|
dynatemp_range: 0
|
||
|
|
dynatemp_exponent: 1
|
||
|
|
top_k: 40
|
||
|
|
min_p: 0.05
|
||
|
|
tfs_z: 1
|
||
|
|
typ_p: 1
|
||
|
|
repeat_last_n: 64
|
||
|
|
repeat_penalty: 1
|
||
|
|
mirostat: false
|
||
|
|
mirostat_tau: 5
|
||
|
|
mirostat_eta: 0.100000001
|
||
|
|
penalize_nl: false
|
||
|
|
ignore_eos: false
|
||
|
|
n_probs: 0
|
||
|
|
min_keep: 0
|
||
|
|
# END OPTIONAL
|
||
|
|
# END INFERENCE PARAMETERS
|
||
|
|
|
||
|
|
# BEGIN MODEL LOAD PARAMETERS
|
||
|
|
# BEGIN REQUIRED
|
||
|
|
engine: llama-cpp # engine to run model
|
||
|
|
prompt_template: "<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\n{system_message}<|eot_id|><|start_header_id|>user<|end_header_id|>\n\n{prompt}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n"
|
||
|
|
# END REQUIRED
|
||
|
|
|
||
|
|
# BEGIN OPTIONAL
|
||
|
|
ctx_len: 4096 # llama.context_length | 0 or undefined = loaded from model
|
||
|
|
ngl: 29 # Undefined = loaded from model
|
||
|
|
# END OPTIONAL
|
||
|
|
# END MODEL LOAD PARAMETERS
|