初始化项目,由ModelHub XC社区提供模型
Model: jsantillana/vectrayx-nano Source: Original Platform
This commit is contained in:
40
configs/base.json
Normal file
40
configs/base.json
Normal file
@@ -0,0 +1,40 @@
|
||||
{
|
||||
"model": {
|
||||
"vocab_size": 16384,
|
||||
"n_layers": 16,
|
||||
"n_heads": 16,
|
||||
"n_kv_heads": 4,
|
||||
"d_model": 1024,
|
||||
"d_ffn": 4096,
|
||||
"max_seq_len": 1024,
|
||||
"rope_theta": 10000.0,
|
||||
"rms_eps": 1e-6,
|
||||
"init_std": 0.02,
|
||||
"dropout": 0.0,
|
||||
"tie_embeddings": true,
|
||||
"qk_norm": true,
|
||||
"z_loss_coef": 1e-4
|
||||
},
|
||||
"tokenizer": {
|
||||
"vocab_size": 16384,
|
||||
"model_type": "bpe",
|
||||
"character_coverage": 1.0,
|
||||
"byte_fallback": true,
|
||||
"normalization": "nmt_nfkc",
|
||||
"split_digits": true,
|
||||
"split_by_unicode_script": true,
|
||||
"add_dummy_prefix": true,
|
||||
"user_defined_symbols": [
|
||||
"<|pad|>", "<|bos|>", "<|eos|>", "<|unk|>", "<|sep|>",
|
||||
"<|system|>", "<|user|>", "<|assistant|>", "<|end|>",
|
||||
"<|tool_call|>", "<|/tool_call|>", "<|tool_result|>", "<|/tool_result|>",
|
||||
"<|cve|>", "<|cvss|>", "<|ioc|>", "<|ttp|>", "<|mitre|>", "<|kev|>",
|
||||
"<|exploit|>", "<|patch|>", "<|alert|>",
|
||||
"<|critical|>", "<|high|>", "<|medium|>", "<|low|>", "<|info|>"
|
||||
],
|
||||
"balance": {
|
||||
"conversational_ratio": 0.5,
|
||||
"technical_ratio": 0.5
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user