21 lines
552 B
Bash
21 lines
552 B
Bash
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
|
|
MODEL_ARGS=(
|
|
--swiglu
|
|
--num-layers 36
|
|
--hidden-size 4096
|
|
--ffn-hidden-size 12288
|
|
--num-attention-heads 32
|
|
--group-query-attention
|
|
--num-query-groups 8
|
|
--use-rotary-position-embeddings
|
|
--disable-bias-linear
|
|
--normalization "RMSNorm"
|
|
--norm-epsilon 1e-6
|
|
--rotary-base 1000000
|
|
--vocab-size 151936
|
|
--kv-channels 128
|
|
--qk-layernorm
|
|
--untie-embeddings-and-output-weights
|
|
) |