初始化项目，由ModelHub XC社区提供模型
Model: build-small-hackathon/figment-finetuned-model-archive Source: Original Platform
This commit is contained in:
ModelHub XC
3
figment_sft_v5/figment-sft-v5-lora-merged-bf16.bf16.gguf
Normal file
3
figment_sft_v5/figment-sft-v5-lora-merged-bf16.bf16.gguf
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:c7f9b38d267c2ab2b791b613e0227ce3d057e61b57b568b16ca501f2e516379c
|
||||
size 7957646688
|
||||
@@ -0,0 +1,204 @@
|
||||
{% macro render_extra_keys(json_dict, handled_keys) %}
|
||||
{%- if json_dict is mapping %}
|
||||
{%- for json_key in json_dict if json_key not in handled_keys %}
|
||||
{%- if json_dict[json_key] is mapping or (json_dict[json_key] is sequence and json_dict[json_key] is not string) %}
|
||||
{{- '\n<' ~ json_key ~ '>' ~ (json_dict[json_key] | tojson | safe) ~ '</' ~ json_key ~ '>' }}
|
||||
{%- else %}
|
||||
{{-'\n<' ~ json_key ~ '>' ~ (json_dict[json_key] | string) ~ '</' ~ json_key ~ '>' }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{% endmacro %}
|
||||
{%- set enable_thinking = enable_thinking if enable_thinking is defined else True %}
|
||||
{%- set truncate_history_thinking = truncate_history_thinking if truncate_history_thinking is defined else True %}
|
||||
|
||||
{%- set ns = namespace(last_user_idx = -1) %}
|
||||
{%- set loop_messages = messages %}
|
||||
{%- for m in loop_messages %}
|
||||
{%- if m["role"] == "user" %}
|
||||
{%- set ns.last_user_idx = loop.index0 %}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
|
||||
{%- if messages[0]["role"] == "system" %}
|
||||
{%- set system_message = messages[0]["content"] %}
|
||||
{%- set loop_messages = messages[1:] %}
|
||||
{%- else %}
|
||||
{%- set system_message = "" %}
|
||||
{%- set loop_messages = messages %}
|
||||
{%- endif %}
|
||||
{%- if not tools is defined %}
|
||||
{%- set tools = [] %}
|
||||
{%- endif %}
|
||||
{# Recompute last_user_idx relative to loop_messages after handling system #}
|
||||
{%- set ns = namespace(last_user_idx = -1) %}
|
||||
{%- for m in loop_messages %}
|
||||
{%- if m["role"] == "user" %}
|
||||
{%- set ns.last_user_idx = loop.index0 %}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- if system_message is defined %}
|
||||
{{- "<|im_start|>system\n" + system_message }}
|
||||
{%- else %}
|
||||
{%- if tools is iterable and tools | length > 0 %}
|
||||
{{- "<|im_start|>system\n" }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- if tools is iterable and tools | length > 0 %}
|
||||
{%- if system_message is defined and system_message | length > 0 %}
|
||||
{{- "\n\n" }}
|
||||
{%- endif %}
|
||||
{{- "# Tools\n\nYou have access to the following functions:\n\n" }}
|
||||
{{- "<tools>" }}
|
||||
{%- for tool in tools %}
|
||||
{%- if tool.function is defined %}
|
||||
{%- set tool = tool.function %}
|
||||
{%- endif %}
|
||||
{{- "\n<function>\n<name>" ~ tool.name ~ "</name>" }}
|
||||
{%- if tool.description is defined %}
|
||||
{{- '\n<description>' ~ (tool.description | trim) ~ '</description>' }}
|
||||
{%- endif %}
|
||||
{{- '\n<parameters>' }}
|
||||
{%- if tool.parameters is defined and tool.parameters is mapping and tool.parameters.properties is defined and tool.parameters.properties is mapping %}
|
||||
{%- for param_name, param_fields in tool.parameters.properties|items %}
|
||||
{{- '\n<parameter>' }}
|
||||
{{- '\n<name>' ~ param_name ~ '</name>' }}
|
||||
{%- if param_fields.type is defined %}
|
||||
{{- '\n<type>' ~ (param_fields.type | string) ~ '</type>' }}
|
||||
{%- endif %}
|
||||
{%- if param_fields.description is defined %}
|
||||
{{- '\n<description>' ~ (param_fields.description | trim) ~ '</description>' }}
|
||||
{%- endif %}
|
||||
{%- if param_fields.enum is defined %}
|
||||
{{- '\n<enum>' ~ (param_fields.enum | tojson | safe) ~ '</enum>' }}
|
||||
{%- endif %}
|
||||
{%- set handled_keys = ['name', 'type', 'description', 'enum'] %}
|
||||
{{- render_extra_keys(param_fields, handled_keys) }}
|
||||
{{- '\n</parameter>' }}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{% set handled_keys = ['type', 'properties', 'required'] %}
|
||||
{{- render_extra_keys(tool.parameters, handled_keys) }}
|
||||
{%- if tool.parameters is defined and tool.parameters.required is defined %}
|
||||
{{- '\n<required>' ~ (tool.parameters.required | tojson | safe) ~ '</required>' }}
|
||||
{%- endif %}
|
||||
{{- '\n</parameters>' }}
|
||||
{%- set handled_keys = ['type', 'name', 'description', 'parameters'] %}
|
||||
{{- render_extra_keys(tool, handled_keys) }}
|
||||
{{- '\n</function>' }}
|
||||
{%- endfor %}
|
||||
{{- "\n</tools>" }}
|
||||
|
||||
{{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n</IMPORTANT>' }}
|
||||
{%- endif %}
|
||||
|
||||
|
||||
{%- if system_message is defined %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- else %}
|
||||
{%- if tools is iterable and tools | length > 0 %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
|
||||
{%- for message in loop_messages %}
|
||||
{%- if message.role == "assistant" %}
|
||||
{# Add reasoning content in to content field for unified processing below. #}
|
||||
{%- if message.reasoning_content is defined and message.reasoning_content is string and message.reasoning_content | trim | length > 0 %}
|
||||
{%- set content = "<think>\n" ~ message.reasoning_content ~ "\n</think>\n" ~ (message.content | default('', true)) %}
|
||||
{%- else %}
|
||||
{%- set content = message.content | default('', true) %}
|
||||
{%- if content is string -%}
|
||||
{# Allow downstream logic to to take care of broken thought, only handle coherent reasoning here. #}
|
||||
{%- if '<think>' not in content and '</think>' not in content -%}
|
||||
{%- set content = "<think></think>" ~ content -%}
|
||||
{%- endif -%}
|
||||
{%- else -%}
|
||||
{%- set content = content -%}
|
||||
{%- endif -%}
|
||||
{%- endif %}
|
||||
{%- if message.tool_calls is defined and message.tool_calls is iterable and message.tool_calls | length > 0 %}
|
||||
{# Assistant message has tool calls. #}
|
||||
{{- '<|im_start|>assistant\n' }}
|
||||
{%- set include_content = not (truncate_history_thinking and loop.index0 < ns.last_user_idx) %}
|
||||
{%- if content is string and content | trim | length > 0 %}
|
||||
{%- if include_content %}
|
||||
{{- (content | trim) ~ '\n' -}}
|
||||
{%- else %}
|
||||
{%- set c = (content | string) %}
|
||||
{%- if '</think>' in c %}
|
||||
{# Keep only content after the last closing think. Also generation prompt causes this. #}
|
||||
{%- set c = c.split('</think>')[-1] %}
|
||||
{%- elif '<think>' in c %}
|
||||
{# If <think> was opened but never closed, drop the trailing think segment #}
|
||||
{%- set c = c.split('<think>')[0] %}
|
||||
{%- endif %}
|
||||
{%- set c = "<think></think>" ~ c | trim %}
|
||||
{%- if c | length > 0 %}
|
||||
{{- c ~ '\n' -}}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- else %}
|
||||
{{- "<think></think>" -}}
|
||||
{%- endif %}
|
||||
{%- for tool_call in message.tool_calls %}
|
||||
{%- if tool_call.function is defined %}
|
||||
{%- set tool_call = tool_call.function %}
|
||||
{%- endif %}
|
||||
{{- '<tool_call>\n<function=' ~ tool_call.name ~ '>\n' -}}
|
||||
{%- if tool_call.arguments is defined %}
|
||||
{%- for args_name, args_value in tool_call.arguments|items %}
|
||||
{{- '<parameter=' ~ args_name ~ '>\n' -}}
|
||||
{%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}
|
||||
{{- args_value ~ '\n</parameter>\n' -}}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{{- '</function>\n</tool_call>\n' -}}
|
||||
{%- endfor %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- else %}
|
||||
{# Assistant message doesn't have tool calls. #}
|
||||
{%- if not (truncate_history_thinking and loop.index0 < ns.last_user_idx) %}
|
||||
{{- '<|im_start|>assistant\n' ~ (content | default('', true) | string | trim) ~ '<|im_end|>\n' }}
|
||||
{%- else %}
|
||||
{%- set c = (content | default('', true) | string) %}
|
||||
{%- if '<think>' in c and '</think>' in c %}
|
||||
{%- set c = "<think></think>" ~ c.split('</think>')[-1] %}
|
||||
{%- endif %}
|
||||
{%- set c = c | trim %}
|
||||
{%- if c | length > 0 %}
|
||||
{{- '<|im_start|>assistant\n' ~ c ~ '<|im_end|>\n' }}
|
||||
{%- else %}
|
||||
{{- '<|im_start|>assistant\n<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- elif message.role == "user" or message.role == "system" %}
|
||||
{{- '<|im_start|>' + message.role + '\n' }}
|
||||
{%- set content = message.content | string %}
|
||||
{{- content }}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- elif message.role == "tool" %}
|
||||
{%- if loop.previtem and loop.previtem.role != "tool" %}
|
||||
{{- '<|im_start|>user\n' }}
|
||||
{%- endif %}
|
||||
{{- '<tool_response>\n' }}
|
||||
{{- message.content }}
|
||||
{{- '\n</tool_response>\n' }}
|
||||
{%- if not loop.last and loop.nextitem.role != "tool" %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- elif loop.last %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- else %}
|
||||
{{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
|
||||
{%- if add_generation_prompt %}
|
||||
{%- if enable_thinking %}
|
||||
{{- '<|im_start|>assistant\n<think>\n' }}
|
||||
{%- else %}
|
||||
{{- '<|im_start|>assistant\n<think></think>' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
59
figment_sft_v5/figment-sft-v5-lora-merged-bf16/config.json
Normal file
59
figment_sft_v5/figment-sft-v5-lora-merged-bf16/config.json
Normal file
@@ -0,0 +1,59 @@
|
||||
{
|
||||
"architectures": [
|
||||
"NemotronHForCausalLM"
|
||||
],
|
||||
"attention_bias": false,
|
||||
"attention_dropout": 0.0,
|
||||
"attention_head_dim": 128,
|
||||
"auto_map": {
|
||||
"AutoConfig": "configuration_nemotron_h.NemotronHConfig",
|
||||
"AutoModelForCausalLM": "modeling_nemotron_h.NemotronHForCausalLM"
|
||||
},
|
||||
"bos_token_id": 1,
|
||||
"chunk_size": 256,
|
||||
"conv_kernel": 4,
|
||||
"dtype": "bfloat16",
|
||||
"eos_token_id": 2,
|
||||
"expand": 2,
|
||||
"head_dim": 128,
|
||||
"hidden_dropout": 0.0,
|
||||
"hidden_size": 3136,
|
||||
"hybrid_override_pattern": "M-M-M-MM-M-M*-M-M*-M-M-M*-M-M-MM*-MMM-M-M-",
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 12544,
|
||||
"layer_norm_epsilon": 1e-05,
|
||||
"mamba_head_dim": 80,
|
||||
"mamba_hidden_act": "silu",
|
||||
"mamba_num_heads": 96,
|
||||
"mamba_proj_bias": false,
|
||||
"max_position_embeddings": 262144,
|
||||
"mlp_bias": false,
|
||||
"mlp_hidden_act": "relu2",
|
||||
"model_type": "nemotron_h",
|
||||
"n_groups": 8,
|
||||
"num_attention_heads": 40,
|
||||
"num_hidden_layers": 42,
|
||||
"num_key_value_heads": 8,
|
||||
"num_logits_to_keep": 1,
|
||||
"pad_token_id": 0,
|
||||
"rescale_prenorm_residual": true,
|
||||
"residual_in_fp32": false,
|
||||
"rms_norm_eps": 1e-05,
|
||||
"sliding_window": null,
|
||||
"ssm_state_size": 128,
|
||||
"tie_word_embeddings": false,
|
||||
"time_step_floor": 0.0001,
|
||||
"time_step_limit": [
|
||||
0.0,
|
||||
Infinity
|
||||
],
|
||||
"time_step_max": 0.1,
|
||||
"time_step_min": 0.001,
|
||||
"time_step_rank": 256,
|
||||
"transformers_version": "4.57.6",
|
||||
"use_bias": false,
|
||||
"use_cache": true,
|
||||
"use_conv_bias": true,
|
||||
"use_mamba_kernels": true,
|
||||
"vocab_size": 131072
|
||||
}
|
||||
@@ -0,0 +1,243 @@
|
||||
# coding=utf-8
|
||||
# Copyright 2024 AI21 Labs Ltd. and the HuggingFace Inc. team. All rights reserved.
|
||||
# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""NemotronH model configuration"""
|
||||
|
||||
import re
|
||||
|
||||
from transformers.configuration_utils import PretrainedConfig
|
||||
from transformers.utils import logging
|
||||
|
||||
|
||||
logger = logging.get_logger(__name__)
|
||||
|
||||
|
||||
class NemotronHConfig(PretrainedConfig):
|
||||
r"""
|
||||
This is the configuration class to store the configuration of a [`NemotronHModel`]. It is used to instantiate a
|
||||
NemotronH model according to the specified arguments, defining the model architecture. Instantiating a configuration
|
||||
with the defaults will yield a similar configuration to that of the NemotronH-v0.1 model.
|
||||
|
||||
[todo](todo)
|
||||
|
||||
Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
|
||||
documentation from [`PretrainedConfig`] for more information.
|
||||
|
||||
|
||||
Args:
|
||||
vocab_size (`int`, *optional*, defaults to 131072):
|
||||
Vocabulary size of the NemotronH model. Defines the number of different tokens that can be represented by the
|
||||
`inputs_ids` passed when calling [`NemotronHModel`]
|
||||
tie_word_embeddings (`bool`, *optional*, defaults to `False`):
|
||||
Whether the model's input and output word embeddings should be tied. Note that this is only relevant if the
|
||||
model has a output word embedding layer.
|
||||
hidden_size (`int`, *optional*, defaults to 4096):
|
||||
Dimension of the hidden representations.
|
||||
intermediate_size (`int`, *optional*, defaults to 21504):
|
||||
Dimension of the MLP representations.
|
||||
num_hidden_layers (`int`, *optional*, defaults to 52):
|
||||
Number of hidden layers in the Transformer encoder.
|
||||
hybrid_override_pattern (`str`, *optional*, defaults to `"M-M-M-M*-M-M-M-M-M*-M-M-M-M-M*-M-M-M-M-M*-M-M-M-M-M-"`):
|
||||
The pattern of the hybrid model. The pattern is a string of characters where each character represents M: Mamba2, *: Attention, -: MLP
|
||||
num_attention_heads (`int`, *optional*, defaults to 32):
|
||||
Number of attention heads for each attention layer in the Transformer encoder.
|
||||
attention_head_dim (`int`, *optional*, defaults to 128):
|
||||
Dimension of each attention head.
|
||||
num_key_value_heads (`int`, *optional*, defaults to 8):
|
||||
This is the number of key_value heads that should be used to implement Grouped Query Attention. If
|
||||
`num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
|
||||
`num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used.
|
||||
mlp_hidden_act (`str`, *optional*, defaults to "relu2"):
|
||||
The non-linear activation function in the MLP layers.
|
||||
attention_bias (`bool`, *optional*, defaults to `False`):
|
||||
Whether to use bias in attention layers.
|
||||
mlp_bias (`bool`, *optional*, defaults to `False`):
|
||||
Whether to use bias in MLP layers.
|
||||
use_bias (`bool`, *optional*, defaults to `False`):
|
||||
Whether to use bias in the model.
|
||||
initializer_range (`float`, *optional*, defaults to 0.02):
|
||||
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
|
||||
layer_norm_epsilon (`float`, *optional*, defaults to 1e-5):
|
||||
The epsilon used by the layer normalization layers.
|
||||
residual_in_fp32 (`bool`, *optional*, defaults to `False`):
|
||||
Whether or not residuals should be in `float32`. If set to `False` residuals will keep the same `dtype` as the rest of the model.
|
||||
use_cache (`bool`, *optional*, defaults to `True`):
|
||||
Whether or not the model should return the last key/values attentions (not used by all models). Only
|
||||
relevant if `config.is_decoder=True`.
|
||||
num_logits_to_keep (`int` or `None`, *optional*, defaults to 1):
|
||||
Number of prompt logits to calculate during generation. If `None`, all logits will be calculated. If an
|
||||
integer value, only last `num_logits_to_keep` logits will be calculated.
|
||||
pad_token_id (`int`, *optional*, defaults to 0):
|
||||
The id of the padding token.
|
||||
bos_token_id (`int`, *optional*, defaults to 1):
|
||||
The id of the "beginning-of-sequence" token.
|
||||
eos_token_id (`int`, *optional*, defaults to 2):
|
||||
The id of the "end-of-sequence" token.
|
||||
sliding_window (`int`, *optional*, defaults to None):
|
||||
Sliding window attention window size.
|
||||
max_position_embeddings (`int`, *optional*, defaults to 4096):
|
||||
The maximum sequence length that this model might ever be used with.
|
||||
attention_dropout (`float`, *optional*, defaults to 0.0):
|
||||
The dropout ratio for the attention probabilities.
|
||||
hidden_dropout (`float`, *optional*, defaults to 0.0):
|
||||
The dropout ratio for the hidden states.
|
||||
use_mamba_kernels (`bool`, *optional*, defaults to `True`):
|
||||
Flag indicating whether or not to use the fast mamba kernels. These are available only if `mamba-ssm` and
|
||||
`causal-conv1d` are installed, and the mamba modules are running on a CUDA device.
|
||||
ssm_state_size (`int`, *optional*, defaults to 128):
|
||||
The dimension of the mamba state space latents.
|
||||
mamba_num_heads (`int`, *optional*, defaults to 128):
|
||||
Number of heads in Mamba layers.
|
||||
mamba_n_groups (`int`, *optional*, defaults to 8):
|
||||
Number of groups in Mamba layers.
|
||||
mamba_head_dim (`int`, *optional*, defaults to 64):
|
||||
Dimension of each Mamba head.
|
||||
mamba_d_conv (`int`, *optional*, defaults to 4):
|
||||
The size of the mamba convolution kernel.
|
||||
mamba_expand (`int`, *optional*, defaults to 2):
|
||||
Expanding factor used to determine the mamba intermediate size.
|
||||
mamba_hidden_act (`str`, *optional*, defaults to "silu"):
|
||||
The non-linear activation function in the Mamba layers.
|
||||
mamba_dt_min (`float`, *optional*, defaults to 0.001):
|
||||
Minimum value for the time step in Mamba.
|
||||
mamba_dt_max (`float`, *optional*, defaults to 0.1):
|
||||
Maximum value for the time step in Mamba.
|
||||
mamba_dt_limit (`tuple`, *optional*, defaults to (0.0, float("inf"))):
|
||||
Limits for the time step in Mamba.
|
||||
mamba_dt_init_floor (`float`, *optional*, defaults to 1e-4):
|
||||
Floor value for time step initialization in Mamba.
|
||||
mamba_conv_bias (`bool`, *optional*, defaults to `True`):
|
||||
Whether to use bias in the convolution layer of the mamba mixer block.
|
||||
mamba_proj_bias (`bool`, *optional*, defaults to `False`):
|
||||
Whether to use bias in the input and output projections of the mamba mixer block.
|
||||
mamba_chunk_size (`int`, *optional*, defaults to 256):
|
||||
Size of chunks for Mamba processing.
|
||||
rescale_prenorm_residual (`bool`, *optional*, defaults to `True`):
|
||||
Whether to rescale the pre-normalization residual connections.
|
||||
"""
|
||||
|
||||
model_type = "nemotron_h"
|
||||
keys_to_ignore_at_inference = ["past_key_values"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vocab_size=131072,
|
||||
tie_word_embeddings=False,
|
||||
hidden_size=4096,
|
||||
intermediate_size=21504,
|
||||
num_hidden_layers=52,
|
||||
hybrid_override_pattern="M-M-M-M*-M-M-M-M-M*-M-M-M-M-M*-M-M-M-M-M*-M-M-M-M-M-",
|
||||
num_attention_heads=32,
|
||||
attention_head_dim=128,
|
||||
num_key_value_heads=8, # nemo: num_query_groups
|
||||
mlp_hidden_act="relu2",
|
||||
attention_bias=False,
|
||||
mlp_bias=False,
|
||||
use_bias=False,
|
||||
initializer_range=0.02, # nemo: init_method_std
|
||||
layer_norm_epsilon=1e-5, # nemo: layernorm_epsilon
|
||||
residual_in_fp32=False, # Megatron Core default value
|
||||
use_cache=True,
|
||||
num_logits_to_keep=1,
|
||||
pad_token_id=0,
|
||||
bos_token_id=1,
|
||||
eos_token_id=2,
|
||||
sliding_window=None,
|
||||
max_position_embeddings=4096,
|
||||
attention_dropout=0.0,
|
||||
hidden_dropout=0.0, # * ADDED
|
||||
use_mamba_kernels=True,
|
||||
ssm_state_size=128, # mamba_state_size
|
||||
mamba_num_heads=128,
|
||||
mamba_n_groups=8, # nemo: mamba_ssm_ngroups = num_heads
|
||||
mamba_head_dim=64,
|
||||
mamba_d_conv=4,
|
||||
mamba_expand=2,
|
||||
mamba_hidden_act="silu",
|
||||
mamba_dt_min=0.001,
|
||||
mamba_dt_max=0.1,
|
||||
mamba_dt_limit=(0.0, float("inf")),
|
||||
mamba_dt_init_floor=1e-4,
|
||||
mamba_conv_bias=True,
|
||||
mamba_proj_bias=False,
|
||||
mamba_chunk_size=256,
|
||||
rescale_prenorm_residual=True,
|
||||
**kwargs,
|
||||
):
|
||||
self.vocab_size = vocab_size
|
||||
self.tie_word_embeddings = tie_word_embeddings
|
||||
self.hidden_size = hidden_size
|
||||
self.intermediate_size = intermediate_size
|
||||
self.num_hidden_layers = num_hidden_layers
|
||||
self.hybrid_override_pattern = hybrid_override_pattern
|
||||
self.num_attention_heads = num_attention_heads
|
||||
self.attention_head_dim = attention_head_dim
|
||||
self.sliding_window = sliding_window
|
||||
self.max_position_embeddings = max_position_embeddings
|
||||
self.attention_dropout = attention_dropout
|
||||
self.hidden_dropout = hidden_dropout
|
||||
|
||||
# Validate hybrid_override_pattern
|
||||
# M: Mamba2, *: Attention, -: MLP
|
||||
assert len(self.hybrid_override_pattern) == self.num_hidden_layers, "hybrid_override_pattern must have the same length as num_hidden_layers"
|
||||
assert re.match(r"^[*-M]+$", self.hybrid_override_pattern), "hybrid_override_pattern must only contain characters 'M', '*', or '-'"
|
||||
|
||||
# for backward compatibility
|
||||
if num_key_value_heads is None:
|
||||
num_key_value_heads = num_attention_heads
|
||||
|
||||
self.num_key_value_heads = num_key_value_heads
|
||||
self.mlp_hidden_act = mlp_hidden_act
|
||||
self.attention_bias = attention_bias
|
||||
self.mlp_bias = mlp_bias
|
||||
self.use_bias = use_bias
|
||||
self.initializer_range = initializer_range
|
||||
self.layer_norm_epsilon = layer_norm_epsilon
|
||||
self.residual_in_fp32 = residual_in_fp32
|
||||
|
||||
self.use_cache = use_cache
|
||||
self.num_logits_to_keep = num_logits_to_keep
|
||||
|
||||
self.use_mamba_kernels = use_mamba_kernels
|
||||
self.n_groups = mamba_n_groups
|
||||
self.mamba_head_dim = mamba_head_dim
|
||||
self.ssm_state_size = ssm_state_size
|
||||
self.mamba_num_heads = mamba_num_heads
|
||||
self.conv_kernel = mamba_d_conv
|
||||
self.expand = mamba_expand
|
||||
self.mamba_hidden_act = mamba_hidden_act
|
||||
self.time_step_min = mamba_dt_min
|
||||
self.time_step_max = mamba_dt_max
|
||||
self.time_step_limit = mamba_dt_limit
|
||||
self.time_step_floor = mamba_dt_init_floor
|
||||
self.use_conv_bias = mamba_conv_bias
|
||||
self.mamba_proj_bias = mamba_proj_bias
|
||||
self.chunk_size = mamba_chunk_size
|
||||
self.rescale_prenorm_residual = rescale_prenorm_residual
|
||||
|
||||
super().__init__(
|
||||
pad_token_id=pad_token_id,
|
||||
bos_token_id=bos_token_id,
|
||||
eos_token_id=eos_token_id,
|
||||
tie_word_embeddings=tie_word_embeddings,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
@property
|
||||
def layers_block_type(self):
|
||||
return [
|
||||
"mamba" if self.hybrid_override_pattern[i] == "M" else
|
||||
"attention" if self.hybrid_override_pattern[i] == "*" else "mlp"
|
||||
for i in range(self.num_hidden_layers)]
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"adapter_dir": "/checkpoints/figment_sft_v5/figment-sft-v5-lora",
|
||||
"base_model_id": "nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16",
|
||||
"dtype": "bfloat16",
|
||||
"files": [
|
||||
"chat_template.jinja",
|
||||
"config.json",
|
||||
"configuration_nemotron_h.py",
|
||||
"generation_config.json",
|
||||
"model-00001-of-00002.safetensors",
|
||||
"model-00002-of-00002.safetensors",
|
||||
"model.safetensors.index.json",
|
||||
"modeling_nemotron_h.py",
|
||||
"special_tokens_map.json",
|
||||
"tokenizer.json",
|
||||
"tokenizer_config.json"
|
||||
],
|
||||
"merge_method": "peft.merge_and_unload(safe_merge=True)",
|
||||
"output_dir": "/checkpoints/figment_sft_v5/figment-sft-v5-lora-merged-bf16"
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"_from_model_config": true,
|
||||
"bos_token_id": 1,
|
||||
"eos_token_id": 2,
|
||||
"pad_token_id": 0,
|
||||
"temperature": null,
|
||||
"top_p": null,
|
||||
"transformers_version": "4.57.6"
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:c52fa4ab8c9bd51ae8485127e5c578aaed686ac32ea57f913953e31919533a87
|
||||
size 3973314912
|
||||
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:f0a8ed1e11d6fd855bc0b41c93cc3d9b4eb09908f58dfda35f0f74ead5d3058c
|
||||
size 3973827728
|
||||
@@ -0,0 +1,271 @@
|
||||
{
|
||||
"metadata": {
|
||||
"total_parameters": 3973556832,
|
||||
"total_size": 7947113664
|
||||
},
|
||||
"weight_map": {
|
||||
"backbone.embeddings.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.0.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.1.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.1.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.1.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.10.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.10.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.10.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.11.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.12.mixer.k_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.12.mixer.o_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.12.mixer.q_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.12.mixer.v_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.12.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.13.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.13.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.13.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.14.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.15.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.15.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.15.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.16.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.17.mixer.k_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.17.mixer.o_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.17.mixer.q_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.17.mixer.v_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.17.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.18.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.18.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.18.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.19.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.2.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.20.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.20.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.20.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.21.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.21.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.21.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.21.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.21.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.21.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.21.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.21.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.21.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.22.mixer.down_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.22.mixer.up_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.22.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.23.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.24.mixer.k_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.24.mixer.o_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.24.mixer.q_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.24.mixer.v_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.24.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.25.mixer.down_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.25.mixer.up_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.25.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.26.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.27.mixer.down_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.27.mixer.up_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.27.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.28.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.29.mixer.down_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.29.mixer.up_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.29.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.3.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.3.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.3.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.30.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.30.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.30.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.30.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.30.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.30.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.30.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.30.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.30.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.31.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.32.mixer.k_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.32.mixer.o_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.32.mixer.q_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.32.mixer.v_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.32.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.33.mixer.down_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.33.mixer.up_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.33.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.34.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.35.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.36.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.37.mixer.down_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.37.mixer.up_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.37.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.38.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.39.mixer.down_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.39.mixer.up_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.39.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.4.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.4.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.4.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.4.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.4.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.4.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.4.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.4.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.4.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.40.mixer.A_log": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.40.mixer.D": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.40.mixer.conv1d.bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.40.mixer.conv1d.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.40.mixer.dt_bias": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.40.mixer.in_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.40.mixer.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.40.mixer.out_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.40.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.41.mixer.down_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.41.mixer.up_proj.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.41.norm.weight": "model-00002-of-00002.safetensors",
|
||||
"backbone.layers.5.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.5.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.5.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.6.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.7.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.8.mixer.down_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.8.mixer.up_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.8.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.mixer.A_log": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.mixer.D": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.mixer.conv1d.bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.mixer.conv1d.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.mixer.dt_bias": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.mixer.in_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.mixer.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.mixer.out_proj.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.layers.9.norm.weight": "model-00001-of-00002.safetensors",
|
||||
"backbone.norm_f.weight": "model-00002-of-00002.safetensors",
|
||||
"lm_head.weight": "model-00002-of-00002.safetensors"
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"bos_token": {
|
||||
"content": "<s>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"eos_token": {
|
||||
"content": "<|im_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"pad_token": {
|
||||
"content": "<|im_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"unk_token": {
|
||||
"content": "<unk>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:623c34567aebb18582765289fbe23d901c62704d6518d71866e0e58db892b5b7
|
||||
size 17077484
|
||||
8019
figment_sft_v5/figment-sft-v5-lora-merged-bf16/tokenizer_config.json
Normal file
8019
figment_sft_v5/figment-sft-v5-lora-merged-bf16/tokenizer_config.json
Normal file
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user