42 lines
1.3 KiB
Python
42 lines
1.3 KiB
Python
"""
|
|
AILO Configuration for HuggingFace Transformers
|
|
"""
|
|
|
|
from transformers import PretrainedConfig
|
|
|
|
|
|
class AILOConfig(PretrainedConfig):
|
|
"""Configuration class for AILO model."""
|
|
|
|
model_type = "ailo"
|
|
|
|
def __init__(
|
|
self,
|
|
vocab_size: int = 50257,
|
|
hidden_size: int = 768,
|
|
num_hidden_layers: int = 12,
|
|
num_attention_heads: int = 12,
|
|
intermediate_size: int = 3072,
|
|
max_position_embeddings: int = 512,
|
|
hidden_dropout_prob: float = 0.1,
|
|
attention_probs_dropout_prob: float = 0.1,
|
|
bos_token_id: int = 50256,
|
|
eos_token_id: int = 50256,
|
|
pad_token_id: int = 50256,
|
|
**kwargs
|
|
):
|
|
super().__init__(
|
|
bos_token_id=bos_token_id,
|
|
eos_token_id=eos_token_id,
|
|
pad_token_id=pad_token_id,
|
|
**kwargs
|
|
)
|
|
self.vocab_size = vocab_size
|
|
self.hidden_size = hidden_size
|
|
self.num_hidden_layers = num_hidden_layers
|
|
self.num_attention_heads = num_attention_heads
|
|
self.intermediate_size = intermediate_size
|
|
self.max_position_embeddings = max_position_embeddings
|
|
self.hidden_dropout_prob = hidden_dropout_prob
|
|
self.attention_probs_dropout_prob = attention_probs_dropout_prob
|