{
  "name": "gpt2_strict_small_gpt2init",
  "model_dim": 768,
  "vocab_size": 16384,
  "seq_len": 514,
  "max_positions": 1024,
  "mlp_mult": 4.0,
  "tie_embeddings": true,
  "norm_type": "layernorm",
  "norm_affine": true,
  "norm_eps": 1e-05,
  "positional_embedding": "learned_absolute",
  "embedding_dropout": 0.1,
  "hidden_dropout": 0.1,
  "attn_dropout": 0.1,
  "hf_export_format": "gpt2",
  "blocks": [
    {
      "type": "plain",
      "heads": 12,
      "attention_mask": "causal",
      "attn_bias": true,
      "ffn_activation": "gelu_new",
      "ffn_pre_norm": true,
      "ffn_bias": true
    },
    {
      "type": "plain",
      "heads": 12,
      "attention_mask": "causal",
      "attn_bias": true,
      "ffn_activation": "gelu_new",
      "ffn_pre_norm": true,
      "ffn_bias": true
    },
    {
      "type": "plain",
      "heads": 12,
      "attention_mask": "causal",
      "attn_bias": true,
      "ffn_activation": "gelu_new",
      "ffn_pre_norm": true,
      "ffn_bias": true
    },
    {
      "type": "plain",
      "heads": 12,
      "attention_mask": "causal",
      "attn_bias": true,
      "ffn_activation": "gelu_new",
      "ffn_pre_norm": true,
      "ffn_bias": true
    },
    {
      "type": "plain",
      "heads": 12,
      "attention_mask": "causal",
      "attn_bias": true,
      "ffn_activation": "gelu_new",
      "ffn_pre_norm": true,
      "ffn_bias": true
    },
    {
      "type": "plain",
      "heads": 12,
      "attention_mask": "causal",
      "attn_bias": true,
      "ffn_activation": "gelu_new",
      "ffn_pre_norm": true,
      "ffn_bias": true
    },
    {
      "type": "plain",
      "heads": 12,
      "attention_mask": "causal",
      "attn_bias": true,
      "ffn_activation": "gelu_new",
      "ffn_pre_norm": true,
      "ffn_bias": true
    },
    {
      "type": "plain",
      "heads": 12,
      "attention_mask": "causal",
      "attn_bias": true,
      "ffn_activation": "gelu_new",
      "ffn_pre_norm": true,
      "ffn_bias": true
    },
    {
      "type": "plain",
      "heads": 12,
      "attention_mask": "causal",
      "attn_bias": true,
      "ffn_activation": "gelu_new",
      "ffn_pre_norm": true,
      "ffn_bias": true
    },
    {
      "type": "plain",
      "heads": 12,
      "attention_mask": "causal",
      "attn_bias": true,
      "ffn_activation": "gelu_new",
      "ffn_pre_norm": true,
      "ffn_bias": true
    },
    {
      "type": "plain",
      "heads": 12,
      "attention_mask": "causal",
      "attn_bias": true,
      "ffn_activation": "gelu_new",
      "ffn_pre_norm": true,
      "ffn_bias": true
    },
    {
      "type": "plain",
      "heads": 12,
      "attention_mask": "causal",
      "attn_bias": true,
      "ffn_activation": "gelu_new",
      "ffn_pre_norm": true,
      "ffn_bias": true
    }
  ],
  "training": {
    "objective": "causal",
    "optimizer": "adamw",
    "lr": 5e-05,
    "weight_decay": 0.0,
    "beta1": 0.9,
    "beta2": 0.999,
    "epsilon": 1e-08,
    "grad_clip": 1.0,
    "steps": 21300,
    "warmup_steps": 213,
    "hold_steps": 0,
    "warmdown_steps": 0,
    "min_lr_fraction": 1e-06,
    "batch_tokens": 8224,
    "compute_dtype": "float32",
    "seed": 42,
    "example_framing": {
      "content_len": 512,
      "bos_id": 1,
      "eos_id": 2
    },
    "weight_init": "gpt2"
  }
}