初始化项目，由ModelHub XC社区提供模型

Model: team-lucid/mptk-1b
Source: Original Platform

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,71 @@
+---
+license: apache-2.0
+language:
+- ko
+---
+# MPTK-1B
+
+MPTK-1B는 한국어/영어코드 데이터셋에서 학습된 1.3B 파라미터의 decoder-only transformer 언어모델입니다.
+
+이 모델은 구글의 [TPU Research Cloud(TRC)](https://sites.research.google/trc/about/)를 통해 지원받은 Cloud TPU로 학습되었습니다.
+
+## Model Details
+
+### Model Description
+
+다른 decoder-only transformer에서 일부 수정된 아키텍처인 MPT를 기반으로 합니다.
+
+- [ALiBi (Attention with Linear Biases)](https://arxiv.org/abs/2108.12409)를 사용합니다
+- bias를 사용하지 않습니다.
+
+| Hyperparameter  | Value |
+|-----------------|-------|
+| n_parameters    | 1.3B  |
+| n_layers        | 24    |
+| n_heads         | 16    |
+| d_model         | 2048  |
+| vocab size      | 50432 |
+| sequence length | 2048  |
+
+## Uses
+
+## How to Get Started with the Model
+
+fp16으로 실행 시 NaN이 발생할 수 있습니다. 따라서 fp32 혹은 bf16로 실행하기를 권장합니다.
+
+```python
+import torch
+from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline
+
+tokenizer = AutoTokenizer.from_pretrained("team-lucid/mptk-1b")
+model = AutoModelForCausalLM.from_pretrained("team-lucid/mptk-1b")
+
+pipe = pipeline('text-generation', model=model, tokenizer=tokenizer, device='cuda:0')
+
+with torch.autocast('cuda', dtype=torch.bfloat16):
+    print(
+        pipe(
+            '대한민국의 수도는',
+            max_new_tokens=100,
+            do_sample=True,
+        )
+    )
+
+```
+
+## Training Details
+
+### Training Data
+
+[OSCAR](https://oscar-project.org/), mC4, wikipedia, namuwiki 등 한국어
+데이터에 [RefinedWeb](https://huggingface.co/datasets/tiiuae/falcon-refinedweb), [The Stack](https://huggingface.co/datasets/bigcode/the-stack)
+에서 일부를 추가해 학습하였습니다.
+
+#### Training Hyperparameters
+
+| **Hyperparameter** | **Value**  |
+|--------------------|------------|
+| Precision          | bfloat16 |
+| Optimizer          | Lion       |
+| Learning rate      | 2e-4       |
+| Batch size         | 1024       |

config.json

@@ -0,0 +1,49 @@
+{
+  "architectures": [
+    "MptForCausalLM"
+  ],
+  "attn_config": {
+    "alibi": true,
+    "alibi_bias_max": 8,
+    "attn_impl": "torch",
+    "attn_pdrop": 0,
+    "attn_type": "multihead_attention",
+    "attn_uses_sequence_id": false,
+    "clip_qkv": null,
+    "prefix_lm": false,
+    "qk_ln": false,
+    "softmax_scale": null
+  },
+  "d_model": 2048,
+  "emb_pdrop": 0,
+  "embedding_fraction": 1.0,
+  "expansion_ratio": 4,
+  "init_config": {
+    "emb_init_std": null,
+    "emb_init_uniform_lim": null,
+    "fan_mode": "fan_in",
+    "init_div_is_residual": true,
+    "init_gain": 0.0,
+    "init_nonlinearity": "relu",
+    "init_std": null,
+    "name": "kaiming_normal_",
+    "verbose": 0
+  },
+  "init_device": "cpu",
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "learned_pos_emb": true,
+  "logit_scale": null,
+  "max_seq_len": 8192,
+  "model_type": "mpt",
+  "n_heads": 16,
+  "n_layers": 24,
+  "no_bias": true,
+  "norm_type": "low_precision_layernorm",
+  "resid_pdrop": 0,
+  "torch_dtype": "float32",
+  "transformers_version": "4.33.0.dev0",
+  "use_cache": false,
+  "verbose": 0,
+  "vocab_size": 50432
+}

flax_model.msgpack

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2bfaeecba8b5780b9f0e278a5e8dc062a91415d9e1d915764e4364b0e1079bea
+size 5245384770

generation_config.json

@@ -0,0 +1,5 @@
+{
+  "_from_model_config": true,
+  "transformers_version": "4.33.0.dev0",
+  "use_cache": false
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:74ea184eb07fc2ae6d16e6c36cb134e7207802e2bcd96eed884deb30834c259a
+size 5245395232

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f75548c5c83818cf1870bbd73b7e3041018d9aeb6963647f18e9df0d4bf3b516
+size 5245425669

special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "unk_token": "<|endoftext|>"
+}

tokenizer_config.json

@@ -0,0 +1,9 @@
+{
+  "add_prefix_space": false,
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|endoftext|>",
+  "model_max_length": 2048,
+  "tokenizer_class": "GPTNeoXTokenizer",
+  "unk_token": "<|endoftext|>"
+}