From 03253c18c517e21ac654137f746a22528126479b Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Sat, 30 May 2026 23:59:20 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: AI4PD/ProtGPT3-1.3B Source: Original Platform --- .gitattributes | 35 ++++++++ README.md | 183 ++++++++++++++++++++++++++++++++++++++++ config.json | 32 +++++++ generation_config.json | 7 ++ model.safetensors | 3 + special_tokens_map.json | 30 +++++++ tokenizer.json | 143 +++++++++++++++++++++++++++++++ tokenizer_config.json | 73 ++++++++++++++++ 8 files changed, 506 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 config.json create mode 100644 generation_config.json create mode 100644 model.safetensors create mode 100644 special_tokens_map.json create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..a6344aa --- /dev/null +++ b/.gitattributes @@ -0,0 +1,35 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..8b59aad --- /dev/null +++ b/README.md @@ -0,0 +1,183 @@ +--- +library_name: transformers +tags: +- biology +- protein-language-model +- protein-generation +- causal-lm +- mixture-of-experts +- transformers +--- + +# Model Card for ProtGPT3-1.3B + +## Model Details + +### Model Description + +ProtGPT3-1.3B is a single-sequence autoregressive protein language model for protein sequence generation. It is part of the ProtGPT3 family, an open-source suite of promptable and aligned protein language models ranging from 112M to 10B parameters. ProtGPT3 models use a causal Mixtral-style Mixture-of-Experts architecture and are trained for causal language modeling on protein sequences. + +The single-sequence ProtGPT3 models can generate proteins in either N-to-C or C-to-N direction using special directional tokens. The model is intended for unconditional or prefix-conditioned protein sequence generation and can be used as a base model for downstream protein design workflows. + + +## Uses + +### Direct Use + +ProtGPT3-1.3B can be used for autoregressive generation of protein sequences. Users can generate sequences unconditionally or condition generation on an amino-acid prefix. + +### Downstream Use + +The model may be fine-tuned or incorporated into protein design workflows, including family-specific generation, protein variant generation, and computational screening pipelines. + +### Out-of-Scope Use + +The model should not be used as the sole basis for experimental, clinical, environmental, or safety-critical decisions. Generated proteins require downstream computational and experimental validation. The model is not guaranteed to generate functional, soluble, safe, or synthesizable proteins. + +## Bias, Risks, and Limitations + +ProtGPT3-1.3B learns from public protein sequence datasets and may reproduce biases present in those datasets. Generated sequences may be low-complexity, nonfunctional, unstable, insoluble, or biologically implausible. Protein generation models may also present dual-use risks if used irresponsibly. + +### Recommendations + +Users should apply appropriate computational filters, expert review, and experimental validation before using generated sequences. Users should also consider responsible-use practices for generative protein design. + +## How to Get Started with the Model + +Install dependencies: + +```bash +pip install transformers accelerate torch +``` + +Load the model and tokenizer: + +```python +import torch +from transformers import AutoTokenizer, AutoModelForCausalLM + +model_id = "protgpt3/ProtGPT3-1.3B" # Replace with the final checkpoint name + +# Load tokenizer for generation +tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True,add_bos_token=True, add_eos_token=False, padding_side="left") + +model = AutoModelForCausalLM.from_pretrained( + model_id, + torch_dtype=torch.bfloat16, + device_map="auto", + trust_remote_code=True, +) + +model.eval() +``` + +### Generate a protein sequence + +```python +import torch + +prompt = "" # Optionally provide an amino-acid prefix or model-specific direction + +inputs = tokenizer(prompt, return_tensors="pt").to(model.device) + +with torch.no_grad(): + output_ids = model.generate( + inputs["input_ids"], + max_new_tokens=512, + do_sample=True, + temperature=0.8, + top_p=0.9, + eos_token_id=tokenizer.eos_token_id, + pad_token_id=tokenizer.pad_token_id, + ) + +sequence = tokenizer.decode(output_ids[0], skip_special_tokens=True) +print(sequence) # output includes directional token "1" or "2" to denote if sequence was generated N-to-C or C-to-N +``` + +### Generate from an amino-acid prefix + +```python +import torch + +# forward N-to-C generation with special token "1" +prefix = "1MKT" # use special token "2" instead of "1" for reverse C-to-N generation + +inputs = tokenizer(prefix, return_tensors="pt").to(model.device) + +with torch.no_grad(): + output_ids = model.generate( + inputs["input_ids"], + max_new_tokens=256, + do_sample=True, + temperature=0.8, + top_p=0.9, + eos_token_id=tokenizer.eos_token_id, + pad_token_id=tokenizer.eos_token_id, + ) + +sequence = tokenizer.decode(output_ids[0], skip_special_tokens=True) +print(sequence) +``` + +### Batch generation + +```python +import torch + +prompts = [ + "", + "1MKT", # N-to-C generation + "2MAV", # C-to-N generation +] + +inputs = tokenizer( + prompts, + return_tensors="pt", + padding=True, +).to(model.device) + +with torch.no_grad(): + output_ids = model.generate( + inputs["input_ids"], + max_new_tokens=256, + do_sample=True, + temperature=0.8, + top_p=0.9, + eos_token_id=tokenizer.eos_token_id, + pad_token_id=tokenizer.bos_token_id, + ) + +sequences = tokenizer.batch_decode(output_ids, skip_special_tokens=True) + +for sequence in sequences: + print(sequence) +``` + +## Technical Specifications + +### Model Architecture and Objective + +ProtGPT3-1.3B is a decoder-only causal language model using a Mixtral-style sparse Mixture-of-Experts architecture. It was trained with a causal language modeling objective on protein sequences. + +### Compute Infrastructure + +#### Software + +Training used FlashAttention-2, online mini-batch packing, Liger Kernel, and DeepSpeed. + +## Citation + +**BibTeX:** + +```bibtex +@article{protgpt3, + title={ProtGPT3: an Open-source family of Promptable and Aligned Protein Language Models}, + author={Anonymous Authors}, + year={2026} +} +``` + +## More Information + +All models and code are released through the Hugging Face ecosystem and accompanying code repository. \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..6d148f8 --- /dev/null +++ b/config.json @@ -0,0 +1,32 @@ +{ + "architectures": [ + "MixtralForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": 1, + "eos_token_id": 2, + "head_dim": null, + "hidden_act": "silu", + "hidden_size": 1024, + "initializer_range": 0.02, + "intermediate_size": 3072, + "max_position_embeddings": 1025, + "model_type": "mixtral", + "num_attention_heads": 16, + "num_experts_per_tok": 2, + "num_hidden_layers": 17, + "num_key_value_heads": 4, + "num_local_experts": 8, + "output_router_logits": false, + "pad_token_id": 0, + "rms_norm_eps": 1e-05, + "rope_theta": 1000000.0, + "router_aux_loss_coef": 0.01, + "router_jitter_noise": 0.0, + "sliding_window": null, + "tie_word_embeddings": false, + "torch_dtype": "bfloat16", + "transformers_version": "4.55.2", + "use_cache": true, + "vocab_size": 31 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..f28b8ca --- /dev/null +++ b/generation_config.json @@ -0,0 +1,7 @@ +{ + "_from_model_config": true, + "bos_token_id": 1, + "eos_token_id": 2, + "pad_token_id": 0, + "transformers_version": "4.55.2" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..26ee3f0 --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91607dfb8c8f826afe2b24b3bbe9518d4f5173f498af3a0004bb648958ed6819 +size 2656587200 diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000..40936f5 --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,30 @@ +{ + "bos_token": { + "content": "<|bos|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "eos_token": { + "content": "<|eos|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "pad_token": { + "content": "<|pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + }, + "unk_token": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false + } +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..2d92c24 --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,143 @@ +{ + "version": "1.0", + "truncation": null, + "padding": null, + "added_tokens": [ + { + "id": 0, + "content": "<|pad|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1, + "content": "<|bos|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2, + "content": "<|eos|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3, + "content": "[UNK]", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 31, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 32, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 33, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + } + ], + "normalizer": null, + "pre_tokenizer": { + "type": "Split", + "pattern": { + "String": "" + }, + "behavior": "Isolated", + "invert": false + }, + "post_processor": { + "type": "TemplateProcessing", + "single": [ + { + "Sequence": { + "id": "A", + "type_id": 0 + } + } + ], + "pair": [ + { + "Sequence": { + "id": "A", + "type_id": 0 + } + }, + { + "Sequence": { + "id": "B", + "type_id": 1 + } + } + ], + "special_tokens": {} + }, + "decoder": null, + "model": { + "type": "WordLevel", + "vocab": { + "<|pad|>": 0, + "<|bos|>": 1, + "<|eos|>": 2, + "[UNK]": 3, + "1": 4, + "2": 5, + "A": 6, + "B": 7, + "C": 8, + "D": 9, + "E": 10, + "F": 11, + "G": 12, + "H": 13, + "I": 14, + "K": 15, + "L": 16, + "M": 17, + "N": 18, + "O": 19, + "P": 20, + "Q": 21, + "R": 22, + "S": 23, + "T": 24, + "U": 25, + "V": 26, + "W": 27, + "X": 28, + "Y": 29, + "Z": 30 + }, + "unk_token": "[UNK]" + } +} \ No newline at end of file diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..92162ad --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,73 @@ +{ + "add_bos_token": false, + "add_eos_token": false, + "add_prefix_space": null, + "added_tokens_decoder": { + "0": { + "content": "<|pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "1": { + "content": "<|bos|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "2": { + "content": "<|eos|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "3": { + "content": "[UNK]", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "31": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "32": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "33": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + } + }, + "bos_token": "<|bos|>", + "clean_up_tokenization_spaces": false, + "eos_token": "<|eos|>", + "extra_special_tokens": {}, + "legacy": true, + "model_max_length": 1025, + "pad_token": "<|pad|>", + "tokenizer_class": "LlamaTokenizerFast", + "unk_token": "", + "use_default_system_prompt": false +}