AI4PD/ZymCTRL
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

初始化项目，由ModelHub XC社区提供模型

Browse Source 
Model: AI4PD/ZymCTRL
Source: Original Platform
This commit is contained in:
ModelHub XC
2026-07-21 18:23:19 +08:00
commit 0eb35a9161
8 changed files with 5182 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
.gitattributes vendored Normal file
View File

@@ -0,0 +1,32 @@
*.7z filter=lfs diff=lfs merge=lfs -text
*.arrow filter=lfs diff=lfs merge=lfs -text
*.bin filter=lfs diff=lfs merge=lfs -text
*.bz2 filter=lfs diff=lfs merge=lfs -text
*.ftz filter=lfs diff=lfs merge=lfs -text
*.gz filter=lfs diff=lfs merge=lfs -text
*.h5 filter=lfs diff=lfs merge=lfs -text
*.joblib filter=lfs diff=lfs merge=lfs -text
*.lfs.* filter=lfs diff=lfs merge=lfs -text
*.mlmodel filter=lfs diff=lfs merge=lfs -text
*.model filter=lfs diff=lfs merge=lfs -text
*.msgpack filter=lfs diff=lfs merge=lfs -text
*.npy filter=lfs diff=lfs merge=lfs -text
*.npz filter=lfs diff=lfs merge=lfs -text
*.onnx filter=lfs diff=lfs merge=lfs -text
*.ot filter=lfs diff=lfs merge=lfs -text
*.parquet filter=lfs diff=lfs merge=lfs -text
*.pb filter=lfs diff=lfs merge=lfs -text
*.pickle filter=lfs diff=lfs merge=lfs -text
*.pkl filter=lfs diff=lfs merge=lfs -text
*.pt filter=lfs diff=lfs merge=lfs -text
*.pth filter=lfs diff=lfs merge=lfs -text
*.rar filter=lfs diff=lfs merge=lfs -text
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
*.tar.* filter=lfs diff=lfs merge=lfs -text
*.tflite filter=lfs diff=lfs merge=lfs -text
*.tgz filter=lfs diff=lfs merge=lfs -text
*.wasm filter=lfs diff=lfs merge=lfs -text
*.xz filter=lfs diff=lfs merge=lfs -text
*.zip filter=lfs diff=lfs merge=lfs -text
*.zst filter=lfs diff=lfs merge=lfs -text
*tfevents* filter=lfs diff=lfs merge=lfs -text

385
5.run_clm-post.py Normal file
View File

@@ -0,0 +1,385 @@
#!/usr/bin/env python
# coding=utf-8
# Copyright 2020 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Fine-tuning the library models for causal language modeling (GPT, GPT-2, CTRL, ...) on a text file or a dataset.
Here is the full list of checkpoints on the hub that can be fine-tuned by this script:
https://huggingface.co/models?filter=causal-lm
"""
# You can also adapt this script on your own causal language modeling task. Pointers for this are left as comments.
import logging
import math
import os
import sys
from dataclasses import dataclass, field
from typing import Optional
import datasets
from datasets import load_dataset
from datasets import load_from_disk
import transformers
from transformers import (
CONFIG_MAPPING,
MODEL_FOR_CAUSAL_LM_MAPPING,
AutoConfig,
AutoModelForCausalLM,
AutoTokenizer,
HfArgumentParser,
Trainer,
TrainingArguments,
default_data_collator,
set_seed,
)
from transformers.testing_utils import CaptureLogger
from transformers.trainer_utils import get_last_checkpoint
from transformers.utils import check_min_version
from transformers.utils.versions import require_version
# Will error if the minimal version of Transformers is not installed. Remove at your own risks.
check_min_version("4.13.0.dev0")
require_version("datasets>=1.8.0", "To fix: pip install -r examples/pytorch/language-modeling/requirements.txt")
logger = logging.getLogger(__name__)
MODEL_CONFIG_CLASSES = list(MODEL_FOR_CAUSAL_LM_MAPPING.keys())
MODEL_TYPES = tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES)
@dataclass
class ModelArguments:
"""
Arguments pertaining to which model/config/tokenizer we are going to fine-tune, or train from scratch.
"""
model_name_or_path: Optional[str] = field(
default=None,
metadata={
"help": "The model checkpoint for weights initialization."
"Don't set if you want to train a model from scratch."
},
)
model_type: Optional[str] = field(
default=None,
metadata={"help": "If training from scratch, pass a model type from the list: " + ", ".join(MODEL_TYPES)},
)
config_overrides: Optional[str] = field(
default=None,
metadata={
"help": "Override some existing default config settings when a model is trained from scratch. Example: "
"n_embd=10,resid_pdrop=0.2,scale_attn_weights=false,summary_type=cls_index"
},
)
config_name: Optional[str] = field(
default=None, metadata={"help": "Pretrained config name or path if not the same as model_name"}
)
tokenizer_name: Optional[str] = field(
default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}
)
cache_dir: Optional[str] = field(
default=None,
metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"},
)
use_fast_tokenizer: bool = field(
default=True,
metadata={"help": "Whether to use one of the fast tokenizer (backed by the tokenizers library) or not."},
)
model_revision: str = field(
default="main",
metadata={"help": "The specific model version to use (can be a branch name, tag name or commit id)."},
)
use_auth_token: bool = field(
default=False,
metadata={
"help": "Will use the token generated when running `transformers-cli login` (necessary to use this script "
"with private models)."
},
)
def __post_init__(self):
if self.config_overrides is not None and (self.config_name is not None or self.model_name_or_path is not None):
raise ValueError(
"--config_overrides can't be used in combination with --config_name or --model_name_or_path"
)
@dataclass
class DataTrainingArguments:
"""
Arguments pertaining to what data we are going to input our model for training and eval.
"""
dataset_name: Optional[str] = field(
default=None, metadata={"help": "The name of the dataset to use (via the datasets library)."}
)
dataset_config_name: Optional[str] = field(
default=None, metadata={"help": "The configuration name of the dataset to use (via the datasets library)."}
)
train_file: Optional[str] = field(default=None, metadata={"help": "The input training data file (a text file)."})
validation_file: Optional[str] = field(
default=None,
metadata={"help": "An optional input evaluation data file to evaluate the perplexity on (a text file)."},
)
max_train_samples: Optional[int] = field(
default=None,
metadata={
"help": "For debugging purposes or quicker training, truncate the number of training examples to this "
"value if set."
},
)
max_eval_samples: Optional[int] = field(
default=None,
metadata={
"help": "For debugging purposes or quicker training, truncate the number of evaluation examples to this "
"value if set."
},
)
block_size: Optional[int] = field(
default=None,
metadata={
"help": "Optional input sequence length after tokenization. "
"The training dataset will be truncated in block of this size for training. "
"Default to the model max input length for single sentence inputs (take into account special tokens)."
},
)
overwrite_cache: bool = field(
default=False, metadata={"help": "Overwrite the cached training and evaluation sets"}
)
validation_split_percentage: Optional[int] = field(
default=5,
metadata={
"help": "The percentage of the train set used as validation set in case there's no validation split"
},
)
preprocessing_num_workers: Optional[int] = field(
default=None,
metadata={"help": "The number of processes to use for the preprocessing."},
)
keep_linebreaks: bool = field(
default=True, metadata={"help": "Whether to keep line breaks when using TXT files or not."}
)
#def __post_init__(self):
# if self.dataset_name is None and self.train_file is None and self.validation_file is None:
# raise ValueError("Need either a dataset name or a training/validation file.")
# else:
# if self.train_file is not None:
# extension = self.train_file.split(".")[-1]
# assert extension in ["csv", "json", "txt"], "`train_file` should be a csv, a json or a txt file."
# if self.validation_file is not None:
# extension = self.validation_file.split(".")[-1]
# assert extension in ["csv", "json", "txt"], "`validation_file` should be a csv, a json or a txt file."
def main():
# See all possible arguments in src/transformers/training_args.py
# or by passing the --help flag to this script.
# We now keep distinct sets of args, for a cleaner separation of concerns.
parser = HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments))
if len(sys.argv) == 2 and sys.argv[1].endswith(".json"):
# If we pass only one argument to the script and it's the path to a json file,
# let's parse it to get our arguments.
model_args, data_args, training_args = parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]))
else:
model_args, data_args, training_args = parser.parse_args_into_dataclasses()
# Setup logging
logging.basicConfig(
format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
datefmt="%m/%d/%Y %H:%M:%S",
handlers=[logging.StreamHandler(sys.stdout)],
)
log_level = training_args.get_process_log_level()
logger.setLevel(log_level)
datasets.utils.logging.set_verbosity(log_level)
transformers.utils.logging.set_verbosity(log_level)
transformers.utils.logging.enable_default_handler()
transformers.utils.logging.enable_explicit_format()
# Log on each process the small summary:
logger.warning(
f"Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}"
+ f"distributed training: {bool(training_args.local_rank != -1)}, 16-bits training: {training_args.fp16}"
)
logger.info(f"Training/evaluation parameters {training_args}")
# Detecting last checkpoint.
last_checkpoint = None
if os.path.isdir(training_args.output_dir) and training_args.do_train and not training_args.overwrite_output_dir:
last_checkpoint = get_last_checkpoint(training_args.output_dir)
if last_checkpoint is None and len(os.listdir(training_args.output_dir)) > 0:
raise ValueError(
f"Output directory ({training_args.output_dir}) already exists and is not empty. "
"Use --overwrite_output_dir to overcome."
)
elif last_checkpoint is not None and training_args.resume_from_checkpoint is None:
logger.info(
f"Checkpoint detected, resuming training at {last_checkpoint}. To avoid this behavior, change "
"the `--output_dir` or add `--overwrite_output_dir` to train from scratch."
)
# Set seed before initializing model.
set_seed(training_args.seed)
# Get the datasets: you can either provide your own CSV/JSON/TXT training and evaluation files (see below)
# or just provide the name of one of the public datasets available on the hub at https://huggingface.co/datasets/
# (the dataset will be downloaded automatically from the datasets Hub).
#
# For CSV/JSON files, this script will use the column called 'text' or the first column if no column called
# 'text' is found. You can easily tweak this behavior (see below).
#
# In distributed training, the load_dataset function guarantee that only one local process can concurrently
# download the dataset.
# See more about loading any type of standard or custom dataset (from files, python dict, pandas DataFrame, etc) at
# https://huggingface.co/docs/datasets/loading_datasets.html.
# Load pretrained model and tokenizer
#
# Distributed training:
# The .from_pretrained methods guarantee that only one local process can concurrently
# download model & vocab.
config_kwargs = {
"cache_dir": model_args.cache_dir,
"revision": model_args.model_revision,
"use_auth_token": True if model_args.use_auth_token else None,
}
if model_args.config_name:
config = AutoConfig.from_pretrained(model_args.config_name, **config_kwargs)
elif model_args.model_name_or_path:
config = AutoConfig.from_pretrained(model_args.model_name_or_path, **config_kwargs)
else:
config = CONFIG_MAPPING[model_args.model_type]()
logger.warning("You are instantiating a new config instance from scratch.")
if model_args.config_overrides is not None:
logger.info(f"Overriding config: {model_args.config_overrides}")
config.update_from_string(model_args.config_overrides)
tokenizer_kwargs = {
"cache_dir": model_args.cache_dir,
"use_fast": model_args.use_fast_tokenizer,
"revision": model_args.model_revision,
"use_auth_token": True if model_args.use_auth_token else None,
}
if model_args.tokenizer_name:
tokenizer = AutoTokenizer.from_pretrained(model_args.tokenizer_name, **tokenizer_kwargs)
elif model_args.model_name_or_path:
tokenizer = AutoTokenizer.from_pretrained(model_args.model_name_or_path, **tokenizer_kwargs)
else:
raise ValueError(
"You are instantiating a new tokenizer from scratch. This is not supported by this script."
"You can do it from another script, save it, and load it from here, using --tokenizer_name."
)
if model_args.model_name_or_path:
model = AutoModelForCausalLM.from_pretrained(
model_args.model_name_or_path,
from_tf=bool(".ckpt" in model_args.model_name_or_path),
config=config,
cache_dir=model_args.cache_dir,
revision=model_args.model_revision,
use_auth_token=True if model_args.use_auth_token else None,
)
else:
model = AutoModelForCausalLM.from_config(config)
n_params = sum(dict((p.data_ptr(), p.numel()) for p in model.parameters()).values())
logger.info(f"Training new model from scratch - Total size={n_params/2**20:.2f}M params")
model.resize_token_embeddings(len(tokenizer))
train_dataset = load_from_disk('dataset/train2')
eval_dataset = load_from_disk('dataset/eval2')
# Initialize our Trainer
trainer = Trainer(
model=model,
args=training_args,
train_dataset=train_dataset if training_args.do_train else None,
eval_dataset=eval_dataset if training_args.do_eval else None,
tokenizer=tokenizer,
# Data collator will default to DataCollatorWithPadding, so we change it.
data_collator=default_data_collator,
)
# Training
if training_args.do_train:
checkpoint = None
if training_args.resume_from_checkpoint is not None:
checkpoint = training_args.resume_from_checkpoint
elif last_checkpoint is not None:
checkpoint = last_checkpoint
train_result = trainer.train(resume_from_checkpoint=checkpoint)
trainer.save_model() # Saves the tokenizer too for easy upload
metrics = train_result.metrics
max_train_samples = (
data_args.max_train_samples if data_args.max_train_samples is not None else len(train_dataset)
)
metrics["train_samples"] = min(max_train_samples, len(train_dataset))
trainer.log_metrics("train", metrics)
trainer.save_metrics("train", metrics)
trainer.save_state()
# Evaluation
if training_args.do_eval:
logger.info("*** Evaluate ***")
metrics = trainer.evaluate()
max_eval_samples = data_args.max_eval_samples if data_args.max_eval_samples is not None else len(eval_dataset)
metrics["eval_samples"] = min(max_eval_samples, len(eval_dataset))
try:
perplexity = math.exp(metrics["eval_loss"])
except OverflowError:
perplexity = float("inf")
metrics["perplexity"] = perplexity
trainer.log_metrics("eval", metrics)
trainer.save_metrics("eval", metrics)
kwargs = {"finetuned_from": model_args.model_name_or_path, "tasks": "text-generation"}
if data_args.dataset_name is not None:
kwargs["dataset_tags"] = data_args.dataset_name
if data_args.dataset_config_name is not None:
kwargs["dataset_args"] = data_args.dataset_config_name
kwargs["dataset"] = f"{data_args.dataset_name} {data_args.dataset_config_name}"
else:
kwargs["dataset"] = data_args.dataset_name
if training_args.push_to_hub:
trainer.push_to_hub(**kwargs)
else:
trainer.create_model_card(**kwargs)
def _mp_fn(index):
# For xla_spawn (TPUs)
main()
if __name__ == "__main__":
main()

374
README.md Normal file
View File

@@ -0,0 +1,374 @@
---
license: apache-2.0
pipeline_tag: text-generation
widget:
- text: 1.1.1.21<sep><start>
inference:
parameters:
top_k: 9
repetition_penalty: 1.2
tags:
- biology
---
# **ZymCTRL**
ZymCTRL (Enzyme Control) ([ see preprint ](https://www.biorxiv.org/content/10.1101/2024.05.03.592223v1))
is a conditional language model for the generation of artificial functional enzymes.
It was trained on the UniProt database of sequences containing (Enzyme Commission) EC annotations, comprising over 37 M sequences.
Given a user-defined Enzymatic Commission (EC) number, the model generates protein sequences that fulfil that catalytic reaction.
The generated sequences are ordered, globular, and distant to natural ones, while their intended catalytic properties match those defined by users.
If you don't know the EC number of your protein of interest, have a look for example here: https://www.brenda-enzymes.org/ecexplorer.php?browser=1
See below for information about the model, how to generate sequences, and how to save and rank them by perplexity.
## **Model description**
ZymCTRL is based on the [CTRL Transformer](https://arxiv.org/abs/1909.05858) architecture (which in turn is very similar to ChatGPT) and contains 36 layers
with a model dimensionality of 1280, totaling 738 million parameters.
ZymCTRL is a decoder-only transformer model pre-trained on the Uniprot subset of enzyme sequences, totalling 37M sequences.
(version July 2022). The pre-training was done on the raw sequences without FASTA headers,
with the EC classes prepended to each sequence. The databases will be uploaded soon.
ZymCTRL was trained with an autoregressive objective, i.e., the model learns to predict
the next token given a sequence context. Because the first tokens on each sequence encode the EC numbers,
the model learns the dependencies among EC classes and their corresponding sequences and is able to _speak_ the enzyme language.
There are stark differences in the number of members among EC classes, and for this reason, we also tokenized the EC numbers.
In this manner, EC numbers '2.7.1.1' and '2.7.1.2' share the first three tokens (six, including separators), and hence the model can infer that
there are relationships between the two classes.
The figure below summarizes the process of training:
![plot](./github1.png)
## **How to use ZymCTRL**
ZymCTRL can be used with the HuggingFace transformer python package.
Detailed installation instructions can be found here: https://huggingface.co/docs/transformers/installation
Since ZymCTRL has been trained on the classical language model objective on enzyme sequences with their EC annotation,
it particularly excels at generating enzyme sequences given a user-defined EC class, such as alcohol dehydrogenases ('1.1.1.2').
The model can generate in two ways: in a zero-shot fashion, i.e., directly generating from the checkpoint weights, or after fine-tuning.
Fine-tuning allows augmenting the specific EC datasets that were used during training, for example,
if you have a curated internal dataset or a set of ancestrally-reconstructed sequences. This is entirely optional. One advantage of
running the model in zero-shot is that it doesn't require any further training.
### **Example 1: Generating nitrilases (EC 3.5.5.1)**
The script below will be used for the generation of any EC class in a zero-shot fashion,
here we showcase the generation of novel nitrilases.
To run this script, you should download ZymCTRL to a local folder in your workstation.
Then replace the placeholders in the script with your actual folder path.
You can run it directly in the command line (once you have hugging face installed),
with the following command: `python generate.py`
The script will write each sequence in a fasta file in the folder you specify. In the fasta header,
it will store the sequence's computed perplexity value. Perplexity is a measure of the model's confidence
in that generation, with lower values being better. The sequences are ordered by perplexity before writing them out,
so those that finish in *_0.fasta and *_1.fasta will be the best ones per batch.
**Given that generation runs so fast, we recommend generating hundreds or thousands and then only picking the best 5% or less.
With the script below, that would mean picking only those that finish in '_0.fasta'. Good perplexity values for this model so be below 1.75-1.5.**
```python
import torch
from transformers import GPT2LMHeadModel, AutoTokenizer
import os
from tqdm import tqdm
import math
def remove_characters(sequence, char_list):
"This function removes special tokens used during training."
columns = sequence.split('<sep>')
seq = columns[1]
for char in char_list:
seq = seq.replace(char, '')
return seq
def calculatePerplexity(input_ids,model,tokenizer):
"This function computes perplexities for the generated sequences"
with torch.no_grad():
outputs = model(input_ids, labels=input_ids)
loss, logits = outputs[:2]
return math.exp(loss)
def main(label, model,special_tokens,device,tokenizer):
# Generating sequences
input_ids = tokenizer.encode(label,return_tensors='pt').to(device)
outputs = model.generate(
input_ids,
top_k=9, #tbd
repetition_penalty=1.2,
max_length=1024,
eos_token_id=1,
pad_token_id=0,
do_sample=True,
num_return_sequences=20) # Depending non your GPU, you'll be able to generate fewer or more sequences. This runs in an A40.
# Check sequence sanity, ensure sequences are not-truncated.
# The model will truncate sequences longer than the specified max_length (1024 above). We want to avoid those sequences.
new_outputs = [ output for output in outputs if output[-1] == 0]
if not new_outputs:
print("not enough sequences with short lengths!!")
# Compute perplexity for every generated sequence in the batch
ppls = [(tokenizer.decode(output), calculatePerplexity(output, model, tokenizer)) for output in new_outputs ]
# Sort the batch by perplexity, the lower the better
ppls.sort(key=lambda i:i[1]) # duplicated sequences?
# Final dictionary with the results
sequences={}
sequences[label] = [(remove_characters(x[0], special_tokens), x[1]) for x in ppls]
return sequences
if __name__=='__main__':
device = torch.device("cuda") # Replace with 'cpu' if you don't have a GPU - but it will be slow
print('Reading pretrained model and tokenizer')
tokenizer = AutoTokenizer.from_pretrained('/path/to/zymCTRL/') # change to ZymCTRL location
model = GPT2LMHeadModel.from_pretrained('/path/to/zymCTRL').to(device) # change to ZymCTRL location
special_tokens = ['<start>', '<end>', '<|endoftext|>','<pad>',' ', '<sep>']
# change to the appropriate EC classes
labels=['3.5.5.1'] # nitrilases. You can put as many labels as you want.
for label in tqdm(labels):
# We'll run 100 batches per label. 20 sequences will be generated per batch.
for i in range(0,100):
sequences = main(label, model, special_tokens, device, tokenizer)
for key,value in sequences.items():
for index, val in enumerate(value):
# Sequences will be saved with the name of the label followed by the batch index,
# and the order of the sequence in that batch.
fn = open(f"/path/to/folder/{label}_{i}_{index}.fasta", "w")
fn.write(f'>{label}_{i}_{index}\t{val[1]}\n{val[0]}')
fn.close()
```
## **Example 2: Fine-tuning on a set of user-defined sequences**
This alternative to the zero-shot generation allows updating ZymCTRL's weights to new sequences.
This strategy is not strictly necessary, in fact, we have observed good generations even for EC classes where there are
only 1-2 representatives in Nature. But you might have an internal set of sequences that you'd like to incorporate into the model.
For example, internal datasets after protein engineering efforts,
ancestrally-reconstructed sets, or after searching against metagenomics databases. In these cases, it is advisable to fine-tune ZymCTRL,
as it will learn new properties from your dataset and potentially improve the generation quality
(especially for poorly populated EC classes).
To fine-tune ZymCTRL, you can use the script below to process your sequences. The only requisite is to start with an input file,
'sequences.fasta' which contains all the sequences in a fasta format. Please follow the format below. There should not be new lines '\n' or
any separator between sequences. In the script, change the variable ec_label to the specific EC class you'd like to fine-tune.
The script will produce a file called {ec_label}_processed.txt and a folder with the training and validation datasets (split 10%)
```
>Sequence1
MMMMYMPLKVCD..
>Sequence2
MQWMXMYMPLKVCD..
>Sequence3
MPLKVCWMXMYMPLD..
```
We recommend using at least 200 sequences to obtain the best results. But we've seen it working with fewer sequences, so if you don't have
that many, give it still a go.
```python
import random
from transformers import AutoTokenizer
from datasets import load_dataset
import transformers
from transformers.testing_utils import CaptureLogger
## DEFINE THESE VARIABLES
tokenizer = AutoTokenizer.from_pretrained('AI4PD/ZymCTRL')
ec_label = '1.1.1.1' # CHANGE TO YOUR LABEL
validation_split_percentage = 10 # change if you want
sequence_file = 'sequence.fasta'
#Load sequences, Read source file
with open(sequence_file, 'r') as fn: #! CHANGE TO SEQUENCES.FASTA
data = fn.readlines()
fn.close()
# Put sequences into dictionary
sequences={}
for line in data:
if '>' in line:
name = line.strip()
sequences[name] = [] #! CHANGE TO corre
continue
sequences[name].append(line.strip())
#Pass sequences to list and shuffle their order randomly
sequences_list = [(key,value[0]) for key,value in sequences.items()]
random.shuffle(sequences_list)
#the objective is to get here strings, that when tokenized, would span a length of 1024.
#for each sequence group its length and untokenized string
print("procesing dataset")
processed_dataset = []
for i in sequences_list:
# length of the control code
sequence = i[1].strip()
separator = '<sep>'
control_code_length = len(tokenizer(ec_label+separator)['input_ids'])
available_space = 1021 - control_code_length # It is not 1024 because '<|endoftext|>', and start and end
# Option 1: the sequence is larger than the available space (3-4% of sequences)
if len(sequence) > available_space:
total_length = control_code_length + len(sequence[:available_space]) + 1
seq = f"{ec_label}{separator}{sequence[:available_space]}<|endoftext|>"
processed_dataset.append((total_length, seq))
# Option 2 & 3: The sequence fits in the block_size space with or without padding
else:
total_length = control_code_length + len(sequence) + 3
# in this case the sequence does not fit with the start/end tokens
seq = f"{ec_label}{separator}<start>{sequence}<end><|endoftext|>"
processed_dataset.append((total_length, seq))
# Group sequences
def grouper(iterable):
prev = None
group = ''
total_sum = 0
for item in iterable:
if prev is None or item[0] + total_sum < 1025:
group += item[1]
total_sum += item[0]
else:
total_sum = item[0]
yield group
group = item[1]
prev = item
if group:
total_sum = 0
yield group
print("grouping processed dataset")
grouped_dataset=dict(enumerate(grouper(processed_dataset),1))
# Write file out for the tokenizer to read
fn = open(f"{ec_label}_processed.txt",'w')
for key,value in grouped_dataset.items():
padding_len = 1024 - len(tokenizer(value)['input_ids'])
padding = "<pad>"*padding_len
fn.write(value+padding)
fn.write
fn.write("\n")
fn.close()
##TOKENIZE
# adapted from the trainer file
data_files = {}
dataset_args = {}
data_files["train"] = f"{ec_label}_processed.txt"
extension = "text"
tok_logger = transformers.utils.logging.get_logger("transformers.tokenization_utils_base")
raw_datasets = load_dataset(extension, data_files=data_files, cache_dir='.', **dataset_args)
raw_datasets["train"] = load_dataset(extension,
data_files=data_files,
split=f"train[{validation_split_percentage}%:]",
cache_dir='.',
**dataset_args,)
raw_datasets["validation"] = load_dataset(extension,
data_files=data_files,
split=f"train[:{validation_split_percentage}%]",
cache_dir='.',
**dataset_args,)
def tokenize_function(examples):
with CaptureLogger(tok_logger) as cl:
output = tokenizer(examples["text"])
# clm input could be much much longer than block_size
if "Token indices sequence length is longer than the" in cl.out:
tok_logger.warning(
"^^^^^^^^^^^^^^^^ Please ignore the warning above - this long input will be chunked into smaller bits before being passed to the model."
)
return output
tokenized_datasets = raw_datasets.map(
tokenize_function,
batched=True,
num_proc=32,
remove_columns=['text'],
load_from_cache_file = False,
desc="Running tokenizer on dataset",
)
block_size = 1024
def group_texts(examples):
# Concatenate all texts.
concatenated_examples = {k: sum(examples[k], []) for k in examples.keys()}
total_length = len(concatenated_examples[list(examples.keys())[0]])
# We drop the small remainder, we could add padding if the model supported it instead of this drop,
# you can customize this part to your needs.
if total_length >= block_size:
total_length = (total_length // block_size) * block_size
# Split by chunks of max_len.
result = {
k: [t[i : i + block_size] for i in range(0, total_length, block_size)]
for k, t in concatenated_examples.items()
}
result["labels"] = result["input_ids"].copy()
return result
lm_datasets = tokenized_datasets.map(
group_texts,
batched=True,
num_proc=124,
load_from_cache_file=False,
desc=f"Grouping texts in chunks of {block_size}",
)
train_dataset = lm_datasets["train"]
eval_dataset = lm_datasets["validation"]
train_dataset.save_to_disk('./dataset/train2')
eval_dataset.save_to_disk('./dataset/eval2')
```
The processed datasets will be inside the folder dataset/, called train2 and eval2.
You could also put the two previous scripts into a single one and run it in one go (that is what we do).
Now you are ready to fine-tune the model.
To do that, you can take the trainer file that we provide in this repository (5.run_clm-post.py), or use the trainer from Hugging Face.
The command below shows an example at an specific learning rate,
but you could try with other hyperparameters to obtain the best training and evaluation losses.
```bash
python 5.run_clm-post.py --tokenizer_name AI4PD/ZymCTRL
--do_train --do_eval --output_dir output --eval_strategy steps --eval_steps 10
--logging_steps 5 --save_steps 500 --num_train_epochs 28 --per_device_train_batch_size 1
--per_device_eval_batch_size 4 --cache_dir '.' --save_total_limit 2 --learning_rate 0.8e-04
--dataloader_drop_last True --model_name_or_path AI4PD/ZymCTRL
```
In any case, the original HuggingFace script run_clm.py can be found here:
https://github.com/huggingface/transformers/blob/master/examples/pytorch/language-modeling/run_clm.py
### **Training specs**
The model was trained on 48 NVIDIA A100 GPUs for eight epochs,
using a block size of 1024 and a total batch size of 768.
The optimizer used was Adam (beta1 = 0.9, beta2 = 0.999)
with a learning rate of 0.8e-04.
### **Contact**
We are the AI for Protein Design group at the Centre for Genomic Regulation (https://www.aiproteindesign.com/).
For any questions post an issue in this repository so that other people can benefit from the feedback, and I'll get back to you shortly.
We are always open for collaborations, send an email to noelia [dot] ferruz [at] crg [dot] eu.

35
config.json Normal file
View File

@@ -0,0 +1,35 @@
{
"activation_function": "gelu_new",
"architectures": [
"GPT2LMHeadModel"
],
"attn_pdrop": 0.1,
"bos_token_id": 1,
"embd_pdrop": 0.1,
"eos_token_id": 1,
"initializer_range": 0.02,
"layer_norm_epsilon": 1e-05,
"model_type": "gpt2",
"n_ctx": 1024,
"n_embd": 1280,
"n_head": 20,
"n_inner": null,
"n_layer": 36,
"n_positions": 1024,
"resid_pdrop": 0.1,
"scale_attn_weights": true,
"summary_activation": null,
"summary_first_dropout": 0.1,
"summary_proj_to_labels": true,
"summary_type": "cls_index",
"summary_use_proj": true,
"task_specific_params": {
"text-generation": {
"do_sample": true,
"max_length": 6092
}
},
"transformers_version": "4.21.1",
"use_cache": true,
"vocab_size": 458
}

BIN
github1.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 52 KiB

3
pytorch_model.bin Normal file
View File

@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:7497973d9b5950dee3d2a97e150fd959098981622f560aea74f1a647fad7e94a
size 2879060617

4352
tokenizer.json Normal file
View File

File diff suppressed because it is too large Load Diff

1
vocab.json Normal file
View File

@@ -0,0 +1 @@
{"<pad>":0,"<|endoftext|>":1,"<sep>":2,"<start>":3,"<end>":4,"[UNK]":5,"0":6,"1":7,"2":8,"3":9,"4":10,"5":11,"6":12,"7":13,"8":14,"9":15,"10":16,"11":17,"12":18,"13":19,"14":20,"15":21,"16":22,"17":23,"18":24,"19":25,"20":26,"21":27,"22":28,"23":29,"24":30,"25":31,"26":32,"27":33,"28":34,"29":35,"30":36,"31":37,"32":38,"33":39,"34":40,"35":41,"36":42,"37":43,"38":44,"39":45,"40":46,"41":47,"42":48,"43":49,"44":50,"45":51,"46":52,"47":53,"48":54,"49":55,"50":56,"51":57,"52":58,"53":59,"54":60,"55":61,"56":62,"57":63,"58":64,"59":65,"60":66,"61":67,"62":68,"63":69,"64":70,"65":71,"66":72,"67":73,"68":74,"69":75,"70":76,"71":77,"72":78,"73":79,"74":80,"75":81,"76":82,"77":83,"78":84,"79":85,"80":86,"81":87,"82":88,"83":89,"84":90,"85":91,"86":92,"87":93,"88":94,"89":95,"90":96,"91":97,"92":98,"93":99,"94":100,"95":101,"96":102,"97":103,"98":104,"99":105,"100":106,"101":107,"102":108,"103":109,"104":110,"105":111,"106":112,"107":113,"108":114,"109":115,"110":116,"111":117,"112":118,"113":119,"114":120,"115":121,"116":122,"117":123,"118":124,"119":125,"120":126,"121":127,"122":128,"123":129,"124":130,"125":131,"126":132,"127":133,"128":134,"129":135,"130":136,"131":137,"132":138,"133":139,"134":140,"135":141,"136":142,"137":143,"138":144,"139":145,"140":146,"141":147,"142":148,"143":149,"144":150,"145":151,"146":152,"147":153,"148":154,"149":155,"150":156,"151":157,"152":158,"153":159,"154":160,"155":161,"156":162,"157":163,"158":164,"159":165,"160":166,"161":167,"162":168,"163":169,"164":170,"165":171,"166":172,"167":173,"168":174,"169":175,"170":176,"171":177,"172":178,"173":179,"174":180,"175":181,"176":182,"177":183,"178":184,"179":185,"180":186,"181":187,"182":188,"183":189,"184":190,"185":191,"186":192,"187":193,"188":194,"189":195,"190":196,"191":197,"192":198,"193":199,"194":200,"195":201,"196":202,"197":203,"198":204,"199":205,"200":206,"201":207,"202":208,"203":209,"204":210,"205":211,"206":212,"207":213,"208":214,"209":215,"210":216,"211":217,"212":218,"213":219,"214":220,"215":221,"216":222,"217":223,"218":224,"219":225,"220":226,"221":227,"222":228,"223":229,"224":230,"225":231,"226":232,"227":233,"228":234,"229":235,"230":236,"231":237,"232":238,"233":239,"234":240,"235":241,"236":242,"237":243,"238":244,"239":245,"240":246,"241":247,"242":248,"243":249,"244":250,"245":251,"246":252,"247":253,"248":254,"249":255,"250":256,"251":257,"252":258,"253":259,"254":260,"255":261,"256":262,"257":263,"258":264,"259":265,"260":266,"261":267,"262":268,"263":269,"264":270,"265":271,"266":272,"267":273,"268":274,"269":275,"270":276,"271":277,"272":278,"273":279,"274":280,"275":281,"276":282,"277":283,"278":284,"279":285,"280":286,"281":287,"282":288,"283":289,"284":290,"285":291,"286":292,"287":293,"288":294,"289":295,"290":296,"291":297,"292":298,"293":299,"294":300,"295":301,"296":302,"297":303,"298":304,"299":305,"300":306,"301":307,"302":308,"303":309,"304":310,"305":311,"306":312,"307":313,"308":314,"309":315,"310":316,"311":317,"312":318,"313":319,"314":320,"315":321,"316":322,"317":323,"318":324,"319":325,"320":326,"321":327,"322":328,"323":329,"324":330,"325":331,"326":332,"327":333,"328":334,"329":335,"330":336,"331":337,"332":338,"333":339,"334":340,"335":341,"336":342,"337":343,"338":344,"339":345,"340":346,"341":347,"342":348,"343":349,"344":350,"345":351,"346":352,"347":353,"348":354,"349":355,"350":356,"351":357,"352":358,"353":359,"354":360,"355":361,"356":362,"357":363,"358":364,"359":365,"360":366,"361":367,"362":368,"363":369,"364":370,"365":371,"366":372,"367":373,"368":374,"369":375,"370":376,"371":377,"372":378,"373":379,"374":380,"375":381,"376":382,"377":383,"378":384,"379":385,"380":386,"381":387,"382":388,"383":389,"384":390,"385":391,"386":392,"387":393,"388":394,"389":395,"390":396,"391":397,"392":398,"393":399,"394":400,"395":401,"396":402,"397":403,"398":404,"399":405,"400":406,"401":407,"402":408,"403":409,"404":410,"405":411,"406":412,"407":413,"408":414,"409":415,"410":416,"411":417,"412":418,"413":419,"414":420,"415":421,"416":422,"417":423,"418":424,"419":425,"420":426,"421":427,"422":428,"423":429,"-":430,".":431,"A":432,"B":433,"C":434,"D":435,"E":436,"F":437,"G":438,"H":439,"I":440,"K":441,"L":442,"M":443,"N":444,"O":445,"P":446,"Q":447,"R":448,"S":449,"T":450,"U":451,"V":452,"W":453,"X":454,"Y":455,"Z":456,"n":457}