misc: add pre-commit config (#637)
This commit is contained in:
@@ -10,16 +10,15 @@ import os
|
||||
|
||||
from transformers import AutoConfig, AutoTokenizer
|
||||
|
||||
|
||||
def add_image_token(model_path: str):
|
||||
tokenizer = AutoTokenizer.from_pretrained(model_path)
|
||||
tokenizer.add_tokens(
|
||||
["<image_placeholder>"],
|
||||
special_tokens=True
|
||||
)
|
||||
tokenizer.add_tokens(["<image_placeholder>"], special_tokens=True)
|
||||
|
||||
print(tokenizer)
|
||||
tokenizer.save_pretrained(model_path)
|
||||
|
||||
|
||||
def edit_model_config(model_path):
|
||||
config = AutoConfig.from_pretrained(model_path)
|
||||
|
||||
@@ -29,10 +28,11 @@ def edit_model_config(model_path):
|
||||
print(config)
|
||||
config.save_pretrained(model_path)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--model-path", type=str)
|
||||
args = parser.parse_args()
|
||||
|
||||
add_image_token(args.model_path)
|
||||
edit_model_config(args.model_path)
|
||||
edit_model_config(args.model_path)
|
||||
|
||||
Reference in New Issue
Block a user