misc: add pre-commit config (#637)

This commit is contained in:
zhyncs
2024-07-18 04:55:39 +10:00
committed by GitHub
parent a8552cb18b
commit 2e341cd493
43 changed files with 481 additions and 299 deletions

View File

@@ -10,16 +10,15 @@ import os
from transformers import AutoConfig, AutoTokenizer
def add_image_token(model_path: str):
tokenizer = AutoTokenizer.from_pretrained(model_path)
tokenizer.add_tokens(
["<image_placeholder>"],
special_tokens=True
)
tokenizer.add_tokens(["<image_placeholder>"], special_tokens=True)
print(tokenizer)
tokenizer.save_pretrained(model_path)
def edit_model_config(model_path):
config = AutoConfig.from_pretrained(model_path)
@@ -29,10 +28,11 @@ def edit_model_config(model_path):
print(config)
config.save_pretrained(model_path)
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--model-path", type=str)
args = parser.parse_args()
add_image_token(args.model_path)
edit_model_config(args.model_path)
edit_model_config(args.model_path)