20 lines
803 B
Python
20 lines
803 B
Python
path = '/usr/local/lib/python3.10/dist-packages/transformers/tokenization_utils_base.py'
|
|
with open(path, 'r') as f:
|
|
content = f.read()
|
|
|
|
old = (' self.SPECIAL_TOKENS_ATTRIBUTES = self.SPECIAL_TOKENS_ATTRIBUTES'
|
|
' + list(special_tokens.keys())')
|
|
new = (' # PATCH: some models have extra_special_tokens as list instead of dict\n'
|
|
' if isinstance(special_tokens, list):\n'
|
|
' special_tokens = {t: t for t in special_tokens}\n'
|
|
' self.SPECIAL_TOKENS_ATTRIBUTES = self.SPECIAL_TOKENS_ATTRIBUTES'
|
|
' + list(special_tokens.keys())')
|
|
|
|
if old in content:
|
|
content = content.replace(old, new)
|
|
with open(path, 'w') as f:
|
|
f.write(content)
|
|
print('Patch applied successfully')
|
|
else:
|
|
print('WARNING: pattern not found')
|