init
This commit is contained in:
187
transformers/tests/models/clip/test_tokenization_clip.py
Normal file
187
transformers/tests/models/clip/test_tokenization_clip.py
Normal file
@@ -0,0 +1,187 @@
|
||||
# Copyright 2021 The HuggingFace Team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
import json
|
||||
import os
|
||||
import unittest
|
||||
|
||||
from transformers import CLIPTokenizer, CLIPTokenizerFast
|
||||
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
|
||||
from transformers.testing_utils import require_ftfy, require_tokenizers
|
||||
|
||||
from ...test_tokenization_common import TokenizerTesterMixin
|
||||
|
||||
|
||||
@require_tokenizers
|
||||
class CLIPTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
|
||||
from_pretrained_id = "openai/clip-vit-base-patch32"
|
||||
tokenizer_class = CLIPTokenizer
|
||||
rust_tokenizer_class = CLIPTokenizerFast
|
||||
test_rust_tokenizer = True
|
||||
from_pretrained_kwargs = {}
|
||||
test_seq2seq = False
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
super().setUpClass()
|
||||
|
||||
vocab = ["l", "o", "w", "e", "r", "s", "t", "i", "d", "n", "lo", "l</w>", "w</w>", "r</w>", "t</w>", "low</w>", "er</w>", "lowest</w>", "newer</w>", "wider", "<unk>", "<|startoftext|>", "<|endoftext|>"] # fmt: skip
|
||||
vocab_tokens = dict(zip(vocab, range(len(vocab))))
|
||||
merges = ["#version: 0.2", "l o", "lo w</w>", "e r</w>"]
|
||||
cls.special_tokens_map = {"unk_token": "<unk>"}
|
||||
|
||||
cls.vocab_file = os.path.join(cls.tmpdirname, VOCAB_FILES_NAMES["vocab_file"])
|
||||
cls.merges_file = os.path.join(cls.tmpdirname, VOCAB_FILES_NAMES["merges_file"])
|
||||
with open(cls.vocab_file, "w", encoding="utf-8") as fp:
|
||||
fp.write(json.dumps(vocab_tokens) + "\n")
|
||||
with open(cls.merges_file, "w", encoding="utf-8") as fp:
|
||||
fp.write("\n".join(merges))
|
||||
|
||||
@classmethod
|
||||
def get_tokenizer(cls, pretrained_name=None, **kwargs):
|
||||
kwargs.update(cls.special_tokens_map)
|
||||
pretrained_name = pretrained_name or cls.tmpdirname
|
||||
return CLIPTokenizer.from_pretrained(pretrained_name, **kwargs)
|
||||
|
||||
@classmethod
|
||||
def get_rust_tokenizer(cls, pretrained_name=None, **kwargs):
|
||||
kwargs.update(cls.special_tokens_map)
|
||||
pretrained_name = pretrained_name or cls.tmpdirname
|
||||
return CLIPTokenizerFast.from_pretrained(pretrained_name, **kwargs)
|
||||
|
||||
def get_input_output_texts(self, tokenizer):
|
||||
input_text = "lower newer"
|
||||
output_text = "lower newer"
|
||||
return input_text, output_text
|
||||
|
||||
def test_full_tokenizer(self):
|
||||
tokenizer = CLIPTokenizer(self.vocab_file, self.merges_file, **self.special_tokens_map)
|
||||
text = "lower newer"
|
||||
bpe_tokens = ["lo", "w", "er</w>", "n", "e", "w", "er</w>"]
|
||||
tokens = tokenizer.tokenize(text)
|
||||
self.assertListEqual(tokens, bpe_tokens)
|
||||
|
||||
input_tokens = tokens + [tokenizer.unk_token]
|
||||
input_bpe_tokens = [10, 2, 16, 9, 3, 2, 16, 20]
|
||||
self.assertListEqual(tokenizer.convert_tokens_to_ids(input_tokens), input_bpe_tokens)
|
||||
|
||||
@require_ftfy
|
||||
def test_check_encoding_slow_fast(self):
|
||||
for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
|
||||
with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"):
|
||||
tokenizer_s = self.get_tokenizer(pretrained_name, **kwargs)
|
||||
tokenizer_r = self.get_rust_tokenizer(pretrained_name, **kwargs)
|
||||
|
||||
text = "A\n'll 11p223RF☆ho!!to?'d'd''d of a cat to-$''d."
|
||||
text_tokenized_s = tokenizer_s.tokenize(text)
|
||||
text_tokenized_r = tokenizer_r.tokenize(text)
|
||||
|
||||
self.assertListEqual(text_tokenized_s, text_tokenized_r)
|
||||
|
||||
# Test that the tokenization is identical on an example containing a character (Latin Small Letter A
|
||||
# with Tilde) encoded in 2 different ways
|
||||
text = "xa\u0303y" + " " + "x\xe3y"
|
||||
text_tokenized_s = tokenizer_s.tokenize(text)
|
||||
text_tokenized_r = tokenizer_r.tokenize(text)
|
||||
|
||||
self.assertListEqual(text_tokenized_s, text_tokenized_r)
|
||||
|
||||
# Test that the tokenization is identical on unicode of space type
|
||||
spaces_unicodes = [
|
||||
"\u0009", # (horizontal tab, '\t')
|
||||
"\u000b", # (vertical tab)
|
||||
"\u000c", # (form feed)
|
||||
"\u0020", # (space, ' ')
|
||||
"\u200e", # (left-to-right mark):w
|
||||
"\u200f", # (right-to-left mark)
|
||||
]
|
||||
for unicode_seq in spaces_unicodes:
|
||||
text_tokenized_s = tokenizer_s.tokenize(unicode_seq)
|
||||
text_tokenized_r = tokenizer_r.tokenize(unicode_seq)
|
||||
|
||||
self.assertListEqual(text_tokenized_s, text_tokenized_r)
|
||||
|
||||
# Test that the tokenization is identical on unicode of line break type
|
||||
line_break_unicodes = [
|
||||
"\u000a", # (line feed, '\n')
|
||||
"\r\n", # (carriage return and line feed, '\r\n')
|
||||
"\u000d", # (carriage return, '\r')
|
||||
"\r", # (carriage return, '\r')
|
||||
"\u000d", # (carriage return, '\r')
|
||||
"\u2028", # (line separator)
|
||||
"\u2029", # (paragraph separator)
|
||||
# "\u0085", # (next line)
|
||||
]
|
||||
|
||||
# The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
|
||||
# it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
|
||||
# space (and thus into an empty list).
|
||||
|
||||
for unicode_seq in line_break_unicodes:
|
||||
text_tokenized_s = tokenizer_s.tokenize(unicode_seq)
|
||||
text_tokenized_r = tokenizer_r.tokenize(unicode_seq)
|
||||
|
||||
self.assertListEqual(text_tokenized_s, text_tokenized_r)
|
||||
|
||||
def test_offsets_mapping_with_different_add_prefix_space_argument(self):
|
||||
# Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space`
|
||||
for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
|
||||
with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"):
|
||||
text_of_1_token = "hello" # `hello` is a token in the vocabulary of `pretrained_name`
|
||||
text = f"{text_of_1_token} {text_of_1_token}"
|
||||
|
||||
tokenizer_r = self.get_rust_tokenizer(
|
||||
pretrained_name,
|
||||
use_fast=True,
|
||||
)
|
||||
encoding = tokenizer_r(text, return_offsets_mapping=True, add_special_tokens=False)
|
||||
self.assertEqual(encoding.offset_mapping[0], (0, len(text_of_1_token)))
|
||||
self.assertEqual(
|
||||
encoding.offset_mapping[1],
|
||||
(len(text_of_1_token) + 1, len(text_of_1_token) + 1 + len(text_of_1_token)),
|
||||
)
|
||||
|
||||
text = f" {text}"
|
||||
|
||||
tokenizer_r = self.get_rust_tokenizer(
|
||||
pretrained_name,
|
||||
use_fast=True,
|
||||
)
|
||||
encoding = tokenizer_r(text, return_offsets_mapping=True, add_special_tokens=False)
|
||||
self.assertEqual(encoding.offset_mapping[0], (1, 1 + len(text_of_1_token)))
|
||||
self.assertEqual(
|
||||
encoding.offset_mapping[1],
|
||||
(1 + len(text_of_1_token) + 1, 1 + len(text_of_1_token) + 1 + len(text_of_1_token)),
|
||||
)
|
||||
|
||||
def test_log_warning(self):
|
||||
# Test related to the breaking change introduced in transformers v4.17.0
|
||||
# We need to check that an error in raised when the user try to load a previous version of the tokenizer.
|
||||
with self.assertRaises(TypeError) as context:
|
||||
self.get_rust_tokenizer("robot-test/old-clip-tokenizer")
|
||||
|
||||
self.assertTrue(
|
||||
context.exception.args[0].startswith(
|
||||
"The `backend_tokenizer` provided does not match the expected format."
|
||||
)
|
||||
)
|
||||
|
||||
@require_ftfy
|
||||
def test_tokenization_python_rust_equals(self):
|
||||
super().test_tokenization_python_rust_equals()
|
||||
|
||||
@unittest.skip(reason="CLIP always lower cases letters")
|
||||
def test_added_tokens_do_lower_case(self):
|
||||
pass
|
||||
Reference in New Issue
Block a user