1 line
2.5 MiB
JSON
1 line
2.5 MiB
JSON
|
{"version":"1.0","truncation":{"max_length":1024,"strategy":"LongestFirst","stride":0},"padding":{"strategy":{"Fixed":1024},"direction":"Right","pad_to_multiple_of":null,"pad_id":1,"pad_type_id":0,"pad_token":"<pad>"},"added_tokens":[{"id":0,"special":true,"content":"<s>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":1,"special":true,"content":"<pad>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":2,"special":true,"content":"</s>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":3,"special":true,"content":"<unk>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":4,"special":true,"content":"<mask>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":5,"special":true,"content":"<|endoftext|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":6,"special":true,"content":"<|startoftext|>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":7,"special":true,"content":"<sep>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":8,"special":true,"content":"<cls>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":9,"special":true,"content":"<nl>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":10,"special":true,"content":"<tab>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":11,"special":true,"content":"<zwnj>","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":12,"special":true,"content":"[U1]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":13,"special":true,"content":"[U2]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":14,"special":true,"content":"[U3]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":15,"special":true,"content":"[U4]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":16,"special":true,"content":"[U5]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":17,"special":true,"content":"[U6]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":18,"special":true,"content":"[U7]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":19,"special":true,"content":"[U8]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":20,"special":true,"content":"[U9]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":21,"special":true,"content":"[U10]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":22,"special":true,"content":"[U11]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":23,"special":true,"content":"[U12]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":24,"special":true,"content":"[U13]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":25,"special":true,"content":"[U14]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":26,"special":true,"content":"[U15]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":27,"special":true,"content":"[U16]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":28,"special":true,"content":"[U17]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":29,"special":true,"content":"[U18]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":30,"special":true,"content":"[U19]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":31,"special":true,"content":"[U20]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false}],"normalizer":null,"pre_tokenizer":{"type":"ByteLevel","add_prefix_space":false,"trim_offsets":true},"post_processor":{"type":"ByteLevel","add_prefix_space":true,"trim_offsets":false},"decoder":{"type":"ByteLevel","add_prefix_space":true,"trim_offsets":true},"model":{"type":"BPE"
|