1 line
747 KiB
JSON
1 line
747 KiB
JSON
|
{"version":"1.0","truncation":{"max_length":512,"strategy":"LongestFirst","stride":0},"padding":{"strategy":"BatchLongest","direction":"Right","pad_to_multiple_of":null,"pad_id":0,"pad_type_id":0,"pad_token":"[PAD]"},"added_tokens":[{"id":0,"special":true,"content":"[PAD]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":1,"special":true,"content":"[UNK]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":2,"special":true,"content":"[CLS]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":3,"special":true,"content":"[SEP]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":4,"special":true,"content":"[MASK]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":5,"special":true,"content":"[NL]","single_word":false,"lstrip":false,"rstrip":false,"normalized":true},{"id":6,"special":true,"content":"[HS]","single_word":false,"lstrip":false,"rstrip":false,"normalized":true},{"id":7,"special":true,"content":"[SOS]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":8,"special":true,"content":"[EOS]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":9,"special":true,"content":"[ZWNJ]","single_word":false,"lstrip":false,"rstrip":false,"normalized":true},{"id":10,"special":true,"content":"[U1]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":11,"special":true,"content":"[U2]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":12,"special":true,"content":"[U3]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":13,"special":true,"content":"[U4]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":14,"special":true,"content":"[U5]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":15,"special":true,"content":"[U6]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":16,"special":true,"content":"[U7]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":17,"special":true,"content":"[U8]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":18,"special":true,"content":"[U9]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":19,"special":true,"content":"[U10]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":20,"special":true,"content":"[U11]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":21,"special":true,"content":"[U12]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":22,"special":true,"content":"[U13]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":23,"special":true,"content":"[U14]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":24,"special":true,"content":"[U15]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":25,"special":true,"content":"[U16]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":26,"special":true,"content":"[U17]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":27,"special":true,"content":"[U18]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":28,"special":true,"content":"[U19]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":29,"special":true,"content":"[U20]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":30,"special":true,"content":"[U21]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":31,"special":true,"content":"[U22]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":32,"special":true,"content":"[U23]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":33,"special":true,"content":"[U24]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":34,"special":true,"content":"[U25]","single_word":false,"lstrip":false
|