dense_encoder-msmarco-distilbert-word2vec256k-MLM_785k_emb_updated/tokenizer.json

{"version":"1.0","truncation":{"max_length":250,"strategy":"LongestFirst","stride":0},"padding":{"strategy":"BatchLongest","direction":"Right","pad_to_multiple_of":null,"pad_id":3,"pad_type_id":0,"pad_token":"[PAD]"},"added_tokens":[{"id":0,"special":true,"content":"[UNK]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":1,"special":true,"content":"[CLS]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":2,"special":true,"content":"[SEP]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":3,"special":true,"content":"[PAD]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":4,"special":true,"content":"[MASK]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false}],"normalizer":{"type":"Sequence","normalizers":[{"type":"NFD"},{"type":"Lowercase"},{"type":"StripAccents"}]},"pre_tokenizer":{"type":"Whitespace"},"post_processor":{"type":"TemplateProcessing","single":[{"SpecialToken":{"id":"[CLS]","type_id":0}},{"Sequence":{"id":"A","type_id":0}},{"SpecialToken":{"id":"[SEP]","type_id":0}}],"pair":[{"SpecialToken":{"id":"[CLS]","type_id":0}},{"Sequence":{"id":"A","type_id":0}},{"SpecialToken":{"id":"[SEP]","type_id":0}},{"Sequence":{"id":"B","type_id":1}},{"SpecialToken":{"id":"[SEP]","type_id":1}}],"special_tokens":{"[CLS]":{"id":"[CLS]","ids":[1],"tokens":["[CLS]"]},"[SEP]":{"id":"[SEP]","ids":[2],"tokens":["[SEP]"]}}},"decoder":null,"model":{"type":"WordPiece","unk_token":"[UNK]","continuing_subword_prefix":"##","max_input_chars_per_word":100,"vocab":{"[UNK]":0,"[CLS]":1,"[SEP]":2,"[PAD]":3,"[MASK]":4,"!":5,"#":6,"$":7,"%":8,"&":9,"'":10,"(":11,")":12,"*":13,"+":14,",":15,"-":16,".":17,"/":18,"0":19,"1":20,"2":21,"3":22,"4":23,"5":24,"6":25,"7":26,"8":27,"9":28,":":29,";":30,"<":31,"=":32,">":33,"?":34,"@":35,"[":36,"\\":37,"]":38,"^":39,"_":40,"`":41,"a":42,"b":43,"c":44,"d":45,"e":46,"f":47,"g":48,"h":49,"i":50,"j":51,"k":52,"l":53,"m":54,"n":55,"o":56,"p":57,"q":58,"r":59,"s":60,"t":61,"u":62,"v":63,"w":64,"x":65,"y":66,"z":67,"{":68,"|":69,"}":70,"~":71,"€":72,"":73,"‚":74,"ƒ":75,"„":76,"†":77,"‡":78,"ˆ":79,"‰":80,"Š":81,"‹":82,"Œ":83,"":84,"Ž":85,"":86,"":87,"‘":88,"’":89,"“":90,"”":91,"•":92,"–":93,"—":94,"˜":95,"™":96,"š":97,"›":98,"œ":99,"":100,"ž":101,"Ÿ":102,"¡":103,"¢":104,"£":105,"¤":106,"¥":107,"¦":108,"§":109,"¨":110,"©":111,"ª":112,"«":113,"¬":114,"­":115,"®":116,"¯":117,"°":118,"±":119,"²":120,"³":121,"´":122,"µ":123,"¶":124,"·":125,"¸":126,"¹":127,"º":128,"»":129,"¼":130,"½":131,"¾":132,"¿":133,"×":134,"ß":135,"æ":136,"ð":137,"ø":138,"þ":139,"##s":140,"##m":141,"##e":142,"##n":143,"##a":144,"##w":145,"##t":146,"##i":147,"##l":148,"##r":149,"##o":150,"##c":151,"##d":152,"##h":153,"##y":154,"##k":155,"##¢":156,"##×":157,"##œ":158,"##”":159,"##p":160,"##u":161,"##g":162,"##b":163,"##¨":164,"##)":165,"##,":166,"##f":167,"##1":168,"##2":169,"##4":170,"##v":171,"##z":172,"##_":173,"##5":174,"##0":175,"##6":176,"##8":177,"##7":178,"##j":179,"##x":180,"##•":181,"##“":182,"##.":183,"##3":184,"##q":185,"##9":186,"##-":187,"##|":188,"##/":189,"##:":190,"##º":191,"##¯":192,"##;":193,"##‰":194,"##*":195,"###":196,"##¡":197,"##´":198,"##!":199,"##˜":200,"##ž":201,"##>":202,"##@":203,"##\\":204,"##‹":205,"##—":206,"##ð":207,"##µ":208,"##=":209,"##$":210,"##'":211,"##{":212,"##+":213,"##}":214,"##¬":215,"##^":216,"##":217,"##š":218,"##²":219,"##ª":220,"##Œ":221,"##]":222,"##¦":223,"##’":224,"##†":225,"##<":226,"##»":227,"##½":228,"##¶":229,"##³":230,"##ƒ":231,"##":232,"##™":233,"##¥":234,"##±":235,"##[":236,"##©":237,"##‘":238,"##¾":239,"##®":240,"##%":241,"##~":242,"##":243,"##?":244,"##Ž":245,"##°":246,"##­":247,"##¸":248,"##›":249,"##–":250,"##¹":251,"##`":252,"##":253,"##¤":254,"##(":255,"##&":256,"##„":257,"##‡":258,"##¿":259,"##Š":260,"##£":261,"##æ":262,"##":263,"##¼":264,"##·":265,"##€":266,"##‚":267,"##§":268,"##Ÿ":269,"##«":270,"##ˆ":271,"##ø":272,"##þ":