66 lines
1.2 KiB
JSON
66 lines
1.2 KiB
JSON
{
|
|
"additional_special_tokens": [
|
|
"[bib_ref]",
|
|
"[/bib_ref]",
|
|
"[fig_ref]",
|
|
"[/fig_ref]",
|
|
"[bib]",
|
|
"[/bib]",
|
|
"[fig]",
|
|
"[/fig]",
|
|
"[table]",
|
|
"[/table]",
|
|
"[formula]",
|
|
"[/formula]"
|
|
],
|
|
"bos_token": {
|
|
"content": "<s>",
|
|
"lstrip": false,
|
|
"normalized": true,
|
|
"rstrip": false,
|
|
"single_word": false
|
|
},
|
|
"cls_token": {
|
|
"content": "<CLS>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false
|
|
},
|
|
"eos_token": {
|
|
"content": "</s>",
|
|
"lstrip": false,
|
|
"normalized": true,
|
|
"rstrip": false,
|
|
"single_word": false
|
|
},
|
|
"mask_token": {
|
|
"content": "<MASK>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false
|
|
},
|
|
"pad_token": {
|
|
"content": "<PAD>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false
|
|
},
|
|
"sep_token": {
|
|
"content": "<SEP>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false
|
|
},
|
|
"unk_token": {
|
|
"content": "<unk>",
|
|
"lstrip": false,
|
|
"normalized": true,
|
|
"rstrip": false,
|
|
"single_word": false
|
|
}
|
|
}
|