初始化项目，由ModelHub XC社区提供模型

Model: nblokker/debatenet-2-cat
Source: Original Platform

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

1_Pooling/config.json

@@ -0,0 +1,7 @@
+{
+  "word_embedding_dimension": 768,
+  "pooling_mode_cls_token": false,
+  "pooling_mode_mean_tokens": true,
+  "pooling_mode_max_tokens": false,
+  "pooling_mode_mean_sqrt_len_tokens": false
+}

License

@@ -0,0 +1,20 @@
+MIT License
+
+Copyright (c) [year] [fullname]
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+----------------------------------------------------------------
+
+This model is based on the `sentence-transformers/paraphrase-multilingual-mpnet-base-v2` model:
+
+Reimers, N., & Gurevych, I. (2019). Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks. In Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing. Association for Computational Linguistics.
+The original model is licensed under the Apache 2.0 License:
+
+Apache License
+Version 2.0, January 2004
+http://www.apache.org/licenses/

README.md

@@ -0,0 +1,176 @@
+---
+pipeline_tag: sentence-similarity
+tags:
+- sentence-transformers
+- feature-extraction
+- sentence-similarity
+- transformers
+license: [mit, apache-2.0] 
+language:
+- multilingual
+- de
+- en
+
+---
+
+# nblokker/debatenet-2-cat
+
+This is a [sentence-transformers](https://www.SBERT.net) model: It maps sentences & paragraphs to a 768 dimensional dense vector space and can be used for tasks like clustering or semantic search.
+
+This model may be used to estimate the similarities between sentences containing migration-related demands and propositions. Check out this [blog post](https://nicoblokker.github.io/PTaD/posts/qca_meets_sbert/dna_meets_sbert.html) for more information and potential use cases.
+
+# Fine-Tuned on `sentence-transformers_paraphrase-multilingual-mpnet-base-v2` Model
+
+This repository contains a fine-tuned version of the `sentence-transformers_paraphrase-multilingual-mpnet-base-v2` model. The original model was created by Nils Reimers and Iryna Gurevych and is available on [Hugging Face](https://huggingface.co/sentence-transformers/paraphrase-multilingual-mpnet-base-v2).
+
+
+## Usage (Sentence-Transformers)
+
+Using this model becomes easy when you have [sentence-transformers](https://www.SBERT.net) installed:
+
+```
+pip install -U sentence-transformers
+```
+
+Then you can use the model like this:
+
+```python
+from sentence_transformers import SentenceTransformer
+sentences = ["This is an example sentence", "Each sentence is converted"]
+
+model = SentenceTransformer('nblokker/debatenet-2-cat')
+embeddings = model.encode(sentences)
+print(embeddings)
+```
+
+
+
+## Usage (HuggingFace Transformers)
+Without [sentence-transformers](https://www.SBERT.net), you can use the model like this: First, you pass your input through the transformer model, then you have to apply the right pooling-operation on-top of the contextualized word embeddings.
+
+```python
+from transformers import AutoTokenizer, AutoModel
+import torch
+
+
+#Mean Pooling - Take attention mask into account for correct averaging
+def mean_pooling(model_output, attention_mask):
+    token_embeddings = model_output[0] #First element of model_output contains all token embeddings
+    input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()
+    return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9)
+
+
+# Sentences we want sentence embeddings for
+sentences = ['This is an example sentence', 'Each sentence is converted']
+
+# Load model from HuggingFace Hub
+tokenizer = AutoTokenizer.from_pretrained('nblokker/debatenet-2-cat')
+model = AutoModel.from_pretrained('nblokker/debatenet-2-cat')
+
+# Tokenize sentences
+encoded_input = tokenizer(sentences, padding=True, truncation=True, return_tensors='pt')
+
+# Compute token embeddings
+with torch.no_grad():
+    model_output = model(**encoded_input)
+
+# Perform pooling. In this case, mean pooling.
+sentence_embeddings = mean_pooling(model_output, encoded_input['attention_mask'])
+
+print("Sentence embeddings:")
+print(sentence_embeddings)
+```
+
+
+
+## Evaluation Results
+
+For an automated evaluation of this model, see the *Sentence Embeddings Benchmark*: [https://seb.sbert.net](https://seb.sbert.net?model_name={MODEL_NAME})
+
+
+## Training
+The model was trained with the parameters:
+
+**DataLoader**:
+
+`torch.utils.data.dataloader.DataLoader` of length 38 with parameters:
+```
+{'batch_size': 32, 'sampler': 'torch.utils.data.sampler.RandomSampler', 'batch_sampler': 'torch.utils.data.sampler.BatchSampler'}
+```
+
+**Loss**:
+
+`sentence_transformers.losses.BatchHardSoftMarginTripletLoss.BatchHardSoftMarginTripletLoss` 
+
+Parameters of the fit()-Method:
+```
+{
+    "epochs": 15,
+    "evaluation_steps": 120.5,
+    "evaluator": "sentence_transformers.evaluation.BinaryClassificationEvaluator.BinaryClassificationEvaluator",
+    "max_grad_norm": 1,
+    "optimizer_class": "<class 'torch.optim.adamw.AdamW'>",
+    "optimizer_params": {
+        "lr": 2e-05
+    },
+    "scheduler": "WarmupLinear",
+    "steps_per_epoch": null,
+    "warmup_steps": 120.5,
+    "weight_decay": 0.01
+}
+```
+
+
+## Full Model Architecture
+```
+SentenceTransformer(
+  (0): Transformer({'max_seq_length': 128, 'do_lower_case': False}) with Transformer model: XLMRobertaModel 
+  (1): Pooling({'word_embedding_dimension': 768, 'pooling_mode_cls_token': False, 'pooling_mode_mean_tokens': True, 'pooling_mode_max_tokens': False, 'pooling_mode_mean_sqrt_len_tokens': False})
+)
+```
+
+## Citing & Authors
+
+```
+@preprint{blokker2023,
+  author = {Blokker, Nico and Blessing, Andre and Dayanik, Erenay and Kuhn, Jonas and Padó, Sebastian and Lapesa, Gabriella},
+  note = {To appear in \textit{Language Resources and Evaluation}},
+  title = {Between welcome culture and border fence: The {E}uropean refugee crisis in {G}erman newspaper reports},
+  url = {https://arxiv.org/abs/2111.10142},
+  year = 2023
+}
+
+@inproceedings{lapesa2020,
+  abstract = {DEbateNet-migr15 is a manually annotated dataset for German which covers the public debate on immigration in 2015. The building block of our annotation is the political science notion of a claim, i.e., a statement made by a political actor (a politician, a party, or a group of citizens) that a specific action should be taken (e.g., vacant flats should be assigned to refugees). We identify claims in newspaper articles, assign them to actors and fine-grained categories and annotate their polarity and date. The aim of this paper is two-fold: first, we release the full DEbateNet-mig15 corpus and document it by means of a quantitative and qualitative analysis; second, we demonstrate its application in a discourse network analysis framework, which enables us to capture the temporal dynamics of the political debate.},
+  address = {Online},
+  author = {Lapesa, Gabriella and Blessing, Andre and Blokker, Nico and Dayanik, Erenay and Haunss, Sebastian and Kuhn, Jonas and Padó, Sebastian},
+  booktitle = {Proceedings of LREC},
+  pages = {919--927},
+  title = {{DEbateNet-mig15}: {T}racing the 2015 Immigration Debate in {G}ermany Over Time},
+  url = {https://www.aclweb.org/anthology/2020.lrec-1.115},
+  year = 2020
+}
+
+
+```
+
+## Acknowledgments
+
+This model is based on the `sentence-transformers/paraphrase-multilingual-mpnet-base-v2` model:
+ ```
+@inproceedings{reimers-2019-sentence-bert,
+    title = "Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks",
+    author = "Reimers, Nils and Gurevych, Iryna",
+    booktitle = "Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing",
+    month = "11",
+    year = "2019",
+    publisher = "Association for Computational Linguistics",
+    url = "http://arxiv.org/abs/1908.10084",
+}
+```
+- Original model URL: https://huggingface.co/sentence-transformers/paraphrase-multilingual-mpnet-base-v2
+- License: Apache 2.0
+
+## License
+
+The fine-tuned parts of this model are released under the MIT License. See the LICENSE file for more details. The original sentence-transformers/paraphrase-multilingual-mpnet-base-v2 model remains under its original Apache 2.0 License.

config.json

@@ -0,0 +1,29 @@
+{
+  "_name_or_path": "/root/.cache/torch/sentence_transformers/sentence-transformers_paraphrase-multilingual-mpnet-base-v2/",
+  "architectures": [
+    "XLMRobertaModel"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 0,
+  "classifier_dropout": null,
+  "eos_token_id": 2,
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 514,
+  "model_type": "xlm-roberta",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "output_past": true,
+  "pad_token_id": 1,
+  "position_embedding_type": "absolute",
+  "torch_dtype": "float32",
+  "transformers_version": "4.26.0",
+  "type_vocab_size": 1,
+  "use_cache": true,
+  "vocab_size": 250002
+}

config_sentence_transformers.json

@@ -0,0 +1,7 @@
+{
+  "__version__": {
+    "sentence_transformers": "2.0.0",
+    "transformers": "4.7.0",
+    "pytorch": "1.9.0+cu102"
+  }
+}

eval/binary_classification_evaluation_results.csv

@@ -0,0 +1,16 @@
+epoch,steps,cossim_accuracy,cossim_accuracy_threshold,cossim_f1,cossim_precision,cossim_recall,cossim_f1_threshold,cossim_ap,manhattan_accuracy,manhattan_accuracy_threshold,manhattan_f1,manhattan_precision,manhattan_recall,manhattan_f1_threshold,manhattan_ap,euclidean_accuracy,euclidean_accuracy_threshold,euclidean_f1,euclidean_precision,euclidean_recall,euclidean_f1_threshold,euclidean_ap,dot_accuracy,dot_accuracy_threshold,dot_f1,dot_precision,dot_recall,dot_f1_threshold,dot_ap
+0,-1,0.977439471753485,0.6852872371673584,0.22887239645678714,0.18642745709828393,0.296342219466832,0.5128152370452881,0.16494330876220628,0.9774676900502286,47.54118347167969,0.2103638368246968,0.16324435318275154,0.2957222566646001,59.05224609375,0.14747785341642275,0.9774959083469722,2.2449405193328857,0.2097873219079083,0.15727554179566564,0.31494110353378796,2.788694381713867,0.14839370767340765,0.9773548168632541,7.260024070739746,0.2064690026954178,0.1826418693371483,0.23744575325480471,4.289517402648926,0.14178830255012298
+1,-1,0.9790620238162424,0.5447380542755127,0.36585365853658536,0.39057002111189304,0.34407935523868566,0.4287317097187042,0.3152793777839269,0.9789068231841526,63.031227111816406,0.34073147858705843,0.34363177805800754,0.337879727216367,72.44259643554688,0.2968048637777491,0.9789350414808962,3.0699684619903564,0.34177997527812115,0.3407270486752927,0.3428394296342219,3.416299819946289,0.2961354099490465,0.9784271121395113,5.970248222351074,0.33756906077348064,0.3044344793223717,0.3787972721636702,4.22919225692749,0.2833282778848849
+2,-1,0.9798380269766916,0.5307826995849609,0.41090555014605656,0.4311989100817439,0.39243645381277126,0.4304565191268921,0.38454473172120207,0.9799791184604097,72.22743225097656,0.3978638184245661,0.4309472161966739,0.36949783013019216,80.00604248046875,0.3741430585825125,0.9800637733506405,3.4823551177978516,0.39925093632958797,0.5042573320719016,0.3304401735895846,3.700659990310669,0.37816350816957833,0.9793159884869349,7.303128242492676,0.40424847119407786,0.4203480589022758,0.38933663980161193,5.500328063964844,0.36283015916308226
+3,-1,0.9814605790394492,0.5218061208724976,0.46227456753772544,0.5688405797101449,0.38933663980161193,0.44186002016067505,0.45209552045241064,0.9814605790394492,86.66259002685547,0.4423347398030943,0.5109666937449229,0.3899566026038438,92.86383056640625,0.43297220726738345,0.9813900332975901,4.045625686645508,0.44497041420118344,0.6117136659436009,0.34965902045877245,4.270045280456543,0.4337843795870024,0.9811925052203849,9.729372024536133,0.45337954939341424,0.5141509433962265,0.40545567265964044,7.416886329650879,0.4418529367812887
+4,-1,0.9824905468705909,0.5798612833023071,0.501892505677517,0.6443148688046647,0.41103533787972724,0.5184295773506165,0.49857954859283615,0.9826175292059371,86.81884765625,0.5064258423063563,0.5758293838862559,0.4519528828270304,96.52304077148438,0.4998542565758241,0.9827445115412834,4.193209648132324,0.5096750638919314,0.6198934280639432,0.43273403595784254,4.489302635192871,0.5018017548441271,0.9814605790394492,11.951271057128906,0.4661168877733955,0.5527210884353742,0.402975821450713,10.558049201965332,0.4589357522210889
+5,-1,0.9828573847282578,0.5792489647865295,0.5107468123861566,0.6192579505300353,0.43459392436453814,0.5011060237884521,0.5226858197381234,0.9834640781082454,95.65682220458984,0.5105148658448151,0.6148471615720524,0.43645381277123374,103.55033874511719,0.5268057077080315,0.9833794232180145,4.5222883224487305,0.5119705340699817,0.6306715063520871,0.43087414755114695,4.865683078765869,0.526106970736683,0.9816016705231673,14.777538299560547,0.48034006376195537,0.5603305785123966,0.4203347799132052,12.306472778320312,0.47371095473029445
+6,-1,0.982843275579886,0.6081464290618896,0.49944751381215463,0.6152450090744102,0.4203347799132052,0.5324764847755432,0.5211788455153079,0.9827304023929115,96.8388671875,0.506656523392925,0.655511811023622,0.41289522628642283,106.29513549804688,0.5215959585865961,0.9827445115412834,4.725986480712891,0.5026399155227034,0.5814332247557004,0.4426534407935524,5.149684429168701,0.5205197942413685,0.9819967266775778,18.029041290283203,0.48339878614780435,0.5698653198653199,0.41971481711097336,14.681556701660156,0.49038992227135325
+7,-1,0.9837744793724251,0.6556621789932251,0.5254428948360347,0.6701923076923076,0.4321140731556107,0.5719858407974243,0.5472918120156424,0.9838732434110277,98.3125,0.5347875141030463,0.6797323135755258,0.4407935523868568,108.6465835571289,0.5485444393276058,0.9837321519273097,4.560788154602051,0.5344011649071716,0.6472663139329806,0.4550526968381897,5.189488410949707,0.5478214295853534,0.9831395676956939,20.612266540527344,0.5076804915514593,0.6670030272452069,0.4097954122752635,18.042613983154297,0.5211650017133908
+8,-1,0.9833794232180145,0.648638129234314,0.5478593804385659,0.6246031746031746,0.4879107253564786,0.539995551109314,0.5586942500751392,0.9833794232180145,104.07281494140625,0.538382099827883,0.6052631578947368,0.4848109113453193,116.29108428955078,0.5460846108869247,0.9834076415147581,4.9742584228515625,0.5364169830859511,0.6051401869158879,0.48171109733415995,5.458589553833008,0.545244642579938,0.9829279304701168,22.148103713989258,0.5195369030390737,0.6238053866203301,0.44513329200247986,18.465391159057617,0.5345101341448621
+9,-1,0.9841272080817202,0.6621180176734924,0.5555555555555557,0.6832579185520362,0.4680719156850589,0.5821796655654907,0.5759450660833354,0.9842682995654383,105.94416809082031,0.567481402763018,0.6619834710743802,0.49659020458772474,115.38886260986328,0.581122288366334,0.9841554263784638,4.93897819519043,0.5654676258992805,0.6735218508997429,0.48729076255424675,5.405415058135986,0.5801213677760783,0.9835346238501044,24.16677474975586,0.5344628417404698,0.7052845528455285,0.4302541847489151,21.125396728515625,0.5483971466248824
+10,-1,0.9838026976691687,0.657180666923523,0.5528962885882761,0.6275590551181103,0.4941103533787973,0.5745883584022522,0.5702456989035126,0.9838591342626559,107.51908111572266,0.5639017443930224,0.6622073578595318,0.49101053936763794,116.48408508300781,0.5782499046446209,0.9838026976691687,4.925640106201172,0.5626865671641791,0.7066541705716963,0.46745195288282704,5.379071235656738,0.5769530865616528,0.9833370957728992,25.30411148071289,0.5265123226288275,0.6619718309859155,0.4370737755734656,22.200775146484375,0.5423113381744178
+11,-1,0.9828997121733732,0.7019469738006592,0.5369572422936691,0.5769230769230769,0.5021698698078115,0.5591743588447571,0.5510631494459538,0.9830831311022067,109.6434326171875,0.5419773095623986,0.5679347826086957,0.51828890266584,121.2200698852539,0.5544506349921243,0.9828997121733732,4.923254013061523,0.5377567140600316,0.5483247422680413,0.527588344699318,5.776214599609375,0.5523131096324556,0.9825893109091935,24.96963119506836,0.5167807103290973,0.5446428571428571,0.4916305021698698,20.23036766052246,0.5170847022586458
+12,-1,0.983492296404989,0.6365242600440979,0.5533737950731882,0.6523569023569024,0.4804711717296962,0.5840268135070801,0.5685857171274472,0.9835487329984762,111.17485046386719,0.555671902268761,0.6357827476038339,0.4934903905765654,119.4373779296875,0.5691928314646528,0.9834358598115017,5.348132610321045,0.5509851365364673,0.62265625,0.4941103533787973,5.650306701660156,0.5679542727723502,0.9832665500310401,24.62594223022461,0.5309190297232662,0.591324200913242,0.48171109733415995,21.309249877929688,0.5456145143600372
+13,-1,0.9832806591794119,0.6357815265655518,0.5477439664218258,0.6284109149277689,0.48543087414755115,0.5822320580482483,0.563611179175558,0.9833229866245273,109.45771789550781,0.5532338308457712,0.5948644793152639,0.5170489770613763,121.76615905761719,0.5640285141950473,0.9832806591794119,5.291834354400635,0.5519785378940308,0.6011687363038715,0.5102293862368258,5.736968517303467,0.562812983274202,0.9831536768440657,25.919189453125,0.5303983228511531,0.6076861489191353,0.47055176689398637,21.98784065246582,0.5405732734594546
+14,-1,0.9834499689598736,0.6525164842605591,0.5483755801499465,0.6464646464646465,0.47613143211407316,0.5876088738441467,0.5659018776624276,0.98342175066313,108.82964324951172,0.5546163849154746,0.5830485304169515,0.5288282703037818,123.06658935546875,0.5655406416092383,0.9834640781082454,5.187602996826172,0.5509732761464863,0.5888575458392101,0.5176689398636082,5.780318260192871,0.5643796212774541,0.9832806591794119,26.09710693359375,0.5301120448179272,0.6090104585679806,0.46931184128952264,21.904300689697266,0.543978384360549

modules.json

@@ -0,0 +1,14 @@
+[
+  {
+    "idx": 0,
+    "name": "0",
+    "path": "",
+    "type": "sentence_transformers.models.Transformer"
+  },
+  {
+    "idx": 1,
+    "name": "1",
+    "path": "1_Pooling",
+    "type": "sentence_transformers.models.Pooling"
+  }
+]

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d49a5744abbd10d64bc1a2298286d3ef97f0997eb8330d49e30d1ccdc3a64014
+size 1112245805

sentence_bert_config.json

@@ -0,0 +1,4 @@
+{
+  "max_seq_length": 128,
+  "do_lower_case": false
+}

sentencepiece.bpe.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cfc8146abe2a0488e9e2a0c56de7952f7c11ab059eca145a0a727afce0db2865
+size 5069051

special_tokens_map.json

@@ -0,0 +1,15 @@
+{
+  "bos_token": "<s>",
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "unk_token": "<unk>"
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b60b6b43406a48bf3638526314f3d232d97058bc93472ff2de930d43686fa441
+size 17082913

tokenizer_config.json

@@ -0,0 +1,20 @@
+{
+  "bos_token": "<s>",
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "mask_token": {
+    "__type": "AddedToken",
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "model_max_length": 512,
+  "name_or_path": "/root/.cache/torch/sentence_transformers/sentence-transformers_paraphrase-multilingual-mpnet-base-v2/",
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "special_tokens_map_file": null,
+  "tokenizer_class": "XLMRobertaTokenizer",
+  "unk_token": "<unk>"
+}