初始化项目，由ModelHub XC社区提供模型

Model: cedpsam/chatbot_fr
Source: Original Platform

.gitattributes

@@ -0,0 +1,10 @@
+*.bin.* filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tar.gz filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+model.safetensors filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,47 @@
+---
+language: fr
+tags:
+- conversational
+widget:
+- text: "bonjour."
+- text: "mais encore"
+- text: "est ce que l'argent achete le bonheur?"
+---
+
+## a dialoggpt model trained on french opensubtitles with custom tokenizer
+trained with this notebook
+https://colab.research.google.com/drive/1pfCV3bngAmISNZVfDvBMyEhQKuYw37Rl#scrollTo=AyImj9qZYLRi&uniqifier=3
+
+config from microsoft/DialoGPT-medium
+dataset generated from 2018 opensubtitle downloaded from opus folowing these guidelines
+https://github.com/PolyAI-LDN/conversational-datasets/tree/master/opensubtitles with this notebook
+https://colab.research.google.com/drive/1uyh3vJ9nEjqOHI68VD73qxt4olJzODxi#scrollTo=deaacv4XfLMk
+### How to use
+
+Now we are ready to try out how the model works as a chatting partner!
+
+```python
+import torch
+from transformers import AutoTokenizer, AutoModelWithLMHead
+
+tokenizer = AutoTokenizer.from_pretrained("cedpsam/chatbot_fr")
+
+model = AutoModelWithLMHead.from_pretrained("cedpsam/chatbot_fr")
+
+for step in range(6):
+    # encode the new user input, add the eos_token and return a tensor in Pytorch
+    new_user_input_ids = tokenizer.encode(input(">> User:") + tokenizer.eos_token, return_tensors='pt')
+    # print(new_user_input_ids)
+
+    # append the new user input tokens to the chat history
+    bot_input_ids = torch.cat([chat_history_ids, new_user_input_ids], dim=-1) if step > 0 else new_user_input_ids
+
+    # generated a response while limiting the total chat history to 1000 tokens, 
+    chat_history_ids = model.generate(
+        bot_input_ids, max_length=1000,
+        pad_token_id=tokenizer.eos_token_id,
+        top_p=0.92, top_k = 50
+    )
+    
+    # pretty print last ouput tokens from bot
+    print("DialoGPT: {}".format(tokenizer.decode(chat_history_ids[:, bot_input_ids.shape[-1]:][0], skip_special_tokens=True)))

config.json

@@ -0,0 +1,27 @@
+{
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.1,
+  "bos_token_id": 50256,
+  "embd_pdrop": 0.1,
+  "eos_token_id": 50256,
+  "gradient_checkpointing": false,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_ctx": 1024,
+  "n_embd": 1024,
+  "n_head": 16,
+  "n_inner": null,
+  "n_layer": 24,
+  "n_positions": 1024,
+  "resid_pdrop": 0.1,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "vocab_size": 50257
+}

flax_model.msgpack

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:949e5d7bd5a3521e13075dc9445fc1f01868200a0a07230ed0ed15de54ec0d77
+size 1419302302

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d8c11f1a5af86f8795efd503533ecfc9b8f140f66700d76640a7d12b11b0878
+size 1444493752

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a047ef1d5761057637002ed9fba77ba1059f01a380c9c471c121105258ff158e
+size 1444586111

special_tokens_map.json

@@ -0,0 +1 @@
+{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "<unk>", "pad_token": "<pad>", "cls_token": "<cls>", "mask_token": "<mask>"}

tokenizer_config.json

@@ -0,0 +1 @@
+{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "<unk>", "pad_token": "<pad>", "cls_token": "<cls>", "mask_token": "<mask>", "special_tokens_map_file": "/gdrive/My Drive/Colab Notebooks/donnees/textgen_fr/chatbot/gpt2-medium-frtok/special_tokens_map.json", "full_tokenizer_file": null}