初始化项目，由ModelHub XC社区提供模型

Model: magicslabnu/clip_softmax_opt125m
Source: Original Platform

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,3 @@
+---
+license: mit
+---

all_results.json

@@ -0,0 +1 @@
+{"perplexity": 15.825365818307334, "model.decoder.final_layer_norm": 107.08700014366714, "model.decoder.layers.0": 35.77059901580118, "model.decoder.layers.1": 40.49228689673655, "model.decoder.layers.2": 42.06121581692331, "model.decoder.layers.3": 42.937613776067195, "model.decoder.layers.4": 43.57463559782768, "model.decoder.layers.5": 43.703882706613705, "model.decoder.layers.6": 43.7385998817424, "model.decoder.layers.7": 43.777486624884695, "model.decoder.layers.8": 43.80256970599777, "model.decoder.layers.9": 43.60152204305745, "model.decoder.layers.10": 42.68628150912729, "model.decoder.layers.11": 10.024581007379194, "model.decoder.layers.0.fc2": 0.4047294884108384, "model.decoder.layers.1.fc2": 0.07164183118889149, "model.decoder.layers.2.fc2": 0.060031713065045046, "model.decoder.layers.3.fc2": 0.07764280836753808, "model.decoder.layers.4.fc2": 0.08722878339786165, "model.decoder.layers.5.fc2": 0.11042726899940525, "model.decoder.layers.6.fc2": 0.13775341703656738, "model.decoder.layers.7.fc2": 0.18352035749051357, "model.decoder.layers.8.fc2": 0.2988111621249665, "model.decoder.layers.9.fc2": 0.5055887425842607, "model.decoder.layers.10.fc2": 0.5475804257446999, "model.decoder.layers.11.fc2": 0.43199118347560694, "model.decoder.layers.0.final_layer_norm": 0.5464058206097175, "model.decoder.layers.1.final_layer_norm": 1.034606302584533, "model.decoder.layers.2.final_layer_norm": 0.7081221116129993, "model.decoder.layers.3.final_layer_norm": 0.5427570727404248, "model.decoder.layers.4.final_layer_norm": 0.59629465032028, "model.decoder.layers.5.final_layer_norm": 0.7175459418216487, "model.decoder.layers.6.final_layer_norm": 0.7895661903727754, "model.decoder.layers.7.final_layer_norm": 0.8847526643356166, "model.decoder.layers.8.final_layer_norm": 1.0245287620083712, "model.decoder.layers.9.final_layer_norm": 1.404656795537524, "model.decoder.layers.10.final_layer_norm": 1.9760857988171199, "model.decoder.layers.11.final_layer_norm": 1.9044236200112898, "model.decoder.layers.0.self_attn.out_proj": 0.19450224048896522, "model.decoder.layers.1.self_attn.out_proj": 0.2538161420018138, "model.decoder.layers.2.self_attn.out_proj": 0.13237796095713258, "model.decoder.layers.3.self_attn.out_proj": 0.12417038498221668, "model.decoder.layers.4.self_attn.out_proj": 0.375805232620734, "model.decoder.layers.5.self_attn.out_proj": 0.2626552359139409, "model.decoder.layers.6.self_attn.out_proj": 0.2987572339722155, "model.decoder.layers.7.self_attn.out_proj": 0.3446044328147799, "model.decoder.layers.8.self_attn.out_proj": 0.5258365141278575, "model.decoder.layers.9.self_attn.out_proj": 1.5080789647677528, "model.decoder.layers.10.self_attn.out_proj": 6.577608219844358, "model.decoder.layers.11.self_attn.out_proj": 41.420152399481196, "model.decoder.layers.0.self_attn_layer_norm": 2.9636064259149375, "model.decoder.layers.1.self_attn_layer_norm": 4.3931188328643405, "model.decoder.layers.2.self_attn_layer_norm": 8.63972928041614, "model.decoder.layers.3.self_attn_layer_norm": 9.806978083615482, "model.decoder.layers.4.self_attn_layer_norm": 8.178833381752405, "model.decoder.layers.5.self_attn_layer_norm": 7.973861183560467, "model.decoder.layers.6.self_attn_layer_norm": 9.047355696539626, "model.decoder.layers.7.self_attn_layer_norm": 9.203052036647822, "model.decoder.layers.8.self_attn_layer_norm": 8.328000256910089, "model.decoder.layers.9.self_attn_layer_norm": 7.248920743335546, "model.decoder.layers.10.self_attn_layer_norm": 6.589432730130493, "model.decoder.layers.11.self_attn_layer_norm": 7.61927829387123, "max_inf_norm": 107.08700014366714, "max_ffn_inf_norm": 0.5475804257446999, "max_layer_inf_norm": 43.80256970599777, "avg_kurtosis": Infinity, "max_kurtosis": Infinity, "max_kurtosis_layers": 117020.92771410912}

checkpoints/checkpoint_120000/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eacea0f0b3576964acb2b7969120b19e706a8703ded02952d1f72ff36435d0bc
+size 496261008

checkpoints/checkpoint_120000/optimizer.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:65340e148789c378403521be3a707f2838ce9376ff8c8b71b50b0ac1a5cbdad8
+size 992641146

checkpoints/checkpoint_120000/random_states_0.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc4e1f682d069ee1e76f8aea55ecddcaa11b5b9461991acaa91c8f2d1c76bc18
+size 14540

checkpoints/checkpoint_120000/scaler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:247d63ecec0b7a41e83930a5ef241649421cf1e19f44dda335e35bb94570cd87
+size 988

checkpoints/checkpoint_120000/scheduler.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0ee874927be5a247eec6da124a3ab9ed18b0ee0029a6ecf267d0739657c6ed72
+size 1064

config.json

@@ -0,0 +1,34 @@
+{
+  "_remove_final_layer_norm": false,
+  "activation_function": "relu",
+  "architectures": [
+    "OPTForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 2,
+  "do_layer_norm_before": true,
+  "dropout": 0.1,
+  "enable_bias": true,
+  "eos_token_id": 2,
+  "ffn_dim": 3072,
+  "hidden_size": 768,
+  "init_std": 0.006,
+  "layer_norm_elementwise_affine": true,
+  "layerdrop": 0.0,
+  "max_position_embeddings": 512,
+  "model_type": "opt",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 1,
+  "torch_dtype": "float32",
+  "transformers_version": "4.31.0",
+  "use_cache": true,
+  "vocab_size": 50272,
+  "word_embed_proj_dim": 768,
+  "auto_map": 
+      {"AutoConfig": "configuration_opt.OPTConfig", 
+        "AutoModel": "modeling_opt.OPTModel", 
+        "AutoModelForCausalLM": "modeling_opt.OPTForCausalLM"
+
+      }
+}

configuration_opt.py

@@ -0,0 +1,143 @@
+# coding=utf-8
+# Copyright 2022 The Metaseq Authors and The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""OPT model configuration"""
+
+from transformers.configuration_utils import PretrainedConfig
+from transformers.utils import logging
+
+
+logger = logging.get_logger(__name__)
+
+
+class OPTConfig(PretrainedConfig):
+    r"""
+    This is the configuration class to store the configuration of a [`OPTModel`]. It is used to instantiate a OPT model
+    according to the specified arguments, defining the model architecture. Instantiating a configuration with the
+    defaults will yield a similar configuration to that of the OPT
+    [facebook/opt-350m](https://huggingface.co/facebook/opt-350m) architecture.
+
+    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+    documentation from [`PretrainedConfig`] for more information.
+
+
+    Args:
+        vocab_size (`int`, *optional*, defaults to 50272):
+            Vocabulary size of the OPT model. Defines the number of different tokens that can be represented by the
+            `inputs_ids` passed when calling [`OPTModel`]
+        hidden_size (`int`, *optional*, defaults to 768):
+            Dimensionality of the layers and the pooler layer.
+        num_hidden_layers (`int`, *optional*, defaults to 12):
+            Number of decoder layers.
+        ffn_dim (`int`, *optional*, defaults to 3072):
+            Dimensionality of the "intermediate" (often named feed-forward) layer in decoder.
+        num_attention_heads (`int`, *optional*, defaults to 12):
+            Number of attention heads for each attention layer in the Transformer decoder.
+        activation_function (`str` or `function`, *optional*, defaults to `"relu"`):
+            The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+            `"relu"`, `"silu"` and `"gelu_new"` are supported.
+        max_position_embeddings (`int`, *optional*, defaults to 2048):
+            The maximum sequence length that this model might ever be used with. Typically set this to something large
+            just in case (e.g., 512 or 1024 or 2048).
+        do_layer_norm_before (`bool`, *optional*, defaults to `True`):
+            Whether to perform layer normalization before the attention block.
+        word_embed_proj_dim (`int`, *optional*):
+            `word_embed_proj_dim` can be set to down-project word embeddings, *e.g.* `opt-350m`. Defaults to
+            `hidden_size`.
+        dropout (`float`, *optional*, defaults to 0.1):
+            The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
+        attention_dropout (`float`, *optional*, defaults to 0.0):
+            The dropout ratio for the attention probabilities.
+        layerdrop (`float`, *optional*, defaults to 0.0):
+            The LayerDrop probability. See the [LayerDrop paper](see https://arxiv.org/abs/1909.11556) for more
+            details.
+        init_std (`float`, *optional*, defaults to 0.02):
+            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
+        use_cache (`bool`, *optional*, defaults to `True`):
+            Whether or not the model should return the last key/values attentions (not used by all models).
+        enable_bias (`bool`, *optional*, defaults to `True`):
+            Whether or not if the linear layers in the attention blocks should use the bias term.
+        layer_norm_elementwise_affine (`bool`, *optional*, defaults to `True`):
+            Whether or not if the layer norms should have learnable parameters.
+
+    Example:
+
+    ```python
+    >>> from transformers import OPTConfig, OPTModel
+
+    >>> # Initializing a OPT facebook/opt-large style configuration
+    >>> configuration = OPTConfig()
+
+    >>> # Initializing a model (with random weights) from the facebook/opt-large style configuration
+    >>> model = OPTModel(configuration)
+
+    >>> # Accessing the model configuration
+    >>> configuration = model.config
+    ```"""
+
+    model_type = "opt"
+    keys_to_ignore_at_inference = ["past_key_values"]
+
+    def __init__(
+        self,
+        vocab_size=50272,
+        hidden_size=768,
+        num_hidden_layers=12,
+        ffn_dim=3072,
+        max_position_embeddings=2048,
+        do_layer_norm_before=True,
+        _remove_final_layer_norm=False,
+        word_embed_proj_dim=None,
+        dropout=0.1,
+        attention_dropout=0.0,
+        num_attention_heads=12,
+        activation_function="relu",
+        layerdrop=0.0,
+        init_std=0.02,
+        use_cache=True,
+        pad_token_id=1,
+        bos_token_id=2,
+        eos_token_id=2,
+        enable_bias=True,
+        layer_norm_elementwise_affine=True,
+        **kwargs,
+    ):
+        super().__init__(
+            pad_token_id=pad_token_id,
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            **kwargs,
+        )
+        self.vocab_size = vocab_size
+        self.max_position_embeddings = max_position_embeddings
+        self.num_attention_heads = num_attention_heads
+        self.word_embed_proj_dim = word_embed_proj_dim if word_embed_proj_dim is not None else hidden_size
+        self.ffn_dim = ffn_dim
+        self.hidden_size = hidden_size
+        self.num_hidden_layers = num_hidden_layers
+        self.dropout = dropout
+        self.attention_dropout = attention_dropout
+        self.activation_function = activation_function
+        self.init_std = init_std
+        self.layerdrop = layerdrop
+        self.use_cache = use_cache
+        self.do_layer_norm_before = do_layer_norm_before
+        # We keep these variables at `True` for backward compatibility.
+        self.enable_bias = enable_bias
+        self.layer_norm_elementwise_affine = layer_norm_elementwise_affine
+
+        # Note that the only purpose of `_remove_final_layer_norm` is to keep backward compatibility
+        # with checkpoints that have been fine-tuned before transformers v4.20.1
+        # see https://github.com/facebookresearch/metaseq/pull/164
+        self._remove_final_layer_norm = _remove_final_layer_norm

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 2,
+  "eos_token_id": 2,
+  "pad_token_id": 1,
+  "transformers_version": "4.31.0"
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8aa7f73e931d356126d5ce5051550f47d757ec080071ffaafea5f8a6a5246a0a
+size 496305242

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,40 @@
+{
+  "add_bos_token": true,
+  "add_prefix_space": false,
+  "bos_token": {
+    "__type": "AddedToken",
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "clean_up_tokenization_spaces": true,
+  "eos_token": {
+    "__type": "AddedToken",
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "errors": "replace",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": {
+    "__type": "AddedToken",
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": {
+    "__type": "AddedToken",
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}