初始化项目，由ModelHub XC社区提供模型

Model: timinar/baby-llama-58m
Source: Original Platform

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,47 @@
+---
+license: unknown
+language:
+- en
+---
+
+# Baby Llama
+
+Our submission to the `strict-small` track of the [BabyLM challenge](https://babylm.github.io/index.html).
+
+Baby Llama is a 58M-parameter model, distilled from an ensemble consisting of LLaMA-360M and GPT2-705M, both trained on the `babylm_10M` dataset.
+
+See the associated [paper](https://arxiv.org/abs/2308.02019) for a detailed discussion of the training procedure and of the model performance.
+The training code is available at [https://github.com/timinar/BabyLlama](https://github.com/timinar/BabyLlama).
+
+### Hyperparameters for the tasks that require fine-tuning
+
+When evaluating the model on the [tasks that require fine-tuning](https://github.com/babylm/evaluation-pipeline/tree/main#fine-tuning),
+we noticed that the [default hyperparameters](https://github.com/babylm/evaluation-pipeline/tree/main#hyperparameters)
+suggested by the BabyLM organizers lead to severe overfitting in a number of tasks.
+To avoid this issue, we have re-tuned those hyperparameters.
+The sets of hyperparameters selected for each task are listed in the table below.
+
+| Task | Maximum learning rate | Batch size | Maximum epochs | Patience | Evaluate every (steps) | Random seed |
+| ---- | ------------- | ---------- | -------- | -------- | ---------- | ---- |
+| CoLA | 4e-5 | 64 | 3 | 10 | 20 | 12 |
+| SST-2 | 5e-5 | 64 | 6 | 10 | 200 | 12 |
+| MRPC | 3e-5 | 64 | 3 | 10 | 20 | 12 |
+| QQP | 4e-5 | 64 | 10 | 10 | 1000 | 12 |
+| MNLI | 5e-5 | 64 | 6 | 10 | 200 | 12 |
+| MNLI-mm |5e-5 | 64 | 6 | 10 | 200 | 12 |
+| QNLI | 5e-5 | 64 | 6 | 10 | 200 | 12 |
+| RTE | 5e-5 | 64 | 6 | 10 | 200 | 12 |
+| BoolQ | 3e-4 | 16 | 10 | 10 | 10| 12 |
+| MultiRC | 1e-4 | 64 | 7 | 10 | 1000 | 42 |
+| WSC | 5e-7 | 1 | 10 | 1000 | 2000 | 12 |
+| CR (Control) | 5e-5 | 64 | 10 | 10 | 100 | 12 |
+| LC (Control) | 1e-3 | 64 | 1 | 2 | 10 | 12 |
+| MV (Control) | 5e-5 | 64 | 6 | 10 | 200 | 12 |
+| RP (Control) | 1e-3 | 64 | 1 | 10 | 10 | 12 |
+| SC (Control) | 1e-3 | 64 | 2 | 10 | 10 | 12 |
+| CR\_LC | 1e-3 | 64 | 2 | 10 | 10 | 12 |
+| CR\_RTP | 5e-5 | 64 | 6 | 10 | 200 | 12 |
+| MV\_LC | 5e-5 | 64 | 6 | 10 | 200 | 12 |
+| MV\_RTP | 5e-5 | 64 | 6 | 10 | 200 | 12 |
+| SC\_LC | 1e-3 | 64 | 2 | 10 | 10 | 12 |
+| SC\_RP | 1e-3 | 64 | 2 | 10 | 10 | 12 |

config.json

@@ -0,0 +1,22 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 512,
+  "initializer_range": 0.02,
+  "intermediate_size": 1024,
+  "max_position_embeddings": 256,
+  "model_type": "llama",
+  "num_attention_heads": 8,
+  "num_hidden_layers": 16,
+  "pad_token_id": 0,
+  "rms_norm_eps": 1e-06,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.29.1",
+  "use_cache": true,
+  "vocab_size": 16000
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "pad_token_id": 0,
+  "transformers_version": "4.29.1"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c395338b407c70d09a063b062de558d209d06e26fd7ecf455ef962b328c1fab3
+size 233395888

mrclean.py

@@ -0,0 +1,95 @@
+import re
+
+# START_TOKEN = '<s>'
+# END_TOKEN = '</s>'
+# PADDING_TOKEN = '<pad>'
+
+START_TOKEN = ''
+END_TOKEN = ''
+PADDING_TOKEN = ''
+
+def _make_padding_sequence(seq_length):
+    return ''.join([END_TOKEN] + seq_length * [PADDING_TOKEN])
+
+def cleanup_simple_wikipedia(text, seq_length):
+    pad_seq = _make_padding_sequence(seq_length)
+    text = START_TOKEN + re.sub(r'\n\n', pad_seq + START_TOKEN, text) + pad_seq
+    return text
+
+def cleanup_wikipedia(text, seq_length):
+    pad_seq = _make_padding_sequence(seq_length)
+    text = re.sub(r'= = = (.+?) = = =\n', r'\1', text)
+    lines = [line.strip() for line in text.splitlines()]
+    text = START_TOKEN + re.sub(r'\n\n', pad_seq + START_TOKEN, '\n'.join(lines)[1:]) + pad_seq
+    return text
+
+def cleanup_qed(text, seq_length):
+    # TODO: this should probably be padded too, but it’s difficult to detect when subtitles start and end
+    # The handling of proper nouns and of parentheses isn’t perfect, but this is still an improvement over the base text
+    punctuation_ex = re.compile(r'([.!?]\s*)')
+    unimportant_chars_ex = re.compile(r'\(.*?\)|[.!?]')
+    lines = []
+    for line in text.splitlines():
+        nchars = len(line)
+        if nchars > 0:
+            line_body = unimportant_chars_ex.sub('', line)
+            f_upper = sum(c.isupper() for c in line_body) / len(line_body)
+            if f_upper >= 0.5: # Mostly uppercase characters
+                # Taken from https://stackoverflow.com/a/41662260
+                split_on_punctuation = punctuation_ex.split(line.replace('l', 'I'))
+                line = ''.join([sentence.capitalize() for sentence in split_on_punctuation])
+        lines.append(line.strip())
+    return START_TOKEN + '\n'.join(lines) + END_TOKEN + ''.join(seq_length * [PADDING_TOKEN])
+
+def cleanup_extra_spaces(text):
+    multiple_spaces_ex = re.compile(r'[ \t\u00A0]+')
+    space_before_punctuation_ex = re.compile(r'[ \t\u00A0]([.,;!?])')
+    text = multiple_spaces_ex.sub(' ', text)
+    text = space_before_punctuation_ex.sub(r'\1', text)
+    return text
+
+def cleanup_bnc_spoken(text, seq_length):
+    pad_seq = _make_padding_sequence(seq_length)
+    text = cleanup_extra_spaces(text)
+    text = START_TOKEN + re.sub(r'\n\n', pad_seq + START_TOKEN, text) + pad_seq
+    return text
+
+def cleanup_aochildes(text, seq_length):
+    text = cleanup_extra_spaces(text)
+    return START_TOKEN + text + _make_padding_sequence(seq_length)
+
+def cleanup_cbt(text, seq_length):
+    text = cleanup_extra_spaces(text)
+    space_before_apostroph = re.compile(r"([\w\d])[ \t\u00A0](['’]\w)")
+    #space_before_quote = re.compile(r"[ \t\u00A0](['’])")
+    #space_after_quote = re.compile(r"([`])[ \t\u00A0]")
+    #text = space_before_quote.sub(r'\1', text)
+    #text = space_after_quote.sub(r'\1', text)
+    text = space_before_apostroph.sub(r'\1\2', text)
+    return START_TOKEN + text + _make_padding_sequence(seq_length)
+
+def cleanup_children_stories(text, seq_length):
+    # Sometimes one skipped line marks the beginning of a new story,
+    # but sometimes it is present within a same story, which doesn’t
+    # make it very useful for separating independent stories.
+    return START_TOKEN + text + _make_padding_sequence(seq_length)
+
+def cleanup_gutenberg(text, seq_length):
+    # Overall, the text is clean, however some entries don’t seem
+    # very useful, e.g. figure captions preceded by a number.
+    # Not sure if we should remove them, because that would also
+    # remove bullet lists which are otherwise consistent with the
+    # surrounding text.
+    # No start or end tokens because the text seems to be cut.
+    return text + ''.join(seq_length * [PADDING_TOKEN])
+
+def cleanup_open_subtitles(text, seq_length):
+    # The text is mostly clean, apart from some subtitle credits
+    # such as "Subtitles by ...".
+    subtitle_credit_ex = re.compile(r'^.*subtitle.*$\n', re.MULTILINE | re.IGNORECASE)
+    text = subtitle_credit_ex.sub('', text)
+    return START_TOKEN + text + _make_padding_sequence(seq_length)
+
+def cleanup_switchboard(text, seq_length):
+    # No start or end tokens because the text seems to be cut.
+    return text + ''.join(seq_length * [PADDING_TOKEN])

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d6e70f468fa96d3315408eb500704ecab718cc4d2afeb44acdd9e4389fec53ab
+size 233432542

special_tokens_map.json

@@ -0,0 +1,6 @@
+{
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "pad_token": "<pad>",
+  "unk_token": "<|endoftext|>"
+}

tokenizer_config.json

@@ -0,0 +1,9 @@
+{
+  "add_prefix_space": false,
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|endoftext|>",
+  "model_max_length": 128,
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:87eba57770d6a732b822fe5b995d45644035322d2cc476b45c94d7d50b111e86
+size 4288