初始化项目，由ModelHub XC社区提供模型
Model: abacusai/bigstral-12b-32k Source: Original Platform
This commit is contained in:
ModelHub XC
53
.gitattributes
vendored
Normal file
53
.gitattributes
vendored
Normal file
@@ -0,0 +1,53 @@
|
||||
*.7z filter=lfs diff=lfs merge=lfs -text
|
||||
*.arrow filter=lfs diff=lfs merge=lfs -text
|
||||
*.bin filter=lfs diff=lfs merge=lfs -text
|
||||
*.bin.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
||||
*.ftz filter=lfs diff=lfs merge=lfs -text
|
||||
*.gz filter=lfs diff=lfs merge=lfs -text
|
||||
*.h5 filter=lfs diff=lfs merge=lfs -text
|
||||
*.joblib filter=lfs diff=lfs merge=lfs -text
|
||||
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
||||
|
||||
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
||||
*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||
*.ot filter=lfs diff=lfs merge=lfs -text
|
||||
*.parquet filter=lfs diff=lfs merge=lfs -text
|
||||
*.pb filter=lfs diff=lfs merge=lfs -text
|
||||
*.pt filter=lfs diff=lfs merge=lfs -text
|
||||
*.pth filter=lfs diff=lfs merge=lfs -text
|
||||
*.rar filter=lfs diff=lfs merge=lfs -text
|
||||
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tflite filter=lfs diff=lfs merge=lfs -text
|
||||
*.tgz filter=lfs diff=lfs merge=lfs -text
|
||||
*.xz filter=lfs diff=lfs merge=lfs -text
|
||||
*.zip filter=lfs diff=lfs merge=lfs -text
|
||||
*.zstandard filter=lfs diff=lfs merge=lfs -text
|
||||
*.tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||
*.db* filter=lfs diff=lfs merge=lfs -text
|
||||
*.ark* filter=lfs diff=lfs merge=lfs -text
|
||||
**/*ckpt*data* filter=lfs diff=lfs merge=lfs -text
|
||||
**/*ckpt*.meta filter=lfs diff=lfs merge=lfs -text
|
||||
**/*ckpt*.index filter=lfs diff=lfs merge=lfs -text
|
||||
|
||||
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
||||
*.gguf* filter=lfs diff=lfs merge=lfs -text
|
||||
*.ggml filter=lfs diff=lfs merge=lfs -text
|
||||
*.llamafile* filter=lfs diff=lfs merge=lfs -text
|
||||
*.pt2 filter=lfs diff=lfs merge=lfs -text
|
||||
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
||||
*.npy filter=lfs diff=lfs merge=lfs -text
|
||||
*.npz filter=lfs diff=lfs merge=lfs -text
|
||||
*.pickle filter=lfs diff=lfs merge=lfs -text
|
||||
*.pkl filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar filter=lfs diff=lfs merge=lfs -text
|
||||
*.wasm filter=lfs diff=lfs merge=lfs -text
|
||||
*.zst filter=lfs diff=lfs merge=lfs -text
|
||||
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||
|
||||
tokenizer.model filter=lfs diff=lfs merge=lfs -text
|
||||
tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
||||
model-00003-of-00003.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||
model-00001-of-00003.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||
model-00002-of-00003.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||
226
README.md
Normal file
226
README.md
Normal file
@@ -0,0 +1,226 @@
|
||||
---
|
||||
license: apache-2.0
|
||||
library_name: transformers
|
||||
tags:
|
||||
- mergekit
|
||||
- merge
|
||||
base_model:
|
||||
- mistralai/Mistral-7B-Instruct-v0.2
|
||||
model-index:
|
||||
- name: bigstral-12b-32k
|
||||
results:
|
||||
- task:
|
||||
type: text-generation
|
||||
name: Text Generation
|
||||
dataset:
|
||||
name: IFEval (0-Shot)
|
||||
type: HuggingFaceH4/ifeval
|
||||
args:
|
||||
num_few_shot: 0
|
||||
metrics:
|
||||
- type: inst_level_strict_acc and prompt_level_strict_acc
|
||||
value: 41.94
|
||||
name: strict accuracy
|
||||
source:
|
||||
url: https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard?query=abacusai/bigstral-12b-32k
|
||||
name: Open LLM Leaderboard
|
||||
- task:
|
||||
type: text-generation
|
||||
name: Text Generation
|
||||
dataset:
|
||||
name: BBH (3-Shot)
|
||||
type: BBH
|
||||
args:
|
||||
num_few_shot: 3
|
||||
metrics:
|
||||
- type: acc_norm
|
||||
value: 25.56
|
||||
name: normalized accuracy
|
||||
source:
|
||||
url: https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard?query=abacusai/bigstral-12b-32k
|
||||
name: Open LLM Leaderboard
|
||||
- task:
|
||||
type: text-generation
|
||||
name: Text Generation
|
||||
dataset:
|
||||
name: MATH Lvl 5 (4-Shot)
|
||||
type: hendrycks/competition_math
|
||||
args:
|
||||
num_few_shot: 4
|
||||
metrics:
|
||||
- type: exact_match
|
||||
value: 0.98
|
||||
name: exact match
|
||||
source:
|
||||
url: https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard?query=abacusai/bigstral-12b-32k
|
||||
name: Open LLM Leaderboard
|
||||
- task:
|
||||
type: text-generation
|
||||
name: Text Generation
|
||||
dataset:
|
||||
name: GPQA (0-shot)
|
||||
type: Idavidrein/gpqa
|
||||
args:
|
||||
num_few_shot: 0
|
||||
metrics:
|
||||
- type: acc_norm
|
||||
value: 5.7
|
||||
name: acc_norm
|
||||
source:
|
||||
url: https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard?query=abacusai/bigstral-12b-32k
|
||||
name: Open LLM Leaderboard
|
||||
- task:
|
||||
type: text-generation
|
||||
name: Text Generation
|
||||
dataset:
|
||||
name: MuSR (0-shot)
|
||||
type: TAUR-Lab/MuSR
|
||||
args:
|
||||
num_few_shot: 0
|
||||
metrics:
|
||||
- type: acc_norm
|
||||
value: 15.86
|
||||
name: acc_norm
|
||||
source:
|
||||
url: https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard?query=abacusai/bigstral-12b-32k
|
||||
name: Open LLM Leaderboard
|
||||
- task:
|
||||
type: text-generation
|
||||
name: Text Generation
|
||||
dataset:
|
||||
name: MMLU-PRO (5-shot)
|
||||
type: TIGER-Lab/MMLU-Pro
|
||||
config: main
|
||||
split: test
|
||||
args:
|
||||
num_few_shot: 5
|
||||
metrics:
|
||||
- type: acc
|
||||
value: 18.24
|
||||
name: accuracy
|
||||
source:
|
||||
url: https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard?query=abacusai/bigstral-12b-32k
|
||||
name: Open LLM Leaderboard
|
||||
---
|
||||
# bigstral-12b-32k
|
||||
|
||||
`ollama run ehartford/bigstral`
|
||||
|
||||
I reached for SOLAR but realized that its only 4k... Decided to make my own 32k version.
|
||||
|
||||
This is [mistralai/Mistral-7B-Instruct-v0.2](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2) with layers interleaved to create a larger 12b model.
|
||||
|
||||
This is a merge of pre-trained language models created using [mergekit](https://github.com/cg123/mergekit).
|
||||
|
||||
## Prompt Format
|
||||
|
||||
```
|
||||
<s>[INST] What is your favourite condiment? [/INST]
|
||||
Well, I'm quite partial to a good squeeze of fresh lemon juice. It adds just the right amount of zesty flavour to whatever I'm cooking up in the kitchen!</s>
|
||||
[INST] Do you have dangerously spicy mayonnaise recipes? [/INST]
|
||||
```
|
||||
|
||||
## Example Output
|
||||
|
||||
|
||||
give me some code that is very simple in javascript but would be very difficult in ANSI-C
|
||||
|
||||
> In JavaScript, functions are first-class values, which means you can assign them to variables, pass them as arguments to other functions, and even nest them inside other functions. This leads to concise and expressive code.
|
||||
>
|
||||
> Here's an example of a simple recursive function that calculates the factorial of a given number in JavaScript:
|
||||
>
|
||||
> ```javascript
|
||||
> const factorial = n => n === 0 ? 1 : n * factorial(n - 1);
|
||||
>
|
||||
> console.log(factorial(5)); // Output: 120
|
||||
>
|
||||
> ```
|
||||
>
|
||||
> Now, let's try to write an equivalent code snippet in ANSI-C with similar functionality:
|
||||
>
|
||||
> ```c
|
||||
> #include <stdio.h>
|
||||
> #define MAX_RECURSION_DEPTH 1000
|
||||
> int factorial(int n) {
|
||||
> if (n == 0)
|
||||
> return 1;
|
||||
> else {
|
||||
> int result;
|
||||
> result = factorial(--n);
|
||||
> if (++depth > MAX_RECURSION_DEPTH)
|
||||
> while(1){} // Recursion stack overflow
|
||||
> return n * result;
|
||||
> }
|
||||
> }
|
||||
> int main() {
|
||||
> int i, result;
|
||||
> int n = 5;
|
||||
> int depth = 0;
|
||||
>
|
||||
> result = factorial(n);
|
||||
> printf("Factorial of %d is: %d\n", n, result);
|
||||
> return 0;
|
||||
> }
|
||||
> ```
|
||||
>
|
||||
> As you can see, the ANSI-C code becomes more verbose and has several disadvantages compared to JavaScript:
|
||||
>
|
||||
> 1. Need to define a maximum recursion depth using a constant or an array index, which can lead to hardcoded limits and potential stack overflow issues.
|
||||
> 2. The use of preprocessor directives (#define) for defining functions and macros can make the code less readable and harder to understand.
|
||||
> 3. The lack of closures in ANSI-C requires using static variables or passing arguments as arguments to nested functions, making the code more verbose and less concise.
|
||||
> ```
|
||||
|
||||
## Merge Details
|
||||
### Merge Method
|
||||
|
||||
This model was merged using the passthrough merge method.
|
||||
|
||||
### Models Merged
|
||||
|
||||
The following models were included in the merge:
|
||||
* [mistralai/Mistral-7B-Instruct-v0.2](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2)
|
||||
|
||||
### Configuration
|
||||
|
||||
The following YAML configuration was used to produce this model:
|
||||
|
||||
```yaml
|
||||
dtype: float16
|
||||
merge_method: passthrough
|
||||
slices:
|
||||
- sources:
|
||||
- layer_range: [0, 8]
|
||||
model: mistralai/Mistral-7B-Instruct-v0.2
|
||||
- sources:
|
||||
- layer_range: [4, 12]
|
||||
model: mistralai/Mistral-7B-Instruct-v0.2
|
||||
- sources:
|
||||
- layer_range: [8, 16]
|
||||
model: mistralai/Mistral-7B-Instruct-v0.2
|
||||
- sources:
|
||||
- layer_range: [12, 20]
|
||||
model: mistralai/Mistral-7B-Instruct-v0.2
|
||||
- sources:
|
||||
- layer_range: [16, 24]
|
||||
model: mistralai/Mistral-7B-Instruct-v0.2
|
||||
- sources:
|
||||
- layer_range: [20, 28]
|
||||
model: mistralai/Mistral-7B-Instruct-v0.2
|
||||
- sources:
|
||||
- layer_range: [24, 32]
|
||||
model: mistralai/Mistral-7B-Instruct-v0.2
|
||||
|
||||
```
|
||||
# [Open LLM Leaderboard Evaluation Results](https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard)
|
||||
Detailed results can be found [here](https://huggingface.co/datasets/open-llm-leaderboard/details_abacusai__bigstral-12b-32k)
|
||||
|
||||
| Metric |Value|
|
||||
|-------------------|----:|
|
||||
|Avg. |18.05|
|
||||
|IFEval (0-Shot) |41.94|
|
||||
|BBH (3-Shot) |25.56|
|
||||
|MATH Lvl 5 (4-Shot)| 0.98|
|
||||
|GPQA (0-shot) | 5.70|
|
||||
|MuSR (0-shot) |15.86|
|
||||
|MMLU-PRO (5-shot) |18.24|
|
||||
|
||||
26
config.json
Normal file
26
config.json
Normal file
@@ -0,0 +1,26 @@
|
||||
{
|
||||
"_name_or_path": "abacusai/bigstral-12b-32k",
|
||||
"architectures": [
|
||||
"MistralForCausalLM"
|
||||
],
|
||||
"attention_dropout": 0.0,
|
||||
"bos_token_id": 1,
|
||||
"eos_token_id": 2,
|
||||
"hidden_act": "silu",
|
||||
"hidden_size": 4096,
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 14336,
|
||||
"max_position_embeddings": 32768,
|
||||
"model_type": "mistral",
|
||||
"num_attention_heads": 32,
|
||||
"num_hidden_layers": 56,
|
||||
"num_key_value_heads": 8,
|
||||
"rms_norm_eps": 1e-05,
|
||||
"rope_theta": 1000000.0,
|
||||
"sliding_window": null,
|
||||
"tie_word_embeddings": false,
|
||||
"torch_dtype": "float16",
|
||||
"transformers_version": "4.36.2",
|
||||
"use_cache": true,
|
||||
"vocab_size": 32000
|
||||
}
|
||||
1
configuration.json
Normal file
1
configuration.json
Normal file
@@ -0,0 +1 @@
|
||||
{"framework": "pytorch", "task": "text-generation", "allow_remote": true}
|
||||
24
mergekit_config.yml
Normal file
24
mergekit_config.yml
Normal file
@@ -0,0 +1,24 @@
|
||||
dtype: float16
|
||||
merge_method: passthrough
|
||||
slices:
|
||||
- sources:
|
||||
- layer_range: [0, 8]
|
||||
model: mistralai/Mistral-7B-Instruct-v0.2
|
||||
- sources:
|
||||
- layer_range: [4, 12]
|
||||
model: mistralai/Mistral-7B-Instruct-v0.2
|
||||
- sources:
|
||||
- layer_range: [8, 16]
|
||||
model: mistralai/Mistral-7B-Instruct-v0.2
|
||||
- sources:
|
||||
- layer_range: [12, 20]
|
||||
model: mistralai/Mistral-7B-Instruct-v0.2
|
||||
- sources:
|
||||
- layer_range: [16, 24]
|
||||
model: mistralai/Mistral-7B-Instruct-v0.2
|
||||
- sources:
|
||||
- layer_range: [20, 28]
|
||||
model: mistralai/Mistral-7B-Instruct-v0.2
|
||||
- sources:
|
||||
- layer_range: [24, 32]
|
||||
model: mistralai/Mistral-7B-Instruct-v0.2
|
||||
3
model-00001-of-00003.safetensors
Normal file
3
model-00001-of-00003.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:a49f4a06543ba58359f7b526c9a309b3d6dae5afd39c4918920fb952f7149f9b
|
||||
size 9909394184
|
||||
3
model-00002-of-00003.safetensors
Normal file
3
model-00002-of-00003.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:cf34b10820f4187f6543a30de5aecd2b57d466601f83e2abc39aeeb5278b93b0
|
||||
size 9976577160
|
||||
3
model-00003-of-00003.safetensors
Normal file
3
model-00003-of-00003.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:9e49d0a525ebf270312ddda08bf3f9ec4c1ea3a7d231d78e0c7aa028441c20a9
|
||||
size 5066927400
|
||||
1
model.safetensors.index.json
Normal file
1
model.safetensors.index.json
Normal file
File diff suppressed because one or more lines are too long
23
special_tokens_map.json
Normal file
23
special_tokens_map.json
Normal file
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"bos_token": {
|
||||
"content": "<s>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"eos_token": {
|
||||
"content": "</s>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"unk_token": {
|
||||
"content": "<unk>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
}
|
||||
}
|
||||
BIN
tokenizer.json
(Stored with Git LFS)
Normal file
BIN
tokenizer.json
(Stored with Git LFS)
Normal file
Binary file not shown.
BIN
tokenizer.model
(Stored with Git LFS)
Normal file
BIN
tokenizer.model
(Stored with Git LFS)
Normal file
Binary file not shown.
43
tokenizer_config.json
Normal file
43
tokenizer_config.json
Normal file
@@ -0,0 +1,43 @@
|
||||
{
|
||||
"add_bos_token": true,
|
||||
"add_eos_token": false,
|
||||
"added_tokens_decoder": {
|
||||
"0": {
|
||||
"content": "<unk>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"1": {
|
||||
"content": "<s>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"2": {
|
||||
"content": "</s>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
}
|
||||
},
|
||||
"additional_special_tokens": [],
|
||||
"bos_token": "<s>",
|
||||
"chat_template": "{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + message['content'] + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ message['content'] + eos_token}}{% else %}{{ raise_exception('Only user and assistant roles are supported!') }}{% endif %}{% endfor %}",
|
||||
"clean_up_tokenization_spaces": false,
|
||||
"eos_token": "</s>",
|
||||
"legacy": true,
|
||||
"model_max_length": 1000000000000000019884624838656,
|
||||
"pad_token": null,
|
||||
"sp_model_kwargs": {},
|
||||
"spaces_between_special_tokens": false,
|
||||
"tokenizer_class": "LlamaTokenizer",
|
||||
"unk_token": "<unk>",
|
||||
"use_default_system_prompt": false
|
||||
}
|
||||
Reference in New Issue
Block a user