初始化项目，由ModelHub XC社区提供模型
Model: exnivo/Echo88-150M-Base Source: Original Platform
This commit is contained in:
ModelHub XC
36
.gitattributes
vendored
Normal file
36
.gitattributes
vendored
Normal file
@@ -0,0 +1,36 @@
|
|||||||
|
*.7z filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.arrow filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.bin filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.ftz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.gz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.h5 filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.joblib filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.model filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.npy filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.npz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.ot filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.parquet filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pb filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pickle filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pkl filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pt filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.pth filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.rar filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||||
|
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.tar filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.tflite filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.tgz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.wasm filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.xz filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.zip filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*.zst filter=lfs diff=lfs merge=lfs -text
|
||||||
|
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||||
|
assets/echo88-150m-base-banner.png filter=lfs diff=lfs merge=lfs -text
|
||||||
574
README.md
Normal file
574
README.md
Normal file
@@ -0,0 +1,574 @@
|
|||||||
|
---
|
||||||
|
license: apache-2.0
|
||||||
|
language:
|
||||||
|
- en
|
||||||
|
library_name: transformers
|
||||||
|
pipeline_tag: text-generation
|
||||||
|
pretty_name: Echo88 150M Base
|
||||||
|
tags:
|
||||||
|
- transformers
|
||||||
|
- safetensors
|
||||||
|
- llama
|
||||||
|
- text-generation
|
||||||
|
- causal-lm
|
||||||
|
- base-model
|
||||||
|
- decoder-only
|
||||||
|
- autoregressive
|
||||||
|
- from-scratch
|
||||||
|
- retro
|
||||||
|
- 1980s
|
||||||
|
- usenet
|
||||||
|
- magazines
|
||||||
|
- books
|
||||||
|
- computer-history
|
||||||
|
- english
|
||||||
|
- small-language-model
|
||||||
|
- tiny-llm
|
||||||
|
datasets:
|
||||||
|
- exnivo/Echo88-150M-Base
|
||||||
|
---
|
||||||
|
|
||||||
|
<p align="center">
|
||||||
|
<img
|
||||||
|
src="https://huggingface.co/exnivo/Echo88-150M-Base/resolve/main/assets/echo88-150m-base-banner.png"
|
||||||
|
alt="Echo88 150M Base — Retro base language model"
|
||||||
|
width="100%"
|
||||||
|
/>
|
||||||
|
</p>
|
||||||
|
|
||||||
|
# Echo88-150M Base
|
||||||
|
|
||||||
|
**A 163M parameter retro English base language model trained from scratch.**
|
||||||
|
|
||||||
|
Echo88-150M Base is a small decoder-only causal language model trained from scratch on the Echo88 pretraining dataset.
|
||||||
|
|
||||||
|
Echo88 is designed as a retro language model inspired by the language, culture, computing, magazines, Usenet discussions, and older book text available up to the late 1980s.
|
||||||
|
|
||||||
|
This is a **base model**, not an instruction-tuned chatbot. It is trained for next-token prediction and is best used for text completion, continued pretraining, or supervised fine-tuning.
|
||||||
|
|
||||||
|
For chat/instruction behavior, use:
|
||||||
|
|
||||||
|
[`exnivo/Echo88-150M-Instruct`](https://huggingface.co/exnivo/Echo88-150M-Instruct)
|
||||||
|
|
||||||
|
## Quick Start
|
||||||
|
|
||||||
|
```python
|
||||||
|
import torch
|
||||||
|
from transformers import AutoTokenizer, AutoModelForCausalLM
|
||||||
|
|
||||||
|
model_id = "exnivo/Echo88-150M-Base"
|
||||||
|
|
||||||
|
tokenizer = AutoTokenizer.from_pretrained(model_id)
|
||||||
|
model = AutoModelForCausalLM.from_pretrained(
|
||||||
|
model_id,
|
||||||
|
torch_dtype=torch.bfloat16,
|
||||||
|
device_map="auto",
|
||||||
|
)
|
||||||
|
|
||||||
|
prompt = "The personal computer revolution of the 1980s"
|
||||||
|
inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
|
||||||
|
|
||||||
|
with torch.no_grad():
|
||||||
|
output = model.generate(
|
||||||
|
**inputs,
|
||||||
|
max_new_tokens=140,
|
||||||
|
do_sample=True,
|
||||||
|
temperature=0.75,
|
||||||
|
top_p=0.9,
|
||||||
|
repetition_penalty=1.08,
|
||||||
|
no_repeat_ngram_size=4,
|
||||||
|
pad_token_id=tokenizer.pad_token_id,
|
||||||
|
eos_token_id=tokenizer.eos_token_id,
|
||||||
|
)
|
||||||
|
|
||||||
|
print(tokenizer.decode(output[0], skip_special_tokens=True))
|
||||||
|
```
|
||||||
|
|
||||||
|
## At a Glance
|
||||||
|
|
||||||
|
| Item | Details |
|
||||||
|
|---|---|
|
||||||
|
| Model type | Base causal language model |
|
||||||
|
| Parameters | 163,606,272 |
|
||||||
|
| Approx. size | 163.6M |
|
||||||
|
| Architecture | LLaMA-style decoder-only transformer |
|
||||||
|
| Language | English |
|
||||||
|
| Context length | 2048 tokens |
|
||||||
|
| Vocabulary size | 32,768 |
|
||||||
|
| Layers | 18 |
|
||||||
|
| Hidden size | 768 |
|
||||||
|
| Intermediate size | 2048 |
|
||||||
|
| Attention heads | 12 |
|
||||||
|
| Key/value heads | 4 |
|
||||||
|
| Tokenizer | Custom Echo88 byte-level BPE tokenizer |
|
||||||
|
| Training type | From scratch |
|
||||||
|
| Theme | Late-1980s / retro computing / historical text |
|
||||||
|
| Training objective | Autoregressive next-token prediction |
|
||||||
|
| Instruction-tuned version | [`exnivo/Echo88-150M-Instruct`](https://huggingface.co/exnivo/Echo88-150M-Instruct) |
|
||||||
|
|
||||||
|
## Model Details
|
||||||
|
|
||||||
|
| Item | Value |
|
||||||
|
|---|---|
|
||||||
|
| Model name | Echo88-150M-Base |
|
||||||
|
| Model type | Decoder-only causal language model |
|
||||||
|
| Architecture | `llama` / `LlamaForCausalLM` |
|
||||||
|
| Training type | From scratch |
|
||||||
|
| Parameters | 163,606,272 |
|
||||||
|
| Language | English |
|
||||||
|
| Context length | 2048 tokens |
|
||||||
|
| Vocabulary size | 32,768 |
|
||||||
|
| Hidden size | 768 |
|
||||||
|
| Intermediate size | 2048 |
|
||||||
|
| Layers | 18 |
|
||||||
|
| Attention heads | 12 |
|
||||||
|
| Key/value heads | 4 |
|
||||||
|
| Activation | SiLU |
|
||||||
|
| RMS norm epsilon | `1e-05` |
|
||||||
|
| Tied embeddings | false |
|
||||||
|
| BOS token | `<|endoftext|>` |
|
||||||
|
| EOS token | `<|endoftext|>` |
|
||||||
|
| PAD token | `<|pad|>` |
|
||||||
|
| UNK token | `<|unk|>` |
|
||||||
|
| Training objective | Autoregressive next-token prediction |
|
||||||
|
|
||||||
|
## Tokenizer
|
||||||
|
|
||||||
|
Echo88 uses a custom tokenizer with a vocabulary size of **32,768**.
|
||||||
|
|
||||||
|
Registered special tokens:
|
||||||
|
|
||||||
|
```text
|
||||||
|
<|endoftext|>
|
||||||
|
<|unk|>
|
||||||
|
<|pad|>
|
||||||
|
<|system|>
|
||||||
|
<|user|>
|
||||||
|
<|assistant|>
|
||||||
|
<|end|>
|
||||||
|
```
|
||||||
|
|
||||||
|
The base model is not trained to behave as a chat assistant. The chat tokens are mainly useful for compatibility with the Echo88 instruction-tuned model.
|
||||||
|
|
||||||
|
## Base Model Usage
|
||||||
|
|
||||||
|
Echo88-150M Base should be prompted as a text completion model.
|
||||||
|
|
||||||
|
Good prompt style:
|
||||||
|
|
||||||
|
```text
|
||||||
|
The personal computer revolution of the 1980s
|
||||||
|
```
|
||||||
|
|
||||||
|
```text
|
||||||
|
A bulletin board system, often called a BBS,
|
||||||
|
```
|
||||||
|
|
||||||
|
```text
|
||||||
|
BYTE Magazine, July 1987:
|
||||||
|
```
|
||||||
|
|
||||||
|
```text
|
||||||
|
From: user@utzoo
|
||||||
|
Subject: learning C on a small machine
|
||||||
|
```
|
||||||
|
|
||||||
|
Avoid expecting direct assistant behavior like:
|
||||||
|
|
||||||
|
```text
|
||||||
|
User: What is a modem?
|
||||||
|
Assistant:
|
||||||
|
```
|
||||||
|
|
||||||
|
For chat behavior, use [`exnivo/Echo88-150M-Instruct`](https://huggingface.co/exnivo/Echo88-150M-Instruct).
|
||||||
|
|
||||||
|
## Recommended Generation Settings
|
||||||
|
|
||||||
|
For retro text completion:
|
||||||
|
|
||||||
|
```python
|
||||||
|
output = model.generate(
|
||||||
|
**inputs,
|
||||||
|
max_new_tokens=140,
|
||||||
|
do_sample=True,
|
||||||
|
temperature=0.75,
|
||||||
|
top_p=0.9,
|
||||||
|
repetition_penalty=1.08,
|
||||||
|
no_repeat_ngram_size=4,
|
||||||
|
pad_token_id=tokenizer.pad_token_id,
|
||||||
|
eos_token_id=tokenizer.eos_token_id,
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
For deterministic testing:
|
||||||
|
|
||||||
|
```python
|
||||||
|
output = model.generate(
|
||||||
|
**inputs,
|
||||||
|
max_new_tokens=140,
|
||||||
|
do_sample=False,
|
||||||
|
repetition_penalty=1.08,
|
||||||
|
no_repeat_ngram_size=4,
|
||||||
|
pad_token_id=tokenizer.pad_token_id,
|
||||||
|
eos_token_id=tokenizer.eos_token_id,
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
Because this is a small base model, long generations may repeat, drift, or turn into magazine/OCR-like fragments.
|
||||||
|
|
||||||
|
## Example Outputs
|
||||||
|
|
||||||
|
These examples are from a local greedy text-completion test. They are raw base-model continuations, not instruction-following answers.
|
||||||
|
|
||||||
|
### Prompt
|
||||||
|
|
||||||
|
```text
|
||||||
|
A bulletin board system, often called a BBS,
|
||||||
|
```
|
||||||
|
|
||||||
|
### Output
|
||||||
|
|
||||||
|
```text
|
||||||
|
A bulletin board system, often called a BBS,
|
||||||
|
is available for $10.00 from:
|
||||||
|
|
||||||
|
The Software Exchange
|
||||||
|
P.O. Box 833
|
||||||
|
[PHONE]
|
||||||
|
|
||||||
|
I have been using the software package for about a year now and am
|
||||||
|
having trouble getting it to work. I have had no problems with it
|
||||||
|
and I have not had any problems with it. I would appreciate any
|
||||||
|
suggestions or suggestions.
|
||||||
|
```
|
||||||
|
|
||||||
|
### Prompt
|
||||||
|
|
||||||
|
```text
|
||||||
|
In 1986, a 2400 baud modem
|
||||||
|
```
|
||||||
|
|
||||||
|
### Output
|
||||||
|
|
||||||
|
```text
|
||||||
|
In 1986, a 2400 baud modem
|
||||||
|
is available. The modem is a
|
||||||
|
modem that can be used with
|
||||||
|
the Hayes Smartmodem 1200
|
||||||
|
and the Hayes Smartmodem 1200.
|
||||||
|
The modem is also available for
|
||||||
|
the Hayes Micromodem II and
|
||||||
|
Apple II Plus.
|
||||||
|
```
|
||||||
|
|
||||||
|
### Prompt
|
||||||
|
|
||||||
|
```text
|
||||||
|
From: user@utzoo
|
||||||
|
Subject: learning C on a small machine
|
||||||
|
```
|
||||||
|
|
||||||
|
### Output
|
||||||
|
|
||||||
|
```text
|
||||||
|
From: user@utzoo
|
||||||
|
Subject: learning C on a small machine
|
||||||
|
|
||||||
|
I have been using the C compiler for about 3 years now, and I have been
|
||||||
|
using it for about 2 years now. It is very easy to use, but I have
|
||||||
|
been using it for about 3 months now.
|
||||||
|
```
|
||||||
|
|
||||||
|
These examples show the intended retro text-completion style. They also show typical small-model repetition and drift.
|
||||||
|
|
||||||
|
## Training Data
|
||||||
|
|
||||||
|
Echo88-150M Base was trained on the Echo88 pretraining dataset.
|
||||||
|
|
||||||
|
The packed training set contains:
|
||||||
|
|
||||||
|
| Item | Value |
|
||||||
|
|---|---:|
|
||||||
|
| Train tokens | 1,470,629,888 |
|
||||||
|
| Eval tokens | 1,454,080 |
|
||||||
|
| Train blocks | 718,081 |
|
||||||
|
| Eval blocks | 710 |
|
||||||
|
| Block size | 2048 tokens |
|
||||||
|
| Packed dtype | uint16 |
|
||||||
|
|
||||||
|
The dataset includes a mixture of:
|
||||||
|
|
||||||
|
- public-domain book text
|
||||||
|
- Gutenberg-style older books
|
||||||
|
- UTZOO Usenet posts
|
||||||
|
- BYTE Magazine text
|
||||||
|
- PC Magazine text
|
||||||
|
- TIME Magazine text
|
||||||
|
- Internet Archive Magazine Rack OCR text
|
||||||
|
- computer and technology magazine text
|
||||||
|
- general historical magazine text
|
||||||
|
|
||||||
|
The dataset emphasizes the 1950s through the late 1980s, with a strong focus on early personal computing, printed magazines, Usenet, and older long-form writing.
|
||||||
|
|
||||||
|
Dataset used:
|
||||||
|
|
||||||
|
[`exnivo/Echo88-150M-Base`](https://huggingface.co/datasets/exnivo/Echo88-150M-Base)
|
||||||
|
|
||||||
|
## Training Configuration
|
||||||
|
|
||||||
|
Echo88-150M Base was trained as a LLaMA-style decoder-only causal language model.
|
||||||
|
|
||||||
|
Main configuration:
|
||||||
|
|
||||||
|
```text
|
||||||
|
vocab_size: 32768
|
||||||
|
hidden_size: 768
|
||||||
|
intermediate_size: 2048
|
||||||
|
num_hidden_layers: 18
|
||||||
|
num_attention_heads: 12
|
||||||
|
num_key_value_heads: 4
|
||||||
|
max_position_embeddings: 2048
|
||||||
|
activation: SiLU
|
||||||
|
normalization: RMSNorm
|
||||||
|
position encoding: RoPE
|
||||||
|
attention: grouped-query attention
|
||||||
|
```
|
||||||
|
|
||||||
|
Training setup:
|
||||||
|
|
||||||
|
```text
|
||||||
|
precision: bf16
|
||||||
|
sequence length: 2048
|
||||||
|
optimizer: AdamW
|
||||||
|
scheduler: cosine
|
||||||
|
weight decay: 0.1
|
||||||
|
gradient clipping: 1.0
|
||||||
|
max steps: 5610
|
||||||
|
training tokens: ~1.47B
|
||||||
|
```
|
||||||
|
|
||||||
|
## Relationship to Echo88
|
||||||
|
|
||||||
|
Echo88 is a retro small-language-model project focused on historical text and late-1980s style language modeling.
|
||||||
|
|
||||||
|
| Stage | Repository | Purpose |
|
||||||
|
|---|---|---|
|
||||||
|
| Base dataset | [`exnivo/Echo88-150M-Base`](https://huggingface.co/datasets/exnivo/Echo88-150M-Base) | Retro pretraining data |
|
||||||
|
| Base model | [`exnivo/Echo88-150M-Base`](https://huggingface.co/exnivo/Echo88-150M-Base) | Small causal LM trained from scratch |
|
||||||
|
| SFT dataset | [`exnivo/Echo88-Instruct-173K`](https://huggingface.co/datasets/exnivo/Echo88-Instruct-173K) | Instruction/chat fine-tuning data |
|
||||||
|
| Instruct model | [`exnivo/Echo88-150M-Instruct`](https://huggingface.co/exnivo/Echo88-150M-Instruct) | Retro assistant model fine-tuned from the base model |
|
||||||
|
|
||||||
|
Pipeline:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Echo88 Pretraining Dataset
|
||||||
|
↓
|
||||||
|
Echo88-150M Base
|
||||||
|
↓
|
||||||
|
Echo88 Instruct 173K
|
||||||
|
↓
|
||||||
|
Echo88-150M Instruct
|
||||||
|
```
|
||||||
|
|
||||||
|
## Intended Use
|
||||||
|
|
||||||
|
Echo88-150M Base is intended for:
|
||||||
|
|
||||||
|
- causal language modeling
|
||||||
|
- retro / historical AI experiments
|
||||||
|
- small language model research
|
||||||
|
- continued pretraining
|
||||||
|
- instruction tuning
|
||||||
|
- 1980s-style assistant experiments
|
||||||
|
- computer-history language modeling
|
||||||
|
- training Echo88-150M-Instruct
|
||||||
|
- studying time-bounded model behavior
|
||||||
|
- studying pre-modern-internet style corpora
|
||||||
|
|
||||||
|
Recommended flow:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Echo88-150M Base
|
||||||
|
→ supervised fine-tuning on Echo88-Instruct-173K
|
||||||
|
→ Echo88-150M Instruct
|
||||||
|
```
|
||||||
|
|
||||||
|
## Not Instruction Tuned
|
||||||
|
|
||||||
|
This model is not instruction tuned.
|
||||||
|
|
||||||
|
It may not reliably follow commands, answer questions directly, or behave like a chat assistant.
|
||||||
|
|
||||||
|
Expected behavior:
|
||||||
|
|
||||||
|
- continues prompts
|
||||||
|
- completes paragraphs
|
||||||
|
- imitates old magazine/book/Usenet style
|
||||||
|
- may produce raw text instead of direct answers
|
||||||
|
- may hallucinate
|
||||||
|
- may repeat phrases
|
||||||
|
- may generate OCR-like artifacts
|
||||||
|
- may continue into fake addresses, phone numbers, article fragments, or listings
|
||||||
|
|
||||||
|
For chat behavior, use:
|
||||||
|
|
||||||
|
[`exnivo/Echo88-150M-Instruct`](https://huggingface.co/exnivo/Echo88-150M-Instruct)
|
||||||
|
|
||||||
|
## Knowledge Boundary
|
||||||
|
|
||||||
|
Echo88 is designed around a historical data mixture ending around the late 1980s.
|
||||||
|
|
||||||
|
The model should not be expected to know modern topics such as:
|
||||||
|
|
||||||
|
- Google
|
||||||
|
- Wikipedia
|
||||||
|
- iPhone
|
||||||
|
- smartphones
|
||||||
|
- modern social media
|
||||||
|
- Windows 95 and later software
|
||||||
|
- COVID-19
|
||||||
|
- modern AI systems
|
||||||
|
- 2000s, 2010s, or 2020s events
|
||||||
|
|
||||||
|
Because this is a **base model**, it does not reliably refuse or explain post-1988 topics. In testing, prompts such as `Google is`, `The iPhone is`, `Windows 95 is`, and `YouTube is` produced hallucinated continuations rather than careful boundary-aware answers.
|
||||||
|
|
||||||
|
The instruction-tuned model is better suited for 1988-boundary behavior, but it is still experimental.
|
||||||
|
|
||||||
|
## Evaluation Notes
|
||||||
|
|
||||||
|
A local quick test confirmed:
|
||||||
|
|
||||||
|
| Metric | Value |
|
||||||
|
|---|---:|
|
||||||
|
| Parameters | 163,606,272 |
|
||||||
|
| Vocabulary size | 32,768 |
|
||||||
|
| Context length | 2048 |
|
||||||
|
| Layers | 18 |
|
||||||
|
| Hidden size | 768 |
|
||||||
|
| Attention heads | 12 |
|
||||||
|
| Key/value heads | 4 |
|
||||||
|
|
||||||
|
The same test showed that the model has a clear retro completion style on prompts about BBS systems, modems, BYTE Magazine, and Usenet-style messages.
|
||||||
|
|
||||||
|
It also showed the model can hallucinate badly on modern or post-1988 prompts, which is expected for a base model.
|
||||||
|
|
||||||
|
## Strengths
|
||||||
|
|
||||||
|
Echo88-150M Base is useful because it is:
|
||||||
|
|
||||||
|
- small and lightweight
|
||||||
|
- trained from scratch
|
||||||
|
- focused on retro/historical text
|
||||||
|
- strong at imitating older magazine and Usenet-style text
|
||||||
|
- useful for computer-history language modeling
|
||||||
|
- suitable as a base for instruction tuning
|
||||||
|
- interesting for time-bounded small-model experiments
|
||||||
|
- part of the Echo88 model family
|
||||||
|
|
||||||
|
## Limitations
|
||||||
|
|
||||||
|
Echo88-150M Base is experimental and small.
|
||||||
|
|
||||||
|
Known limitations:
|
||||||
|
|
||||||
|
- not instruction tuned
|
||||||
|
- may hallucinate
|
||||||
|
- may repeat text
|
||||||
|
- may produce OCR-like artifacts
|
||||||
|
- may reflect outdated historical language or views
|
||||||
|
- may struggle with complex reasoning
|
||||||
|
- may not reliably refuse post-1988 topics
|
||||||
|
- may produce incomplete or strange continuations
|
||||||
|
- may mix unrelated historical/computer facts
|
||||||
|
- may generate fake addresses, phone numbers, product listings, or citations
|
||||||
|
- may continue modern prompts with unrelated old-book or magazine-like text
|
||||||
|
|
||||||
|
This model is intended for research, experimentation, and creative retro AI work. It is not intended for high-stakes use.
|
||||||
|
|
||||||
|
## Bias and Historical Content
|
||||||
|
|
||||||
|
The training data includes historical books, magazines, and Usenet text. As a result, the model may reproduce outdated language, assumptions, stereotypes, or viewpoints present in older source material.
|
||||||
|
|
||||||
|
Users should review outputs carefully.
|
||||||
|
|
||||||
|
## Suggested Evaluation
|
||||||
|
|
||||||
|
Recommended checks:
|
||||||
|
|
||||||
|
- retro computing completions
|
||||||
|
- magazine-style continuations
|
||||||
|
- Usenet-style continuations
|
||||||
|
- repetition behavior
|
||||||
|
- OCR artifact frequency
|
||||||
|
- modern-topic hallucination checks
|
||||||
|
- base vs instruct comparison
|
||||||
|
- continued pretraining experiments
|
||||||
|
- SFT experiments with Echo88-Instruct-173K
|
||||||
|
|
||||||
|
Example prompts:
|
||||||
|
|
||||||
|
```text
|
||||||
|
The personal computer revolution of the 1980s
|
||||||
|
```
|
||||||
|
|
||||||
|
```text
|
||||||
|
A bulletin board system, often called a BBS,
|
||||||
|
```
|
||||||
|
|
||||||
|
```text
|
||||||
|
In 1986, a 2400 baud modem
|
||||||
|
```
|
||||||
|
|
||||||
|
```text
|
||||||
|
BYTE Magazine, July 1987:
|
||||||
|
```
|
||||||
|
|
||||||
|
```text
|
||||||
|
From: user@utzoo
|
||||||
|
Subject: learning C on a small machine
|
||||||
|
```
|
||||||
|
|
||||||
|
```text
|
||||||
|
Google is
|
||||||
|
```
|
||||||
|
|
||||||
|
```text
|
||||||
|
The iPhone is
|
||||||
|
```
|
||||||
|
|
||||||
|
## Model Family
|
||||||
|
|
||||||
|
Echo88 model family:
|
||||||
|
|
||||||
|
| Repository | Type |
|
||||||
|
|---|---|
|
||||||
|
| [`exnivo/Echo88-150M-Base`](https://huggingface.co/exnivo/Echo88-150M-Base) | Base model |
|
||||||
|
| [`exnivo/Echo88-150M-Instruct`](https://huggingface.co/exnivo/Echo88-150M-Instruct) | Instruction-tuned model |
|
||||||
|
| [`exnivo/Echo88-Instruct-173K`](https://huggingface.co/datasets/exnivo/Echo88-Instruct-173K) | SFT dataset |
|
||||||
|
| [`exnivo/Echo88-150M-Base`](https://huggingface.co/datasets/exnivo/Echo88-150M-Base) | Pretraining dataset |
|
||||||
|
|
||||||
|
## Citation
|
||||||
|
|
||||||
|
If you use this model, you can cite it as:
|
||||||
|
|
||||||
|
```bibtex
|
||||||
|
@misc{echo88_150m_base,
|
||||||
|
title = {Echo88-150M Base},
|
||||||
|
author = {exnivo},
|
||||||
|
year = {2026},
|
||||||
|
publisher = {Hugging Face},
|
||||||
|
howpublished = {\url{https://huggingface.co/exnivo/Echo88-150M-Base}}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## License
|
||||||
|
|
||||||
|
The model weights are released under the Apache 2.0 license.
|
||||||
|
|
||||||
|
The training dataset is mixed-source and released separately. Users are responsible for checking dataset source rights, licensing, and suitability for their own use case.
|
||||||
|
|
||||||
|
## Disclaimer
|
||||||
|
|
||||||
|
Echo88-150M Base is an experimental small base language model. It may produce incorrect, biased, outdated, unsafe, nonsensical, or misleading outputs.
|
||||||
|
|
||||||
|
Do not use this model for high-stakes decisions or as a reliable source of factual information.
|
||||||
3
assets/echo88-150m-base-banner.png
Normal file
3
assets/echo88-150m-base-banner.png
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:a479db16d717b5e91f9296f4f88c176e929ba643894942495df2a2993d11b743
|
||||||
|
size 1371963
|
||||||
32
config.json
Normal file
32
config.json
Normal file
@@ -0,0 +1,32 @@
|
|||||||
|
{
|
||||||
|
"architectures": [
|
||||||
|
"LlamaForCausalLM"
|
||||||
|
],
|
||||||
|
"attention_bias": false,
|
||||||
|
"attention_dropout": 0.0,
|
||||||
|
"bos_token_id": 0,
|
||||||
|
"dtype": "float32",
|
||||||
|
"eos_token_id": 0,
|
||||||
|
"head_dim": 64,
|
||||||
|
"hidden_act": "silu",
|
||||||
|
"hidden_size": 768,
|
||||||
|
"initializer_range": 0.02,
|
||||||
|
"intermediate_size": 2048,
|
||||||
|
"max_position_embeddings": 2048,
|
||||||
|
"mlp_bias": false,
|
||||||
|
"model_type": "llama",
|
||||||
|
"num_attention_heads": 12,
|
||||||
|
"num_hidden_layers": 18,
|
||||||
|
"num_key_value_heads": 4,
|
||||||
|
"pad_token_id": 1,
|
||||||
|
"pretraining_tp": 1,
|
||||||
|
"rms_norm_eps": 1e-05,
|
||||||
|
"rope_parameters": {
|
||||||
|
"rope_theta": 10000.0,
|
||||||
|
"rope_type": "default"
|
||||||
|
},
|
||||||
|
"tie_word_embeddings": false,
|
||||||
|
"transformers_version": "5.7.0",
|
||||||
|
"use_cache": false,
|
||||||
|
"vocab_size": 32768
|
||||||
|
}
|
||||||
10
generation_config.json
Normal file
10
generation_config.json
Normal file
@@ -0,0 +1,10 @@
|
|||||||
|
{
|
||||||
|
"_from_model_config": true,
|
||||||
|
"bos_token_id": 0,
|
||||||
|
"eos_token_id": 0,
|
||||||
|
"output_attentions": false,
|
||||||
|
"output_hidden_states": false,
|
||||||
|
"pad_token_id": 1,
|
||||||
|
"transformers_version": "5.7.0",
|
||||||
|
"use_cache": false
|
||||||
|
}
|
||||||
3
model.safetensors
Normal file
3
model.safetensors
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:bdf1bb5b75ebcbb878eb8c0f263e817eb3d9293d8a86536438cac369b05542c3
|
||||||
|
size 654443456
|
||||||
162891
tokenizer.json
Normal file
162891
tokenizer.json
Normal file
File diff suppressed because it is too large
Load Diff
17
tokenizer_config.json
Normal file
17
tokenizer_config.json
Normal file
@@ -0,0 +1,17 @@
|
|||||||
|
{
|
||||||
|
"backend": "tokenizers",
|
||||||
|
"bos_token": "<|endoftext|>",
|
||||||
|
"eos_token": "<|endoftext|>",
|
||||||
|
"extra_special_tokens": [
|
||||||
|
"<|system|>",
|
||||||
|
"<|user|>",
|
||||||
|
"<|assistant|>",
|
||||||
|
"<|end|>"
|
||||||
|
],
|
||||||
|
"is_local": true,
|
||||||
|
"local_files_only": false,
|
||||||
|
"model_max_length": 1000000000000000019884624838656,
|
||||||
|
"pad_token": "<|pad|>",
|
||||||
|
"tokenizer_class": "TokenizersBackend",
|
||||||
|
"unk_token": "<|unk|>"
|
||||||
|
}
|
||||||
3
training_args.bin
Normal file
3
training_args.bin
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
version https://git-lfs.github.com/spec/v1
|
||||||
|
oid sha256:9a666c5a83e07052074e2eb38f2941469f93bb2b8f3c2e7ae626ffed605df488
|
||||||
|
size 5265
|
||||||
14
training_info.json
Normal file
14
training_info.json
Normal file
@@ -0,0 +1,14 @@
|
|||||||
|
{
|
||||||
|
"model_name": "Echo88-150M-Base",
|
||||||
|
"repo_id": "guus4324343/Echo88-150M-Base",
|
||||||
|
"seq_len": 2048,
|
||||||
|
"micro_batch": 64,
|
||||||
|
"grad_accum": 2,
|
||||||
|
"tokens_per_step": 262144,
|
||||||
|
"max_steps": 5610,
|
||||||
|
"learning_rate": 0.0004,
|
||||||
|
"weight_decay": 0.1,
|
||||||
|
"warmup_steps": 112,
|
||||||
|
"total_params": 163606272,
|
||||||
|
"trainable_params": 163606272
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user