初始化项目，由ModelHub XC社区提供模型

Model: FlameF0X/ChessSLM-Nano Source: Original Platform
2026-07-02 23:42:17 +08:00
commit 21723aaf19
13 changed files with 5518 additions and 0 deletions
--- a/.gitattributes
+++ b/.gitattributes
@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
--- a/README.md
+++ b/README.md
@@ -0,0 +1,99 @@
+---
+license: apache-2.0
+datasets:
+- mlabonne/chessllm
+library_name: transformers
+tags:
+- chess
+pipeline_tag: text-generation
+---
+
+# ChessSLM
+
+**ChessSLM-Nano** is a small language model designed to play chess using natural language move generation.  
+Despite having only **20M parameters**, it is capable of competing with and occasionally outperforming larger language models in chess-playing tasks.
+
+The model is based on the **GPT-2 architecture** and was pre-trained from scratch on **200,000 chess games** from the `mlabonne/chessllm` dataset using **SAN (Standard Algebraic Notation)**.
+
+Play against ChessSLM [here](https://flamef0x.github.io/other/chess/chess).
+
+---
+
+## Overview
+
+- **Architecture:** GPT-2
+- **Costume Tokenizer**
+- **Parameters:** ~20M  
+- **Training data:** 200k chess games  
+- **Notation:** SAN (Standard Algebraic Notation)  
+- **Task:** Autoregressive chess move generation
+
+ChessSLM demonstrates that **specialized small language models can perform competitively in narrow domains** such as chess.
+
+---
+
+## Capabilities
+
+ChessSLM can play chess by generating moves sequentially in SAN notation.  
+It has been evaluated in matches against several language models, including:
+
+- Claude
+- Gemini
+- Qwen
+- GPT-2
+- GPT-Neo
+- Pythia
+- LLaMA
+- Mistral
+- other small chess-oriented models
+
+The model achieves an averaging rating of **around ~{TBD} Elo** against other language models despite its small size.
+
+---
+
+## Benchmark Results
+
+| Model | Elo Rating |
+|------|------------|
+| EleutherAI/pythia-70m-deduped | 1111 |
+| mlabonne/chesspythia-70m | 1101 |
+| nlpguy/amdchess-v9 | 1094 |
+| nlpguy/smolchess-v2 | 1093 |
+| DedeProGames/mini-chennus | 1083 |
+| distilbert/distilgpt2 | 1061 |
+| DedeProGames/dialochess | 1059 |
+| facebook/opt-125m | 1057 |
+| **FlameF0X/ChessSLM** | **1054** |
+| **FlameF0X/ChessSLM-RL** | **1054** |
+| mlabonne/grandpythia-200k-70m | 1050 |
+| DedeProGames/Chesser-248K-Mini | 1048 |
+
+---
+
+## Limitations
+
+Like many language-model-based chess systems, ChessSLM has several limitations:
+
+- **Illegal move hallucinations:** The model may occasionally generate moves that violate chess rules.
+- **No board-state verification:** Moves are generated purely from learned patterns rather than a validated game state.
+- **Limited strategic depth:** While competitive at lower Elo levels, it cannot match dedicated chess engines.
+
+These limitations are common for **pure language-model chess agents** that do not use external rule engines.
+
+---
+
+## Future Improvements
+
+Potential improvements include:
+
+- Adding **move legality filtering**
+- Integrating **board-state validation**
+- Training on **larger datasets**
+- Reinforcement learning through **self-play**
+
+---
+
+## Summary
+
+ChessSLM shows that **very small language models can achieve meaningful chess performance** when trained on domain-specific data.  
+It serves as a lightweight baseline for exploring **LLM-based chess agents** and **specialized small language models (SLMs)**.
--- a/config.json
+++ b/config.json
@@ -0,0 +1,34 @@
+{
+  "activation_function": "gelu_new",
+  "add_cross_attention": false,
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.1,
+  "bos_token_id": 2,
+  "dtype": "float32",
+  "embd_pdrop": 0.1,
+  "eos_token_id": 3,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_embd": 512,
+  "n_head": 8,
+  "n_inner": null,
+  "n_layer": 8,
+  "n_positions": 1024,
+  "pad_token_id": null,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.1,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "tie_word_embeddings": true,
+  "transformers_version": "5.0.0",
+  "use_cache": false,
+  "vocab_size": 1024
+}
--- a/generation_config.json
+++ b/generation_config.json
@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 2,
+  "eos_token_id": 3,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "transformers_version": "5.0.0",
+  "use_cache": false
+}
--- a/model.safetensors
+++ b/model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c2038b7d186dcf5681c4d83de9768682356029a22590cad51687295c3994b772
+size 105084648
--- a/optimizer.pt
+++ b/optimizer.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:593fb2c6e4858080c601758c9635d147ce263203bca363c3dfc105c7a402733c
+size 210233675
--- a/rng_state.pth
+++ b/rng_state.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:878ed4344f40441f51be4ac14226427439b6dc5bb61077e19d16d91c52b3678a
+size 14645
--- a/scaler.pt
+++ b/scaler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c0033c7745b46bdca3ecab5787678834ca68f7f7e1288869dceeb38812abc253
+size 1383
--- a/scheduler.pt
+++ b/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:caad6e0f583a07a0c98d029dca8047c09991d00e348a78110b876e78a7d60992
+size 1465
--- a/tokenizer.json
+++ b/tokenizer.json
--- a/tokenizer_config.json
+++ b/tokenizer_config.json
@@ -0,0 +1,9 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<GAME>",
+  "eos_token": "<END>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<PAD>",
+  "tokenizer_class": "TokenizersBackend",
+  "unk_token": "<UNK>"
+}
--- a/trainer_state.json
+++ b/trainer_state.json
@@ -0,0 +1,314 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.4454976303317535,
+  "eval_steps": 500,
+  "global_step": 4000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.08616975441619991,
+      "grad_norm": 1.8284112215042114,
+      "learning_rate": 1.98e-05,
+      "loss": 5.544659423828125,
+      "step": 100
+    },
+    {
+      "epoch": 0.17233950883239982,
+      "grad_norm": 0.5319015383720398,
+      "learning_rate": 3.979999999999999e-05,
+      "loss": 3.90488037109375,
+      "step": 200
+    },
+    {
+      "epoch": 0.25850926324859974,
+      "grad_norm": 1.3977950811386108,
+      "learning_rate": 5.98e-05,
+      "loss": 3.39756103515625,
+      "step": 300
+    },
+    {
+      "epoch": 0.34467901766479964,
+      "grad_norm": 1.9291573762893677,
+      "learning_rate": 7.98e-05,
+      "loss": 3.019991149902344,
+      "step": 400
+    },
+    {
+      "epoch": 0.4308487720809996,
+      "grad_norm": 1.4095340967178345,
+      "learning_rate": 9.979999999999999e-05,
+      "loss": 2.815445861816406,
+      "step": 500
+    },
+    {
+      "epoch": 0.5170185264971995,
+      "grad_norm": 3.2716641426086426,
+      "learning_rate": 0.00011979999999999998,
+      "loss": 2.6590045166015623,
+      "step": 600
+    },
+    {
+      "epoch": 0.6031882809133994,
+      "grad_norm": 1.3838716745376587,
+      "learning_rate": 0.00013979999999999998,
+      "loss": 2.543310089111328,
+      "step": 700
+    },
+    {
+      "epoch": 0.6893580353295993,
+      "grad_norm": 1.069161057472229,
+      "learning_rate": 0.00015979999999999998,
+      "loss": 2.396273651123047,
+      "step": 800
+    },
+    {
+      "epoch": 0.7755277897457993,
+      "grad_norm": 0.8585665822029114,
+      "learning_rate": 0.0001798,
+      "loss": 2.242165985107422,
+      "step": 900
+    },
+    {
+      "epoch": 0.8616975441619992,
+      "grad_norm": 0.7467069625854492,
+      "learning_rate": 0.0001998,
+      "loss": 2.1027012634277344,
+      "step": 1000
+    },
+    {
+      "epoch": 0.9478672985781991,
+      "grad_norm": 0.5805935859680176,
+      "learning_rate": 0.00021979999999999998,
+      "loss": 2.037454376220703,
+      "step": 1100
+    },
+    {
+      "epoch": 1.033606204222318,
+      "grad_norm": 0.5948718786239624,
+      "learning_rate": 0.00023979999999999997,
+      "loss": 1.9681085205078126,
+      "step": 1200
+    },
+    {
+      "epoch": 1.1197759586385179,
+      "grad_norm": 0.5413378477096558,
+      "learning_rate": 0.00025979999999999997,
+      "loss": 1.9135774230957032,
+      "step": 1300
+    },
+    {
+      "epoch": 1.2059457130547178,
+      "grad_norm": 0.5196030139923096,
+      "learning_rate": 0.00027979999999999997,
+      "loss": 1.8392716979980468,
+      "step": 1400
+    },
+    {
+      "epoch": 1.2921154674709177,
+      "grad_norm": 0.49619364738464355,
+      "learning_rate": 0.00029979999999999997,
+      "loss": 1.8049734497070313,
+      "step": 1500
+    },
+    {
+      "epoch": 1.3782852218871176,
+      "grad_norm": 0.44414839148521423,
+      "learning_rate": 0.000299991068233357,
+      "loss": 1.7638165283203124,
+      "step": 1600
+    },
+    {
+      "epoch": 1.4644549763033177,
+      "grad_norm": 0.46444711089134216,
+      "learning_rate": 0.0002999639122316208,
+      "loss": 1.7137832641601562,
+      "step": 1700
+    },
+    {
+      "epoch": 1.5506247307195173,
+      "grad_norm": 0.5176238417625427,
+      "learning_rate": 0.0002999185343831476,
+      "loss": 1.675589599609375,
+      "step": 1800
+    },
+    {
+      "epoch": 1.6367944851357175,
+      "grad_norm": 0.4177858829498291,
+      "learning_rate": 0.0002998549402017187,
+      "loss": 1.6349491882324219,
+      "step": 1900
+    },
+    {
+      "epoch": 1.7229642395519171,
+      "grad_norm": 0.42198434472084045,
+      "learning_rate": 0.0002997731374145493,
+      "loss": 1.596505126953125,
+      "step": 2000
+    },
+    {
+      "epoch": 1.8091339939681172,
+      "grad_norm": 0.4523915946483612,
+      "learning_rate": 0.0002996731359613498,
+      "loss": 1.5908058166503907,
+      "step": 2100
+    },
+    {
+      "epoch": 1.8953037483843171,
+      "grad_norm": 0.3901713788509369,
+      "learning_rate": 0.0002995549479931178,
+      "loss": 1.5610142517089844,
+      "step": 2200
+    },
+    {
+      "epoch": 1.981473502800517,
+      "grad_norm": 0.41816478967666626,
+      "learning_rate": 0.00029941858787066206,
+      "loss": 1.5319706726074218,
+      "step": 2300
+    },
+    {
+      "epoch": 2.067212408444636,
+      "grad_norm": 0.3872755765914917,
+      "learning_rate": 0.00029926407216285706,
+      "loss": 1.5055549621582032,
+      "step": 2400
+    },
+    {
+      "epoch": 2.1533821628608356,
+      "grad_norm": 0.4193103611469269,
+      "learning_rate": 0.0002990914196446301,
+      "loss": 1.4792218017578125,
+      "step": 2500
+    },
+    {
+      "epoch": 2.2395519172770357,
+      "grad_norm": 0.4024358093738556,
+      "learning_rate": 0.00029890065129467986,
+      "loss": 1.4786280822753906,
+      "step": 2600
+    },
+    {
+      "epoch": 2.325721671693236,
+      "grad_norm": 0.37588468194007874,
+      "learning_rate": 0.0002986917902929273,
+      "loss": 1.4545697021484374,
+      "step": 2700
+    },
+    {
+      "epoch": 2.4118914261094355,
+      "grad_norm": 0.39736974239349365,
+      "learning_rate": 0.0002984648620176991,
+      "loss": 1.4498170471191407,
+      "step": 2800
+    },
+    {
+      "epoch": 2.4980611805256356,
+      "grad_norm": 0.42380592226982117,
+      "learning_rate": 0.00029821989404264424,
+      "loss": 1.4262150573730468,
+      "step": 2900
+    },
+    {
+      "epoch": 2.5842309349418353,
+      "grad_norm": 0.411803662776947,
+      "learning_rate": 0.00029795691613338307,
+      "loss": 1.417086181640625,
+      "step": 3000
+    },
+    {
+      "epoch": 2.6704006893580354,
+      "grad_norm": 0.3662901818752289,
+      "learning_rate": 0.000297675960243891,
+      "loss": 1.3942941284179688,
+      "step": 3100
+    },
+    {
+      "epoch": 2.756570443774235,
+      "grad_norm": 0.3642771244049072,
+      "learning_rate": 0.00029737706051261557,
+      "loss": 1.38471923828125,
+      "step": 3200
+    },
+    {
+      "epoch": 2.842740198190435,
+      "grad_norm": 0.4138600826263428,
+      "learning_rate": 0.00029706025325832857,
+      "loss": 1.3765927124023438,
+      "step": 3300
+    },
+    {
+      "epoch": 2.9289099526066353,
+      "grad_norm": 0.3687536418437958,
+      "learning_rate": 0.0002967255769757127,
+      "loss": 1.3617820739746094,
+      "step": 3400
+    },
+    {
+      "epoch": 3.014648858250754,
+      "grad_norm": 0.3252148926258087,
+      "learning_rate": 0.0002963730723306845,
+      "loss": 1.3490205383300782,
+      "step": 3500
+    },
+    {
+      "epoch": 3.100818612666954,
+      "grad_norm": 0.3874260187149048,
+      "learning_rate": 0.0002960027821554529,
+      "loss": 1.3380169677734375,
+      "step": 3600
+    },
+    {
+      "epoch": 3.1869883670831536,
+      "grad_norm": 0.37778887152671814,
+      "learning_rate": 0.00029561475144331467,
+      "loss": 1.3190237426757812,
+      "step": 3700
+    },
+    {
+      "epoch": 3.2731581214993537,
+      "grad_norm": 0.37266016006469727,
+      "learning_rate": 0.00029520902734318766,
+      "loss": 1.313209991455078,
+      "step": 3800
+    },
+    {
+      "epoch": 3.359327875915554,
+      "grad_norm": 0.3792646527290344,
+      "learning_rate": 0.00029478565915388153,
+      "loss": 1.3055996704101562,
+      "step": 3900
+    },
+    {
+      "epoch": 3.4454976303317535,
+      "grad_norm": 0.3583495318889618,
+      "learning_rate": 0.00029434469831810764,
+      "loss": 1.301021728515625,
+      "step": 4000
+    }
+  ],
+  "logging_steps": 100,
+  "max_steps": 30000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 26,
+  "save_steps": 2000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.9825523114901504e+16,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}
--- a/training_args.bin
+++ b/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cbd27128033a9a2ac4f386cfb4bd5b45cc1f0e6fca9bed9203b8cd754d6cbe99
+size 5201