初始化项目，由ModelHub XC社区提供模型
Model: EPFL-LARA/lean-standard-prover-qwen3-8b-grpo Source: Original Platform
This commit is contained in:
ModelHub XC
36
.gitattributes
vendored
Normal file
36
.gitattributes
vendored
Normal file
@@ -0,0 +1,36 @@
|
||||
*.7z filter=lfs diff=lfs merge=lfs -text
|
||||
*.arrow filter=lfs diff=lfs merge=lfs -text
|
||||
*.bin filter=lfs diff=lfs merge=lfs -text
|
||||
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
||||
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
||||
*.ftz filter=lfs diff=lfs merge=lfs -text
|
||||
*.gz filter=lfs diff=lfs merge=lfs -text
|
||||
*.h5 filter=lfs diff=lfs merge=lfs -text
|
||||
*.joblib filter=lfs diff=lfs merge=lfs -text
|
||||
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
||||
*.model filter=lfs diff=lfs merge=lfs -text
|
||||
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
||||
*.npy filter=lfs diff=lfs merge=lfs -text
|
||||
*.npz filter=lfs diff=lfs merge=lfs -text
|
||||
*.onnx filter=lfs diff=lfs merge=lfs -text
|
||||
*.ot filter=lfs diff=lfs merge=lfs -text
|
||||
*.parquet filter=lfs diff=lfs merge=lfs -text
|
||||
*.pb filter=lfs diff=lfs merge=lfs -text
|
||||
*.pickle filter=lfs diff=lfs merge=lfs -text
|
||||
*.pkl filter=lfs diff=lfs merge=lfs -text
|
||||
*.pt filter=lfs diff=lfs merge=lfs -text
|
||||
*.pth filter=lfs diff=lfs merge=lfs -text
|
||||
*.rar filter=lfs diff=lfs merge=lfs -text
|
||||
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
||||
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
||||
*.tar filter=lfs diff=lfs merge=lfs -text
|
||||
*.tflite filter=lfs diff=lfs merge=lfs -text
|
||||
*.tgz filter=lfs diff=lfs merge=lfs -text
|
||||
*.wasm filter=lfs diff=lfs merge=lfs -text
|
||||
*.xz filter=lfs diff=lfs merge=lfs -text
|
||||
*.zip filter=lfs diff=lfs merge=lfs -text
|
||||
*.zst filter=lfs diff=lfs merge=lfs -text
|
||||
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
||||
tokenizer.json filter=lfs diff=lfs merge=lfs -text
|
||||
3
Example.lean
Normal file
3
Example.lean
Normal file
@@ -0,0 +1,3 @@
|
||||
import Mathlib
|
||||
|
||||
theorem demo (n : ℕ) : n + 0 = n := by sorry
|
||||
170
README.md
Normal file
170
README.md
Normal file
@@ -0,0 +1,170 @@
|
||||
---
|
||||
library_name: transformers
|
||||
pipeline_tag: text-generation
|
||||
base_model: Qwen/Qwen3-8B-Base
|
||||
base_model_relation: finetune
|
||||
language:
|
||||
- en
|
||||
tags:
|
||||
- lean
|
||||
- lean4
|
||||
- theorem-proving
|
||||
- qwen3
|
||||
- standard-prover
|
||||
- grpo
|
||||
datasets:
|
||||
- EPFL-LARA/lean-world-model-cpt
|
||||
- EPFL-LARA/lean-world-model-sft
|
||||
- Goedel-LM/RL_dataset_V2
|
||||
---
|
||||
|
||||
# Lean Standard Prover — Qwen3-8B
|
||||
|
||||
**Paper model:** S8 · **Stage:** After GRPO
|
||||
|
||||
Generates complete Lean proofs from theorem statements using the standard prover response format.
|
||||
|
||||
## Hugging Face Transformers
|
||||
|
||||
Load the published weights from the Hub and generate on your GPU:
|
||||
|
||||
```bash
|
||||
pip install 'transformers==4.57.3' accelerate torch
|
||||
hf download EPFL-LARA/lean-standard-prover-qwen3-8b-grpo example_transformers.py inference.py prompt_utils.py Example.lean --local-dir lean-standard-prover-qwen3-8b-grpo
|
||||
cd lean-standard-prover-qwen3-8b-grpo
|
||||
python example_transformers.py
|
||||
```
|
||||
|
||||
The included `Example.lean` is the input; replace it with your own imports,
|
||||
context, and theorem ending in `by sorry`:
|
||||
|
||||
```lean4
|
||||
import Mathlib
|
||||
|
||||
theorem demo (n : ℕ) : n + 0 = n := by sorry
|
||||
```
|
||||
|
||||
The complete Transformers example is:
|
||||
|
||||
```python
|
||||
from pathlib import Path
|
||||
|
||||
import torch
|
||||
from transformers import AutoModelForCausalLM, AutoTokenizer, set_seed
|
||||
|
||||
from inference import build_prompt
|
||||
|
||||
MODEL_ID = 'EPFL-LARA/lean-standard-prover-qwen3-8b-grpo'
|
||||
|
||||
source = Path("Example.lean").read_text(encoding="utf-8")
|
||||
prompt = build_prompt(source)
|
||||
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
|
||||
model = AutoModelForCausalLM.from_pretrained(
|
||||
MODEL_ID, dtype=torch.bfloat16, device_map="auto", attn_implementation="sdpa",
|
||||
).eval()
|
||||
inputs = tokenizer(prompt, return_tensors="pt", add_special_tokens=False).to(model.device)
|
||||
prompt_length = inputs["input_ids"].shape[1]
|
||||
budget = min(2048, 32768 - prompt_length)
|
||||
if budget <= 0:
|
||||
raise ValueError("The prompt must leave room for a proof in the 32,768-token context.")
|
||||
|
||||
set_seed(0)
|
||||
with torch.inference_mode():
|
||||
output = model.generate(
|
||||
**inputs, max_new_tokens=budget, do_sample=True,
|
||||
temperature=0.3, top_p=0.95, top_k=0, min_p=0.0,
|
||||
repetition_penalty=1.0, num_beams=1,
|
||||
pad_token_id=tokenizer.pad_token_id,
|
||||
)
|
||||
response = tokenizer.decode(output[0, prompt_length:], skip_special_tokens=True)
|
||||
print(response, end="")
|
||||
```
|
||||
|
||||
`build_prompt` supplies this checkpoint's exact raw completion prompt.
|
||||
Tokenization uses `add_special_tokens=False`; the context budget is 32,768
|
||||
tokens. This compact example allows up to 2,048 generated tokens and prints
|
||||
only the generated response.
|
||||
|
||||
## Example output
|
||||
|
||||
Recorded with the script above on an H100 in BF16, using seed 0,
|
||||
temperature 0.3, and top-p 0.95. This is the extracted final proof, checked
|
||||
with Lean 4.15 and Mathlib:
|
||||
|
||||
```lean4
|
||||
import Mathlib
|
||||
|
||||
|
||||
theorem demo (n : ℕ) : n + 0 = n := by
|
||||
|
||||
induction n with
|
||||
|
||||
| zero => rfl
|
||||
|
||||
| succ n ih => simp_all
|
||||
```
|
||||
|
||||
<details>
|
||||
<summary>Full generated response</summary>
|
||||
|
||||
````text
|
||||
The complete typechecked Lean 4 proof is:
|
||||
|
||||
```lean4
|
||||
import Mathlib
|
||||
|
||||
|
||||
theorem demo (n : ℕ) : n + 0 = n := by
|
||||
|
||||
induction n with
|
||||
|
||||
| zero => rfl
|
||||
|
||||
| succ n ih => simp_all
|
||||
```
|
||||
````
|
||||
|
||||
</details>
|
||||
|
||||
[Full response](https://huggingface.co/EPFL-LARA/lean-standard-prover-qwen3-8b-grpo/blob/main/example_response.txt) ·
|
||||
[Input/output JSON](https://huggingface.co/EPFL-LARA/lean-standard-prover-qwen3-8b-grpo/blob/main/example_output.json) ·
|
||||
[Lean proof](https://huggingface.co/EPFL-LARA/lean-standard-prover-qwen3-8b-grpo/blob/main/example_proof.lean)
|
||||
|
||||
## vLLM
|
||||
|
||||
For multiple samples and longer output budgets:
|
||||
|
||||
```bash
|
||||
pip install 'vllm==0.12.0'
|
||||
hf download EPFL-LARA/lean-standard-prover-qwen3-8b-grpo inference.py prompt_utils.py Example.lean --local-dir lean-standard-prover-qwen3-8b-grpo
|
||||
python lean-standard-prover-qwen3-8b-grpo/inference.py lean-standard-prover-qwen3-8b-grpo/Example.lean --samples 1
|
||||
```
|
||||
|
||||
The vLLM runner uses the same raw prompt. Its default temperature is 0.3
|
||||
for one sample and 0.7 for 32 samples; top-p is 0.95. Output tokens fit
|
||||
within the context remaining after the prompt. Use `--print-prompt` to
|
||||
inspect the exact input without loading weights.
|
||||
|
||||
## Prompt template
|
||||
|
||||
The standard template follows `evaluate_prover_model_run.py`.
|
||||
Replace the placeholder with the imports, context, and theorem ending in
|
||||
`by sorry`, and keep the final newline:
|
||||
|
||||
````text
|
||||
Solve the following problem with Lean 4 code and explanatory comments:
|
||||
|
||||
```lean4
|
||||
{formal_statement_with_imports}
|
||||
```
|
||||
|
||||
Replace every sorry statement with an appropriate proof. Provide a complete solution in the lean4 code block.
|
||||
````
|
||||
|
||||
## Expected response
|
||||
|
||||
The response contains a complete proof in a `lean4` code fence, together with explanatory text or comments.
|
||||
|
||||
Use the **last complete Lean code block** as the final proof candidate for Lean verification. The vLLM runner returns the full `response`, extracted `lean_code`, and `finish_reason` as JSONL.
|
||||
|
||||
[Formal Theorem Proving with Lean World Models](https://huggingface.co/collections/EPFL-LARA/formal-theorem-proving-with-lean-world-models-6ad2b4aa4400b3e2933f380d)
|
||||
28
added_tokens.json
Normal file
28
added_tokens.json
Normal file
@@ -0,0 +1,28 @@
|
||||
{
|
||||
"</think>": 151668,
|
||||
"</tool_call>": 151658,
|
||||
"</tool_response>": 151666,
|
||||
"<think>": 151667,
|
||||
"<tool_call>": 151657,
|
||||
"<tool_response>": 151665,
|
||||
"<|box_end|>": 151649,
|
||||
"<|box_start|>": 151648,
|
||||
"<|endoftext|>": 151643,
|
||||
"<|file_sep|>": 151664,
|
||||
"<|fim_middle|>": 151660,
|
||||
"<|fim_pad|>": 151662,
|
||||
"<|fim_prefix|>": 151659,
|
||||
"<|fim_suffix|>": 151661,
|
||||
"<|im_end|>": 151645,
|
||||
"<|im_start|>": 151644,
|
||||
"<|image_pad|>": 151655,
|
||||
"<|object_ref_end|>": 151647,
|
||||
"<|object_ref_start|>": 151646,
|
||||
"<|quad_end|>": 151651,
|
||||
"<|quad_start|>": 151650,
|
||||
"<|repo_name|>": 151663,
|
||||
"<|video_pad|>": 151656,
|
||||
"<|vision_end|>": 151653,
|
||||
"<|vision_pad|>": 151654,
|
||||
"<|vision_start|>": 151652
|
||||
}
|
||||
1
chat_template.jinja
Normal file
1
chat_template.jinja
Normal file
@@ -0,0 +1 @@
|
||||
{% for message in messages %}{% if message["role"] == "user" %}{{ message["content"] }}{% endif %}{% endfor %}
|
||||
68
config.json
Normal file
68
config.json
Normal file
@@ -0,0 +1,68 @@
|
||||
{
|
||||
"architectures": [
|
||||
"Qwen3ForCausalLM"
|
||||
],
|
||||
"attention_bias": false,
|
||||
"attention_dropout": 0.0,
|
||||
"dtype": "bfloat16",
|
||||
"eos_token_id": 151643,
|
||||
"head_dim": 128,
|
||||
"hidden_act": "silu",
|
||||
"hidden_size": 4096,
|
||||
"initializer_range": 0.02,
|
||||
"intermediate_size": 12288,
|
||||
"layer_types": [
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention",
|
||||
"full_attention"
|
||||
],
|
||||
"max_position_embeddings": 32768,
|
||||
"max_window_layers": 36,
|
||||
"model_type": "qwen3",
|
||||
"num_attention_heads": 32,
|
||||
"num_hidden_layers": 36,
|
||||
"num_key_value_heads": 8,
|
||||
"pad_token_id": 151643,
|
||||
"rms_norm_eps": 1e-06,
|
||||
"rope_scaling": null,
|
||||
"rope_theta": 1000000,
|
||||
"sliding_window": null,
|
||||
"tie_word_embeddings": false,
|
||||
"transformers_version": "4.57.6",
|
||||
"use_cache": false,
|
||||
"use_sliding_window": false,
|
||||
"vocab_size": 151936
|
||||
}
|
||||
35
example_output.json
Normal file
35
example_output.json
Normal file
@@ -0,0 +1,35 @@
|
||||
{
|
||||
"model_id": "EPFL-LARA/lean-standard-prover-qwen3-8b-grpo",
|
||||
"backend": "transformers",
|
||||
"input": "import Mathlib\n\ntheorem demo (n : ℕ) : n + 0 = n := by sorry\n",
|
||||
"prompt": "Solve the following problem with Lean 4 code and explanatory comments:\n\n```lean4\nimport Mathlib\n\n\ntheorem demo (n : ℕ) : n + 0 = n := by sorry\n```\n\nReplace every sorry statement with an appropriate proof. Provide a complete solution in the lean4 code block.\n",
|
||||
"response": "The complete typechecked Lean 4 proof is:\n\n```lean4\nimport Mathlib\n\n\ntheorem demo (n : ℕ) : n + 0 = n := by \n \n induction n with\n \n | zero => rfl\n \n | succ n ih => simp_all\n```\n",
|
||||
"lean_code": "import Mathlib\n\n\ntheorem demo (n : ℕ) : n + 0 = n := by \n \n induction n with\n \n | zero => rfl\n \n | succ n ih => simp_all",
|
||||
"generation": {
|
||||
"seed": 0,
|
||||
"temperature": 0.3,
|
||||
"top_p": 0.95,
|
||||
"top_k": 0,
|
||||
"min_p": 0.0,
|
||||
"max_new_tokens": 2048,
|
||||
"add_special_tokens": false
|
||||
},
|
||||
"environment": {
|
||||
"torch": "2.9.0+cu129",
|
||||
"transformers": "4.57.3",
|
||||
"gpu": "NVIDIA H100 80GB",
|
||||
"dtype": "bfloat16"
|
||||
},
|
||||
"verification": {
|
||||
"lean_version": "4.15.0",
|
||||
"compiled": true,
|
||||
"original_theorem_type_checked": true,
|
||||
"contains_sorry": false
|
||||
},
|
||||
"weight_sha256": {
|
||||
"model-00001-of-00004.safetensors": "1b6ae09e332fc2054cb264598a331514bb21637bfb3d805a3c42d825aa18b401",
|
||||
"model-00002-of-00004.safetensors": "576d535fbdbbc4d8ff4f5c2f1edf036ce56a8b48a5944d9d335c7f465bc03e59",
|
||||
"model-00003-of-00004.safetensors": "0b76c9b9eade33b10781fe0a24a341ff42f6c857b33c57fdeefc643a294f0857",
|
||||
"model-00004-of-00004.safetensors": "671dce81c1ed04bb18552be55256050a1bc040cbe5dbb0b5ea674b71c81f2408"
|
||||
}
|
||||
}
|
||||
10
example_prompt.txt
Normal file
10
example_prompt.txt
Normal file
@@ -0,0 +1,10 @@
|
||||
Solve the following problem with Lean 4 code and explanatory comments:
|
||||
|
||||
```lean4
|
||||
import Mathlib
|
||||
|
||||
|
||||
theorem demo (n : ℕ) : n + 0 = n := by sorry
|
||||
```
|
||||
|
||||
Replace every sorry statement with an appropriate proof. Provide a complete solution in the lean4 code block.
|
||||
10
example_proof.lean
Normal file
10
example_proof.lean
Normal file
@@ -0,0 +1,10 @@
|
||||
import Mathlib
|
||||
|
||||
|
||||
theorem demo (n : ℕ) : n + 0 = n := by
|
||||
|
||||
induction n with
|
||||
|
||||
| zero => rfl
|
||||
|
||||
| succ n ih => simp_all
|
||||
14
example_response.txt
Normal file
14
example_response.txt
Normal file
@@ -0,0 +1,14 @@
|
||||
The complete typechecked Lean 4 proof is:
|
||||
|
||||
```lean4
|
||||
import Mathlib
|
||||
|
||||
|
||||
theorem demo (n : ℕ) : n + 0 = n := by
|
||||
|
||||
induction n with
|
||||
|
||||
| zero => rfl
|
||||
|
||||
| succ n ih => simp_all
|
||||
```
|
||||
31
example_transformers.py
Normal file
31
example_transformers.py
Normal file
@@ -0,0 +1,31 @@
|
||||
from pathlib import Path
|
||||
|
||||
import torch
|
||||
from transformers import AutoModelForCausalLM, AutoTokenizer, set_seed
|
||||
|
||||
from inference import build_prompt
|
||||
|
||||
MODEL_ID = 'EPFL-LARA/lean-standard-prover-qwen3-8b-grpo'
|
||||
|
||||
source = Path("Example.lean").read_text(encoding="utf-8")
|
||||
prompt = build_prompt(source)
|
||||
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
|
||||
model = AutoModelForCausalLM.from_pretrained(
|
||||
MODEL_ID, dtype=torch.bfloat16, device_map="auto", attn_implementation="sdpa",
|
||||
).eval()
|
||||
inputs = tokenizer(prompt, return_tensors="pt", add_special_tokens=False).to(model.device)
|
||||
prompt_length = inputs["input_ids"].shape[1]
|
||||
budget = min(2048, 32768 - prompt_length)
|
||||
if budget <= 0:
|
||||
raise ValueError("The prompt must leave room for a proof in the 32,768-token context.")
|
||||
|
||||
set_seed(0)
|
||||
with torch.inference_mode():
|
||||
output = model.generate(
|
||||
**inputs, max_new_tokens=budget, do_sample=True,
|
||||
temperature=0.3, top_p=0.95, top_k=0, min_p=0.0,
|
||||
repetition_penalty=1.0, num_beams=1,
|
||||
pad_token_id=tokenizer.pad_token_id,
|
||||
)
|
||||
response = tokenizer.decode(output[0, prompt_length:], skip_special_tokens=True)
|
||||
print(response, end="")
|
||||
8
generation_config.json
Normal file
8
generation_config.json
Normal file
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"eos_token_id": [
|
||||
151643
|
||||
],
|
||||
"max_new_tokens": 2048,
|
||||
"pad_token_id": 151643,
|
||||
"transformers_version": "4.57.6"
|
||||
}
|
||||
82
inference.py
Normal file
82
inference.py
Normal file
@@ -0,0 +1,82 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate Lean proofs with the paper's prompt and last-complete-block parser."""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from prompt_utils import (
|
||||
LEAN4_DEFAULT_HEADER, DEFAULT_GENERATION_TOP_P,
|
||||
build_prover_prompt_for_example, default_generation_temperature,
|
||||
default_generation_max_tokens, default_lean_stop_sequences,
|
||||
parse_generated_lean, split_context_and_command,
|
||||
)
|
||||
|
||||
DEFAULT_MODEL = 'EPFL-LARA/lean-standard-prover-qwen3-8b-grpo'
|
||||
FEEDBACK = False
|
||||
REASONING = False
|
||||
|
||||
|
||||
def build_prompt(source: str) -> str:
|
||||
context, declaration, found = split_context_and_command(source)
|
||||
if not found:
|
||||
raise ValueError("Supply a Lean theorem, lemma, or example ending in 'by sorry'.")
|
||||
example = {"formal_statement": declaration, "header": context or LEAN4_DEFAULT_HEADER}
|
||||
_, prompt = build_prover_prompt_for_example(
|
||||
example, use_informal_statement=False, is_deepseek=False,
|
||||
tgat_prompt=FEEDBACK, reasoning=REASONING,
|
||||
)
|
||||
return prompt
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("files", nargs="+", type=Path, help="Lean problem files, including imports and a theorem to complete")
|
||||
parser.add_argument("--model", default=DEFAULT_MODEL)
|
||||
parser.add_argument("--samples", type=int, default=1)
|
||||
parser.add_argument("--temperature", type=float)
|
||||
parser.add_argument("--top-p", type=float, default=DEFAULT_GENERATION_TOP_P)
|
||||
parser.add_argument("--max-new-tokens", type=int, default=default_generation_max_tokens(False))
|
||||
parser.add_argument("--tensor-parallel-size", type=int, default=1)
|
||||
parser.add_argument("--seed", type=int, default=0)
|
||||
parser.add_argument("--print-prompt", action="store_true", help="Show the exact raw prompt without loading the model")
|
||||
args = parser.parse_args()
|
||||
if args.samples < 1 or args.max_new_tokens < 1:
|
||||
parser.error("--samples and --max-new-tokens must be positive")
|
||||
prompts = [build_prompt(path.read_text(encoding="utf-8")) for path in args.files]
|
||||
if args.print_prompt:
|
||||
for prompt in prompts:
|
||||
print(prompt, end="")
|
||||
return
|
||||
|
||||
from vllm import LLM, SamplingParams
|
||||
|
||||
llm = LLM(model=args.model, tokenizer=args.model,
|
||||
tensor_parallel_size=args.tensor_parallel_size, dtype="auto",
|
||||
max_model_len=32768, trust_remote_code=False, seed=args.seed,
|
||||
gpu_memory_utilization=0.90)
|
||||
tokenizer = llm.get_tokenizer()
|
||||
tokenized = [tokenizer.encode(prompt, add_special_tokens=False) for prompt in prompts]
|
||||
if any(len(ids) >= 32768 for ids in tokenized):
|
||||
raise ValueError("The problem prompt must leave space in the 32,768-token context for a proof.")
|
||||
temperature = args.temperature if args.temperature is not None else default_generation_temperature(args.samples)
|
||||
params = [SamplingParams(n=args.samples, temperature=temperature,
|
||||
top_p=args.top_p,
|
||||
max_tokens=min(args.max_new_tokens, 32768-len(ids)),
|
||||
stop=default_lean_stop_sequences(False),
|
||||
include_stop_str_in_output=True, seed=args.seed)
|
||||
for ids in tokenized]
|
||||
outputs = llm.generate([{"prompt_token_ids": ids} for ids in tokenized], params)
|
||||
for path, output in zip(args.files, outputs, strict=True):
|
||||
for i, completion in enumerate(output.outputs):
|
||||
print(json.dumps({
|
||||
"source": str(path), "sample_id": i,
|
||||
"response": completion.text,
|
||||
"lean_code": parse_generated_lean(completion.text, is_deepseek=False),
|
||||
"finish_reason": completion.finish_reason,
|
||||
}, ensure_ascii=False), flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
151388
merges.txt
Normal file
151388
merges.txt
Normal file
File diff suppressed because it is too large
Load Diff
3
model-00001-of-00004.safetensors
Normal file
3
model-00001-of-00004.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:1b6ae09e332fc2054cb264598a331514bb21637bfb3d805a3c42d825aa18b401
|
||||
size 4924279320
|
||||
3
model-00002-of-00004.safetensors
Normal file
3
model-00002-of-00004.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:576d535fbdbbc4d8ff4f5c2f1edf036ce56a8b48a5944d9d335c7f465bc03e59
|
||||
size 4919113376
|
||||
3
model-00003-of-00004.safetensors
Normal file
3
model-00003-of-00004.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:0b76c9b9eade33b10781fe0a24a341ff42f6c857b33c57fdeefc643a294f0857
|
||||
size 4927438496
|
||||
3
model-00004-of-00004.safetensors
Normal file
3
model-00004-of-00004.safetensors
Normal file
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:671dce81c1ed04bb18552be55256050a1bc040cbe5dbb0b5ea674b71c81f2408
|
||||
size 1610685616
|
||||
407
model.safetensors.index.json
Normal file
407
model.safetensors.index.json
Normal file
@@ -0,0 +1,407 @@
|
||||
{
|
||||
"metadata": {
|
||||
"total_parameters": 8190735360,
|
||||
"total_size": 16381470720
|
||||
},
|
||||
"weight_map": {
|
||||
"lm_head.weight": "model-00002-of-00004.safetensors",
|
||||
"model.embed_tokens.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.0.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.0.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.0.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.0.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.0.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.0.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.0.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.1.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.1.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.1.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.1.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.1.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.1.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.1.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.10.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.10.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.10.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.10.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.10.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.10.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.10.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.10.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.10.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.11.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.self_attn.k_norm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.11.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.11.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.12.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.12.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.12.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.12.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.12.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.12.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.12.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.12.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.12.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.13.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.13.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.13.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.14.input_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.14.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.14.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.14.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.14.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.14.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.14.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.14.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.14.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.15.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.15.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.15.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.15.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.16.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.16.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.16.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.16.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.16.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.16.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.16.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.17.input_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.17.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.17.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.17.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.17.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.17.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.17.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.17.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.18.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.18.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.18.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.18.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.18.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.18.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.19.input_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.19.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.19.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.19.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.19.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.19.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.19.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.19.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.19.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.19.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.2.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.2.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.2.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.2.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.2.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.2.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.2.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.20.input_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.20.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.20.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.20.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.20.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.20.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.20.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.20.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.20.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.20.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.20.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.21.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.21.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.21.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.21.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.21.self_attn.k_norm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.21.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.21.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.21.self_attn.q_norm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.21.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.21.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.22.input_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.22.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.22.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.22.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.22.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.22.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.22.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.23.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.23.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.23.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.23.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.23.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.23.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.24.input_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.24.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.24.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.24.self_attn.k_norm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.24.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.25.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.25.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.26.input_layernorm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.26.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.26.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.26.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.26.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.26.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.26.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.26.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.26.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.26.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.27.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.27.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.27.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.27.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.27.self_attn.k_norm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.27.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.27.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.27.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.27.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.27.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.28.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.28.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.28.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.28.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.28.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.28.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.28.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.29.input_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.29.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.29.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.29.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.29.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.29.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.29.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.29.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.29.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.29.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.3.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.3.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.3.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.3.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.3.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.3.self_attn.q_norm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.3.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.3.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.30.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.30.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.30.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.30.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.30.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.30.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.30.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.31.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.31.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.31.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.31.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.31.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.31.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.31.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.31.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.31.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.31.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.31.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.32.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.32.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.32.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.32.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.32.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.32.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.32.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.32.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.32.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.32.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.32.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.33.input_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.33.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.33.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.33.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.33.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.33.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.33.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.33.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.33.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.33.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.33.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.34.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.34.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.34.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.34.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.34.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.34.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.34.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.34.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.34.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.34.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.34.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.35.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.35.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.35.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.35.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.35.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.35.self_attn.k_norm.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.35.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.35.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.35.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.35.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.35.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.4.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.4.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.4.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.4.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.4.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.4.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.4.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.4.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.4.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.5.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.5.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.5.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.5.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.5.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.5.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.5.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.6.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.6.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.6.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.6.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.6.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.6.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.7.input_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.7.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.7.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.7.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.7.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.7.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.7.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.7.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.8.input_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.8.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.8.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.8.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.8.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.8.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.8.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.8.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.8.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.8.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.9.input_layernorm.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.9.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.9.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.9.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.9.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
|
||||
"model.layers.9.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.9.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
|
||||
"model.layers.9.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
|
||||
"model.layers.9.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
|
||||
"model.layers.9.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
|
||||
"model.norm.weight": "model-00002-of-00004.safetensors"
|
||||
}
|
||||
}
|
||||
303
prompt_utils.py
Normal file
303
prompt_utils.py
Normal file
@@ -0,0 +1,303 @@
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
|
||||
LEAN4_DEFAULT_HEADER = (
|
||||
"import Mathlib\nimport Aesop\n\n"
|
||||
"set_option maxHeartbeats 0\n\n"
|
||||
"open BigOperators Real Nat Topology Rat\n\n"
|
||||
)
|
||||
DEFAULT_LEAN_VERSION = "v4.15.0"
|
||||
DEFAULT_TIMEOUT_PER_CMD = 60
|
||||
DEFAULT_MISSING_LEAN_CODE = "sorry, no code found in output"
|
||||
DEFAULT_DEEPSEEK_GENERATION_MAX_TOKENS = 32768
|
||||
DEFAULT_STANDARD_GENERATION_MAX_TOKENS = 32768
|
||||
DEFAULT_GENERATION_TOP_P = 0.95
|
||||
DEEPSEEK_STYLE_MODEL_MARKERS = (
|
||||
"deepseek-ai",
|
||||
"goedel-lm",
|
||||
)
|
||||
DECLARATION_START_RE = re.compile(
|
||||
r"(?m)^\s*(?:@\[[^\n]*\]\s*)*"
|
||||
r"(?:(?:private|protected|noncomputable|unsafe|local)\s+)*"
|
||||
r"(theorem|lemma|example)\b"
|
||||
)
|
||||
|
||||
def remove_imports(text: str) -> str:
|
||||
"""Remove all lines starting with 'import' (ignoring leading spaces)."""
|
||||
cleaned_lines = [
|
||||
line for line in text.splitlines()
|
||||
if not re.match(r'^\s*import\s+', line)
|
||||
]
|
||||
return "\n".join(cleaned_lines).strip()
|
||||
|
||||
def strip_trailing_backticks(text: str) -> str:
|
||||
return re.sub(r"\s*```(?:\s*)$", "", text).strip()
|
||||
|
||||
_LEAN_FENCE_OPEN_RE = re.compile(r"```lean4?(?![A-Za-z0-9_])")
|
||||
_LEAN_FENCE_RE = re.compile(r"```")
|
||||
|
||||
|
||||
def closed_lean_blocks(text: str) -> list[str]:
|
||||
"""Bodies of every *closed* ```lean4 block, left to right.
|
||||
|
||||
A fence that itself opens a new lean block does not close the current one --
|
||||
the earlier block is an abandoned draft and is skipped. A plain regex would
|
||||
read ```` ```lean4 A ```lean4 B ``` ```` as a single block bodied "A ```lean4 B".
|
||||
"""
|
||||
blocks: list[str] = []
|
||||
pos = 0
|
||||
while True:
|
||||
opener = _LEAN_FENCE_OPEN_RE.search(text, pos)
|
||||
if opener is None:
|
||||
return blocks
|
||||
body_start = opener.end()
|
||||
closer = _LEAN_FENCE_RE.search(text, body_start)
|
||||
if closer is None:
|
||||
return blocks
|
||||
if _LEAN_FENCE_OPEN_RE.match(text, closer.start()) is not None:
|
||||
pos = closer.start()
|
||||
continue
|
||||
blocks.append(text[body_start : closer.start()])
|
||||
pos = closer.end()
|
||||
|
||||
|
||||
def extract_lean_code(text: str, idx: int = 0) -> str | None:
|
||||
matches = closed_lean_blocks(text or "")
|
||||
if not matches:
|
||||
return None
|
||||
lean_code = matches[idx].strip()
|
||||
return strip_trailing_backticks(lean_code)
|
||||
|
||||
def is_deepseek_style_model(
|
||||
model_id: str | None,
|
||||
loadable_model_id: str | None = None,
|
||||
) -> bool:
|
||||
candidates = [candidate for candidate in (model_id, loadable_model_id) if candidate]
|
||||
return any(
|
||||
any(marker in candidate.lower() for marker in DEEPSEEK_STYLE_MODEL_MARKERS)
|
||||
for candidate in candidates
|
||||
)
|
||||
|
||||
def split_context_and_command(code: str) -> tuple[str, str, bool]:
|
||||
text = (code or "").strip()
|
||||
if not text:
|
||||
return "", "", False
|
||||
|
||||
match = DECLARATION_START_RE.search(text)
|
||||
if match is None:
|
||||
return "", text, False
|
||||
|
||||
return text[: match.start()].strip(), text[match.start() :].strip(), True
|
||||
|
||||
def build_formal_statement(formal_statement: str, header: str = LEAN4_DEFAULT_HEADER, informal_statement: str = "") -> str:
|
||||
"""
|
||||
Prepare the prompt for the model by combining the header, informal statement, and formal statement.
|
||||
The formal statement is wrapped in a lean4 code block for clarity.
|
||||
"""
|
||||
header = (header or LEAN4_DEFAULT_HEADER).strip()
|
||||
|
||||
return f"""
|
||||
{header.strip()}
|
||||
|
||||
{informal_statement.strip()}
|
||||
{formal_statement.strip()}
|
||||
""".strip()
|
||||
|
||||
|
||||
def preprocess_formal_statement(formal_statement: str) -> str:
|
||||
"""
|
||||
Preprocess the formal statement to ensure it ends with 'sorry' if it ends with 'by' or ':='.
|
||||
This is a safety measure to ensure the prompt is well-formed for the model.
|
||||
"""
|
||||
formal_statement = formal_statement.strip()
|
||||
if formal_statement.endswith("by"):
|
||||
formal_statement += " sorry"
|
||||
elif formal_statement.endswith(":="):
|
||||
formal_statement += " by sorry"
|
||||
elif not formal_statement.endswith("sorry"):
|
||||
raise ValueError(f"Formal statement must end with 'sorry' or 'by' for safety. Got:\n{formal_statement}")
|
||||
|
||||
return formal_statement
|
||||
|
||||
PROMPT_INTRO = "Solve the following problem with Lean 4 code and explanatory comments:"
|
||||
# Phase-1 instructions: formalmathatepfl/sft_classic and formalmathatepfl/feedback_data_training.
|
||||
PLAIN_INSTRUCTION = (
|
||||
"Replace every sorry statement with an appropriate proof. "
|
||||
"Provide a complete solution in the lean4 code block."
|
||||
)
|
||||
FEEDBACK_CLAUSE = (
|
||||
", annotated with Lean 4 compiler feedback blocks in the form "
|
||||
"`/- <feedback> ... </feedback> -/` at the appropriate locations"
|
||||
)
|
||||
FEEDBACK_INSTRUCTION = PLAIN_INSTRUCTION[:-1] + FEEDBACK_CLAUSE + "."
|
||||
# Phase-2 (reasoning) instructions: `question` / `question_feedback` of formalmathatepfl/reasoning_data
|
||||
# and formalmathatepfl/rl_data. Byte-identical to LEARNING_PLAIN_TAIL / LEARNING_FEEDBACK_TAIL in
|
||||
# annotate_numina/unify_reasoning_data.py (tests/test_prompt_utils_reasoning.py pins the text).
|
||||
REASONING_SECTIONS = (
|
||||
"[Problem Restatement], [Useful Observations], [Reasoning Path], [Key Derivation], "
|
||||
"[Proof Type], [Formal Anchors] and [Final Solution]"
|
||||
)
|
||||
REPAIR_SECTIONS = "[Error Analysis], [Revised Reasoning] and [Updated Proof Plan]"
|
||||
REASONING_INSTRUCTION = (
|
||||
"Replace every sorry statement with an appropriate proof. Before writing any Lean code, give an "
|
||||
f"informal solution with the sections {REASONING_SECTIONS}. Then provide the complete solution in "
|
||||
f"the lean4 code block. If a previous attempt failed, first give {REPAIR_SECTIONS}, then provide "
|
||||
"the corrected solution."
|
||||
)
|
||||
REASONING_FEEDBACK_INSTRUCTION = (
|
||||
"Replace every sorry statement with an appropriate proof. Before writing any Lean code, give an "
|
||||
f"informal solution with the sections {REASONING_SECTIONS}. Then provide the complete solution in "
|
||||
f"the lean4 code block{FEEDBACK_CLAUSE}. If a previous attempt failed, first give {REPAIR_SECTIONS}, "
|
||||
f"then provide the corrected solution{FEEDBACK_CLAUSE}."
|
||||
)
|
||||
INSTRUCTIONS = (REASONING_FEEDBACK_INSTRUCTION, REASONING_INSTRUCTION, FEEDBACK_INSTRUCTION, PLAIN_INSTRUCTION)
|
||||
|
||||
|
||||
def instruction_for(tgat_prompt: bool = False, reasoning: bool = False) -> str:
|
||||
if reasoning:
|
||||
return REASONING_FEEDBACK_INSTRUCTION if tgat_prompt else REASONING_INSTRUCTION
|
||||
return FEEDBACK_INSTRUCTION if tgat_prompt else PLAIN_INSTRUCTION
|
||||
|
||||
|
||||
def prompt_with_instruction(formal_statement: str, instruction: str) -> str:
|
||||
"""The SFT prompt layout: intro, the statement in a lean4 block, the instruction, and the
|
||||
trailing newline every training prompt ends with (the model starts its answer at column 0)."""
|
||||
return f"{PROMPT_INTRO}\n\n```lean4\n{formal_statement}\n```\n\n{instruction}\n"
|
||||
|
||||
|
||||
def with_instruction(prompt: str, tgat_prompt: bool = False, reasoning: bool = False) -> str:
|
||||
"""Re-target a prebuilt SFT-style prompt (e.g. a dataset `question`) to the requested
|
||||
instruction. Prompts without a known instruction sentence are returned unchanged."""
|
||||
wanted = instruction_for(tgat_prompt=tgat_prompt, reasoning=reasoning)
|
||||
for present in INSTRUCTIONS: # longest first: the plain sentence is a prefix of the others
|
||||
if present in prompt:
|
||||
return prompt.replace(present, wanted, 1)
|
||||
return prompt
|
||||
|
||||
|
||||
def prompt_template(formal_statement: str) -> str:
|
||||
return prompt_with_instruction(formal_statement, PLAIN_INSTRUCTION)
|
||||
|
||||
|
||||
def prompt_template_feedback(formal_statement: str) -> str:
|
||||
return prompt_with_instruction(formal_statement, FEEDBACK_INSTRUCTION)
|
||||
|
||||
|
||||
def prompt_template_reasoning(formal_statement: str) -> str:
|
||||
return prompt_with_instruction(formal_statement, REASONING_INSTRUCTION)
|
||||
|
||||
|
||||
def prompt_template_reasoning_feedback(formal_statement: str) -> str:
|
||||
return prompt_with_instruction(formal_statement, REASONING_FEEDBACK_INSTRUCTION)
|
||||
|
||||
|
||||
def deepseek_prompt_template(formal_statement: str) -> str:
|
||||
return f"""Complete the following Lean 4 code:
|
||||
|
||||
```lean4
|
||||
{formal_statement}
|
||||
```
|
||||
|
||||
Before producing the Lean 4 code to formally prove the given theorem, provide a detailed proof plan outlining the main proof steps and strategies.
|
||||
The plan should highlight key ideas, intermediate lemmas, and proof structures that will guide the construction of the final formal proof.
|
||||
""".strip()
|
||||
|
||||
|
||||
def informal_statement_from_example(
|
||||
example: dict[str, Any],
|
||||
use_informal_statement: bool,
|
||||
) -> str:
|
||||
if not use_informal_statement:
|
||||
return ""
|
||||
for key in ("informal_prefix", "informal_statement"):
|
||||
value = example.get(key)
|
||||
if isinstance(value, str) and value.strip():
|
||||
return value.strip()
|
||||
return ""
|
||||
|
||||
|
||||
def prover_prompt_template(
|
||||
formal_statement: str,
|
||||
is_deepseek: bool = False,
|
||||
tgat_prompt: bool = False,
|
||||
reasoning: bool = False,
|
||||
) -> str:
|
||||
"""`reasoning` asks for the phase-2 informal solution sections before the proof and
|
||||
`tgat_prompt` for the feedback-annotated proof; the two combine. The DeepSeek layout is
|
||||
used only when neither is requested."""
|
||||
if reasoning or tgat_prompt:
|
||||
return prompt_with_instruction(formal_statement, instruction_for(tgat_prompt=tgat_prompt, reasoning=reasoning))
|
||||
if is_deepseek:
|
||||
return deepseek_prompt_template(formal_statement=formal_statement)
|
||||
return prompt_template(formal_statement=formal_statement)
|
||||
|
||||
|
||||
def build_prover_prompt_for_example(
|
||||
example: dict[str, Any],
|
||||
use_informal_statement: bool,
|
||||
is_deepseek: bool,
|
||||
tgat_prompt: bool = False,
|
||||
reasoning: bool = False,
|
||||
) -> tuple[str, str]:
|
||||
formal_statement = build_formal_statement(
|
||||
formal_statement=preprocess_formal_statement(example["formal_statement"]),
|
||||
header=example.get("header", None),
|
||||
informal_statement=informal_statement_from_example(
|
||||
example,
|
||||
use_informal_statement=use_informal_statement,
|
||||
),
|
||||
)
|
||||
prompt = prover_prompt_template(
|
||||
formal_statement=formal_statement,
|
||||
is_deepseek=is_deepseek,
|
||||
tgat_prompt=tgat_prompt,
|
||||
reasoning=reasoning,
|
||||
)
|
||||
return formal_statement, prompt
|
||||
|
||||
|
||||
def parse_generated_lean(
|
||||
raw_text: str,
|
||||
example: dict[str, Any] | None = None,
|
||||
is_deepseek: bool = False,
|
||||
missing_code: str | None = None,
|
||||
) -> str | None:
|
||||
lean_code = extract_lean_code(raw_text, idx=-1 if is_deepseek else -1)
|
||||
if lean_code is None:
|
||||
if missing_code is None:
|
||||
return None
|
||||
lean_code = missing_code
|
||||
|
||||
if is_deepseek and isinstance(example, dict):
|
||||
header = example.get("header")
|
||||
if isinstance(header, str) and header.strip():
|
||||
return f"{header.strip()}\n\n{lean_code.strip()}".strip()
|
||||
|
||||
return lean_code.strip()
|
||||
|
||||
|
||||
def default_lean_stop_sequences(is_deepseek: bool) -> list[str] | None:
|
||||
return None if is_deepseek else None #["``` ", "```\n"]
|
||||
|
||||
|
||||
def default_generation_max_tokens(
|
||||
is_deepseek: bool,
|
||||
deepseek_max_tokens: int = DEFAULT_DEEPSEEK_GENERATION_MAX_TOKENS,
|
||||
standard_max_tokens: int = DEFAULT_STANDARD_GENERATION_MAX_TOKENS,
|
||||
) -> int:
|
||||
return deepseek_max_tokens if is_deepseek else standard_max_tokens
|
||||
|
||||
|
||||
def default_generation_temperature(max_attempts: int) -> float:
|
||||
if max_attempts <= 0:
|
||||
raise ValueError(f"max_attempts must be positive, got {max_attempts}")
|
||||
if max_attempts == 1:
|
||||
return 0.3
|
||||
if max_attempts <= 16:
|
||||
return 0.5
|
||||
if max_attempts <= 32:
|
||||
return 0.7
|
||||
if max_attempts <= 64:
|
||||
return 0.8
|
||||
return 1.0
|
||||
31
special_tokens_map.json
Normal file
31
special_tokens_map.json
Normal file
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"additional_special_tokens": [
|
||||
"<|im_start|>",
|
||||
"<|im_end|>",
|
||||
"<|object_ref_start|>",
|
||||
"<|object_ref_end|>",
|
||||
"<|box_start|>",
|
||||
"<|box_end|>",
|
||||
"<|quad_start|>",
|
||||
"<|quad_end|>",
|
||||
"<|vision_start|>",
|
||||
"<|vision_end|>",
|
||||
"<|vision_pad|>",
|
||||
"<|image_pad|>",
|
||||
"<|video_pad|>"
|
||||
],
|
||||
"eos_token": {
|
||||
"content": "<|endoftext|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
},
|
||||
"pad_token": {
|
||||
"content": "<|endoftext|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false
|
||||
}
|
||||
}
|
||||
BIN
tokenizer.json
(Stored with Git LFS)
Normal file
BIN
tokenizer.json
(Stored with Git LFS)
Normal file
Binary file not shown.
240
tokenizer_config.json
Normal file
240
tokenizer_config.json
Normal file
@@ -0,0 +1,240 @@
|
||||
{
|
||||
"add_bos_token": false,
|
||||
"add_prefix_space": false,
|
||||
"added_tokens_decoder": {
|
||||
"151643": {
|
||||
"content": "<|endoftext|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151644": {
|
||||
"content": "<|im_start|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151645": {
|
||||
"content": "<|im_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151646": {
|
||||
"content": "<|object_ref_start|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151647": {
|
||||
"content": "<|object_ref_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151648": {
|
||||
"content": "<|box_start|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151649": {
|
||||
"content": "<|box_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151650": {
|
||||
"content": "<|quad_start|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151651": {
|
||||
"content": "<|quad_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151652": {
|
||||
"content": "<|vision_start|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151653": {
|
||||
"content": "<|vision_end|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151654": {
|
||||
"content": "<|vision_pad|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151655": {
|
||||
"content": "<|image_pad|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151656": {
|
||||
"content": "<|video_pad|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": true
|
||||
},
|
||||
"151657": {
|
||||
"content": "<tool_call>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151658": {
|
||||
"content": "</tool_call>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151659": {
|
||||
"content": "<|fim_prefix|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151660": {
|
||||
"content": "<|fim_middle|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151661": {
|
||||
"content": "<|fim_suffix|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151662": {
|
||||
"content": "<|fim_pad|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151663": {
|
||||
"content": "<|repo_name|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151664": {
|
||||
"content": "<|file_sep|>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151665": {
|
||||
"content": "<tool_response>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151666": {
|
||||
"content": "</tool_response>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151667": {
|
||||
"content": "<think>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
},
|
||||
"151668": {
|
||||
"content": "</think>",
|
||||
"lstrip": false,
|
||||
"normalized": false,
|
||||
"rstrip": false,
|
||||
"single_word": false,
|
||||
"special": false
|
||||
}
|
||||
},
|
||||
"additional_special_tokens": [
|
||||
"<|im_start|>",
|
||||
"<|im_end|>",
|
||||
"<|object_ref_start|>",
|
||||
"<|object_ref_end|>",
|
||||
"<|box_start|>",
|
||||
"<|box_end|>",
|
||||
"<|quad_start|>",
|
||||
"<|quad_end|>",
|
||||
"<|vision_start|>",
|
||||
"<|vision_end|>",
|
||||
"<|vision_pad|>",
|
||||
"<|image_pad|>",
|
||||
"<|video_pad|>"
|
||||
],
|
||||
"bos_token": null,
|
||||
"clean_up_tokenization_spaces": false,
|
||||
"eos_token": "<|endoftext|>",
|
||||
"errors": "replace",
|
||||
"extra_special_tokens": {},
|
||||
"model_max_length": 131072,
|
||||
"pad_token": "<|endoftext|>",
|
||||
"padding_side": "right",
|
||||
"split_special_tokens": false,
|
||||
"tokenizer_class": "Qwen2Tokenizer",
|
||||
"unk_token": null
|
||||
}
|
||||
1
vocab.json
Normal file
1
vocab.json
Normal file
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user