From 07eef1532f2e975f127e3b347be282c387da62cd Mon Sep 17 00:00:00 2001 From: ModelHub XC Date: Sun, 19 Jul 2026 09:43:09 +0800 Subject: [PATCH] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=EF=BC=8C=E7=94=B1ModelHub=20XC=E7=A4=BE=E5=8C=BA=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=A8=A1=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Model: TIGER-Lab/FIM-8B Source: Original Platform --- .gitattributes | 36 +++++++++++++ README.md | 115 +++++++++++++++++++++++++++++++++++++++++ chat_template.jinja | 7 +++ config.json | 73 ++++++++++++++++++++++++++ generation_config.json | 12 +++++ model.safetensors | 3 ++ tokenizer.json | 3 ++ tokenizer_config.json | 31 +++++++++++ 8 files changed, 280 insertions(+) create mode 100644 .gitattributes create mode 100644 README.md create mode 100644 chat_template.jinja create mode 100644 config.json create mode 100644 generation_config.json create mode 100644 model.safetensors create mode 100644 tokenizer.json create mode 100644 tokenizer_config.json diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52373fe --- /dev/null +++ b/.gitattributes @@ -0,0 +1,36 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000..8b61cbb --- /dev/null +++ b/README.md @@ -0,0 +1,115 @@ +--- +license: apache-2.0 +library_name: transformers +pipeline_tag: text-generation +base_model: +- Qwen/Qwen3-8B +datasets: +- TIGER-Lab/FIM-Midtraining-400K +- SWE-Lego/SWE-Lego-Synthetic-Data +tags: +- code +- software-engineering +- agent +--- + +# FIM-8B + +[๐Ÿ“„ Paper](https://arxiv.org/abs/2607.12463) ยท [๐Ÿ’ป GitHub](https://github.com/TIGER-AI-Lab/FIM-Midtraining) ยท [๐Ÿค— Dataset](https://huggingface.co/datasets/TIGER-Lab/FIM-Midtraining-400K) ยท [๐Ÿค— Collection](https://huggingface.co/collections/TIGER-Lab/fim-midtraining) + +**FIM-8B** is the strongest released model of *"Function-Aware Fill-in-the-Middle as Mid-Training for Coding Agent Foundation Models"*: `Qwen3-8B`, mid-trained on function-aware FIM data, then post-trained on SWE-Lego agent trajectories. The mid-training stage is the only difference from a standard SWE-Lego reproduction โ€” worth **+3.2 points on SWE-Bench-Verified and +5.4 on SWE-Bench-Lite**. Unlike FIM-7B and FIM-14B (R2E-Gym scaffold), this model is evaluated with the SWE-Lego setup: OpenHands `CodeActAgent` for inference and the official SWE-bench harness for scoring. + +## Training pipeline + +- **Base model**: [`Qwen/Qwen3-8B`](https://huggingface.co/Qwen/Qwen3-8B) +- **FIM mid-training**: [`midtraining/configs/fim_midtrain.yaml`](https://github.com/TIGER-AI-Lab/FIM-Midtraining/blob/main/midtraining/configs/fim_midtrain.yaml) on [TIGER-Lab/FIM-Midtraining-400K](https://huggingface.co/datasets/TIGER-Lab/FIM-Midtraining-400K) (as-run copy: [`FIM_Midtrain_8B.yaml`](https://github.com/TIGER-AI-Lab/FIM-Midtraining/blob/main/midtraining/configs/FIM_Midtrain_8B.yaml)) โ†’ intermediate checkpoint released as [TIGER-Lab/FIM-Mid-8B](https://huggingface.co/TIGER-Lab/FIM-Mid-8B) +- **Post-training**: SFT on SWE-Lego trajectories (real + synthetic, `resolved`-filtered, 2 epochs) โ€” [`posttraining/swe_lego/`](https://github.com/TIGER-AI-Lab/FIM-Midtraining/tree/main/posttraining/swe_lego) (as-run copy: [`FIM_Posttrain_8B.yaml`](https://github.com/TIGER-AI-Lab/FIM-Midtraining/blob/main/posttraining/swe_lego/FIM_Posttrain_8B.yaml)) + +## Results + +Means over three evaluation seeds, identical harness for both arms (paper Table 1): + +| Setting | SWE-Bench-Verified | SWE-Bench-Lite | +|---|---|---| +| Qwen3-8B + SWE-Lego (reproduced) | 31.80 | 27.30 | +| **FIM-8B (+ FIM mid-training)** | **35.00** | **32.70** | +| ฮ” | +3.20 | +5.40 | + +## Evaluate on SWE-Bench Verified + +The complete pinned walkthrough lives at [`evaluation/swebench/released_checkpoints.md`](https://github.com/TIGER-AI-Lab/FIM-Midtraining/blob/main/evaluation/swebench/released_checkpoints.md). + +### 1. Serve the model with vLLM + +The checkpoint ships `max_position_embeddings: 163840` and its own chat template, so no rope or template overrides are needed: + +```bash +CUDA_VISIBLE_DEVICES=0 \ +python -m vllm.entrypoints.openai.api_server \ + --model TIGER-Lab/FIM-8B \ + --served-model-name FIM-8B \ + --host 127.0.0.1 \ + --port 8400 \ + --tensor-parallel-size 1 \ + --max-model-len 163840 \ + --max-num-seqs 16 \ + --gpu-memory-utilization 0.9 \ + > vllm_fim8b.log 2>&1 & +``` + +Wait until the server is up (model load takes ~1 minute): + +```bash +curl -s http://127.0.0.1:8400/v1/models +``` + +### 2. Run the agent on SWE-Bench Verified + +Inference uses OpenHands 0.53.0 with `CodeActAgent`. Define the LLM in `config.toml`: + +```toml +[llm.eval_fim] +model = "openai/FIM-8B" +base_url = "http://127.0.0.1:8400/v1" +api_key = "EMPTY" +temperature = 0.0 +max_input_tokens = 147456 +max_output_tokens = 16384 +native_tool_calling = false +``` + +From the OpenHands checkout: + +```bash +env USE_HINT_TEXT=false \ + INSTRUCTION_TEMPLATE_NAME=swe_default.j2 \ + ENABLE_PLAN_MODE=false \ + ADD_IN_CONTEXT_LEARNING_EXAMPLE=false \ +poetry run python evaluation/benchmarks/swe_bench/run_infer.py \ + --config-file config.toml \ + --agent-cls CodeActAgent \ + --llm-config llm.eval_fim \ + --max-iterations 100 \ + --eval-num-workers 1 \ + --eval-output-dir ./eval_out \ + --dataset princeton-nlp/SWE-bench_Verified \ + --split test \ + --mode swe +``` + +For SWE-Bench Lite, use `--dataset princeton-nlp/SWE-bench_Lite`. + +### 3. Score with the SWE-bench harness + +Convert the OpenHands `output.jsonl` to a predictions file with `evaluation/benchmarks/swe_bench/scripts/eval/convert_oh_output_to_swe_json.py`, then evaluate it with the official SWE-bench harness (`python -m swebench.harness.run_evaluation`). The reported score is `resolved_instances / total_instances`. + +## Citation + +```bibtex +@article{wang2026fim, + title={Function-Aware Fill-in-the-Middle as Mid-Training for Coding Agent Foundation Models}, + author={Wang, Yubo and Liang, Jiarong and Zhang, Yuxuan and Liu, Xuye and Wei, Cong and Zhang, Yuyu and Nie, Ping and Chen, Wenhu}, + journal={arXiv preprint arXiv:2607.12463}, + year={2026} +} +``` diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000..85e8f5d --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,7 @@ +{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% endif %}{% if system_message is defined %}{{ '<|im_start|>system +' + system_message + '<|im_end|> +' }}{% endif %}{% for message in loop_messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ '<|im_start|>user +' + content + '<|im_end|> +<|im_start|>assistant +' }}{% elif message['role'] == 'assistant' %}{{ content + '<|im_end|> +' }}{% endif %}{% endfor %} \ No newline at end of file diff --git a/config.json b/config.json new file mode 100644 index 0000000..2e869e7 --- /dev/null +++ b/config.json @@ -0,0 +1,73 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 12288, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 163840.0, + "max_window_layers": 36, + "model_type": "qwen3", + "num_attention_heads": 32, + "num_hidden_layers": 36, + "num_key_value_heads": 8, + "pad_token_id": 151643, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "factor": 4.0, + "original_max_position_embeddings": 40960, + "rope_theta": 1000000, + "rope_type": "yarn" + }, + "sliding_window": null, + "tie_word_embeddings": false, + "transformers_version": "5.0.0", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000..c33fb76 --- /dev/null +++ b/generation_config.json @@ -0,0 +1,12 @@ +{ + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "5.0.0" +} diff --git a/model.safetensors b/model.safetensors new file mode 100644 index 0000000..3683d5e --- /dev/null +++ b/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:86a69ce2ed5edd997d442770367c0f767924cefcd5797ef32dbc778116a159ee +size 16381517208 diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000..c7afbed --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be75606093db2094d7cd20f3c2f385c212750648bd6ea4fb2bf507a6a4c55506 +size 11422650 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000..3ad48f2 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "extra_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "is_local": true, + "model_max_length": 131072, + "model_specific_special_tokens": {}, + "pad_token": "<|endoftext|>", + "padding_side": "right", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +}