sandeeprdy1729/TIMPS-Coder-7B
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity
Files
main
TIMPS-Coder-7B/benchmark_evaluation.ipynb

519 lines
32 KiB
Plaintext
Raw Permalink Normal View History

初始化项目，由ModelHub XC社区提供模型 Model: sandeeprdy1729/TIMPS-Coder-7B Source: Original Platform
2026-07-20 12:51:10 +08:00
{
"metadata": {
"kernelspec": {
"language": "python",
"display_name": "Python 3",
"name": "python3"
},
"language_info": {
"name": "python",
"version": "3.12.13",
"mimetype": "text/x-python",
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"pygments_lexer": "ipython3",
"nbconvert_exporter": "python",
"file_extension": ".py"
},
"kaggle": {
"accelerator": "nvidiaTeslaT4",
"dataSources": [],
"isInternetEnabled": true,
"language": "python",
"sourceType": "notebook",
"isGpuEnabled": false
}
},
"nbformat_minor": 4,
"nbformat": 4,
"cells": [
{
"cell_type": "code",
"source": "# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n for filename in filenames:\n print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n\n# Use the kagglehub client library to attach Kaggle resources like competitions, datasets, and models to your session\n# Learn more about kagglehub: https://github.com/Kaggle/kagglehub/blob/main/README.md\n\nimport kagglehub\n# kagglehub.dataset_download('<owner>/<dataset-slug>')",
"metadata": {
"_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
"_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
"trusted": true
},
"outputs": [],
"execution_count": null
},
{
"cell_type": "code",
"source": "!pip uninstall -y torchao -q\n!pip install -q transformers accelerate datasets evalplus sentencepiece\nprint(\"Done\")",
"metadata": {
"trusted": true,
"execution": {
"iopub.status.busy": "2026-07-16T18:34:10.22367Z",
"iopub.execute_input": "2026-07-16T18:34:10.223899Z",
"iopub.status.idle": "2026-07-16T18:34:27.408138Z",
"shell.execute_reply.started": "2026-07-16T18:34:10.223877Z",
"shell.execute_reply": "2026-07-16T18:34:27.407041Z"
}
},
"outputs": [
{
"name": "stdout",
"text": " Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n\u001b[2K \u001b[90m\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u001b[0m \u001b[32m68.6/68.6 kB\u001b[0m \u001b[31m1.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n\u001b[2K \u001b[90m\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u001b[0m \u001b[32m956.9/956.9 kB\u001b[0m \u001b[31m12.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n\u001b[2K \u001b[90m\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u001b[0m \u001b[32m115.9/115.9 kB\u001b[0m \u001b[31m6.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n\u001b[2K \u001b[90m\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u001b[0m \u001b[32m667.5/667.5 kB\u001b[0m \u001b[31m35.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n\u001b[2K \u001b[90m\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u001b[0m \u001b[32m108.1/108.1 kB\u001b[0m \u001b[31m4.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n\u001b[?25h Building wheel for stop-sequencer (setup.py) ... \u001b[?25l\u001b[?25hdone\n Building wheel for tempdir (setup.py) ... \u001b[?25l\u001b[?25hdone\n Building wheel for wget (setup.py) ... \u001b[?25l\u001b[?25hdone\nDone\n",
"output_type": "stream"
}
],
"execution_count": 1
},
{
"cell_type": "code",
"source": "import os, gc, json, sys, subprocess, tempfile, time, traceback, warnings\nfrom datetime import datetime, timezone\nfrom typing import Any, Dict, List, Optional\nfrom collections import OrderedDict\nwarnings.filterwarnings(\"ignore\")\n\nimport torch\nos.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\nos.environ[\"PYTORCH_CUDA_ALLOC_CONF\"] = \"expandable_segments:True\"\n\nfrom kaggle_secrets import UserSecretsClient\nos.environ[\"HF_TOKEN\"] = UserSecretsClient().get_secret(\"HF_TOKEN\")\n\nfrom huggingface_hub import login\nlogin(token=os.environ[\"HF_TOKEN\"])\n\nfor i in range(torch.cuda.device_count()):\n p = torch.cuda.get_device_properties(i)\n print(f\"GPU {i}: {p.name} ({p.total_memory/1e9:.1f} GB)\")",
"metadata": {
"trusted": true,
"execution": {
"iopub.status.busy": "2026-07-16T18:34:51.597293Z",
"iopub.execute_input": "2026-07-16T18:34:51.598206Z",
"iopub.status.idle": "2026-07-16T18:34:53.274328Z",
"shell.execute_reply.started": "2026-07-16T18:34:51.598169Z",
"shell.execute_reply": "2026-07-16T18:34:53.273346Z"
}
},
"outputs": [
{
"name": "stderr",
"text": "Note: Environment variable`HF_TOKEN` is set and is the current active token independently from the token you've just configured.\n",
"output_type": "stream"
},
{
"name": "stdout",
"text": "GPU 0: Tesla T4 (15.6 GB)\nGPU 1: Tesla T4 (15.6 GB)\n",
"output_type": "stream"
}
],
"execution_count": 3
},
{
"cell_type": "code",
"source": "MODEL_ID = \"sandeeprdy1729/TIMPS-Coder-7B\"\nOUTPUT_DIR = \"/kaggle/working/timps-benchmarks\"\nos.makedirs(OUTPUT_DIR, exist_ok=True)\n\nMAX_NEW_TOKENS = 512\nTEMPERATURE = 0.1\nTOP_P = 0.95\n\nfrom transformers import AutoModelForCausalLM, AutoTokenizer\n\nprint(f\"Loading {MODEL_ID} ...\")\nt0 = time.time()\ndtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16\nmodel = AutoModelForCausalLM.from_pretrained(\n MODEL_ID, torch_dtype=dtype, device_map=\"auto\", trust_remote_code=True,\n)\ntokenizer = AutoTokenizer.from_pretrained(MODEL_ID, trust_remote_code=True)\nif tokenizer.pad_token is None:\n tokenizer.pad_token = tokenizer.eos_token\nmodel.eval()\nprint(f\"Loaded in {time.time()-t0:.1f}s | VRAM: {torch.cuda.memory_allocated()/1e9:.2f} GB\")",
"metadata": {
"trusted": true,
"execution": {
"iopub.status.busy": "2026-07-16T18:35:00.552166Z",
"iopub.execute_input": "2026-07-16T18:35:00.552989Z",
"iopub.status.idle": "2026-07-16T18:37:05.693429Z",
"shell.execute_reply.started": "2026-07-16T18:35:00.552957Z",
"shell.execute_reply": "2026-07-16T18:37:05.692577Z"
}
},
"outputs": [
{
"name": "stdout",
"text": "Loading sandeeprdy1729/TIMPS-Coder-7B ...\n",
"output_type": "stream"
},
{
"output_type": "display_data",
"data": {
"text/plain": "config.json: 0.00B [00:00, ?B/s]",
"application/vnd.jupyter.widget-view+json": {
"version_major": 2,
"version_minor": 0,
"model_id": "8b01012f34244751800d730837b69bbc"
}
},
"metadata": {}
},
{
"name": "stderr",
"text": "`torch_dtype` is deprecated! Use `dtype` instead!\n",
"output_type": "stream"
},
{
"output_type": "display_data",
"data": {
"text/plain": "model.safetensors.index.json: 0.00B [00:00, ?B/s]",
"application/vnd.jupyter.widget-view+json": {
"version_major": 2,
"version_minor": 0,
"model_id": "3cb7862321904ca1a3bdd5a505f9df38"
}
},
"metadata": {}
},
{
"output_type": "display_data",
"data": {
"text/plain": "Downloading (incomplete total...): 0.00B [00:00, ?B/s]",
"application/vnd.jupyter.widget-view+json": {
"version_major": 2,
"version_minor": 0,
"model_id": "e02ebb8474854f1389f711f429ac0346"
}
},
"metadata": {}
},
{
"output_type": "display_data",
"data": {
"text/plain": "Fetching 4 files: 0%| | 0/4 [00:00<?, ?it/s]",
"application/vnd.jupyter.widget-view+json": {
"version_major": 2,
"version_minor": 0,
"model_id": "cfa086a1396a45aaab0c5da893a660b7"
}
},
"metadata": {}
},
{
"output_type": "display_data",
"data": {
"text/plain": "Loading weights: 0%| | 0/339 [00:00<?, ?it/s]",
"application/vnd.jupyter.widget-view+json": {
"version_major": 2,
"version_minor": 0,
"model_id": "9589f9668fe6443ab292a3ad08a46d8a"
}
},
"metadata": {}
},
{
"output_type": "display_data",
"data": {
"text/plain": "tokenizer_config.json: 0%| | 0.00/693 [00:00<?, ?B/s]",
"application/vnd.jupyter.widget-view+json": {
"version_major": 2,
"version_minor": 0,
"model_id": "fb3bd05b1218437a9abc4d06650f1b79"
}
},
"metadata": {}
},
{
"output_type": "display_data",
"data": {
"text/plain": "tokenizer.json: 0%| | 0.00/11.4M [00:00<?, ?B/s]",
"application/vnd.jupyter.widget-view+json": {
"version_major": 2,
"version_minor": 0,
"model_id": "87d446c568d541ad8bbb57e8917ad8db"
}
},
"metadata": {}
},
{
"output_type": "display_data",
"data": {
"text/plain": "chat_template.jinja: 0.00B [00:00, ?B/s]",
"application/vnd.jupyter.widget-view+json": {
"version_major": 2,
"version_minor": 0,
"model_id": "3a49f6d0f9b14ee587a5dc98dc60a401"
}
},
"metadata": {}
},
{
"name": "stdout",
"text": "Loaded in 95.5s | VRAM: 6.68 GB\n",
"output_type": "stream"
}
],
"execution_count": 4
},
{
"cell_type": "code",
"source": "def extract_code(generation: str) -> str:\n text = generation.strip()\n if \"```python\" in text:\n parts = text.split(\"```python\")\n if len(parts) > 1:\n return parts[1].split(\"```\")[0].strip()\n if \"```\" in text:\n parts = text.split(\"```\")\n for part in parts[1:]:\n code = part.strip()\n first = code.split(\"\\n\")[0].strip()\n if first and not first.startswith((\"def \", \"class \", \"import \", \"from \")):\n code = \"\\n\".join(code.split(\"\\n\")[1:]).strip()\n if code:\n return code\n lines = text.split(\"\\n\")\n code_lines, started = [], False\n for line in lines:\n s = line.strip()\n if s.startswith((\"def \", \"class \", \"import \", \"from \")):\n started = True\n if started:\n code_lines.append(line)\n return \"\\n\".join(code_lines).strip() if code_lines else text\n\n\ndef run_test(code: str, test_code: str, timeout: float = 10.0) -> Dict[str, Any]:\n full = code + \"\\n\\n\" + test_code + \"\\n\\nprint('TEST_PASSED')\\n\"\n tmp = None\n try:\n with tempfile.NamedTemporaryFile(mode=\"w\", suffix=\".py\", delete=False) as f:\n f.write(full); tmp = f.name\n proc = subprocess.run(\n [sys.executable, tmp], capture_output=True, text=True, timeout=timeout,\n env={**os.environ, \"PYTHONDONTWRITEBYTECODE\": \"1\"},\n )\n passed = \"TEST_PASSED\" in proc.stdout\n return {\"passed\": passed, \"error\": None if passed else (proc.stderr[-300:] or \"tests did not pass\")}\n except subprocess.TimeoutExpired:\n return {\"passed\": False, \"error\": \"timeout\"}\n except Exception as e:\n return {\"passed\": False, \"error\": str(e)[:200]}\n finally:\n if tmp:\n try: os.unlink(tmp)\n except OSError: pass\n\n\ndef generate_one(prompt: str) -> str:\n if hasattr(tokenizer, \"apply_chat_template\"):\n messages = [\n {\"role\": \"system\", \"content\": \"You are an expert Python programmer. Write a complete, correct, and efficient solution. Output ONLY the code, no explanation.\"},\n {\"role\": \"user\", \"content\": prompt},\n ]\n try:\n chat = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)\n except Exception:\n chat = f\"<|im_start|>system\\nYou are an expert Python programmer. Write a complete, correct, and efficient solution. Output ONLY the code, no explanation.<|im_end|>\\n<|im_start|>user\\n{prompt}<|im_end|>\\n<|im_start|>assistant\\n\"\n else:\n chat = f\"<|im_start|>system\\nYou are an expert Python programmer. Write a complete, correct, and efficient solution. Output ONLY the code, no explanation.<|im_end|>\\n<|im_start|>user\\n{prompt}<|im_end|>\\n<|im_start|>assistant\\n\"\n\n inputs = tokenizer(chat, return_tensors=\"pt\", truncation=True, max_length=1536)\n inputs = {k: v.to(model.device) for k, v in inputs.items()}\n with torch.no_grad():\n out = model.generate(\n **inputs, max_new_tokens=MAX_NEW_TOKENS, temperature=TEMPERATURE,\n top_p=TOP_P, do_sample=(TEMPERATURE > 0),\n pad_token_id=tokenizer.pad_token_id, eos_token_id=tokenizer.eos_token_id,\n )\n new_tokens = out[0][inputs[\"input_ids\"].shape[1]:]\n return tokenizer.decode(new_tokens, skip_special_tokens=True)",
"metadata": {
"trusted": true,
"execution": {
"iopub.status.busy": "2026-07-16T18:37:33.497179Z",
"iopub.execute_input": "2026-07-16T18:37:33.497638Z",
"iopub.status.idle": "2026-07-16T18:37:33.51044Z",
"shell.execute_reply.started": "2026-07-16T18:37:33.497608Z",
"shell.execute_reply": "2026-07-16T18:37:33.509737Z"
}
},
"outputs": [],
"execution_count": 9
},
{
"cell_type": "code",
"source": [
"from datasets import load_dataset\n",
"\n",
"print(\"=\" * 60)\n",
"print(\" BENCHMARK: HumanEval (164 problems)\")\n",
"print(\"=\" * 60)\n",
"\n",
"ds = load_dataset(\"openai/openai_humaneval\", split=\"test\")\n",
"problems = list(ds)\n",
"total = len(problems)\n",
"passed = failed = errors = 0\n",
"he_details = []\n",
"\n",
"t0 = time.time()\n",
"for i, prob in enumerate(problems):\n",
" prompt, test, entry_point = prob[\"prompt\"], prob[\"test\"], prob[\"entry_point\"]\n",
" try:\n",
" gen = generate_one(prompt)\n",
" code = extract_code(gen)\n",
" res = run_test(code, test)\n",
" ok = res[\"passed\"]\n",
" except Exception as e:\n",
" ok, res, code = False, {\"passed\": False, \"error\": str(e)[:200]}, \"\"\n",
"\n",
" if ok: passed += 1; tag = \"PASS\"\n",
" elif res.get(\"error\") and (\"Error\" in str(res[\"error\"]) or \"timeout\" in str(res.get(\"error\",\"\")).lower()):\n",
" errors += 1; tag = \"ERR\"\n",
" else: failed += 1; tag = \"FAIL\"\n",
"\n",
" # NOTE: we now store the generated `code` so HumanEval+ can reuse it\n",
" # instead of re-running generation for all 164 problems a second time.\n",
" he_details.append({\"idx\": i, \"task_id\": prob[\"task_id\"], \"entry_point\": entry_point,\n",
" \"passed\": ok, \"tag\": tag, \"error\": res.get(\"error\"), \"code\": code})\n",
" if (i + 1) % 10 == 0 or i == total - 1:\n",
" print(f\" [{i+1:>3}/{total}] {tag:<5} running pass@1 = {passed/(i+1)*100:.1f}%\")\n",
"\n",
"he_result = {\"benchmark\": \"HumanEval\", \"total\": total, \"passed\": passed, \"failed\": failed,\n",
" \"errors\": errors, \"pass@1\": round(passed/total*100, 1), \"per_problem\": he_details}\n",
"print(f\"\\n HumanEval done in {(time.time()-t0)/60:.1f} min | pass@1 = {he_result['pass@1']}%\")"
],
"metadata": {
"trusted": true,
"execution": {
"iopub.status.busy": "2026-07-16T18:37:38.550056Z",
"iopub.execute_input": "2026-07-16T18:37:38.551101Z",
"execution_failed": "2026-07-16T18:44:48.67Z"
}
},
"outputs": [],
"execution_count": null
},
{
"cell_type": "code",
"source": [
"print(\"=\" * 60)\n",
"print(\" BENCHMARK: HumanEval+ (extended tests via evalplus)\")\n",
"print(\"=\" * 60)\n",
"\n",
"from evalplus.data import get_human_eval_plus\n",
"from evalplus.eval import PASS\n",
"\n",
"t0 = time.time()\n",
"hep_result = None\n",
"\n",
"try:\n",
" hep_problems = get_human_eval_plus() # same 164 problems, with extra \"plus\" tests\n",
"\n",
" # Reuse the code already generated during the HumanEval pass above \u2014\n",
" # no need to regenerate (this alone halves the GPU time this cell used to burn).\n",
" samples_path = f\"{OUTPUT_DIR}/humaneval_plus_samples.jsonl\"\n",
" result_path = samples_path.replace(\".jsonl\", \"_eval_results.json\")\n",
" for p in (samples_path, result_path):\n",
" if os.path.exists(p):\n",
" os.remove(p) # evalplus prompts interactively if a stale result file exists\n",
"\n",
" covered = set()\n",
" with open(samples_path, \"w\") as f:\n",
" for d in he_details:\n",
" task_id = d[\"task_id\"]\n",
" if task_id in hep_problems and d[\"code\"]:\n",
" f.write(json.dumps({\"task_id\": task_id, \"solution\": d[\"code\"]}) + \"\\n\")\n",
" covered.add(task_id)\n",
"\n",
" missing = set(hep_problems) - covered\n",
" if missing:\n",
" print(f\" Filling {len(missing)} problem(s) missing/empty from the base run ...\")\n",
" with open(samples_path, \"a\") as f:\n",
" for task_id in missing:\n",
" try:\n",
" gen = generate_one(hep_problems[task_id][\"prompt\"])\n",
" code = extract_code(gen)\n",
" except Exception:\n",
" code = \"\"\n",
" f.write(json.dumps({\"task_id\": task_id, \"solution\": code}) + \"\\n\")\n",
"\n",
" from evalplus.evaluate import evaluate as ev_eval\n",
" ev_eval(dataset=\"humaneval\", samples=samples_path, i_just_wanna_run=True)\n",
"\n",
" with open(result_path) as f:\n",
" raw = json.load(f)\n",
" n = len(raw[\"eval\"])\n",
" base_pass = sum(1 for r in raw[\"eval\"].values() if r[0][\"base_status\"] == PASS)\n",
" plus_pass = sum(1 for r in raw[\"eval\"].values()\n",
" if r[0][\"base_status\"] == PASS and r[0][\"plus_status\"] == PASS)\n",
" hep_result = {\n",
" \"benchmark\": \"HumanEval+\", \"total\": n, \"passed\": plus_pass,\n",
" \"pass@1\": round(plus_pass / n * 100, 1),\n",
" \"base_pass@1\": round(base_pass / n * 100, 1),\n",
" \"evalplus_raw_path\": result_path,\n",
" }\n",
" print(f\" HumanEval+ pass@1 = {hep_result['pass@1']}% (base-only pass@1 = {hep_result['base_pass@1']}%)\")\n",
"\n",
"except Exception as e:\n",
" print(f\"[humaneval+] evalplus failed: {e}\")\n",
" traceback.print_exc()\n",
" passed = sum(1 for d in he_details if d[\"passed\"])\n",
" hep_result = {\"benchmark\": \"HumanEval+\", \"total\": len(he_details), \"passed\": passed,\n",
" \"pass@1\": round(passed/len(he_details)*100, 1), \"note\": f\"evalplus error: {e}\",\n",
" \"per_problem\": he_details}\n",
"\n",
"print(f\" Done in {(time.time()-t0)/60:.1f} min\")"
],
"metadata": {
"trusted": true,
"execution": {
"execution_failed": "2026-07-16T18:44:48.67Z"
}
},
"outputs": [],
"execution_count": null
},
{
"cell_type": "code",
"source": [
"print(\"=\" * 60)\n",
"print(\" BENCHMARK: MBPP (974 problems)\")\n",
"print(\"=\" * 60)\n",
"\n",
"ds_mbpp = load_dataset(\"google-research-datasets/mbpp\", split=\"test\")\n",
"mbpp_problems = list(ds_mbpp)\n",
"total = len(mbpp_problems)\n",
"passed = failed = errors = 0\n",
"mbpp_details = []\n",
"\n",
"t0 = time.time()\n",
"for i, prob in enumerate(mbpp_problems):\n",
" # BUG FIX: this dataset's field is \"text\", not \"prompt\" (that key doesn't exist\n",
" # and was throwing KeyError: 'prompt' before generating a single sample).\n",
" prompt = prob.get(\"text\", prob.get(\"prompt\", \"\"))\n",
" test_list = prob.get(\"test_list\", [])\n",
" test_code = \"\\n\".join(test_list) if test_list else \"\"\n",
" try:\n",
" gen = generate_one(prompt)\n",
" code = extract_code(gen)\n",
" res = run_test(code, test_code)\n",
" ok = res[\"passed\"]\n",
" except Exception as e:\n",
" ok, res, code = False, {\"passed\": False, \"error\": str(e)[:200]}, \"\"\n",
"\n",
" if ok: passed += 1; tag = \"PASS\"\n",
" elif res.get(\"error\") and (\"Error\" in str(res[\"error\"]) or \"timeout\" in str(res.get(\"error\",\"\")).lower()):\n",
" errors += 1; tag = \"ERR\"\n",
" else: failed += 1; tag = \"FAIL\"\n",
"\n",
" mbpp_details.append({\"idx\": i, \"task_id\": prob.get(\"task_id\", f\"mbpp_{i}\"),\n",
" \"passed\": ok, \"tag\": tag, \"error\": res.get(\"error\"), \"code\": code})\n",
" if (i + 1) % 50 == 0 or i == total - 1:\n",
" print(f\" [{i+1:>3}/{total}] {tag:<5} running pass@1 = {passed/(i+1)*100:.1f}%\")\n",
"\n",
"mbpp_result = {\"benchmark\": \"MBPP\", \"total\": total, \"passed\": passed, \"failed\": failed,\n",
" \"errors\": errors, \"pass@1\": round(passed/total*100, 1), \"per_problem\": mbpp_details}\n",
"print(f\"\\n MBPP done in {(time.time()-t0)/60:.1f} min | pass@1 = {mbpp_result['pass@1']}%\")"
],
"metadata": {
"trusted": true
},
"outputs": [],
"execution_count": null
},
{
"cell_type": "code",
"source": [
"print(\"=\" * 60)\n",
"print(\" BENCHMARK: MBPP+ (extended tests via evalplus)\")\n",
"print(\"=\" * 60)\n",
"\n",
"from evalplus.data import get_mbpp_plus\n",
"from evalplus.eval import PASS\n",
"\n",
"t0 = time.time()\n",
"mbppp_result = None\n",
"\n",
"try:\n",
" # NOTE: evalplus's MBPP+ is a separate, sanitized ~399-problem subset with its\n",
" # own task_ids and prompts \u2014 it does NOT line up 1:1 with the raw 974-problem\n",
" # MBPP test split used in the cell above, so we can't reuse those completions.\n",
" # We generate fresh (but only for this smaller set, not all 974 again).\n",
" mbppp_problems = get_mbpp_plus()\n",
" mbppp_task_ids = list(mbppp_problems.keys())\n",
" n_tasks = len(mbppp_task_ids)\n",
" print(f\" evalplus MBPP+ has {n_tasks} sanitized problems (independent of the raw MBPP split above)\")\n",
"\n",
" samples_path = f\"{OUTPUT_DIR}/mbpp_plus_samples.jsonl\"\n",
" result_path = samples_path.replace(\".jsonl\", \"_eval_results.json\")\n",
" for p in (samples_path, result_path):\n",
" if os.path.exists(p):\n",
" os.remove(p)\n",
"\n",
" with open(samples_path, \"w\") as f:\n",
" for i, task_id in enumerate(mbppp_task_ids):\n",
" prompt = mbppp_problems[task_id][\"prompt\"]\n",
" try:\n",
" gen = generate_one(prompt)\n",
" code = extract_code(gen)\n",
" except Exception:\n",
" code = \"\"\n",
" f.write(json.dumps({\"task_id\": task_id, \"solution\": code}) + \"\\n\")\n",
" if (i + 1) % 50 == 0 or i == n_tasks - 1:\n",
" print(f\" Generated {i+1}/{n_tasks} solutions ...\")\n",
"\n",
" from evalplus.evaluate import evaluate as ev_eval\n",
" ev_eval(dataset=\"mbpp\", samples=samples_path, i_just_wanna_run=True)\n",
"\n",
" with open(result_path) as f:\n",
" raw = json.load(f)\n",
" n = len(raw[\"eval\"])\n",
" base_pass = sum(1 for r in raw[\"eval\"].values() if r[0][\"base_status\"] == PASS)\n",
" plus_pass = sum(1 for r in raw[\"eval\"].values()\n",
" if r[0][\"base_status\"] == PASS and r[0][\"plus_status\"] == PASS)\n",
" mbppp_result = {\n",
" \"benchmark\": \"MBPP+\", \"total\": n, \"passed\": plus_pass,\n",
" \"pass@1\": round(plus_pass / n * 100, 1),\n",
" \"base_pass@1\": round(base_pass / n * 100, 1),\n",
" \"evalplus_raw_path\": result_path,\n",
" }\n",
" print(f\" MBPP+ pass@1 = {mbppp_result['pass@1']}% (base-only pass@1 = {mbppp_result['base_pass@1']}%)\")\n",
"\n",
"except Exception as e:\n",
" print(f\"[mbpp+] evalplus failed: {e}\")\n",
" traceback.print_exc()\n",
" mbppp_result = {**mbpp_result, \"benchmark\": \"MBPP+\", \"note\": f\"evalplus error: {e}\"}\n",
"\n",
"print(f\" Done in {(time.time()-t0)/60:.1f} min\")"
],
"metadata": {
"trusted": true,
"execution": {
"execution_failed": "2026-07-16T18:44:48.67Z"
}
},
"outputs": [],
"execution_count": null
},
{
"cell_type": "code",
"source": "BASELINES = {\n \"Model\": [\"TIMPS-Coder-7B (this model)\", \"Qwen2.5-Coder-7B-Instruct\", \"Qwen2.5-Coder-7B\",\n \"DeepSeek-Coder-7B-Instruct-v1.5\", \"CodeLlama-7B-Instruct\", \"CodeGemma-7B-it\",\n \"StarCoder2-7B\", \"Llama-3.1-8B-Instruct\", \"Phi-3.5-mini-instruct (3.8B)\", \"Gemma-2-9B-it\"],\n \"HumanEval\": [\"\u2014\", \"86.6\", \"89.6\", \"84.1\", \"53.7\", \"56.1\", \"40.2\", \"72.6\", \"68.8\", \"54.3\"],\n \"HumanEval+\": [\"\u2014\", \"71.3\", \"76.2\", \"70.8\", \"44.5\", \"46.9\", \"32.9\", \"61.0\", \"57.9\", \"44.5\"],\n \"MBPP\": [\"\u2014\", \"82.0\", \"84.0\", \"79.6\", \"55.6\", \"61.8\", \"46.0\", \"70.8\", \"73.0\", \"59.6\"],\n \"MBPP+\": [\"\u2014\", \"69.6\", \"72.0\", \"68.4\", \"45.0\", \"50.6\", \"36.5\", \"58.7\", \"61.3\", \"49.3\"],\n \"Params\": [\"7B\", \"7.6B\", \"7.6B\", \"7.1B\", \"6.7B\", \"7.0B\", \"7.0B\", \"8.0B\", \"3.8B\", \"9.2B\"],\n}\n\nresults = {\"humaneval\": he_result, \"humaneval_plus\": hep_result, \"mbpp\": mbpp_result, \"mbpp_plus\": mbppp_result}\n\n# Fill in TIMPS-Coder scores\nbl = {k: list(v) for k, v in BASELINES.items()}\nbl[\"HumanEval\"][0] = str(he_result.get(\"pass@1\", \"\u2014\"))\nbl[\"HumanEval+\"][0] = str(hep_result.get(\"pass@1\", \"\u2014\"))\nbl[\"MBPP\"][0] = str(mbpp_result.get(\"pass@1\", \"\u2014\"))\nbl[\"MBPP+\"][0] = str(mbppp_result.get(\"pass@1\", \"\u2014\"))\n\ncols = list(bl.keys())\nhdr = \"| \" + \" | \".join(cols) + \" |\"\nsep = \"|\" + \"|\".join([\"---\"] * len(cols)) + \"|\"\nrows = [\"| \" + \" | \".join(str(bl[c][i]) for c in cols) + \" |\" for i in range(len(bl[\"Model\"]))]\ntable = \"\\n\".join([hdr, sep] + rows)\n\nprint(table)\n\n# Save results\npayload = {\n \"model\": MODEL_ID, \"timestamp\": datetime.now(timezone.utc).isoformat(),\n \"results_summary\": {name: {\"pass@1\": r.get(\"pass@1\"), \"total\": r.get(\"total\"), \"passed\": r.get(\"passed\")} for name, r in results.items()},\n \"detailed_results\": results,\n}\nwith open(f\"{OUTPUT_DIR}/benchmark_results.json\", \"w\") as f:\n json.dump(payload, f, indent=2, default=str)\nprint(f\"\\nResults saved to {OUTPUT_DIR}/benchmark_results.json\")",
"metadata": {
"trusted": true
},
"outputs": [],
"execution_count": null
},
{
"cell_type": "code",
"source": "he = he_result.get(\"pass@1\", \"\u2014\")\nhep = hep_result.get(\"pass@1\", \"\u2014\")\nmb = mbpp_result.get(\"pass@1\", \"\u2014\")\nmbp = mbppp_result.get(\"pass@1\", \"\u2014\")\n\nMODEL_CARD = f\"\"\"---\nlicense: apache-2.0\nlanguage: [en]\nbase_model: Qwen/Qwen2.5-Coder-7B-Instruct\ntags: [code, qwen2.5, lora, merged, sft, dpo, grpo]\nlibrary_name: transformers\npipeline_tag: text-generation\nmodel-index:\n- name: TIMPS-Coder-7B\n results:\n - task: {{\"type\": \"text-generation\", \"name\": \"Code Generation\"}}\n dataset: {{\"type\": \"openai_humaneval\", \"name\": \"HumanEval\"}}\n metrics:\n - {{type: \"pass@1\", \"value\": {he}, \"name\": \"pass@1\"}}\n - task: {{\"type\": \"text-generation\", \"name\": \"Code Generation\"}}\n dataset: {{\"type\": \"evalplus/humanevalplus\", \"name\": \"HumanEval+\"}}\n metrics:\n - {{type: \"pass@1\", \"value\": {hep}, \"name\": \"pass@1\"}}\n - task: {{\"type\": \"text-generation\", \"name\": \"Code Generation\"}}\n dataset: {{\"type\": \"mbpp\", \"name\": \"MBPP\"}}\n metrics:\n - {{type: \"pass@1\", \"value\": {mb}, \"name\": \"pass@1\"}}\n - task: {{\"type\": \"text-generation\", \"name\": \"Code Generation\"}}\n dataset: {{\"type\": \"evalplus/mbppplus\", \"name\": \"MBPP+\"}}\n metrics:\n - {{type: \"pass@1\", \"value\": {mbp}, \"name\": \"pass@1\"}}\n---\n\n# TIMPS-Coder-7B\n\nTIMPS-Coder-7B is a code-generation model built by fine-tuning **Qwen2.5-Coder-7B-Instruct**\nthrough a 4-step pipeline: SFT, GRPO, DPO, and Self-Guided Self-Play.\n\n## Benchmark Results\n\n| Benchmark | Score |\n|-----------|-------|\n| **HumanEval pass@1** | **{he}%** |\n| **HumanEval+ pass@1** | **{hep}%** |\n| **MBPP pass@1** | **{mb}%** |\n| **MBPP+ pass@1** | **{mbp}%** |\n\n### Comparison with 7B-9B Code Models\n\n{table}\n\n## Usage\n\n```python\nfrom transformers import AutoModelForCausalLM, AutoTokenizer\nmodel = AutoModelForCausalLM.from_pretrained(\"sandeeprdy1729/TIMPS-Coder-7B\", device_map=\"auto\", torch_dtype=\"auto\")\ntokenizer = AutoTokenizer.from_pretrained(\"sandeeprdy1729/TIMPS-Coder-7B\")\nmessages = [{{\"role\": \"user\", \"content\": \"Write a fibonacci function.\"}}]\ninputs = tokenizer.apply_chat_template(messages, return_tensors=\"pt\").to(model.device)\nprint(tokenizer.decode(model.generate(inputs, max_new_tokens=512)[0]))\n\"\"\"\n\nwith open(f\"{OUTPUT_DIR}/model_card_results.md\", \"w\") as f:\n f.write(MODEL_CARD)\nprint(f\"Model card saved to {OUTPUT_DIR}/model_card_results.md\")\nprint(MODEL_CARD)",
"metadata": {
"trusted": true
},
"outputs": [],
"execution_count": null
},
{
"cell_type": "code",
"source": "\n#**Cell 11 \u2014 Push scores to HuggingFace model card (optional):**\n# ```python\nfrom huggingface_hub import HfApi\n\napi = HfApi()\napi.upload_file(\n path_or_fileobj=f\"{OUTPUT_DIR}/model_card_results.md\",\n path_in_repo=\"README.md\",\n repo_id=MODEL_ID,\n repo_type=\"model\",\n commit_message=f\"Add benchmark results: HE={he}% HE+={hep}% MBPP={mb}% MBPP+={mbp}%\",\n)\nprint(f\"Model card updated at https://huggingface.co/{MODEL_ID}\")",
"metadata": {
"trusted": true
},
"outputs": [],
"execution_count": null
}
]
}
Reference in New Issue Copy Permalink