diff --git a/.agents/README.md b/.agents/README.md index 6d4f1707..591be9f3 100644 --- a/.agents/README.md +++ b/.agents/README.md @@ -8,6 +8,7 @@ Note: Please copy the skills directory `.agents/skills` to `.claude/skills` if y - [vLLM Ascend Model Adapter Skill](#vllm-ascend-model-adapter-skill) - [vLLM Ascend main2main Skill](#vllm-ascend-main2main-skill) +- [vLLM Ascend Release Note Writer Skill](#vllm-ascend-release-note-writer-skill) ## vLLM Ascend Model Adapter Skill @@ -74,3 +75,41 @@ This skill facilitates the process of: 1. Open a conversation with the AI agent inside the vllm-ascend dev container. 2. Invoke the skill (e.g. `/main2main`). 3. The agent follows the playbook and produces a ready-to-merge commit. + +## vLLM Ascend Release Note Writer Skill + +You just need to say: `Please help me write a 0.13.0 release note based on commits from v0.11.0 and releases/v0.13.0` + +### What it does + +This skill guides you through a structured workflow to: + +1. Fetch commits between two versions using the provided script. +2. Analyze and categorize each commit in a CSV workspace. +3. Draft highlights and write polished release notes. +4. Generate release notes organized by category (Features, Hardware Support, Performance, Dependencies, etc.). + +### File layout + +| File | Purpose | +| ---- | ------- | +| `SKILL.md` | Skill definition, workflow, and writing guidelines | +| `references/ref-past-release-notes-highlight.md` | Style and category reference for release notes | +| `scripts/fetch_commits-optimize.py` | Script to fetch commits between versions | + +### Quick start + +1. Open a conversation with the AI agent. +2. Invoke the skill (e.g. `/vllm-ascend-release-note-writer`). +3. Follow the workflow steps: + - Fetch commits between versions + - Analyze commits in CSV format + - Draft and edit highlights +4. Output files are saved to `vllm-ascend-release-note/output/$version` + +### Key guidelines + +- Use one-level headings (###) for sections in a specific order: Highlights, Features, Hardware and Operator Support, Performance, Dependencies, Deprecation & Breaking Changes, Documentation, Others. +- Focus on user-facing impact and include context for practical usage. +- Verify details by checking linked PRs (use GitHub API for descriptions if needed). +- Keep notes concise and avoid unnecessary technical details. diff --git a/.agents/skills/vllm-ascend-release-note-writer/SKILL.md b/.agents/skills/vllm-ascend-release-note-writer/SKILL.md new file mode 100644 index 00000000..34bbf6a1 --- /dev/null +++ b/.agents/skills/vllm-ascend-release-note-writer/SKILL.md @@ -0,0 +1,79 @@ +--- +name: vLLM Ascend Release Note Writer +description: You are a release note writer for vLLM Ascend project (vllm-project/vllm-ascend). You are responsible for writing release notes for vLLM Ascend. +--- + +# vLLM Ascend release Note Writer Skill + +## Overview + +You should use the `ref-past-release-notes-highlight.md` as style and category reference. Always read these first. + +## When to use this skill + +When a new version of vLLM Ascend is released, you should use this skill to write the release notes. + +## How to use it + +0. all output files should be saved under `vllm-ascend-release-note/output/$version` folder + +1. Use the `fetch_commits-optimize.py` script to fetch the commits between the previous and current version. + +```bash +uv run python fetch_commits-optimize.py --base-tag $LAST_TAG --head-tag $NEW_TAG --output 0-current-raw-commits.md +``` + +`0-current-raw-commits.md` is your raw data input. + +2. Use the `commit-analysis-draft.csv` tool to analyze the commits and put them into the correct section. +`1-commit-analysis-draft.csv` is your workspace for commit by commit analysis for which commit goes into which section, whether can be ignored, and why. You can create auxilariy files in `tmp` folder. + * You should check each commit. They are put into rows in the CSV file. + * The CSV should have headers `title`, `pr number`, `user facing impact/summary`, `category`, `decision`, `reason`. Please brainstorm other fields as you see fit. + +3. Draft the highlights note, and save it to `2-highlights-note-draft.md`. +4. Edit the draft highlights note in `2-highlights-note-draft.md`, and save it to `3-highlights-note-edit.md`. You should double and triple check with the raw commits + analysis. You can leave any uncertainty and doubts in the file, and we will discuss them together. +5. Use the format `This is the $NUMBER release candidate of $VERSION for vLLM Ascend. Please follow the [official doc](https://docs.vllm.ai/projects/ascend/en/latest) to get started.`. + +## Writing style + +1. To keep simple, you should only save one level of headings, starting with ###, which may include the following categories follow below order: + +### Highlights + +### Features + +### Hardware and Operator Support + +### Performance + +### Dependencies + +### Deprecation & Breaking Changes + +### Documentation + +### Others + +2. Additional Inclusion Criteria + +* User experience improvements (CLI enhancements, better error messages, configuration flexibility) +* Core feature (PD Disaggregation, KVCaceh, Graph mode, CP/SP, quantization) +* Breaking changes and deprecations (always include with clear impact description) +* Significant infrastructure changes (elastic scaling, distributed serving, hardware support) +* Major dependency updates (CANN/torch_npu/triton-ascend/MoonCake/Ray/transformers versions, critical library updates) +* Binary/deployment improvements (size reductions, Docker enhancements) +* Default behavior changes (default models, configuration changes that affect all users) +* Hardware compatibility expansions (310P, A2, A3, A5 support) +In the end we don't want to miss any important changes. But also don't want to spam the notes with unnecessary details. + +3. Section Organization Guidelines + +* **Model Support first**: Most immediately visible to users, should lead the highlights +* **Group by user impact**: Hardware/performance should focus on what users experience, not internal optimizations +* **Provide usage context**: Include relevant flags, configuration options, and practical usage information +* **Technical detail level**: Explain what features enable rather than just listing technical changes + +4. Writing Tips + +* Look up the PR if you are not sure about the details. The PR number at the end (#12345) can be looked up via vllm-project/vllm#12345. To get the description, you just need to call and look at the body field. +* When writing the highlights, don't be too verbose. Focus exclusively on what users should know. diff --git a/.agents/skills/vllm-ascend-release-note-writer/references/ref-past-release-notes-highlight.md b/.agents/skills/vllm-ascend-release-note-writer/references/ref-past-release-notes-highlight.md new file mode 100644 index 00000000..3c275952 --- /dev/null +++ b/.agents/skills/vllm-ascend-release-note-writer/references/ref-past-release-notes-highlight.md @@ -0,0 +1,198 @@ +## v0.14.0rc1 - 2026.01.26 + +This is the first release candidate of v0.14.0 for vLLM Ascend. Please follow the [official doc](https://docs.vllm.ai/projects/ascend/en/latest) to get started. This release includes all the changes in v0.13.0rc2. So We just list the differences from v0.13.0rc2. If you are upgrading from v0.13.0rc1, please read both v0.14.0rc1 and v0.13.0rc2 release notes. + +### Highlights + +- 310P support is back now. In this release, only basic dense and vl models are supported with eager mode. We'll keep improving and maintaining the support for 310P. [#5776](https://github.com/vllm-project/vllm-ascend/pull/5776) +- Support compressed tensors moe w8a8-int8 quantization. [#5718](https://github.com/vllm-project/vllm-ascend/pull/5718) +- Support Medusa speculative decoding. [#5668](https://github.com/vllm-project/vllm-ascend/pull/5668) +- Support Eagle3 speculative decoding for Qwen3vl. [#4848](https://github.com/vllm-project/vllm-ascend/pull/4848) + +### Features + +- Xlite Backend supports Qwen3 MoE now. [#5951](https://github.com/vllm-project/vllm-ascend/pull/5951) +- Support DSA-CP for PD-mix deployment case. [#5702](https://github.com/vllm-project/vllm-ascend/pull/5702) +- Add support of new W4A4_LAOS_DYNAMIC quantization method. [#5143](https://github.com/vllm-project/vllm-ascend/pull/5143) + +### Performance + +- The performance of Qwen3-next has been improved. [#5664](https://github.com/vllm-project/vllm-ascend/pull/5664) [#5984](https://github.com/vllm-project/vllm-ascend/pull/5984) [#5765](https://github.com/vllm-project/vllm-ascend/pull/5765) +- The CPU bind logic and performance has been improved. [#5555](https://github.com/vllm-project/vllm-ascend/pull/5555) +- Merge Q/K split to simplify AscendApplyRotaryEmb for better performance. [#5799](https://github.com/vllm-project/vllm-ascend/pull/5799) +- Add Matmul Allreduce Rmsnorm fusion Pass. It's disabled by default. Set `fuse_allreduce_rms=True` in `--additional_config` to enable it. [#5034](https://github.com/vllm-project/vllm-ascend/pull/5034) +- Optimize rope embedding with triton kernel for huge performance gain. [#5918](https://github.com/vllm-project/vllm-ascend/pull/5918) +- support advanced apply_top_k_top_p without top_k constraint. [#6098](https://github.com/vllm-project/vllm-ascend/pull/6098) +- Parallelize Q/K/V padding in AscendMMEncoderAttention for better performance. [#6204](https://github.com/vllm-project/vllm-ascend/pull/6204) + +### Others + +- model runner v2 support triton of penalty. [#5854](https://github.com/vllm-project/vllm-ascend/pull/5854) +- model runner v2 support eagle spec decoding. [#5840](https://github.com/vllm-project/vllm-ascend/pull/5840) +- Fix multi-modal inference OOM issues by setting `expandable_segments:True` by default. [#5855](https://github.com/vllm-project/vllm-ascend/pull/5855) +- `VLLM_ASCEND_ENABLE_MLAPO` is set to `True` by default. It's enabled automatically on decode node in PD deployment case. Please note that this feature will cost more memory. If you are memory sensitive, please set it to False. [#5952](https://github.com/vllm-project/vllm-ascend/pull/5952) +- SSL config can be set to kv_extra_config for PD deployment with mooncake layerwise connector. [#5875](https://github.com/vllm-project/vllm-ascend/pull/5875) +- support `--max_model_len=auto`. [#6193](https://github.com/vllm-project/vllm-ascend/pull/6193) + +### Dependencies + +- torch-npu is upgraded to 2.9.0 [#6112](https://github.com/vllm-project/vllm-ascend/pull/6112) + +### Deprecation & Breaking Changes + +- EPLB config options is moved to `eplb_config` in [additional config](https://docs.vllm.ai/projects/ascend/en/latest/user_guide/configuration/additional_config.html). The old ones are removed in this release. +- The profiler envs, such as `VLLM_TORCH_PROFILER_DIR` and `VLLM_TORCH_PROFILER_WITH_PROFILE_MEMORY` do not work with vLLM Ascend now. Please use vLLM `--profiler-config` parameters instead. [#5928](https://github.com/vllm-project/vllm-ascend/pull/5928) + +### Known Issues + +- If you hit the pickle error from `EngineCore` process sometimes, please cherry-pick the [PR](https://github.com/vllm-project/vllm/pull/32022) into your local vLLM code. This known issue will be fixed in vLLM in the next release. + +## v0.13.0rc2 - 2026.01.24 + +This is the second release candidate of v0.13.0 for vLLM Ascend. In this rc release, we fixed lots of bugs and improved the performance of many models. Please follow the [official doc](https://docs.vllm.ai/projects/ascend/en/v0.13.0/) to get started. Any feedback is welcome to help us to improve the final version of v0.13.0. + +### Highlights + +We mainly focus on quality and performance improvement in this release. The spec decode, graph mode, context parallel and EPLB have been improved significantly. A lot of bugs have been fixed and the performance has been improved for DeepSeek3.1/3.2, Qwen3 Dense/MOE models. + +### Features + +- implement basic framework for batch invariant [#5517](https://github.com/vllm-project/vllm-ascend/pull/5517) +- Eagle spec decode feature now works with full graph mode. [#5118](https://github.com/vllm-project/vllm-ascend/pull/5118) +- Context Parallel(PCP&DCP) feature is more stable now. And it works for most case. Please try it out. +- MTP and eagle spec decode feature now works in most cases. And it's suggested to use them in most cases. +- EPLB feature more stable now. Many bugs have been fixed. Mix placement works now [#6086](https://github.com/vllm-project/vllm-ascend/pull/6086) +- Support kv nz feature for DeepSeek decode node in disagg-prefill scenario [#3072](https://github.com/vllm-project/vllm-ascend/pull/3072) + +### Model Support + +- LongCat-Flash is supported now.[#3833](https://github.com/vllm-project/vllm-ascend/pull/3833) +- minimax_m2 is supported now. [#5624](https://github.com/vllm-project/vllm-ascend/pull/5624) +- Support for cross-attention and whisper models [#5592](https://github.com/vllm-project/vllm-ascend/pull/5592) + +### Performance + +- Many custom ops and triton kernels are added in this release to speed up the performance of models. Such as `RejectSampler`, `MoeInitRoutingCustom`, `DispatchFFNCombine` and so on. +- Improved the performance of Layerwise Connector [#5303](https://github.com/vllm-project/vllm-ascend/pull/5303) + +### Others + +- Basic support Model Runner v2. Model Runner V2 is the next generation of vLLM. It will be used by default in the future release. [#5210](https://github.com/vllm-project/vllm-ascend/pull/5210) +- Fixed a bug that the zmq send/receive may failed [#5503](https://github.com/vllm-project/vllm-ascend/pull/5503) +- Supported to use full-graph with Qwen3-Next-MTP [#5477](https://github.com/vllm-project/vllm-ascend/pull/5477) +- Fix weight transpose in RL scenarios [#5567](https://github.com/vllm-project/vllm-ascend/pull/5567) +- Adapted SP to eagle3 [#5562](https://github.com/vllm-project/vllm-ascend/pull/5562) +- Context Parallel(PCP&DCP) support mlapo [#5672](https://github.com/vllm-project/vllm-ascend/pull/5672) +- GLM4.6 support mtp with fullgraph [#5460](https://github.com/vllm-project/vllm-ascend/pull/5460) +- Flashcomm2 now works with oshard generalized feature [#4723](https://github.com/vllm-project/vllm-ascend/pull/4723) +- Support setting tp=1 for the Eagle draft model [#5804](https://github.com/vllm-project/vllm-ascend/pull/5804) +- Flashcomm1 feature now works with qwen3-vl [#5848](https://github.com/vllm-project/vllm-ascend/pull/5848) +- Support fine-grained shared expert overlap [#5962](https://github.com/vllm-project/vllm-ascend/pull/5962) + +### Dependencies + +- CANN is upgraded to 8.5.0 +- torch-npu is upgraded to 2.8.0.post1. Please note that the post version will not be installed by default. Please install it by hand from [pypi mirror](https://mirrors.huaweicloud.com/ascend/repos/pypi/torch-npu/). +- triton-ascend is upgraded to 3.2.0 + +### Deprecation & Breaking Changes + +- `CPUOffloadingConnector` is deprecated. We'll remove it in the next release. It'll be replaced by CPUOffload feature from vLLM in the future. +- eplb config options is moved to `eplb_config` in [additional config](https://docs.vllm.ai/projects/ascend/en/latest/user_guide/configuration/additional_config.html). The old ones will be removed in the next release. +- `ProfileExecuteDuration` [feature](https://docs.vllm.ai/projects/ascend/en/latest/developer_guide/performance_and_debug/profile_execute_duration.html) is deprecated. It's replaced by `ObservabilityConfig` from vLLM. +- The value of `VLLM_ASCEND_ENABLE_MLAPO` env will be set to True by default in the next release. It'll be enabled in decode node by default. Please note that this feature will cost more memory. If you are memory sensitive, please set it to False. + +## v0.13.0rc1 - 2025.12.27 + +This is the first release candidate of v0.13.0 for vLLM Ascend. We landed lots of bug fix, performance improvement and feature support in this release. Any feedback is welcome to help us to improve vLLM Ascend. Please follow the [official doc](https://docs.vllm.ai/projects/ascend/en/latest) to get started. + +### Highlights + +- Improved the performance of DeepSeek V3.2, please refer to [tutorials](https://docs.vllm.ai/projects/ascend/en/latest/tutorials/DeepSeek-V3.2.html) +- Qwen3-Next MTP with chunked prefill is supported now [#4770](https://github.com/vllm-project/vllm-ascend/pull/4770), please refer to [tutorials](https://docs.vllm.ai/projects/ascend/en/latest/tutorials/Qwen3-Next.html) +- [Experimental] Prefill Context Parallel and Decode Context Parallel are supported, but notice that it is an experimental feature now, welcome any feedback. please refer to [context parallel feature guide](https://docs.vllm.ai/projects/ascend/en/latest/user_guide/feature_guide/context_parallel.html) + +### Features + +- Support openPangu Ultra MoE [4615](https://github.com/vllm-project/vllm-ascend/pull/4615) +- A new quantization method W8A16 is supported now. [#4541](https://github.com/vllm-project/vllm-ascend/pull/4541) +- Cross-machine Disaggregated Prefill is supported now. [#5008](https://github.com/vllm-project/vllm-ascend/pull/5008) +- Add UCMConnector for KV Cache Offloading. [#4411](https://github.com/vllm-project/vllm-ascend/pull/4411) +- Support async_scheduler and disable_padded_drafter_batch in eagle. [#4893](https://github.com/vllm-project/vllm-ascend/pull/4893) +- Support pcp + mtp in full graph mode. [#4572](https://github.com/vllm-project/vllm-ascend/pull/4572) +- Enhance all-reduce skipping logic for MoE models in NPUModelRunner [#5329](https://github.com/vllm-project/vllm-ascend/pull/5329) + +### Performance + +Some general performance improvement: + +- Add l2norm triton kernel [#4595](https://github.com/vllm-project/vllm-ascend/pull/4595) +- Add new pattern for AddRmsnormQuant with SP, which could only take effect in graph mode. [#5077](https://github.com/vllm-project/vllm-ascend/pull/5077) +- Add async exponential while model executing. [#4501](https://github.com/vllm-project/vllm-ascend/pull/4501) +- Remove the transpose step after attention and switch to transpose_batchmatmul [#5390](https://github.com/vllm-project/vllm-ascend/pull/5390) +- To optimize the performance in small batch size scenario, an attention operator with flash decoding function is offered, please refer to item 22 in [FAQs](https://docs.vllm.ai/projects/ascend/en/latest/faqs.html) to enable it. + +### Other + +- OOM error on VL models is fixed now. We're keeping observing it, if you hit OOM problem again, please submit an issue. [#5136](https://github.com/vllm-project/vllm-ascend/pull/5136) +- Fixed an accuracy bug of Qwen3-Next-MTP when batched inferring. [#4932](https://github.com/vllm-project/vllm-ascend/pull/4932) +- Fix npu-cpu offloading interface change bug. [#5290](https://github.com/vllm-project/vllm-ascend/pull/5290) +- Fix MHA model runtime error in aclgraph mode [#5397](https://github.com/vllm-project/vllm-ascend/pull/5397) +- Fix unsuitable moe_comm_type under ep=1 scenario [#5388](https://github.com/vllm-project/vllm-ascend/pull/5388) + +### Deprecation & Breaking Changes + +- `VLLM_ASCEND_ENABLE_DENSE_OPTIMIZE` is removed and `VLLM_ASCEND_ENABLE_PREFETCH_MLP` is recommend to replace as they always be enabled together. [#5272](https://github.com/vllm-project/vllm-ascend/pull/5272) +- `VLLM_ENABLE_FUSED_EXPERTS_ALLGATHER_EP` is dropped now. [#5270](https://github.com/vllm-project/vllm-ascend/pull/5270) +- `VLLM_ASCEND_ENABLE_NZ` is disabled for float weight case, since we notice that the performance is not good in some float case. Feel free to set it to 2 if you make sure it works for your case. [#4878](https://github.com/vllm-project/vllm-ascend/pull/4878) +- `chunked_prefill_for_mla` in `additional_config` is dropped now. [#5296](https://github.com/vllm-project/vllm-ascend/pull/5296) +- `dump_config` in `additional_config` is renamed to `dump_config_path` and the type is change from `dict` to `string`. [#5296](https://github.com/vllm-project/vllm-ascend/pull/5296) + +### Dependencies + +- vLLM version has been upgraded to 0.13.0 and drop 0.12.0 support. [#5146](https://github.com/vllm-project/vllm-ascend/pull/5146) +- Transformer version has been upgraded >= 4.57.3 [#5250](https://github.com/vllm-project/vllm-ascend/pull/5250) + +### Known Issues + +- Qwen3-Next doesn't support long sequence scenario, and we should limit `gpu-memory-utilization` according to the doc to run Qwen3-Next. We'll improve it in the next release +- The functional break on Qwen3-Next when the input/output is around 3.5k/1.5k is fixed, but it introduces a regression on performance. We'll fix it in next release. [#5357](https://github.com/vllm-project/vllm-ascend/issues/5357) +- There is a precision issue with curl on ultra-short sequences in DeepSeek-V3.2. We'll fix it in next release. [#5370](https://github.com/vllm-project/vllm-ascend/issues/5370) + +## v0.11.0 - 2025.12.16 + +We're excited to announce the release of v0.11.0 for vLLM Ascend. This is the official release for v0.11.0. Please follow the [official doc](https://docs.vllm.ai/projects/ascend/en/v0.11.0) to get started. We'll consider to release post version in the future if needed. This release note will only contain the important change and note from v0.11.0rc3. + +### Highlights + +- Improved the performance for deepseek 3/3.1. [#3995](https://github.com/vllm-project/vllm-ascend/pull/3995) +- Fixed the accuracy bug for qwen3-vl. [#4811](https://github.com/vllm-project/vllm-ascend/pull/4811) +- Improved the performance of sample. [#4153](https://github.com/vllm-project/vllm-ascend/pull/4153) +- Eagle3 is back now. [#4721](https://github.com/vllm-project/vllm-ascend/pull/4721) + +### Other + +- Improved the performance for kimi-k2. [#4555](https://github.com/vllm-project/vllm-ascend/pull/4555) +- Fixed a quantization bug for deepseek3.2-exp. [#4797](https://github.com/vllm-project/vllm-ascend/pull/4797) +- Fixed qwen3-vl-moe bug under high concurrency. [#4658](https://github.com/vllm-project/vllm-ascend/pull/4658) +- Fixed an accuracy bug for Prefill Decode disaggregation case. [#4437](https://github.com/vllm-project/vllm-ascend/pull/4437) +- Fixed some bugs for EPLB [#4576](https://github.com/vllm-project/vllm-ascend/pull/4576) [#4777](https://github.com/vllm-project/vllm-ascend/pull/4777) +- Fixed the version incompatibility issue for openEuler docker image. [#4745](https://github.com/vllm-project/vllm-ascend/pull/4745) + +### Deprecation announcement + +- LLMdatadist connector has been deprecated, it'll be removed in v0.12.0rc1 +- Torchair graph has been deprecated, it'll be removed in v0.12.0rc1 +- Ascend scheduler has been deprecated, it'll be removed in v0.12.0rc1 + +### Upgrade notice + +- torch-npu is upgraded to 2.7.1.post1. Please note that the package is pushed to [pypi mirror](https://mirrors.huaweicloud.com/ascend/repos/pypi/torch-npu/). So it's hard to add it to auto dependence. Please install it by yourself. +- CANN is upgraded to 8.3.rc2. + +### Known Issues + +- Qwen3-Next doesn't support expert parallel and MTP features in this release. And it'll be oom if the input is too long. We'll improve it in the next release +- Deepseek 3.2 only work with torchair graph mode in this release. We'll make it work with aclgraph mode in the next release. +- Qwen2-audio doesn't work by default. Temporary solution is to set `--gpu-memory-utilization` to a suitable value, such as 0.8. +- CPU bind feature doesn't work if more than one vLLM instance is running on the same node. diff --git a/.agents/skills/vllm-ascend-release-note-writer/scripts/fetch_commits-optimize.py b/.agents/skills/vllm-ascend-release-note-writer/scripts/fetch_commits-optimize.py new file mode 100644 index 00000000..9a8b07cb --- /dev/null +++ b/.agents/skills/vllm-ascend-release-note-writer/scripts/fetch_commits-optimize.py @@ -0,0 +1,1546 @@ +#!/usr/bin/env python3 +""" +Fetch all commits between two tags from a GitHub repository. +Usage: python fetch_commits.py [--token YOUR_GITHUB_TOKEN] +""" + +import argparse +import os +import re + +import dotenv +import requests + +# Load .env.local first (higher priority), then .env as fallback +dotenv.load_dotenv(".env.local") +dotenv.load_dotenv() # .env as fallback + + +def get_github_token(): + """Get GitHub token from environment or argument.""" + return os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN") + + +def resolve_tag_to_sha(base_url: str, tag: str, headers: dict) -> str: + """Resolve a tag name to its commit SHA.""" + print(f"Resolving tag {tag}...") + + tag_resp = requests.get(f"{base_url}/git/refs/tags/{tag}", headers=headers) + if tag_resp.status_code != 200: + raise Exception(f"Failed to get tag {tag}: {tag_resp.text}") + + tag_data = tag_resp.json() + sha = tag_data["object"]["sha"] + + # If it's an annotated tag, we need to get the commit it points to + if tag_data["object"]["type"] == "tag": + tag_obj_resp = requests.get(f"{base_url}/git/tags/{sha}", headers=headers) + if tag_obj_resp.status_code == 200: + sha = tag_obj_resp.json()["object"]["sha"] + + return sha + + +def resolve_commit_sha(base_url: str, commit_ref: str, headers: dict) -> str: + """Resolve a commit reference (SHA or short SHA) to full SHA.""" + print(f"Resolving commit {commit_ref}...") + + commit_resp = requests.get(f"{base_url}/commits/{commit_ref}", headers=headers) + if commit_resp.status_code != 200: + raise Exception(f"Failed to get commit {commit_ref}: {commit_resp.text}") + + return commit_resp.json()["sha"] + + +def get_default_branch_head(base_url: str, headers: dict) -> tuple[str, str]: + """ + Get the HEAD commit of the default branch. + + Returns: + Tuple of (branch_name, head_sha) + """ + print("Getting default branch HEAD...") + + # Get repository info to find default branch + repo_resp = requests.get(base_url, headers=headers) + if repo_resp.status_code != 200: + raise Exception(f"Failed to get repository info: {repo_resp.text}") + + default_branch = repo_resp.json()["default_branch"] + print(f" Default branch: {default_branch}") + + # Get the HEAD commit of the default branch + branch_resp = requests.get(f"{base_url}/branches/{default_branch}", headers=headers) + if branch_resp.status_code != 200: + raise Exception(f"Failed to get branch {default_branch}: {branch_resp.text}") + + head_sha = branch_resp.json()["commit"]["sha"] + print(f" HEAD: {head_sha[:8]}") + + return (default_branch, head_sha) + + +def get_all_tags(base_url: str, headers: dict) -> list[dict]: + """Get all tags from the repository with their commit SHAs and dates.""" + print("Fetching all tags...") + + all_tags = [] + page = 1 + per_page = 100 + + while True: + resp = requests.get( + f"{base_url}/tags", + headers=headers, + params={"per_page": per_page, "page": page}, + ) + + if resp.status_code != 200: + raise Exception(f"Failed to get tags: {resp.text}") + + tags = resp.json() + if not tags: + break + + all_tags.extend(tags) + page += 1 + + if len(tags) < per_page: + break + + print(f" Found {len(all_tags)} tags") + return all_tags + + +def get_commit_date(base_url: str, sha: str, headers: dict) -> str: + """Get the commit date for a given SHA.""" + commit_resp = requests.get(f"{base_url}/commits/{sha}", headers=headers) + if commit_resp.status_code != 200: + return None + return commit_resp.json()["commit"]["committer"]["date"] + + +def find_previous_tag( + base_url: str, head_sha: str, headers: dict, tag_pattern: str | None = None +) -> tuple[str, str] | None: + """ + Find the most recent tag that is an ancestor of the given commit. + + Uses git history to find tags that are reachable from the commit. + + Args: + base_url: GitHub API base URL + head_sha: The commit SHA to search from + headers: Request headers + tag_pattern: Optional regex pattern to filter tags (e.g., r'^v\\d+\\.\\d+\\.\\d+$') + + Returns: + Tuple of (tag_name, tag_sha) or None if no tag found + """ + print(f"Finding previous tag before commit {head_sha[:8]}...") + + # Get the date of the head commit + head_date = get_commit_date(base_url, head_sha, headers) + if not head_date: + print(" Warning: Could not get head commit date") + return None + + print(f" Head commit date: {head_date}") + + # Get all tags + all_tags = get_all_tags(base_url, headers) + + # Filter tags by pattern if provided + if tag_pattern: + import re + + pattern = re.compile(tag_pattern) + all_tags = [t for t in all_tags if pattern.match(t["name"])] + print(f" After pattern filter: {len(all_tags)} tags") + + # For each tag, check if it's an ancestor of head_sha and get its date + tag_candidates = [] + + for tag in all_tags: + tag_name = tag["name"] + tag_commit_sha = tag["commit"]["sha"] + + # Skip if this is the same commit as head + if tag_commit_sha == head_sha: + continue + + # Check if this tag's commit is an ancestor of head + compare_resp = requests.get(f"{base_url}/compare/{tag_commit_sha}...{head_sha}", headers=headers) + + if compare_resp.status_code != 200: + continue + + compare_data = compare_resp.json() + + # If tag is behind head (status = "behind" or "ahead"), it's an ancestor + # We want tags where the comparison shows head is ahead + if compare_data.get("status") in ["ahead", "diverged"]: + # Get the tag's commit date + tag_date = get_commit_date(base_url, tag_commit_sha, headers) + if tag_date and tag_date < head_date: + tag_candidates.append( + { + "name": tag_name, + "sha": tag_commit_sha, + "date": tag_date, + "ahead_by": compare_data.get("ahead_by", 0), + } + ) + print(f" Found candidate: {tag_name} ({compare_data.get('ahead_by', 0)} commits behind)") + + if not tag_candidates: + print(" No previous tag found") + return None + + # Sort by date (most recent first) or by ahead_by (smallest first) + # Using ahead_by gives us the closest tag + tag_candidates.sort(key=lambda x: x["ahead_by"]) + + best_tag = tag_candidates[0] + print(f" Selected: {best_tag['name']} ({best_tag['ahead_by']} commits behind)") + + return (best_tag["name"], best_tag["sha"]) + + +def fetch_commits_between_tags( + owner: str, repo: str, base_tag: str, head_tag: str, token: str | None = None +) -> list[dict]: + """ + Fetch all commits between two tags by walking the commit graph. + + This method traverses from head_tag back to base_tag, collecting all commits. + It properly handles the commit history and doesn't rely on date filtering. + + Args: + owner: Repository owner (e.g., 'vllm-project') + repo: Repository name (e.g., 'vllm') + base_tag: Base tag (older, e.g., 'v0.11.2') + head_tag: Head tag (newer, e.g., 'v0.12.0') + token: Optional GitHub token for higher rate limits + + Returns: + List of commit dictionaries + """ + headers = { + "Accept": "application/vnd.github.v3+json", + } + if token: + headers["Authorization"] = f"token {token}" + + base_url = f"https://api.github.com/repos/{owner}/{repo}" + + # Resolve tags to commit SHAs + base_sha = resolve_tag_to_sha(base_url, base_tag, headers) + head_sha = resolve_tag_to_sha(base_url, head_tag, headers) + + print(f"\nBase SHA ({base_tag}): {base_sha}") + print(f"Head SHA ({head_tag}): {head_sha}") + + # First, use Compare API to get total commit count (for progress info) + print(f"\nComparing {base_tag}...{head_tag}...") + compare_resp = requests.get(f"{base_url}/compare/{base_sha}...{head_sha}", headers=headers) + if compare_resp.status_code == 200: + compare_data = compare_resp.json() + total_commits = compare_data.get("total_commits", "unknown") + print(f"Total commits to fetch: {total_commits}") + + # Walk the commit history from head to base + # We use the commits API starting from head_sha and stop when we reach base_sha + all_commits = [] + seen_shas = set() + seen_shas.add(base_sha) # Don't include the base commit itself + + # BFS traversal of commit graph + to_visit = [head_sha] + page_count = 0 + + print(f"\nFetching commits from {head_tag} back to {base_tag}...") + + while to_visit: + current_sha = to_visit.pop(0) + + if current_sha in seen_shas: + continue + + seen_shas.add(current_sha) + + # Fetch commit details + commit_resp = requests.get(f"{base_url}/commits/{current_sha}", headers=headers) + + if commit_resp.status_code != 200: + print(f" Warning: Failed to fetch commit {current_sha[:8]}") + continue + + commit = commit_resp.json() + all_commits.append(commit) + + # Add parent commits to visit queue + for parent in commit.get("parents", []): + parent_sha = parent["sha"] + if parent_sha not in seen_shas: + to_visit.append(parent_sha) + + # Progress logging + if len(all_commits) % 50 == 0: + page_count += 1 + print(f" Fetched {len(all_commits)} commits...") + + print(f" Completed: {len(all_commits)} commits fetched") + + return all_commits + + +def fetch_commits_by_date_range( + owner: str, + repo: str, + since: str, + until: str, + token: str | None = None, + branch: str | None = None, +) -> list[dict]: + """ + Fetch all commits within a date range. + + Args: + owner: Repository owner (e.g., 'vllm-project') + repo: Repository name (e.g., 'vllm') + since: Start date (ISO 8601 format, e.g., '2025-01-01' or '2025-01-01T00:00:00Z') + until: End date (ISO 8601 format, e.g., '2025-01-31' or '2025-01-31T23:59:59Z') + token: Optional GitHub token for higher rate limits + branch: Optional branch name (defaults to repository's default branch) + + Returns: + List of commit dictionaries + """ + headers = { + "Accept": "application/vnd.github.v3+json", + } + if token: + headers["Authorization"] = f"token {token}" + + base_url = f"https://api.github.com/repos/{owner}/{repo}" + per_page = 100 + + # Normalize date format - add time if not present + if len(since) == 10: # YYYY-MM-DD format + since = f"{since}T00:00:00Z" + if len(until) == 10: # YYYY-MM-DD format + until = f"{until}T23:59:59Z" + + print(f"\nFetching commits from {since} to {until}...") + if branch: + print(f" Branch: {branch}") + + all_commits = [] + page = 1 + + while True: + params = {"since": since, "until": until, "per_page": per_page, "page": page} + if branch: + params["sha"] = branch + + response = requests.get(f"{base_url}/commits", headers=headers, params=params) + + if response.status_code != 200: + raise Exception(f"Failed to fetch commits: {response.text}") + + commits = response.json() + if not commits: + break + + all_commits.extend(commits) + print(f" Page {page}: fetched {len(commits)} commits (total: {len(all_commits)})") + + if len(commits) < per_page: + break + + page += 1 + + print(f" Completed: {len(all_commits)} commits fetched") + return all_commits + + +def get_merge_base(base_url: str, base_sha: str, head_sha: str, headers: dict) -> str | None: + """ + Get the merge base (common ancestor) of two commits. + + Args: + base_url: GitHub API base URL for the repo + base_sha: First commit SHA + head_sha: Second commit SHA + headers: Request headers + + Returns: + Merge base commit SHA, or None if not found + """ + # GitHub Compare API returns merge_base_commit + compare_resp = requests.get( + f"{base_url}/compare/{base_sha}...{head_sha}", + headers=headers, + ) + + if compare_resp.status_code != 200: + return None + + compare_data = compare_resp.json() + merge_base = compare_data.get("merge_base_commit", {}).get("sha") + return merge_base + + +def fetch_commits_by_walking_history( + base_url: str, + base_sha: str, + head_sha: str, + base_tag: str, + head_tag: str, + headers: dict, + stop_sha: str | None = None, +) -> list[dict]: + """ + Fetch commits by walking the commit history from head to a stop point. + + This method correctly handles release branches with cherry-picks. + It walks the head's commit history until it reaches the stop commit. + + Args: + base_url: GitHub API base URL for the repo + base_sha: Base commit SHA (for display purposes) + head_sha: Head commit SHA (newer) + base_tag: Display name for base reference + head_tag: Display name for head reference + headers: Request headers + stop_sha: SHA to stop at (if None, uses base_sha) + + Returns: + List of commit dictionaries (excluding stop commit) + """ + per_page = 100 + all_commits = [] + page = 1 + target_sha = stop_sha or base_sha + + print(f"\nWalking commit history from {head_tag} back to {base_tag}...") + print(f" Stop SHA: {target_sha[:8]}") + + while True: + response = requests.get( + f"{base_url}/commits", + headers=headers, + params={"sha": head_sha, "per_page": per_page, "page": page}, + ) + + if response.status_code != 200: + print(f" Warning: API error on page {page}, stopping") + break + + commits = response.json() + if not commits: + print(f" No more commits found on page {page}") + break + + found_stop = False + for commit in commits: + if commit["sha"] == target_sha: + # Reached stop commit, stop (don't include it) + found_stop = True + break + all_commits.append(commit) + + print(f" Page {page}: fetched {len(commits)} commits (total: {len(all_commits)})") + + if found_stop: + print(f" Reached stop commit ({target_sha[:8]})") + break + + if len(commits) < per_page: + print(" Warning: Reached end of history without finding stop commit") + break + + page += 1 + + return all_commits + + +def fetch_commits_between_tags_fast( + owner: str, + repo: str, + base_tag: str, + head_tag: str, + token: str | None = None, + head_is_commit: bool = False, + base_is_commit: bool = False, +) -> list[dict]: + """ + Fetch all commits between two tags using GitHub Compare API with pagination. + + This properly fetches only the commits between the two tags. + Automatically handles diverged branches (e.g., release branches with cherry-picks) + by falling back to walking the commit history. + + Args: + owner: Repository owner (e.g., 'vllm-project') + repo: Repository name (e.g., 'vllm') + base_tag: Base tag (older, e.g., 'v0.11.2') or commit SHA if base_is_commit=True + head_tag: Head tag (newer, e.g., 'v0.12.0') or commit SHA if head_is_commit=True + token: Optional GitHub token for higher rate limits + head_is_commit: If True, treat head_tag as a commit SHA instead of a tag + base_is_commit: If True, treat base_tag as a commit SHA instead of a tag + + Returns: + List of commit dictionaries + """ + headers = { + "Accept": "application/vnd.github.v3+json", + } + if token: + headers["Authorization"] = f"token {token}" + + base_url = f"https://api.github.com/repos/{owner}/{repo}" + per_page = 100 + + # Resolve to commit SHAs + if base_is_commit: + base_sha = resolve_commit_sha(base_url, base_tag, headers) + else: + base_sha = resolve_tag_to_sha(base_url, base_tag, headers) + + if head_is_commit: + head_sha = resolve_commit_sha(base_url, head_tag, headers) + else: + head_sha = resolve_tag_to_sha(base_url, head_tag, headers) + + print(f"\nBase SHA ({base_tag}): {base_sha}") + print(f"Head SHA ({head_tag}): {head_sha}") + + # Use Compare API to check relationship and get commits + print(f"\nComparing {base_tag}...{head_tag}...") + compare_resp = requests.get( + f"{base_url}/compare/{base_sha}...{head_sha}", + headers=headers, + params={"per_page": per_page}, + ) + + if compare_resp.status_code != 200: + raise Exception(f"Failed to compare: {compare_resp.text}") + + compare_data = compare_resp.json() + status = compare_data.get("status", "unknown") + total_commits = compare_data.get("total_commits", 0) + + print(f" Comparison status: {status}") + print(f" Total commits: {total_commits}") + + # Get merge_base for potential fallback + merge_base = compare_data.get("merge_base_commit", {}).get("sha") + + # If branches have diverged (e.g., release branch with cherry-picks), + # we need to filter by PR numbers to avoid duplicates + is_diverged = status == "diverged" + + if is_diverged: + print("\n Branches have diverged (likely a release branch scenario)") + print(" Will filter by PR numbers to handle cherry-picks...") + if merge_base: + print(f" Merge base: {merge_base[:8]}") + + # Use Compare API results + all_commits = compare_data.get("commits", []) + print(f" Initial fetch: {len(all_commits)} commits") + + if len(all_commits) >= total_commits: + print(" All commits fetched in initial response") + return all_commits + + # Need to paginate - try Compare API pagination first + page = 1 + while len(all_commits) < total_commits: + page += 1 + print(f" Fetching page {page}...") + + compare_resp = requests.get( + f"{base_url}/compare/{base_sha}...{head_sha}", + headers=headers, + params={"per_page": per_page, "page": page}, + ) + + if compare_resp.status_code != 200: + # Compare API doesn't support pagination well for large diffs + print(" Compare API pagination not supported, using commit walk...") + break + + page_data = compare_resp.json() + page_commits = page_data.get("commits", []) + + if not page_commits: + break + + all_commits.extend(page_commits) + print(f" Page {page}: got {len(page_commits)} commits (total: {len(all_commits)})") + + # If we still don't have all commits, walk the history + if len(all_commits) < total_commits: + print(f"\n Need to fetch remaining {total_commits - len(all_commits)} commits via history walk...") + + # For diverged branches, use merge_base as stop point + # For non-diverged, use base_sha + stop_sha = merge_base if (status == "diverged" and merge_base) else base_sha + + # Get commits we already have + seen_shas = {c["sha"] for c in all_commits} + + # Walk from head, collecting commits not already seen, until we reach stop point + walk_commits = [] + walk_page = 1 + found_stop = False + + while len(all_commits) + len(walk_commits) < total_commits and not found_stop: + response = requests.get( + f"{base_url}/commits", + headers=headers, + params={"sha": head_sha, "per_page": per_page, "page": walk_page}, + ) + + if response.status_code != 200: + print(f" Warning: API error on page {walk_page}") + break + + commits = response.json() + if not commits: + break + + for commit in commits: + sha = commit["sha"] + if sha == stop_sha: + found_stop = True + break + if sha not in seen_shas: + seen_shas.add(sha) + walk_commits.append(commit) + + print(f" Walk page {walk_page}: found {len(walk_commits)} additional commits") + walk_page += 1 + + # Combine: Compare API commits first (they're in order), then walk commits + # Actually, we should return all unique commits + all_commits.extend(walk_commits) + print(f" Total after walk: {len(all_commits)} commits") + + # For diverged branches, filter out commits whose PRs are already in base release + # This handles cherry-picks that exist in both releases + if is_diverged: + print(f"\n Filtering out PRs already in {base_tag}...") + + # Get base release commits to extract PR numbers + print(f" Fetching {base_tag} commits...") + base_commits = [] + base_page = 1 + while True: + response = requests.get( + f"{base_url}/commits", + headers=headers, + params={"sha": base_sha, "per_page": per_page, "page": base_page}, + ) + if response.status_code != 200: + break + commits = response.json() + if not commits: + break + + for commit in commits: + if merge_base and commit["sha"] == merge_base: + break + base_commits.append(commit) + else: + base_page += 1 + continue + break + + print(f" Found {len(base_commits)} commits in {base_tag}") + + # Extract PR numbers from base commits + base_pr_numbers = set() + for commit in base_commits: + message = commit.get("commit", {}).get("message", "") + pr_num = extract_pr_number(message) + if pr_num: + base_pr_numbers.add(pr_num) + print(f" Found {len(base_pr_numbers)} unique PRs in {base_tag}") + + # Filter out commits whose PR is already in base + filtered_commits = [] + skipped_count = 0 + for commit in all_commits: + message = commit.get("commit", {}).get("message", "") + pr_num = extract_pr_number(message) + if pr_num and pr_num in base_pr_numbers: + skipped_count += 1 + continue + filtered_commits.append(commit) + + print(f" Skipped {skipped_count} commits (PRs already in {base_tag})") + print(f" Final count: {len(filtered_commits)} new commits in {head_tag}") + return filtered_commits + + return all_commits + + +def extract_contributors(commits: list[dict]) -> dict: + """ + Extract unique contributors from commits. + + Returns a dict with: + - contributors: set of (login, name) tuples + - by_login: dict mapping login -> contributor info + - by_email: dict mapping email -> contributor info (for commits without GitHub user) + """ + contributors_by_login = {} + contributors_by_email = {} + + for commit in commits: + # Try to get GitHub user info first (author field) + author = commit.get("author") + if author and author.get("login"): + login = author["login"] + if login not in contributors_by_login: + contributors_by_login[login] = { + "login": login, + "name": commit.get("commit", {}).get("author", {}).get("name", ""), + "email": commit.get("commit", {}).get("author", {}).get("email", ""), + "avatar_url": author.get("avatar_url", ""), + "html_url": author.get("html_url", ""), + "commits": 0, + } + contributors_by_login[login]["commits"] += 1 + else: + # Fallback to git author info + git_author = commit.get("commit", {}).get("author", {}) + email = git_author.get("email", "") + name = git_author.get("name", "") + + if email and email not in contributors_by_email: + contributors_by_email[email] = { + "login": None, + "name": name, + "email": email, + "avatar_url": "", + "html_url": "", + "commits": 0, + } + if email: + contributors_by_email[email]["commits"] += 1 + + return { + "by_login": contributors_by_login, + "by_email": contributors_by_email, + "total": len(contributors_by_login) + len(contributors_by_email), + } + + +def get_tag_date(base_url: str, tag: str, headers: dict) -> str: + """Get the date of a tag's commit.""" + # First resolve the tag to a commit SHA + tag_resp = requests.get(f"{base_url}/git/refs/tags/{tag}", headers=headers) + if tag_resp.status_code != 200: + return None + + tag_data = tag_resp.json() + sha = tag_data["object"]["sha"] + + # If it's an annotated tag, get the underlying commit + if tag_data["object"]["type"] == "tag": + tag_obj_resp = requests.get(f"{base_url}/git/tags/{sha}", headers=headers) + if tag_obj_resp.status_code == 200: + sha = tag_obj_resp.json()["object"]["sha"] + + # Get the commit date + commit_resp = requests.get(f"{base_url}/commits/{sha}", headers=headers) + if commit_resp.status_code == 200: + return commit_resp.json()["commit"]["committer"]["date"] + + return None + + +def check_contributor_is_new(owner: str, repo: str, login: str, before_date: str, headers: dict) -> bool: + """ + Check if a contributor has any commits before a given date. + + Returns True if this is their first contribution (no commits before the date). + """ + base_url = f"https://api.github.com/repos/{owner}/{repo}" + + # Search for commits by this author before the base tag date + response = requests.get( + f"{base_url}/commits", + headers=headers, + params={"author": login, "until": before_date, "per_page": 1}, + ) + + if response.status_code == 200: + commits = response.json() + # If no commits found before the date, they're a new contributor + return len(commits) == 0 + + return False + + +def find_first_contribution(commits: list[dict], login: str) -> dict | None: + """ + Find the first (earliest) contribution by a user in the commit list. + + Returns the commit dict or None. + """ + user_commits = [] + for commit in commits: + author = commit.get("author") + if author and author.get("login") == login: + user_commits.append(commit) + + # Commits are usually newest first, so reverse to get oldest first + if user_commits: + return user_commits[-1] # Last one is the oldest/first contribution + return None + + +def calculate_new_contributors_via_generate_notes( + owner: str, + repo: str, + base_tag: str, + head_tag: str, + token: str | None = None, +) -> list[dict]: + """ + Calculate new contributors using GitHub's generate-notes API. + + This is more accurate than checking commit history because GitHub + tracks contributor status internally. + + Args: + owner: Repository owner + repo: Repository name + base_tag: The base tag (older version) + head_tag: The head tag (newer version) + token: GitHub token + + Returns: + List of new contributor info dicts with login and first_pr fields + """ + import re + import subprocess + + print("\nGetting new contributors via GitHub generate-notes API...") + + # Use gh CLI to call the generate-notes API + cmd = [ + "gh", + "api", + f"repos/{owner}/{repo}/releases/generate-notes", + "-f", + f"tag_name={head_tag}", + "-f", + f"target_commitish={head_tag}", + "-f", + f"previous_tag_name={base_tag}", + "--jq", + ".body", + ] + + try: + result = subprocess.run(cmd, capture_output=True, text=True, timeout=60) + if result.returncode != 0: + print(f" Warning: gh CLI failed: {result.stderr}") + return [] + + body = result.stdout + + # Parse new contributors from the generated notes + # Format: "* @username made their first contribution in https://github.com/owner/repo/pull/12345" + pattern = r"\* @(\S+) made their first contribution in https://github\.com/[^/]+/[^/]+/pull/(\d+)" + matches = re.findall(pattern, body) + + new_contributors = [] + for login, pr_number in matches: + new_contributors.append( + { + "login": login, + "first_pr": pr_number, + } + ) + + print(f" Found {len(new_contributors)} new contributors") + return new_contributors + + except subprocess.TimeoutExpired: + print(" Warning: gh CLI timed out") + return [] + except FileNotFoundError: + print(" Warning: gh CLI not found, falling back to legacy method") + return [] + + +def calculate_new_contributors( + commits: list[dict], + current_contributors: dict, + owner: str, + repo: str, + base_tag: str, + head_tag: str = "", + token: str | None = None, +) -> list[dict]: + """ + Calculate which contributors are new (first-time) in this release. + + First tries GitHub's generate-notes API (more accurate), then falls back + to checking commit history if that fails. + + Args: + commits: List of commits in the current release + current_contributors: Output from extract_contributors() + owner: Repository owner + repo: Repository name + base_tag: The base tag (older version) + head_tag: The head tag (newer version) + token: GitHub token + + Returns: + List of new contributor info dicts with first_pr field + """ + # Try the accurate method first (via generate-notes API) + if head_tag: + new_contributors = calculate_new_contributors_via_generate_notes( + owner=owner, + repo=repo, + base_tag=base_tag, + head_tag=head_tag, + token=token, + ) + if new_contributors: + return new_contributors + + # Fall back to legacy method (checking commit history) + print("\nFalling back to legacy new contributor detection...") + + headers = { + "Accept": "application/vnd.github.v3+json", + } + if token: + headers["Authorization"] = f"token {token}" + + base_url = f"https://api.github.com/repos/{owner}/{repo}" + + # Get the date of the base tag + print("Getting base tag date...") + base_date = get_tag_date(base_url, base_tag, headers) + if not base_date: + print(f" Warning: Could not get date for tag {base_tag}") + return [] + + print(f" Base tag date: {base_date}") + + new_contributors = [] + logins = list(current_contributors["by_login"].keys()) + total = len(logins) + + print(f"\nChecking {total} contributors for first-time status...") + + for i, login in enumerate(logins): + if (i + 1) % 20 == 0: + print(f" Checked {i + 1}/{total} contributors...") + + is_new = check_contributor_is_new(owner, repo, login, base_date, headers) + + if is_new: + info = current_contributors["by_login"][login].copy() + + # Find their first PR in this release + first_commit = find_first_contribution(commits, login) + if first_commit: + message = first_commit.get("commit", {}).get("message", "") + pr_number = extract_pr_number(message) + info["first_pr"] = pr_number + info["first_commit_sha"] = first_commit.get("sha", "")[:8] + + new_contributors.append(info) + + print(f" Found {len(new_contributors)} new contributors (legacy method)") + + return new_contributors + + +def generate_contributor_stats( + commits: list[dict], + owner: str, + repo: str, + base_tag: str, + head_tag: str, + token: str | None = None, + check_new: bool = True, +) -> dict: + """ + Generate contributor statistics for the release. + + Returns a dict with all statistics data. + """ + print("\n" + "=" * 60) + print("CONTRIBUTOR STATISTICS") + print("=" * 60) + + # Extract contributors from current commits + contributors = extract_contributors(commits) + + print(f"\nTotal commits: {len(commits)}") + print(f"Total contributors: {contributors['total']}") + print(f" - With GitHub account: {len(contributors['by_login'])}") + print(f" - Without GitHub account (by email): {len(contributors['by_email'])}") + + new_count = 0 + new_contributors_list = [] + + if check_new: + # Calculate new contributors (tries GitHub generate-notes API first, then falls back to commit history) + new_contributors_list = calculate_new_contributors( + commits=commits, + current_contributors=contributors, + owner=owner, + repo=repo, + base_tag=base_tag, + head_tag=head_tag, + token=token, + ) + new_count = len(new_contributors_list) + + print(f"\nNew contributors (first-time): {new_count}") + + if new_contributors_list: + print("\nNew contributors list:") + for c in sorted(new_contributors_list, key=lambda x: x["login"].lower()): + pr_info = f" in #{c['first_pr']}" if c.get("first_pr") else "" + print(f" - @{c['login']} made their first contribution{pr_info}") + + # Print summary line for release notes + print("\n" + "-" * 60) + print("RELEASE NOTES SUMMARY LINE:") + print("-" * 60) + if check_new: + summary_line = ( + f"This release features {len(commits)} commits from {contributors['total']} contributors ({new_count} new)!" + ) + else: + summary_line = f"This release features {len(commits)} commits from {contributors['total']} contributors!" + print(summary_line) + print("-" * 60) + + # Get all contributors sorted by commit count + all_contributors_list = list(contributors["by_login"].values()) + list(contributors["by_email"].values()) + sorted_contributors = sorted(all_contributors_list, key=lambda x: x["commits"], reverse=True) + + # Print top contributors + print("\nTop contributors by commit count:") + for i, c in enumerate(sorted_contributors[:20], 1): + if c.get("login"): + print(f" {i:2}. @{c['login']:20} - {c['commits']:3} commits") + else: + print(f" {i:2}. {c['name']:20} - {c['commits']:3} commits (no GitHub account)") + + return { + "total_commits": len(commits), + "total_contributors": contributors["total"], + "new_contributors": new_count if check_new else None, + "new_contributors_list": new_contributors_list, + "contributors": contributors, + "sorted_contributors": sorted_contributors, + "summary_line": summary_line, + "base_tag": base_tag, + "head_tag": head_tag, + "owner": owner, + "repo": repo, + } + + +def save_contributor_stats(stats: dict, output_file: str, owner: str, repo: str): + """ + Save contributor statistics to a markdown file. + + Args: + stats: Statistics dict from generate_contributor_stats() + output_file: Output file path + owner: Repository owner + repo: Repository name + """ + lines = [] + + # Header + lines.append(f"# Contributor Statistics: {stats['base_tag']} → {stats['head_tag']}") + lines.append("") + + # Summary for release notes + lines.append("## Release Notes Summary") + lines.append("") + lines.append(f"> {stats['summary_line']}") + lines.append("") + + # Overview stats + lines.append("## Overview") + lines.append("") + lines.append(f"- **Total Commits**: {stats['total_commits']}") + lines.append(f"- **Total Contributors**: {stats['total_contributors']}") + if stats["new_contributors"] is not None: + lines.append(f"- **New Contributors**: {stats['new_contributors']}") + lines.append("") + + # Top contributors table + lines.append("## Top Contributors") + lines.append("") + lines.append("| Rank | Contributor | Commits |") + lines.append("|------|-------------|---------|") + + for i, c in enumerate(stats["sorted_contributors"][:30], 1): + if c.get("login"): + contributor_link = f"[@{c['login']}](https://github.com/{c['login']})" + else: + contributor_link = c["name"] + lines.append(f"| {i} | {contributor_link} | {c['commits']} |") + + lines.append("") + + # New contributors section + if stats["new_contributors_list"]: + lines.append("## New Contributors 🎉") + lines.append("") + + sorted_new = sorted(stats["new_contributors_list"], key=lambda x: x["login"].lower()) + for c in sorted_new: + pr_num = c.get("first_pr") + if pr_num: + pr_link = f"https://github.com/{owner}/{repo}/pull/{pr_num}" + lines.append(f"* @{c['login']} made their first contribution in {pr_link}") + else: + lines.append(f"* @{c['login']} made their first contribution") + lines.append("") + + # All contributors section (collapsed) + lines.append("## All Contributors") + lines.append("") + lines.append("
") + lines.append("Click to expand full list") + lines.append("") + lines.append("| Contributor | Commits |") + lines.append("|-------------|---------|") + + for c in stats["sorted_contributors"]: + if c.get("login"): + contributor_link = f"[@{c['login']}](https://github.com/{c['login']})" + else: + contributor_link = c["name"] + lines.append(f"| {contributor_link} | {c['commits']} |") + + lines.append("") + lines.append("
") + lines.append("") + + # Write to file + with open(output_file, "w", encoding="utf-8") as f: + f.write("\n".join(lines)) + + print(f"\nSaved contributor statistics to {output_file}") + + +def extract_pr_number(message: str) -> str | None: + """Extract PR number from commit message.""" + # Common patterns: (#12345), (https://github.com/.../pull/12345) + patterns = [ + r"\(#(\d+)\)", # (#12345) + r"pull/(\d+)", # https://github.com/.../pull/12345 + r"#(\d+)$", # #12345 at end + ] + + for pattern in patterns: + match = re.search(pattern, message) + if match: + return match.group(1) + return None + + +def format_commit_message( + commit: dict, + owner: str, + repo: str, + include_sha: bool = False, + include_date: bool = False, +) -> str: + """ + Format a commit message for the output file. + + Format: [Category] Description in https://github.com/owner/repo/pull/XXXX + or: [Category] Description (#XXXX) + + If include_sha is True, prepends the full SHA: `sha` Message (#XXXX) + If include_date is True, prepends the date: [YYYY-MM-DD] Message (#XXXX) + """ + message = commit["commit"]["message"] + sha = commit.get("sha", "") + + # Get commit date (use committer date for when it was merged) + commit_date = "" + if include_date: + date_str = commit.get("commit", {}).get("committer", {}).get("date", "") + if date_str: + # Parse ISO format and extract date part (YYYY-MM-DD) + commit_date = date_str[:10] + + # Get the first line of the commit message + first_line = message.split("\n")[0].strip() + + # Extract PR number if present + pr_number = extract_pr_number(first_line) + + # Clean up the message - remove existing PR references for reformatting + clean_message = first_line + clean_message = re.sub(r"\s*\(#\d+\)\s*$", "", clean_message) + clean_message = re.sub(r"\s*https://github\.com/[^/]+/[^/]+/pull/\d+\s*", "", clean_message) + clean_message = re.sub(r"\s+in\s*$", "", clean_message) + clean_message = clean_message.strip() + + # Format output + if pr_number: + # Check if message already contains the full URL pattern + if f"https://github.com/{owner}/{repo}/pull/" in first_line: + formatted = first_line + else: + formatted = f"{clean_message} (#{pr_number})" + else: + formatted = clean_message + + # Prepend metadata if requested + prefix_parts = [] + if include_date and commit_date: + prefix_parts.append(f"[{commit_date}]") + if include_sha and sha: + prefix_parts.append(f"`{sha}`") + + if prefix_parts: + formatted = f"{' '.join(prefix_parts)} {formatted}" + + return formatted + + +def save_commits_to_file( + commits: list[dict], + output_file: str, + owner: str, + repo: str, + sort_mode: str = "chronological", + include_sha: bool = False, + include_date: bool = False, +): + """ + Save formatted commits to a markdown file. + + Args: + commits: List of commit dictionaries + output_file: Output file path + owner: Repository owner + repo: Repository name + sort_mode: "chronological" (newest first, like GitHub), + "alphabetical" (by commit message), + "reverse" (oldest first) + include_sha: If True, include full commit SHA in output + include_date: If True, include commit date in output + """ + print(f"\nFormatting and saving {len(commits)} commits to {output_file}...") + + formatted_lines = [] + for commit in commits: + formatted = format_commit_message(commit, owner, repo, include_sha=include_sha, include_date=include_date) + formatted_lines.append(formatted) + + # Sort based on mode + if sort_mode == "alphabetical": + formatted_lines.sort(key=lambda x: x.lower()) + print(" Sorted alphabetically by commit message") + elif sort_mode == "reverse": + formatted_lines.reverse() + print(" Sorted chronologically (oldest first)") + else: + # chronological - keep original order (newest first, as returned by API) + print(" Keeping chronological order (newest first)") + + with open(output_file, "w", encoding="utf-8") as f: + for line in formatted_lines: + f.write(line + "\n") + + print(f"Saved {len(formatted_lines)} commits to {output_file}") + + +def main(): + parser = argparse.ArgumentParser(description="Fetch commits between two GitHub tags or between a tag and a commit") + parser.add_argument( + "--owner", + default="vllm-project", + help="Repository owner (default: vllm-project)", + ) + parser.add_argument("--repo", default="vllm", help="Repository name (default: vllm)") + parser.add_argument( + "--base-tag", + help="Base tag (older, e.g., v0.11.2). If not provided with --head-commit, will auto-detect previous tag.", + ) + parser.add_argument( + "--head-tag", + help="Head tag (newer, e.g., v0.12.0). Use this OR --head-commit. If neither specified, uses " + "HEAD of default branch.", + ) + parser.add_argument( + "--head-commit", + help="Head commit SHA (can be short or full). If not specified and no --head-tag, uses HEAD of default branch.", + ) + parser.add_argument( + "--tag-pattern", + default=r"^v\d+\.\d+\.\d+$", + help="Regex pattern to filter tags when auto-detecting previous tag (default: ^v\\d+\\.\\d+\\.\\d+$)", + ) + parser.add_argument( + "--output", + default="0-current-raw-commits.md", + help="Output file (default: 0-current-raw-commits.md)", + ) + parser.add_argument("--token", help="GitHub token (or set GITHUB_TOKEN env var)") + parser.add_argument( + "--slow", + action="store_true", + help="Use slower but more thorough commit-by-commit fetching", + ) + parser.add_argument( + "--sort", + choices=["chronological", "alphabetical", "reverse"], + default="chronological", + help="Sort mode: chronological (newest first, like GitHub), alphabetical (by message), reverse (oldest first)", + ) + parser.add_argument("--stats", action="store_true", help="Generate and save contributor statistics") + parser.add_argument( + "--stats-output", + default="0-contributor-stats.md", + help="Output file for contributor statistics (default: 0-contributor-stats.md)", + ) + parser.add_argument( + "--no-new-check", + action="store_true", + help="Skip checking for new contributors (faster, avoids extra API calls)", + ) + parser.add_argument( + "--include-sha", + action="store_true", + help="Include full commit SHA in output (format: `sha` message)", + ) + parser.add_argument( + "--include-date", + action="store_true", + help="Include commit date in output (format: [YYYY-MM-DD] message)", + ) + parser.add_argument( + "--since", + help="Fetch commits since this date (ISO 8601: YYYY-MM-DD or YYYY-MM-DDTHH:MM:SSZ). " + "Use with --until for date range mode.", + ) + parser.add_argument( + "--until", + help="Fetch commits until this date (ISO 8601: YYYY-MM-DD or YYYY-MM-DDTHH:MM:SSZ). " + "Use with --since for date range mode.", + ) + parser.add_argument( + "--branch", + help="Branch to fetch commits from (only used with --since/--until date range mode)", + ) + + args = parser.parse_args() + + # Validate arguments + if args.head_tag and args.head_commit: + parser.error("Cannot specify both --head-tag and --head-commit") + + # Check for date range mode + date_range_mode = args.since is not None or args.until is not None + if date_range_mode: + if not args.since or not args.until: + parser.error("Both --since and --until must be specified for date range mode") + if args.head_tag or args.head_commit or args.base_tag: + parser.error("Cannot use --since/--until with --head-tag, --head-commit, or --base-tag") + + token = args.token or get_github_token() + + if not token: + print("Warning: No GitHub token provided. Rate limits will be stricter.") + print("Set GITHUB_TOKEN environment variable or use --token argument.") + print() + + headers = { + "Accept": "application/vnd.github.v3+json", + } + if token: + headers["Authorization"] = f"token {token}" + + base_url = f"https://api.github.com/repos/{args.owner}/{args.repo}" + + try: + # Date range mode + if date_range_mode: + print(f"\n{'=' * 60}") + print(f"Fetching commits by date range: {args.since} → {args.until}") + if args.branch: + print(f"Branch: {args.branch}") + print(f"{'=' * 60}") + + commits = fetch_commits_by_date_range( + owner=args.owner, + repo=args.repo, + since=args.since, + until=args.until, + token=token, + branch=args.branch, + ) + + print(f"\nTotal commits found: {len(commits)}") + + save_commits_to_file( + commits=commits, + output_file=args.output, + owner=args.owner, + repo=args.repo, + sort_mode=args.sort, + include_sha=args.include_sha, + include_date=args.include_date, + ) + + # Stats not fully supported in date range mode (no base_tag for new contributor check) + if args.stats: + print("\nNote: Contributor statistics in date range mode won't check for new contributors.") + stats = generate_contributor_stats( + commits=commits, + owner=args.owner, + repo=args.repo, + base_tag=args.since, + head_tag=args.until, + token=token, + check_new=False, # Can't check new contributors without a base tag + ) + save_contributor_stats( + stats=stats, + output_file=args.stats_output, + owner=args.owner, + repo=args.repo, + ) + + return + + # Tag/commit mode (existing logic) + # Determine head reference + head_is_commit = False + head_ref = None + head_display_name = None + + if args.head_tag: + head_ref = args.head_tag + head_is_commit = False + head_display_name = args.head_tag + elif args.head_commit: + head_ref = args.head_commit + head_is_commit = True + head_display_name = args.head_commit[:8] if len(args.head_commit) > 8 else args.head_commit + else: + # Auto-detect HEAD of default branch + branch_name, head_sha = get_default_branch_head(base_url, headers) + head_ref = head_sha + head_is_commit = True + head_display_name = f"{branch_name} ({head_sha[:8]})" + + base_tag = args.base_tag + base_is_commit = False + + # Auto-detect previous tag if needed + if not base_tag and head_is_commit: + print("Auto-detecting previous tag...") + head_sha = resolve_commit_sha(base_url, head_ref, headers) + + result = find_previous_tag( + base_url=base_url, + head_sha=head_sha, + headers=headers, + tag_pattern=args.tag_pattern, + ) + + if result is None: + raise Exception("Could not find a previous tag. Please specify --base-tag manually.") + + base_tag, _ = result + print(f"\nUsing auto-detected base tag: {base_tag}") + elif not base_tag: + parser.error("Must specify --base-tag when using --head-tag") + + print(f"\n{'=' * 60}") + print(f"Fetching commits: {base_tag} → {head_display_name}") + print(f"{'=' * 60}") + + if args.slow: + # Note: slow mode doesn't support commit SHA yet, only tags + if head_is_commit: + print("Warning: --slow mode with --head-commit not fully supported, using fast mode") + commits = fetch_commits_between_tags_fast( + owner=args.owner, + repo=args.repo, + base_tag=base_tag, + head_tag=head_ref, + token=token, + head_is_commit=head_is_commit, + base_is_commit=base_is_commit, + ) + else: + commits = fetch_commits_between_tags_fast( + owner=args.owner, + repo=args.repo, + base_tag=base_tag, + head_tag=head_ref, + token=token, + head_is_commit=head_is_commit, + base_is_commit=base_is_commit, + ) + + print(f"\nTotal commits found: {len(commits)}") + + save_commits_to_file( + commits=commits, + output_file=args.output, + owner=args.owner, + repo=args.repo, + sort_mode=args.sort, + include_sha=args.include_sha, + include_date=args.include_date, + ) + + # Generate and save contributor statistics if requested + if args.stats: + stats = generate_contributor_stats( + commits=commits, + owner=args.owner, + repo=args.repo, + base_tag=base_tag, + head_tag=head_display_name, + token=token, + check_new=not args.no_new_check, + ) + save_contributor_stats( + stats=stats, + output_file=args.stats_output, + owner=args.owner, + repo=args.repo, + ) + + except Exception as e: + print(f"Error: {e}") + raise + + +if __name__ == "__main__": + main()