From 803c02db0aea8310c3e2e43cfbecf0a0680ebcf4 Mon Sep 17 00:00:00 2001 From: Fangjun Kuang Date: Wed, 17 Jul 2024 14:41:27 +0800 Subject: [PATCH] publish all pre-built wheels to huggingface (#1142) pypi.org provides only 10GB of free space for open-source projects. Each new release of sherpa-onnx occupies about 800MB, so we have to delete previous releases otherwise pypi.org refuses to accept new releases due to limited spaces. To let users install previous versions, we also publish wheels to huggingface and users can find them at https://k2-fsa.github.io/sherpa/onnx/cpu.html and https://k2-fsa.github.io/sherpa/onnx/cpu-cn.html (for users without access to huggingface.co) --- .github/workflows/build-wheels-aarch64.yaml | 10 ++++++++-- .github/workflows/build-wheels-armv7l.yaml | 10 ++++++++-- .github/workflows/build-wheels-linux.yaml | 10 ++++++++-- .github/workflows/build-wheels-macos-arm64.yaml | 10 ++++++++-- .github/workflows/build-wheels-macos-universal2.yaml | 10 ++++++++-- .github/workflows/build-wheels-macos-x64.yaml | 10 ++++++++-- .github/workflows/build-wheels-win32.yaml | 10 ++++++++-- .github/workflows/build-wheels-win64.yaml | 9 ++++++++- 8 files changed, 64 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build-wheels-aarch64.yaml b/.github/workflows/build-wheels-aarch64.yaml index 7221d0c0..0822dbd8 100644 --- a/.github/workflows/build-wheels-aarch64.yaml +++ b/.github/workflows/build-wheels-aarch64.yaml @@ -81,7 +81,6 @@ jobs: mv ./wheels ./wheelhouse - name: Publish to huggingface - if: (matrix.python-version == 'cp38' || matrix.python-version == 'cp39' ) && matrix.manylinux == 'manylinux2014' env: HF_TOKEN: ${{ secrets.HF_TOKEN }} uses: nick-fields/retry@v3 @@ -97,13 +96,20 @@ jobs: export GIT_LFS_SKIP_SMUDGE=1 export GIT_CLONE_PROTECTION_ACTIVE=false + SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) + echo "SHERPA_ONNX_VERSION $SHERPA_ONNX_VERSION" + + d=cpu/$SHERPA_ONNX_VERSION + git clone https://huggingface.co/csukuangfj/sherpa-onnx-wheels huggingface cd huggingface git fetch git pull git merge -m "merge remote" --ff origin main - cp -v ../wheelhouse/*.whl . + mkdir -p $d + + cp -v ../wheelhouse/*.whl $d/ git status git add . diff --git a/.github/workflows/build-wheels-armv7l.yaml b/.github/workflows/build-wheels-armv7l.yaml index a2a2a49a..0e5f1088 100644 --- a/.github/workflows/build-wheels-armv7l.yaml +++ b/.github/workflows/build-wheels-armv7l.yaml @@ -84,7 +84,6 @@ jobs: ls -lh ./wheelhouse/ - name: Publish to huggingface - if: matrix.python-version == '3.8' env: HF_TOKEN: ${{ secrets.HF_TOKEN }} uses: nick-fields/retry@v3 @@ -100,13 +99,20 @@ jobs: export GIT_LFS_SKIP_SMUDGE=1 export GIT_CLONE_PROTECTION_ACTIVE=false + SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) + echo "SHERPA_ONNX_VERSION $SHERPA_ONNX_VERSION" + + d=cpu/$SHERPA_ONNX_VERSION + git clone https://huggingface.co/csukuangfj/sherpa-onnx-wheels huggingface cd huggingface git fetch git pull git merge -m "merge remote" --ff origin main - cp -v ../wheelhouse/*.whl . + mkdir -p $d + + cp -v ../wheelhouse/*.whl $d/ git status git add . diff --git a/.github/workflows/build-wheels-linux.yaml b/.github/workflows/build-wheels-linux.yaml index 557d94f5..fe302827 100644 --- a/.github/workflows/build-wheels-linux.yaml +++ b/.github/workflows/build-wheels-linux.yaml @@ -80,7 +80,6 @@ jobs: path: ./wheelhouse/*.whl - name: Publish to huggingface - if: matrix.python-version == 'cp38' env: HF_TOKEN: ${{ secrets.HF_TOKEN }} uses: nick-fields/retry@v3 @@ -96,13 +95,20 @@ jobs: export GIT_LFS_SKIP_SMUDGE=1 export GIT_CLONE_PROTECTION_ACTIVE=false + SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) + echo "SHERPA_ONNX_VERSION $SHERPA_ONNX_VERSION" + + d=cpu/$SHERPA_ONNX_VERSION + git clone https://huggingface.co/csukuangfj/sherpa-onnx-wheels huggingface cd huggingface git fetch git pull git merge -m "merge remote" --ff origin main - cp -v ../wheelhouse/*.whl . + mkdir -p $d + + cp -v ../wheelhouse/*.whl $d/ git status git add . diff --git a/.github/workflows/build-wheels-macos-arm64.yaml b/.github/workflows/build-wheels-macos-arm64.yaml index e47a9c30..506a5c8a 100644 --- a/.github/workflows/build-wheels-macos-arm64.yaml +++ b/.github/workflows/build-wheels-macos-arm64.yaml @@ -50,7 +50,6 @@ jobs: path: ./wheelhouse/*.whl - name: Publish to huggingface - if: matrix.python-version == 'cp39' env: HF_TOKEN: ${{ secrets.HF_TOKEN }} uses: nick-fields/retry@v3 @@ -66,13 +65,20 @@ jobs: export GIT_LFS_SKIP_SMUDGE=1 export GIT_CLONE_PROTECTION_ACTIVE=false + SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) + echo "SHERPA_ONNX_VERSION $SHERPA_ONNX_VERSION" + + d=cpu/$SHERPA_ONNX_VERSION + git clone https://huggingface.co/csukuangfj/sherpa-onnx-wheels huggingface cd huggingface git fetch git pull git merge -m "merge remote" --ff origin main - cp -v ../wheelhouse/*.whl . + mkdir -p $d + + cp -v ../wheelhouse/*.whl $d/ git status git add . diff --git a/.github/workflows/build-wheels-macos-universal2.yaml b/.github/workflows/build-wheels-macos-universal2.yaml index 2ffaf93c..b149e9ae 100644 --- a/.github/workflows/build-wheels-macos-universal2.yaml +++ b/.github/workflows/build-wheels-macos-universal2.yaml @@ -50,7 +50,6 @@ jobs: path: ./wheelhouse/*.whl - name: Publish to huggingface - if: matrix.python-version == 'cp39' env: HF_TOKEN: ${{ secrets.HF_TOKEN }} uses: nick-fields/retry@v3 @@ -66,13 +65,20 @@ jobs: export GIT_LFS_SKIP_SMUDGE=1 export GIT_CLONE_PROTECTION_ACTIVE=false + SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) + echo "SHERPA_ONNX_VERSION $SHERPA_ONNX_VERSION" + + d=cpu/$SHERPA_ONNX_VERSION + git clone https://huggingface.co/csukuangfj/sherpa-onnx-wheels huggingface cd huggingface git fetch git pull git merge -m "merge remote" --ff origin main - cp -v ../wheelhouse/*.whl . + mkdir -p $d + + cp -v ../wheelhouse/*.whl $d/ git status git add . diff --git a/.github/workflows/build-wheels-macos-x64.yaml b/.github/workflows/build-wheels-macos-x64.yaml index f8f0a7e8..95146629 100644 --- a/.github/workflows/build-wheels-macos-x64.yaml +++ b/.github/workflows/build-wheels-macos-x64.yaml @@ -65,7 +65,6 @@ jobs: path: ./wheelhouse/*.whl - name: Publish to huggingface - if: matrix.python-version == 'cp39' env: HF_TOKEN: ${{ secrets.HF_TOKEN }} uses: nick-fields/retry@v3 @@ -81,13 +80,20 @@ jobs: export GIT_LFS_SKIP_SMUDGE=1 export GIT_CLONE_PROTECTION_ACTIVE=false + SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) + echo "SHERPA_ONNX_VERSION $SHERPA_ONNX_VERSION" + + d=cpu/$SHERPA_ONNX_VERSION + git clone https://huggingface.co/csukuangfj/sherpa-onnx-wheels huggingface cd huggingface git fetch git pull git merge -m "merge remote" --ff origin main - cp -v ../wheelhouse/*.whl . + mkdir -p $d + + cp -v ../wheelhouse/*.whl $d/ git status git add . diff --git a/.github/workflows/build-wheels-win32.yaml b/.github/workflows/build-wheels-win32.yaml index b2dbd157..692bb816 100644 --- a/.github/workflows/build-wheels-win32.yaml +++ b/.github/workflows/build-wheels-win32.yaml @@ -49,7 +49,6 @@ jobs: path: ./wheelhouse/*.whl - name: Publish to huggingface - if: matrix.python-version == 'cp38' env: HF_TOKEN: ${{ secrets.HF_TOKEN }} uses: nick-fields/retry@v3 @@ -65,13 +64,20 @@ jobs: export GIT_LFS_SKIP_SMUDGE=1 export GIT_CLONE_PROTECTION_ACTIVE=false + SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) + echo "SHERPA_ONNX_VERSION $SHERPA_ONNX_VERSION" + + d=cpu/$SHERPA_ONNX_VERSION + git clone https://huggingface.co/csukuangfj/sherpa-onnx-wheels huggingface cd huggingface git fetch git pull git merge -m "merge remote" --ff origin main - cp -v ../wheelhouse/*.whl . + mkdir -p $d + + cp -v ../wheelhouse/*.whl $d/ git status git add . diff --git a/.github/workflows/build-wheels-win64.yaml b/.github/workflows/build-wheels-win64.yaml index b883ba28..710322e3 100644 --- a/.github/workflows/build-wheels-win64.yaml +++ b/.github/workflows/build-wheels-win64.yaml @@ -70,13 +70,20 @@ jobs: export GIT_LFS_SKIP_SMUDGE=1 export GIT_CLONE_PROTECTION_ACTIVE=false + SHERPA_ONNX_VERSION=$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2) + echo "SHERPA_ONNX_VERSION $SHERPA_ONNX_VERSION" + + d=cpu/$SHERPA_ONNX_VERSION + git clone https://huggingface.co/csukuangfj/sherpa-onnx-wheels huggingface cd huggingface git fetch git pull git merge -m "merge remote" --ff origin main - cp -v ../wheelhouse/*.whl . + mkdir -p $d + + cp -v ../wheelhouse/*.whl $d/ git status git add .