From b2b25773c1cca6dd7df1a3aa3b05fb42c62c4acf Mon Sep 17 00:00:00 2001 From: Huy Do Date: Fri, 12 Jan 2024 16:28:42 -0800 Subject: [PATCH] Switch to Nova build jobs and cleanup AWS credentials (#1217) * Enable Python 3.12 build for torchdata * Minor push to run all builds * Build conda 3.12 with conda-build in 3.9 * Install setuptools for 3.12 * Use the correct conda env * Figure it out * Clean up obsolete workflows * Clean up AWS credentials --- .github/workflows/_build_test_upload.yml | 555 ----------------------- .github/workflows/build-conda-m1.yml | 2 +- .github/workflows/build-wheels-m1.yml | 9 +- .github/workflows/build_conda_linux.yml | 4 +- .github/workflows/build_wheels_linux.yml | 10 +- .github/workflows/nightly_release.yml | 20 - .github/workflows/pull_release.yml | 22 - .github/workflows/release.yml | 24 - .github/workflows/test_release.yml | 36 -- 9 files changed, 12 insertions(+), 670 deletions(-) delete mode 100644 .github/workflows/_build_test_upload.yml delete mode 100644 .github/workflows/nightly_release.yml delete mode 100644 .github/workflows/pull_release.yml delete mode 100644 .github/workflows/release.yml delete mode 100644 .github/workflows/test_release.yml diff --git a/.github/workflows/_build_test_upload.yml b/.github/workflows/_build_test_upload.yml deleted file mode 100644 index 5b7811aa1..000000000 --- a/.github/workflows/_build_test_upload.yml +++ /dev/null @@ -1,555 +0,0 @@ -name: Build, Test and Upload Wheel - -on: - workflow_call: - inputs: - branch: - required: true - type: string - pre_dev_release: - required: true - type: boolean - pytorch_version: - required: true - type: string - do-upload: - required: false - default: true - type: boolean - secrets: - PYTORCH_BINARY_AWS_ACCESS_KEY_ID: - required: true - PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY: - required: true - PYPI_TOKEN: - required: false - CONDA_PYTORCHBOT_TOKEN: - required: false - CONDA_TEST_PYTORCHBOT_TOKEN: - required: false - CONDA_NIGHTLY_PYTORCHBOT_TOKEN: - required: false - -jobs: - get_release_type: - runs-on: ubuntu-latest - outputs: - type: ${{ steps.get_release_type.outputs.type }} - steps: - - name: Get Release Type - run: | - if [[ "${{ inputs.branch }}" == v* ]] && [[ ${{ inputs.pre_dev_release }} == false ]]; then - RELEASE_TYPE=official - elif [[ "${{ inputs.branch }}" == release/* ]] && [[ ${{ inputs.pre_dev_release }} == true ]]; then - RELEASE_TYPE=test - else - if [[ "${{ github.base_ref }}" == release/* ]]; then - RELEASE_TYPE=test - else - RELEASE_TYPE=nightly - fi - fi - echo "Release Type: $RELEASE_TYPE" - echo "type=$RELEASE_TYPE" >> $GITHUB_OUTPUT - id: get_release_type - - wheel_build_test: - needs: get_release_type - runs-on: ${{ matrix.os }} - container: ${{ startsWith( matrix.os, 'ubuntu' ) && 'pytorch/manylinux-cpu' || null }} - strategy: - fail-fast: false - matrix: - os: - - macos-latest - - ubuntu-latest - - windows-latest - - macos-m1-12 - python-version: - - 3.8 - - 3.9 - - "3.10" - - "3.11" - - pure - exclude: - - os: macos-latest - python-version: pure - - os: windows-latest - python-version: pure - - os: macos-m1-12 - python-version: pure - steps: - - name: Checkout Source Repository - uses: actions/checkout@v3 - with: - ref: ${{ inputs.branch }} - submodules: recursive - - name: Setup Python ${{ matrix.python-version }} for Windows - if: ${{ startsWith( matrix.os, 'windows' ) }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Add temp runner environment variables - if: ${{ startsWith( matrix.os, 'macos' ) }} - shell: bash -l {0} - run: | - echo "MINICONDA_INSTALL_PATH_MACOS=${RUNNER_TEMP}/miniconda" >> "${GITHUB_ENV}" - - name: Install Conda on MacOS - if: ${{ startsWith( matrix.os, 'macos' ) }} - shell: bash -l {0} - run: | - mkdir -p "${MINICONDA_INSTALL_PATH_MACOS}" - if ${{ startsWith( matrix.os, 'macos-m1' ) }}; then - curl https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-arm64.sh -o "${MINICONDA_INSTALL_PATH_MACOS}/miniconda.sh" - else - curl https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -o "${MINICONDA_INSTALL_PATH_MACOS}/miniconda.sh" - fi - bash "${MINICONDA_INSTALL_PATH_MACOS}/miniconda.sh" -b -u -p "${MINICONDA_INSTALL_PATH_MACOS}" - rm -rf "${MINICONDA_INSTALL_PATH_MACOS}/miniconda.sh" - echo "${MINICONDA_INSTALL_PATH_MACOS}/bin" >> "$GITHUB_PATH" - - name: Setup Python ${{ matrix.python-version }} on MacOS - if: ${{ startsWith( matrix.os, 'macos' ) }} - shell: bash -l {0} - run: | - conda init bash - conda create -y --name wheel_build_env_${{ matrix.python-version }} python=${{ matrix.python-version }} - - name: Setup msbuild on Windows - if: startsWith( matrix.os, 'windows' ) - uses: microsoft/setup-msbuild@v1.1 - - name: Set up Visual Studio shell - if: startsWith( matrix.os, 'windows' ) - uses: egor-tensin/vs-shell@v2 - with: - arch: x64 - - name: Install Build Dependency - shell: bash -l {0} - env: - PYTHON_VERSION: ${{ matrix.python-version }} - run: | - if ${{ startsWith( matrix.os, 'ubuntu' ) }}; then - if ${{ matrix.python-version != 'pure' }}; then - source packaging/manylinux/python_helper.sh - yum -y install ninja-build zlib-static - # Docker path is /__w by default - export WORKSPACE="/__w" - # Install static OpenSSL/libcrypto library - ./packaging/manylinux/install_openssl_curl.sh - fi - else - if ${{ startsWith( matrix.os, 'macos' ) }}; then - conda activate wheel_build_env_${{ matrix.python-version }} - fi - pip install cmake ninja - echo "/home/runner/.local/bin" >> $GITHUB_PATH - fi - - name: Install PyTorch and Build TorchData Wheel - shell: bash -l {0} - env: - PYTHON_VERSION: ${{ matrix.python-version }} - PYTORCH_VERSION: ${{ inputs.pytorch_version }} - BUILD_S3: ${{ matrix.python-version != 'pure' }} - run: | - set -ex - if ${{ startsWith( matrix.os, 'ubuntu' ) }}; then - # Use Python 3.8 for pure python build - if ${{ matrix.python-version == 'pure' }}; then - export PYTHON_VERSION=3.8 - fi - source packaging/manylinux/python_helper.sh - # Docker path is /__w by default - export WORKSPACE="/__w" - # See: https://github.com/actions/checkout/issues/760 - git config --global --add safe.directory "$WORKSPACE/data/data" - if ${{ matrix.python-version != 'pure' }}; then - # AWSSDK uses $CMAKE_PREFIX_PATH to find openssl - export OPENSSL_ROOT_DIR="$WORKSPACE/ssl" - export CURL_ROOT_DIR="$WORKSPACE/curl" - export CMAKE_PREFIX_PATH="$OPENSSL_ROOT_DIR:$CURL_ROOT_DIR:$CMAKE_PREFIX_PATH" - export STATIC_DEPS=TRUE - fi - fi - if ${{ startsWith( matrix.os, 'macos' ) }}; then - conda activate wheel_build_env_${{ matrix.python-version }} - conda run -n wheel_build_env_${{ matrix.python-version }} packaging/build_wheel.sh - else - packaging/build_wheel.sh - fi - - name: Validate TorchData Wheel - shell: bash -l {0} - env: - PYTHON_VERSION: ${{ matrix.python-version }} - run: | - if ${{ startsWith( matrix.os, 'ubuntu' ) }}; then - if ${{ matrix.python-version == 'pure' }}; then - export PYTHON_VERSION=3.8 - fi - source packaging/manylinux/python_helper.sh - pip3 install auditwheel - fi - if ${{ startsWith( matrix.os, 'macos' ) }}; then - conda activate wheel_build_env_${{ matrix.python-version }} - fi - pip3 install pkginfo - for pkg in dist/torchdata*.whl; do - echo "PkgInfo of $pkg:" - pkginfo $pkg - if ${{ startsWith( matrix.os, 'ubuntu' ) && matrix.python-version != 'pure' }}; then - auditwheel show $pkg - auditwheel repair $pkg --plat manylinux2014_x86_64 -w wheelhouse - fi - done - if ${{ startsWith( matrix.os, 'ubuntu' ) && matrix.python-version != 'pure' }}; then - mv dist temp - mv wheelhouse dist - fi - - name: Install TorchData Wheel - shell: bash -l {0} - env: - PYTHON_VERSION: ${{ matrix.python-version }} - run: | - if ${{ startsWith( matrix.os, 'ubuntu' ) }}; then - if ${{ matrix.python-version == 'pure' }}; then - export PYTHON_VERSION=3.8 - fi - source packaging/manylinux/python_helper.sh - fi - if ${{ startsWith( matrix.os, 'macos' ) }}; then - conda activate wheel_build_env_${{ matrix.python-version }} - conda run -n wheel_build_env_${{ matrix.python-version }} pip3 install dist/torchdata*.whl - else - pip3 install dist/torchdata*.whl - fi - - name: Run Smoke Tests - shell: bash -l {0} - env: - PYTHON_VERSION: ${{ matrix.python-version }} - run: | - if ${{ startsWith( matrix.os, 'ubuntu' ) }}; then - if ${{ matrix.python-version == 'pure' }}; then - export PYTHON_VERSION=3.8 - fi - source packaging/manylinux/python_helper.sh - fi - if ${{ startsWith( matrix.os, 'macos' ) }}; then - conda activate wheel_build_env_${{ matrix.python-version }} - conda run -n wheel_build_env_${{ matrix.python-version }} python test/smoke_test/smoke_test.py - else - if ${{ matrix.python-version == 'pure' }}; then - python test/smoke_test/smoke_test.py --no-s3 - else - python test/smoke_test/smoke_test.py - fi - fi - - name: Upload Wheels to Github - if: always() - uses: actions/upload-artifact@v3 - with: - name: torchdata-artifact - path: dist/torchdata*.whl - - wheel_upload: - if: always() && inputs.branch != '' && inputs.do-upload == true - needs: [get_release_type, wheel_build_test] - runs-on: ubuntu-latest - environment: pytorchbot-env - outputs: - upload: ${{ steps.trigger_upload.outputs.value }} - steps: - - name: Download Artifacts from Github - continue-on-error: true - uses: actions/download-artifact@v3 - with: - name: torchdata-artifact - - name: Determine if Wheel Uploading is needed - run: | - upload=false - for txt in torchdata*.whl; do - upload=true - break - done - echo "value=$upload" >> $GITHUB_OUTPUT - id: trigger_upload - - name: Display All TorchData Wheels - if: steps.trigger_upload.outputs.value == 'true' - run: ls -lh torchdata*.whl - - name: Upload Wheels to S3 Storage - if: steps.trigger_upload.outputs.value == 'true' - env: - AWS_ACCESS_KEY_ID: ${{ secrets.PYTORCH_BINARY_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY }} - run: | - if [[ ${{ inputs.branch }} == 'main' ]]; then - S3_PATH=s3://pytorch/whl/nightly/ - else - S3_PATH=s3://pytorch/whl/test/ - fi - pip3 install awscli - set -x - for pkg in torchdata*.whl; do - aws s3 cp "$pkg" "$S3_PATH" --acl public-read - done - - name: Upload Official Wheels to PYPI - if: | - steps.trigger_upload.outputs.value == 'true' && - needs.get_release_type.outputs.type == 'official' - env: - PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} - run: | - pip3 install twine - python -m twine upload \ - --username __token__ \ - --password "$PYPI_TOKEN" \ - torchdata*.whl - - conda_build_test: - needs: get_release_type - runs-on: ${{ matrix.os }} - container: ${{ startsWith( matrix.os, 'ubuntu' ) && 'pytorch/manylinux-cpu' || null }} - strategy: - fail-fast: false - matrix: - os: - - macos-latest - - ubuntu-latest - - windows-latest - - macos-m1-12 - python-version: - - 3.8 - - 3.9 - - "3.10" - - "3.11" - steps: - - name: Checkout Source Repository - uses: actions/checkout@v3 - with: - ref: ${{ inputs.branch }} - submodules: recursive - - name: Determine if build AWSSDK - shell: bash -l {0} - run: | - if ${{ startsWith( matrix.os, 'windows' ) }}; then - BUILD_S3=0 - else - BUILD_S3=1 - fi - echo "value=$BUILD_S3" >> $GITHUB_OUTPUT - id: build_s3 - - name: Add temp runner environment variables - shell: bash -l {0} - run: | - echo "MINICONDA_INSTALL_PATH=${RUNNER_TEMP}/miniconda" >> "${GITHUB_ENV}" - echo "CONDA_ENV_PATH=${RUNNER_TEMP}/conda_build_env" >> "${GITHUB_ENV}" - - name: Create Conda Env on Windows - if: ${{ startsWith( matrix.os, 'windows' ) }} - uses: conda-incubator/setup-miniconda@v2 - with: - python-version: ${{ matrix.python-version }} - activate-environment: ${{ env.CONDA_ENV_PATH }} - - name: Install Conda on MacOS and Linux - if: ${{ ! startsWith( matrix.os, 'windows' ) }} - shell: bash -l {0} - run: | - mkdir -p "${MINICONDA_INSTALL_PATH}" - if ${{ startsWith( matrix.os, 'ubuntu' ) }}; then - curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -o "${MINICONDA_INSTALL_PATH}/miniconda.sh" - elif ${{ startsWith( matrix.os, 'macos-m1' ) }}; then - curl https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-arm64.sh -o "${MINICONDA_INSTALL_PATH}/miniconda.sh" - else - curl https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -o "${MINICONDA_INSTALL_PATH}/miniconda.sh" - fi - bash "${MINICONDA_INSTALL_PATH}/miniconda.sh" -b -u -p "${MINICONDA_INSTALL_PATH}" - rm -rf "${MINICONDA_INSTALL_PATH}/miniconda.sh" - echo "${MINICONDA_INSTALL_PATH}/bin" >> "$GITHUB_PATH" - ${MINICONDA_INSTALL_PATH}/bin/conda init bash - - name: Create Conda Env on MacOS and Linux - if: ${{ ! startsWith( matrix.os, 'windows' ) }} - shell: bash -l {0} - run: | - source "${MINICONDA_INSTALL_PATH}/etc/profile.d/conda.sh" - - if [[ ${{ matrix.python-version }} = "3.11" ]]; then - conda create -y -p "${CONDA_ENV_PATH}" python=3.8 - else - conda create -y -p "${CONDA_ENV_PATH}" python=${{ matrix.python-version }} - fi - - name: Build TorchData for Conda - shell: bash -l {0} - env: - PYTHON_VERSION: ${{ matrix.python-version }} - PYTORCH_VERSION: ${{ inputs.pytorch_version }} - BUILD_S3: ${{ steps.build_s3.outputs.value }} - run: | - set -ex - if ${{ ! startsWith( matrix.os, 'windows' ) }}; then - source "${MINICONDA_INSTALL_PATH}/etc/profile.d/conda.sh" - fi - conda activate "${CONDA_ENV_PATH}" - if [[ "${PYTHON_VERSION}" = "3.11" ]]; then - conda install -yq conda-build -c malfet -c conda-forge - export CONDA_CHANNEL_FLAGS="${CONDA_CHANNEL_FLAGS} -c malfet" - else - conda install -yq conda-build - fi - packaging/build_conda.sh - conda index ./conda-bld - - name: Upload Conda Package to Github - if: always() - uses: actions/upload-artifact@v3 - with: - name: torchdata-artifact - path: conda-bld/*/torchdata-*.tar.bz2 - - conda_upload: - if: always() && inputs.branch != '' && inputs.do-upload == true - needs: [get_release_type, conda_build_test] - runs-on: ubuntu-latest - container: continuumio/miniconda3 - environment: pytorchbot-env - outputs: - upload: ${{ steps.trigger_upload.outputs.value }} - steps: - - name: Download Artifacts from Github - continue-on-error: true - uses: actions/download-artifact@v3 - with: - name: torchdata-artifact - - name: Determine if Conda Uploading is needed - run: | - upload=false - for pkg in ./*/torchdata-*.tar.bz2; do - upload=true - break - done - echo "value=$upload" >> $GITHUB_OUTPUT - id: trigger_upload - - name: Display All TorchData Conda Package - if: steps.trigger_upload.outputs.value == 'true' - run: ls -lh ./*/torchdata-*.tar.bz2 - - name: Upload Packages to Conda - if: steps.trigger_upload.outputs.value == 'true' - shell: bash -l {0} - env: - CONDA_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }} - CONDA_NIGHTLY_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_NIGHTLY_PYTORCHBOT_TOKEN }} - CONDA_TEST_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_TEST_PYTORCHBOT_TOKEN }} - run: | - conda create -y --name conda_upload_env - conda activate conda_upload_env - - conda install -yq anaconda-client - conda install -c conda-forge -yq jq - - if [[ ${{ needs.get_release_type.outputs.type }} == 'official' ]]; then - CONDA_CHANNEL=pytorch - CONDA_TOKEN=${CONDA_PYTORCHBOT_TOKEN} - elif [[ ${{ needs.get_release_type.outputs.type }} == 'test' ]]; then - CONDA_CHANNEL=pytorch-${{ needs.get_release_type.outputs.type }} - CONDA_TOKEN=${CONDA_TEST_PYTORCHBOT_TOKEN} - else - CONDA_CHANNEL=pytorch-${{ needs.get_release_type.outputs.type }} - CONDA_TOKEN=${CONDA_NIGHTLY_PYTORCHBOT_TOKEN} - fi - - if [[ ${{ needs.get_release_type.outputs.type }} == 'nightly' ]]; then - # Loop over all platforms [win-64, osx-64, linux-64] - for subdir in $(find . -type f -name '*torchdata*.tar.bz2' | sed -r 's|/[^/]+$||' | uniq | cut -f2 -d/) ; do - version="" - # Find existing conda packages on pytorch nightly - for val in $(conda search --json torchdata --channel=$CONDA_CHANNEL --subdir=$subdir | jq -r '.[][] | .version, .build'); do - if [[ -z $version ]]; then - version=$val - else - build=$val - # Check if a new conda package built based on the same Python version and platform exists - for new_package in ./$subdir/torchdata-*-$build.tar.bz2; do - if [[ -f "$new_package" ]]; then - echo "Removing $CONDA_CHANNEL/torchdata/$version/$subdir/torchdata-$version-$build.tar.bz2" - anaconda -t "${CONDA_TOKEN}" remove -f "$CONDA_CHANNEL/torchdata/$version/$subdir/torchdata-$version-$build.tar.bz2" - break - fi - done - version="" - build="" - fi - done - # Upload new conda packages per subdir - anaconda -t "${CONDA_TOKEN}" upload ./$subdir/torchdata-*.tar.bz2 -u "$CONDA_CHANNEL" --label main --no-progress --force - done - else - anaconda -t "${CONDA_TOKEN}" upload ./*/torchdata-*.tar.bz2 -u "$CONDA_CHANNEL" --label main --no-progress --force - fi - - build_docs: - if: | - always() && inputs.branch != '' && - ( needs.wheel_upload.outputs.upload == 'true' || needs.conda_upload.outputs.upload == 'true' ) - needs: [get_release_type, wheel_upload, conda_upload] - runs-on: ubuntu-latest - steps: - - name: Setup Python 3.8 - uses: actions/setup-python@v4 - with: - python-version: 3.8 - - name: Checkout - uses: actions/checkout@v3 - with: - ref: ${{ inputs.branch }} - submodules: recursive - - name: Install Dependencies - run: | - echo `python3 --version` - sudo apt-get install -y python-setuptools - python3 -m pip install --upgrade pip - python3 -m pip install setuptools - python3 -m pip install matplotlib - sudo apt-get install -y yarn - - name: Install PyTorch & TorchData - run: | - pip3 install numpy - # Add version requirement to PyTorch except nightly release - if [[ -z "${{ inputs.pytorch_version }}" ]]; then - PYTORCH_VERSION=torch - else - PYTORCH_VERSION=torch==${{ inputs.pytorch_version }} - fi - - PIP_CHANNEL=${{ needs.get_release_type.outputs.type }} - if [[ $PIP_CHANNEL == 'official' ]]; then - pip3 install "$PYTORCH_VERSION" -f https://download.pytorch.org/whl/torch_stable.html - else - pip3 install --pre "$PYTORCH_VERSION" -f "https://download.pytorch.org/whl/$PIP_CHANNEL/torch_$PIP_CHANNEL.html" - fi - - pip3 install -r requirements.txt - pip3 install . - - name: Check env - run: echo `which spinx-build` - - name: Build the docset - run: | - cd ./docs - sudo apt-get install -y graphviz - pip3 install -r requirements.txt - make html - cd .. - - name: Export Target Folder - run: | - TARGET_FOLDER=${{ inputs.branch }} - if [[ $TARGET_FOLDER == release/* ]]; then - TARGET_FOLDER=${TARGET_FOLDER:8} - elif [[ $TARGET_FOLDER == tags/* ]]; then - TARGET_FOLDER=${TARGET_FOLDER:5} - elif [[ $TARGET_FOLDER == v* ]] && [[ ${{ inputs.pre_dev_release }} == false ]]; then - if [[ $TARGET_FOLDER == v*.*.* ]]; then - TARGET_FOLDER=${TARGET_FOLDER%.*} - fi - TARGET_FOLDER=${TARGET_FOLDER:1} - fi - echo "value=$TARGET_FOLDER" >> $GITHUB_OUTPUT - id: target_folder - - name: Deploy - uses: JamesIves/github-pages-deploy-action@v4.4.1 - with: - token: ${{ secrets.GITHUB_TOKEN }} - branch: gh-pages # The branch the action should deploy to. - folder: docs/build/html # The folder the action should deploy. - target-folder: ${{ steps.target_folder.outputs.value }} # The destination folder the action should deploy to. diff --git a/.github/workflows/build-conda-m1.yml b/.github/workflows/build-conda-m1.yml index b61649b87..c565b7b08 100644 --- a/.github/workflows/build-conda-m1.yml +++ b/.github/workflows/build-conda-m1.yml @@ -38,6 +38,6 @@ jobs: env-var-script: packaging/env-var-script.txt smoke-test-script: test/smoke_test/smoke_test.py runner-type: macos-m1-12 - trigger-event: dev + trigger-event: ${{ github.event_name }} secrets: CONDA_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }} diff --git a/.github/workflows/build-wheels-m1.yml b/.github/workflows/build-wheels-m1.yml index d66875933..d75f0dac0 100644 --- a/.github/workflows/build-wheels-m1.yml +++ b/.github/workflows/build-wheels-m1.yml @@ -13,6 +13,10 @@ on: - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ workflow_dispatch: +permissions: + id-token: write + contents: read + jobs: generate-matrix: uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main @@ -37,7 +41,4 @@ jobs: env-var-script: packaging/env-var-script.txt runner-type: macos-m1-12 smoke-test-script: test/smoke_test/smoke_test.py - trigger-event: dev - secrets: - AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID: ${{ secrets.AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID }} - AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY: ${{ secrets.AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY }} + trigger-event: ${{ github.event_name }} diff --git a/.github/workflows/build_conda_linux.yml b/.github/workflows/build_conda_linux.yml index c3ac3ff8a..9ed2b4916 100644 --- a/.github/workflows/build_conda_linux.yml +++ b/.github/workflows/build_conda_linux.yml @@ -49,8 +49,6 @@ jobs: smoke-test-script: ${{ matrix.smoke-test-script }} package-name: ${{ matrix.package-name }} env-var-script: packaging/env-var-script.txt - # Using "development" as trigger event so these binaries are not uploaded - # to official channels yet - trigger-event: development + trigger-event: ${{ github.event_name }} secrets: CONDA_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }} diff --git a/.github/workflows/build_wheels_linux.yml b/.github/workflows/build_wheels_linux.yml index cf8022a47..c8be185c5 100644 --- a/.github/workflows/build_wheels_linux.yml +++ b/.github/workflows/build_wheels_linux.yml @@ -13,6 +13,10 @@ on: - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ workflow_dispatch: +permissions: + id-token: write + contents: read + jobs: generate-matrix: uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main @@ -47,8 +51,4 @@ jobs: smoke-test-script: ${{ matrix.smoke-test-script }} package-name: ${{ matrix.package-name }} env-var-script: packaging/env-var-script.txt - # Using "development" as trigger event so these binaries are not uploaded to official channels yet - trigger-event: development - secrets: - AWS_PYTORCH_UPLOADER_ACCESS_KEY_ID: ${{ secrets.PYTORCH_BINARY_AWS_ACCESS_KEY_ID }} - AWS_PYTORCH_UPLOADER_SECRET_ACCESS_KEY: ${{ secrets.PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY }} + trigger-event: ${{ github.event_name }} diff --git a/.github/workflows/nightly_release.yml b/.github/workflows/nightly_release.yml deleted file mode 100644 index 02771077a..000000000 --- a/.github/workflows/nightly_release.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: Push Nightly Release - -on: - workflow_dispatch: - schedule: - - cron: 00 11 * * * - -jobs: - build_test_upload: - if: | - github.repository == 'pytorch/data' && github.ref_name == 'main' - uses: ./.github/workflows/_build_test_upload.yml - with: - branch: "main" - pre_dev_release: true - pytorch_version: "" - secrets: - PYTORCH_BINARY_AWS_ACCESS_KEY_ID: ${{ secrets.PYTORCH_BINARY_AWS_ACCESS_KEY_ID }} - PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY: ${{ secrets.PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY }} - CONDA_NIGHTLY_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_NIGHTLY_PYTORCHBOT_TOKEN }} diff --git a/.github/workflows/pull_release.yml b/.github/workflows/pull_release.yml deleted file mode 100644 index c73b12f04..000000000 --- a/.github/workflows/pull_release.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: Test Release Pipelines - -on: - workflow_dispatch: - pull_request: - paths: - - .github/workflows/pull_release.yml - - .github/workflows/_build_test_upload.yml - -jobs: - build_test_upload: - if: github.repository == 'pytorch/data' - uses: ./.github/workflows/_build_test_upload.yml - with: - branch: "" - pre_dev_release: true - pytorch_version: "" - do-upload: false - secrets: - PYTORCH_BINARY_AWS_ACCESS_KEY_ID: "" - PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY: "" - CONDA_TEST_PYTORCHBOT_TOKEN: "" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index 38637cb2b..000000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,24 +0,0 @@ -name: Push Official Release - -on: - # [ Note: Manually Trigger the Workflow ] - # 1. Go to Actions under pytorch/data repo - # 2. In the left sidebar, click the workflow you want to run - # 3. Above the list of workflow runs, select Run workflow - # 4. Use the Branch dropdown to select the release/* branch - # 5. Click Run workflow - workflow_dispatch: - -jobs: - build_test_upload: - if: github.repository == 'pytorch/data' && github.ref_type == 'tag' - uses: ./.github/workflows/_build_test_upload.yml - with: - branch: "v0.7.0" - pre_dev_release: false - pytorch_version: "2.1.0" - secrets: - PYTORCH_BINARY_AWS_ACCESS_KEY_ID: ${{ secrets.PYTORCH_BINARY_AWS_ACCESS_KEY_ID }} - PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY: ${{ secrets.PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY }} - PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} - CONDA_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }} diff --git a/.github/workflows/test_release.yml b/.github/workflows/test_release.yml deleted file mode 100644 index 9ef91b692..000000000 --- a/.github/workflows/test_release.yml +++ /dev/null @@ -1,36 +0,0 @@ -name: Push Test Release - -on: - # [ Note: Manually Trigger the Workflow ] - # 1. Go to Actions under pytorch/data repo - # 2. In the left sidebar, click the workflow you want to run - # 3. Above the list of workflow runs, select Run workflow - # 4. Use the Branch dropdown to select the release/* branch - # 5. Click Run workflow - workflow_dispatch: - # Automatically trigger test/official release - # Requred Feature of GHA: Run schedule on specific branch - # Otherwise, all changes for release need to be landed into main branch - # See: https://github.community/t/scheduled-builds-of-non-default-branch/16306 - # schedule: - # - cron: 30 23 * * * - -# [ Note: Workflow/Job Level ENV ] -# Workflow/Job level env doesn't work even though document indicates this feature -# https://github.com/actions/runner/issues/480 -# https://github.community/t/how-to-set-and-access-a-workflow-variable/17335 -# env: -# RELEASE_BRANCH: "" - -jobs: - build_test_upload: - if: github.repository == 'pytorch/data' && startsWith(github.ref_name, 'release/') - uses: ./.github/workflows/_build_test_upload.yml - with: - branch: "release/0.7" - pre_dev_release: true - pytorch_version: "2.1.0" - secrets: - PYTORCH_BINARY_AWS_ACCESS_KEY_ID: ${{ secrets.PYTORCH_BINARY_AWS_ACCESS_KEY_ID }} - PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY: ${{ secrets.PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY }} - CONDA_TEST_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_TEST_PYTORCHBOT_TOKEN }}