From 10a487b6b859362558817f0e1859d0602403dff2 Mon Sep 17 00:00:00 2001 From: Ankita Katiyar Date: Thu, 31 Aug 2023 14:38:47 +0100 Subject: [PATCH] update GHA --- .github/workflows/check-plugin.yml | 183 -------------------------- .github/workflows/check-release.yml | 52 +------- .github/workflows/e2e-tests.yml | 43 ++++++ .github/workflows/kedro-airflow.yml | 29 +++- .github/workflows/kedro-datasets.yml | 42 +++++- .github/workflows/kedro-docker.yml | 29 +++- .github/workflows/kedro-telemetry.yml | 29 +++- .github/workflows/lint.yml | 44 +++++++ .github/workflows/unit-tests.yml | 63 +++++++++ 9 files changed, 272 insertions(+), 242 deletions(-) delete mode 100644 .github/workflows/check-plugin.yml create mode 100644 .github/workflows/e2e-tests.yml create mode 100644 .github/workflows/lint.yml create mode 100644 .github/workflows/unit-tests.yml diff --git a/.github/workflows/check-plugin.yml b/.github/workflows/check-plugin.yml deleted file mode 100644 index e0df1114c..000000000 --- a/.github/workflows/check-plugin.yml +++ /dev/null @@ -1,183 +0,0 @@ -name: Running tests and linter - -on: - workflow_call: - inputs: - plugin: - type: string - -jobs: - - setup-matrix: - # kedro-datasets is the only plugin that supports python 3.11 - runs-on: ubuntu-latest - outputs: - python-versions: ${{ steps.set-matrix.outputs.matrix }} - steps: - - id: set-matrix - run: | - if [[ "${{ inputs.plugin }}" == "kedro-datasets" ]]; then - MATRIX='["3.7", "3.8", "3.9", "3.10", "3.11"]' - else - MATRIX='["3.7", "3.8", "3.9", "3.10"]' - fi - echo "matrix=${MATRIX}" >> $GITHUB_OUTPUT - - unit-tests: - needs: setup-matrix - defaults: - run: - shell: bash - strategy: - matrix: - os: [ ubuntu-latest, windows-latest ] - python-version: ${{fromJson(needs.setup-matrix.outputs.python-versions)}} - runs-on: ${{ matrix.os }} - steps: - - name: Checkout code - uses: actions/checkout@v3 - - name: Set up Python ${{matrix.python-version}} - uses: actions/setup-python@v3 - with: - python-version: ${{matrix.python-version}} - - name: Cache python packages for Linux - if: matrix.os == 'ubuntu-latest' - uses: actions/cache@v3 - with: - path: ~/.cache/pip - key: ${{inputs.plugin}}-${{matrix.os}}-python-${{matrix.python-version}} - restore-keys: ${{inputs.plugin}} - - name: Cache python packages for Windows - if: matrix.os == 'windows-latest' - uses: actions/cache@v3 - with: - path: ~\AppData\Local\pip\Cache - key: ${{inputs.plugin}}-${{matrix.os}}-python-${{matrix.python-version}} - restore-keys: ${{inputs.plugin}} - - name: Install Kedro - run: pip install git+https://github.com/kedro-org/kedro@main - - name: Add MSBuild to PATH - if: matrix.os == 'windows-latest' - uses: microsoft/setup-msbuild@v1 - - name: Install dependencies - run: | - cd ${{ inputs.plugin }} - pip install ".[test]" - - name: pip freeze - run: pip freeze - - name: Run unit tests for Linux / all plugins - if: matrix.os != 'windows-latest' - run: make plugin=${{ inputs.plugin }} test - - name: Run unit tests for Windows / kedro-airflow, kedro-docker, kedro-telemetry - if: matrix.os == 'windows-latest' && inputs.plugin != 'kedro-datasets' - run: | - cd ${{ inputs.plugin }} - pytest tests - - name: Run unit tests for Windows / kedro-datasets / no spark parallel - if: matrix.os == 'windows-latest' && inputs.plugin == 'kedro-datasets' - run: | - make test-no-spark - - lint: - defaults: - run: - shell: bash - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v3 - # kedro-datasets is the only plugin that supports python 3.11 - - name: Determine Python version for linting - id: get-python-version - run: | - if [[ "${{ inputs.plugin }}" == "kedro-datasets" ]]; then - echo "version=3.11" >> $GITHUB_OUTPUT - else - echo "version=3.8" >> $GITHUB_OUTPUT - fi - - name: Set up Python - uses: actions/setup-python@v3 - with: - python-version: ${{ steps.get-python-version.outputs.version }} - - name: Cache python packages - uses: actions/cache@v3 - with: - path: ~/.cache/pip - key: ${{inputs.plugin}}-${{matrix.os}}-python-${{matrix.python-version}} - restore-keys: ${{inputs.plugin}} - - name: Install dependencies - run: | - cd ${{ inputs.plugin }} - pip install git+https://github.com/kedro-org/kedro@main - pip install ".[test]" - pip freeze - - name: Install pre-commit hooks - run: | - pre-commit install --install-hooks - pre-commit install --hook-type pre-push - - name: Run linter - run: make plugin=${{ inputs.plugin }} lint - - RTD-build: - if: inputs.plugin == 'kedro-datasets' - defaults: - run: - shell: bash - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v3 - - name: Set up Python 3.8 - uses: actions/setup-python@v3 - with: - python-version: "3.8" - - name: Cache python packages - uses: actions/cache@v3 - with: - path: ~/.cache/pip - key: ${{inputs.plugin}}-ubuntu-latest-python-"3.8" - restore-keys: ${{inputs.plugin}} - - name: Install dependencies - run: | - cd ${{ inputs.plugin }} - pip install ".[docs]" - pip install ".[test]" - - name: RTD build for kedro-datasets - run: | - make rtd - - e2e-tests: - if: inputs.plugin != 'kedro-datasets' - defaults: - run: - shell: bash - strategy: - matrix: - os: [ ubuntu-latest ] - python-version: [ "3.7", "3.8", "3.9", "3.10" ] - runs-on: ${{ matrix.os }} - steps: - - name: Checkout code - uses: actions/checkout@v3 - - name: Set up Python ${{matrix.python-version}} - uses: actions/setup-python@v3 - with: - python-version: ${{matrix.python-version}} - - name: Cache python packages - uses: actions/cache@v3 - with: - path: ~/.cache/pip - key: ${{inputs.plugin}}-${{matrix.os}}-python-${{matrix.python-version}} - restore-keys: ${{inputs.plugin}} - - name: Install dependencies - run: | - cd ${{ inputs.plugin }} - pip install git+https://github.com/kedro-org/kedro@main - pip install ".[test]" - - name: pip freeze - run: pip freeze - - name: Run end to end tests - # Custom shell to run kedro-docker e2e-tests because -it flag for `docker run` - # isn't supported on Github Actions. See https://github.com/actions/runner/issues/241 - shell: 'script -q -e -c "bash {0}"' - run: make plugin=${{ inputs.plugin }} e2e-tests diff --git a/.github/workflows/check-release.yml b/.github/workflows/check-release.yml index 51036d260..cd200cde9 100644 --- a/.github/workflows/check-release.yml +++ b/.github/workflows/check-release.yml @@ -34,56 +34,6 @@ jobs: test: needs: check-version if: ${{ needs.check-version.outputs.new_release == 'true' }} - uses: ./.github/workflows/check-plugin.yml + uses: ./.github/workflows/${{ steps.version_check.outputs.package_name }}.yml with: plugin: ${{ needs.check-version.outputs.package_name }} - - build-publish: - needs: [check-version, test] - if: ${{ needs.check-version.outputs.new_release == 'true' }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Set up Python - uses: actions/setup-python@v3 - with: - python-version: '3.10' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install build - - name: Build package - run: | - export plugin=${{ needs.check-version.outputs.package_name }} - make package - - name: Extract release notes from ${{needs.check-version.outputs.package_name}}/RELEASE.md - id: extract - run: | - python tools/github_actions/extract_release_notes.py \ - "${{needs.check-version.outputs.package_name}}/RELEASE.md" \ - "Release ${{needs.check-version.outputs.package_version}}" - - name: Create GitHub Release - uses: softprops/action-gh-release@v1 - with: - tag_name: ${{needs.check-version.outputs.package_name}}-${{needs.check-version.outputs.package_version}} - name: ${{needs.check-version.outputs.package_name}}-${{needs.check-version.outputs.package_version}} - body_path: release_body.txt - draft: false - prerelease: false - token: ${{ secrets.GH_TAGGING_TOKEN }} - - name: Set PyPI token - run: | - if [ "${{ needs.check-version.outputs.PACKAGE_NAME }}" == "kedro-airflow" ]; then - echo 'PYPI_TOKEN=${{ secrets.AIRFLOW_PYPI_TOKEN }}' >> $GITHUB_ENV - elif [ "${{ needs.check-version.outputs.PACKAGE_NAME }}" == "kedro-datasets" ]; then - echo 'PYPI_TOKEN=${{ secrets.DATASETS_PYPI_TOKEN }}' >> $GITHUB_ENV - elif [ "${{ needs.check-version.outputs.PACKAGE_NAME }}" == "kedro-docker" ]; then - echo 'PYPI_TOKEN=${{ secrets.DOCKER_PYPI_TOKEN }}' >> $GITHUB_ENV - elif [ "${{ needs.check-version.outputs.PACKAGE_NAME }}" == "kedro-telemetry" ]; then - echo 'PYPI_TOKEN=${{ secrets.TELEMETRY_PYPI_TOKEN }}' >> $GITHUB_ENV - fi - - name: Publish distribution 📦 to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 - with: - packages-dir: ${{ needs.check-version.outputs.package_name }}/dist - password: ${{ env.PYPI_TOKEN }} diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml new file mode 100644 index 000000000..d0482c138 --- /dev/null +++ b/.github/workflows/e2e-tests.yml @@ -0,0 +1,43 @@ +name: Running e2e tests + +on: + workflow_call: + inputs: + plugin: + type: string + os: + type: string + python-version: + type: string + +jobs: + e2e-tests: + defaults: + run: + shell: bash + runs-on: ${{ inputs.os }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + - name: Set up Python ${{inputs.python-version}} + uses: actions/setup-python@v3 + with: + python-version: ${{inputs.python-version}} + - name: Cache python packages + uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{inputs.plugin}}-${{inputs.os}}-python-${{inputs.python-version}} + restore-keys: ${{inputs.plugin}} + - name: Install dependencies + run: | + cd ${{ inputs.plugin }} + pip install git+https://github.com/kedro-org/kedro@main + pip install ".[test]" + - name: pip freeze + run: pip freeze + - name: Run end to end tests + # Custom shell to run kedro-docker e2e-tests because -it flag for `docker run` + # isn't supported on Github Actions. See https://github.com/actions/runner/issues/241 + shell: 'script -q -e -c "bash {0}"' + run: make plugin=${{ inputs.plugin }} e2e-tests diff --git a/.github/workflows/kedro-airflow.yml b/.github/workflows/kedro-airflow.yml index fecb91db2..51462f550 100644 --- a/.github/workflows/kedro-airflow.yml +++ b/.github/workflows/kedro-airflow.yml @@ -1,6 +1,7 @@ name: Run checks on Kedro-Airflow on: + workflow_call: push: branches: - main @@ -20,7 +21,31 @@ on: - cron: '0 0 * * *' jobs: - airflow-test: - uses: ./.github/workflows/check-plugin.yml + unit-tests: + strategy: + matrix: + os: [ ubuntu-latest, windows-latest ] + python-version: [ "3.7", "3.8", "3.9", "3.10" ] + uses: ./.github/workflows/unit-tests.yml with: plugin: kedro-airflow + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} + + lint: + uses: ./.github/workflows/lint.yml + with: + plugin: kedro-airflow + os: ubuntu-latest + python-version: "3.8" + + e2e-tests: + strategy: + matrix: + os: [ ubuntu-latest, windows-latest ] + python-version: [ "3.7", "3.8", "3.9", "3.10" ] + uses: ./.github/workflows/e2e-tests.yml + with: + plugin: kedro-airflow + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/kedro-datasets.yml b/.github/workflows/kedro-datasets.yml index be4315cd9..45dea3707 100644 --- a/.github/workflows/kedro-datasets.yml +++ b/.github/workflows/kedro-datasets.yml @@ -1,6 +1,7 @@ name: Run checks on Kedro-Datasets on: + workflow_call: push: branches: - main @@ -20,7 +21,44 @@ on: - cron: '0 0 * * *' jobs: - datasets-test: - uses: ./.github/workflows/check-plugin.yml + unit-tests: + strategy: + matrix: + os: [ ubuntu-latest, windows-latest ] + python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11" ] + uses: ./.github/workflows/unit-tests.yml with: plugin: kedro-datasets + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} + + lint: + uses: ./.github/workflows/lint.yml + with: + plugin: kedro-datasets + os: ubuntu-latest + python-version: "3.11" + + RTD-build: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v3 + - name: Set up Python 3.8 + uses: actions/setup-python@v3 + with: + python-version: "3.8" + - name: Cache python packages + uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: kedro-datasets-ubuntu-latest-python-"3.8" + restore-keys: kedro-datasets + - name: Install dependencies + run: | + cd kedro-datasets + pip install ".[docs]" + pip install ".[test]" + - name: RTD build for kedro-datasets + run: | + make rtd diff --git a/.github/workflows/kedro-docker.yml b/.github/workflows/kedro-docker.yml index 430058513..60a88ca3c 100644 --- a/.github/workflows/kedro-docker.yml +++ b/.github/workflows/kedro-docker.yml @@ -1,6 +1,7 @@ name: Run checks on Kedro-Docker on: + workflow_call: push: branches: - main @@ -20,7 +21,31 @@ on: - cron: '0 0 * * *' jobs: - docker-test: - uses: ./.github/workflows/check-plugin.yml + unit-tests: + strategy: + matrix: + os: [ ubuntu-latest, windows-latest ] + python-version: [ "3.7", "3.8", "3.9", "3.10" ] + uses: ./.github/workflows/unit-tests.yml with: plugin: kedro-docker + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} + + lint: + uses: ./.github/workflows/lint.yml + with: + plugin: kedro-docker + os: ubuntu-latest + python-version: "3.8" + + e2e-tests: + strategy: + matrix: + os: [ ubuntu-latest, windows-latest ] + python-version: [ "3.7", "3.8", "3.9", "3.10" ] + uses: ./.github/workflows/e2e-tests.yml + with: + plugin: kedro-docker + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/kedro-telemetry.yml b/.github/workflows/kedro-telemetry.yml index 88fb8ae7a..f402944ef 100644 --- a/.github/workflows/kedro-telemetry.yml +++ b/.github/workflows/kedro-telemetry.yml @@ -1,6 +1,7 @@ name: Run checks on Kedro-Telemetry on: + workflow_call: push: branches: - main @@ -20,7 +21,31 @@ on: - cron: '0 0 * * *' jobs: - telemetry-test: - uses: ./.github/workflows/check-plugin.yml + unit-tests: + strategy: + matrix: + os: [ ubuntu-latest, windows-latest ] + python-version: [ "3.7", "3.8", "3.9", "3.10" ] + uses: ./.github/workflows/unit-tests.yml with: plugin: kedro-telemetry + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} + + lint: + uses: ./.github/workflows/lint.yml + with: + plugin: kedro-telemetry + os: ubuntu-latest + python-version: "3.8" + + e2e-tests: + strategy: + matrix: + os: [ ubuntu-latest, windows-latest ] + python-version: [ "3.7", "3.8", "3.9", "3.10" ] + uses: ./.github/workflows/e2e-tests.yml + with: + plugin: kedro-telemetry + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 000000000..fe83fed95 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,44 @@ +name: Run linter + +on: + workflow_call: + inputs: + plugin: + type: string + os: + type: string + python-version: + type: string + +jobs: + lint: + defaults: + run: + shell: bash + runs-on: ${{ inputs.os }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + # kedro-datasets is the only plugin that supports python 3.11 + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: ${{ inputs.python-version }} + - name: Cache python packages + uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{inputs.plugin}}-${{inputs.os}}-python-${{inputs.python-version}} + restore-keys: ${{inputs.plugin}} + - name: Install dependencies + run: | + cd ${{ inputs.plugin }} + pip install git+https://github.com/kedro-org/kedro@main + pip install ".[test]" + pip freeze + - name: Install pre-commit hooks + run: | + pre-commit install --install-hooks + pre-commit install --hook-type pre-push + - name: Run linter + run: make plugin=${{ inputs.plugin }} lint diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml new file mode 100644 index 000000000..5f479afa5 --- /dev/null +++ b/.github/workflows/unit-tests.yml @@ -0,0 +1,63 @@ +name: Run unit tests + +on: + workflow_call: + inputs: + plugin: + type: string + os: + type: string + python-version: + type: string + +jobs: + + unit-tests: + runs-on: ${{ inputs.os }} + defaults: + run: + shell: bash + steps: + - name: Checkout code + uses: actions/checkout@v3 + - name: Set up Python ${{inputs.python-version}} + uses: actions/setup-python@v3 + with: + python-version: ${{inputs.python-version}} + - name: Cache python packages for Linux + if: inputs.os == 'ubuntu-latest' + uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{inputs.plugin}}-${{inputs.os}}-python-${{inputs.python-version}} + restore-keys: ${{inputs.plugin}} + - name: Cache python packages for Windows + if: inputs.os == 'windows-latest' + uses: actions/cache@v3 + with: + path: ~\AppData\Local\pip\Cache + key: ${{inputs.plugin}}-${{inputs.os}}-python-${{inputs.python-version}} + restore-keys: ${{inputs.plugin}} + - name: Install Kedro + run: pip install git+https://github.com/kedro-org/kedro@main + - name: Add MSBuild to PATH + if: inputs.os == 'windows-latest' + uses: microsoft/setup-msbuild@v1 + - name: Install dependencies + run: | + cd ${{ inputs.plugin }} + pip install ".[test]" + - name: pip freeze + run: pip freeze + - name: Run unit tests for Linux / all plugins + if: inputs.os != 'windows-latest' + run: make plugin=${{ inputs.plugin }} test + - name: Run unit tests for Windows / kedro-airflow, kedro-docker, kedro-telemetry + if: inputs.os == 'windows-latest' && inputs.plugin != 'kedro-datasets' + run: | + cd ${{ inputs.plugin }} + pytest tests + - name: Run unit tests for Windows / kedro-datasets / no spark parallel + if: inputs.os == 'windows-latest' && inputs.plugin == 'kedro-datasets' + run: | + make test-no-spark