update GHA

ankatiyar · Aug 31, 2023 · 10a487b · 10a487b
1 parent dad3edd
commit 10a487b
Show file tree

Hide file tree

Showing 9 changed files with 272 additions and 242 deletions.
diff --git a/.github/workflows/check-plugin.yml b/.github/workflows/check-plugin.yml
diff --git a/.github/workflows/check-release.yml b/.github/workflows/check-release.yml
@@ -34,56 +34,6 @@ jobs:
   test:
     needs: check-version
     if: ${{ needs.check-version.outputs.new_release == 'true' }}
-    uses: ./.github/workflows/check-plugin.yml
+    uses: ./.github/workflows/${{ steps.version_check.outputs.package_name }}.yml
     with:
       plugin: ${{ needs.check-version.outputs.package_name }}
-
-  build-publish:
-    needs: [check-version, test]
-    if: ${{ needs.check-version.outputs.new_release == 'true' }}
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v3
-    - name: Set up Python
-      uses: actions/setup-python@v3
-      with:
-        python-version: '3.10'
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install build
-    - name: Build package
-      run: |
-        export plugin=${{ needs.check-version.outputs.package_name }}
-        make package
-    - name: Extract release notes from ${{needs.check-version.outputs.package_name}}/RELEASE.md
-      id: extract
-      run: |
-        python tools/github_actions/extract_release_notes.py \
-        "${{needs.check-version.outputs.package_name}}/RELEASE.md" \
-        "Release ${{needs.check-version.outputs.package_version}}"
-    - name: Create GitHub Release
-      uses: softprops/action-gh-release@v1
-      with:
-        tag_name: ${{needs.check-version.outputs.package_name}}-${{needs.check-version.outputs.package_version}}
-        name: ${{needs.check-version.outputs.package_name}}-${{needs.check-version.outputs.package_version}}
-        body_path: release_body.txt
-        draft: false
-        prerelease: false
-        token: ${{ secrets.GH_TAGGING_TOKEN }}
-    - name: Set PyPI token
-      run: |
-        if [ "${{ needs.check-version.outputs.PACKAGE_NAME }}" == "kedro-airflow" ]; then
-          echo 'PYPI_TOKEN=${{ secrets.AIRFLOW_PYPI_TOKEN }}' >> $GITHUB_ENV
-        elif [ "${{ needs.check-version.outputs.PACKAGE_NAME }}" == "kedro-datasets" ]; then
-          echo 'PYPI_TOKEN=${{ secrets.DATASETS_PYPI_TOKEN }}' >> $GITHUB_ENV
-        elif [ "${{ needs.check-version.outputs.PACKAGE_NAME }}" == "kedro-docker" ]; then
-          echo 'PYPI_TOKEN=${{ secrets.DOCKER_PYPI_TOKEN }}' >> $GITHUB_ENV
-        elif [ "${{ needs.check-version.outputs.PACKAGE_NAME }}" == "kedro-telemetry" ]; then
-          echo 'PYPI_TOKEN=${{ secrets.TELEMETRY_PYPI_TOKEN }}' >> $GITHUB_ENV
-        fi
-    - name: Publish distribution 📦 to PyPI
-      uses: pypa/gh-action-pypi-publish@release/v1
-      with:
-        packages-dir: ${{ needs.check-version.outputs.package_name }}/dist
-        password: ${{ env.PYPI_TOKEN }}
diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml
@@ -0,0 +1,43 @@
+name: Running e2e tests
+
+on:
+  workflow_call:
+    inputs:
+      plugin:
+        type: string
+      os:
+        type: string
+      python-version:
+        type: string
+
+jobs:
+  e2e-tests:
+    defaults:
+      run:
+        shell: bash
+    runs-on: ${{ inputs.os }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Set up Python ${{inputs.python-version}}
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{inputs.python-version}}
+      - name: Cache python packages
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pip
+          key: ${{inputs.plugin}}-${{inputs.os}}-python-${{inputs.python-version}}
+          restore-keys: ${{inputs.plugin}}
+      - name: Install dependencies
+        run: |
+          cd ${{ inputs.plugin }}
+          pip install git+https://github.com/kedro-org/kedro@main
+          pip install ".[test]"
+      - name: pip freeze
+        run: pip freeze
+      - name: Run end to end tests
+        # Custom shell to run kedro-docker e2e-tests because -it flag for `docker run`
+        # isn't supported on Github Actions. See https://github.com/actions/runner/issues/241
+        shell: 'script -q -e -c "bash {0}"'
+        run: make plugin=${{ inputs.plugin }} e2e-tests
diff --git a/.github/workflows/kedro-airflow.yml b/.github/workflows/kedro-airflow.yml
@@ -1,6 +1,7 @@
 name: Run checks on Kedro-Airflow
 
 on:
+  workflow_call:
   push:
     branches:
       - main
@@ -20,7 +21,31 @@ on:
     - cron: '0 0 * * *'
 
 jobs:
-  airflow-test:
-    uses: ./.github/workflows/check-plugin.yml
+  unit-tests:
+    strategy:
+      matrix:
+        os: [ ubuntu-latest, windows-latest ]
+        python-version: [ "3.7", "3.8", "3.9", "3.10" ]
+    uses: ./.github/workflows/unit-tests.yml
     with:
       plugin: kedro-airflow
+      os: ${{ matrix.os }}
+      python-version: ${{ matrix.python-version }}
+
+  lint:
+    uses: ./.github/workflows/lint.yml
+    with:
+      plugin: kedro-airflow
+      os: ubuntu-latest
+      python-version: "3.8"
+
+  e2e-tests:
+    strategy:
+      matrix:
+        os: [ ubuntu-latest, windows-latest ]
+        python-version: [ "3.7", "3.8", "3.9", "3.10" ]
+    uses: ./.github/workflows/e2e-tests.yml
+    with:
+      plugin: kedro-airflow
+      os: ${{ matrix.os }}
+      python-version: ${{ matrix.python-version }}
diff --git a/.github/workflows/kedro-datasets.yml b/.github/workflows/kedro-datasets.yml
@@ -1,6 +1,7 @@
 name: Run checks on Kedro-Datasets
 
 on:
+  workflow_call:
   push:
     branches:
       - main
@@ -20,7 +21,44 @@ on:
     - cron: '0 0 * * *'
 
 jobs:
-  datasets-test:
-    uses: ./.github/workflows/check-plugin.yml
+  unit-tests:
+    strategy:
+      matrix:
+        os: [ ubuntu-latest, windows-latest ]
+        python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11" ]
+    uses: ./.github/workflows/unit-tests.yml
     with:
       plugin: kedro-datasets
+      os: ${{ matrix.os }}
+      python-version: ${{ matrix.python-version }}
+
+  lint:
+    uses: ./.github/workflows/lint.yml
+    with:
+      plugin: kedro-datasets
+      os: ubuntu-latest
+      python-version: "3.11"
+
+  RTD-build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v3
+        with:
+          python-version: "3.8"
+      - name: Cache python packages
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pip
+          key: kedro-datasets-ubuntu-latest-python-"3.8"
+          restore-keys: kedro-datasets
+      - name: Install dependencies
+        run: |
+          cd kedro-datasets
+          pip install ".[docs]"
+          pip install ".[test]"
+      - name: RTD build for kedro-datasets
+        run: |
+          make rtd