release.yml

name: Release

on:
  workflow_dispatch:
  push:
    branches: ['*']
    tags: ['*']
    paths-ignore:
      - 'docs/**'
  pull_request:
    branches: [main]
    paths-ignore:
      - 'docs/**'

permissions:
  # needs write permission at workflow level to create release and upload artifacts
  contents: write

env:
  MPLBACKEND: 'agg'
  # pin pytest to 7.4.4 to get the test suite of pyxem to work
  TEST_DEPS: pytest==7.4.4 pytest-qt pytest-xdist pytest-rerunfailures pytest-mpl filelock

jobs:
  create_release_job:
    name: Create Release (on tag only)
    runs-on: ubuntu-latest
    outputs: 
      ID: ${{ steps.create_release.outputs.id }}
      VERSION: ${{ env.VERSION }}
    steps:
      - uses: actions/checkout@v4

      - name: Get version (on tag)
        if: startsWith(github.ref, 'refs/tags/')
        run: |
          echo "VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV

      - name: Get version (short hash)
        if: ${{ startsWith(github.ref, 'refs/tags/') == false }}
        run: |
          # Use git short hash instead of tag
          echo "VERSION=$(git rev-parse --short HEAD)" >> $GITHUB_ENV

      - name: Display version
        run: |
          echo ${{ env.VERSION }}

      - name: Find and Replace
        uses: jacobtomlinson/gha-find-replace@a51bbcd94d000df9ca0fcb54ec8be69aad8374b0
        if: startsWith(github.ref, 'refs/tags/')
        with:
          find: "__TAG__"
          replace: ${{ env.VERSION }}
          include: "RELEASE_TEXT.md" # Will match all RELEASE_TEXT.md files in any nested directory

      - name: Create Release
        if: startsWith(github.ref, 'refs/tags/')
        id: create_release
        uses: softprops/action-gh-release@9d7c94cfd0a1f3ed45544c887983e9fa900f0564
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # This token is provided by Actions
        with:
          body_path: "RELEASE_TEXT.md"
          draft: true


  build:
    name: ${{ matrix.TARGET_PLATFORM }}-${{ matrix.BLAS_IMPL }}
    needs: create_release_job
    runs-on: ${{ matrix.os }}-${{ matrix.os_version }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - os: ubuntu
            os_version: latest
            INSTALLER_EXTENSION: sh
            BLAS_IMPL: mkl
            ARCH: x86_64
            TARGET_PLATFORM: linux-64
          - os: ubuntu
            os_version: latest
            INSTALLER_EXTENSION: sh
            BLAS_IMPL: openblas
            ARCH: x86_64
            TARGET_PLATFORM: linux-64
          - os: windows
            os_version: latest
            INSTALLER_EXTENSION: exe
            BLAS_IMPL: mkl
            ARCH: x86_64
            TARGET_PLATFORM: win-64
          - os: windows
            os_version: latest
            INSTALLER_EXTENSION: exe
            BLAS_IMPL: openblas
            ARCH: x86_64
            TARGET_PLATFORM: win-64
          - os: macos
            os_version: '13'
            INSTALLER_EXTENSION: pkg
            BLAS_IMPL: mkl
            ARCH: x86_64
            TARGET_PLATFORM: osx-64
          - os: macos
            os_version: latest
            INSTALLER_EXTENSION: pkg
            BLAS_IMPL: accelerate
            ARCH: arm64
            TARGET_PLATFORM: osx-arm64
    env:
      DISPLAY: ':0'
      MICROMAMBA_VERSION: '1.5.8'

    steps:
      - uses: actions/checkout@v4
      - uses: conda-incubator/setup-miniconda@v3
        with:
          miniforge-variant: Mambaforge
          channels: conda-forge
          channel-priority: strict

      - shell: bash -l {0}
        name: Conda info
        run: |
          conda info
          conda list

      - shell: bash -l {0}
        name: Install constructor
        run: |
          mamba install constructor jinja2 conda-libmamba-solver

      - shell: bash -l {0}
        if: matrix.ARCH == 'arm64'
        name: Install micromamba
        run: |
          TEMP_DIR=$(mktemp -d)
          mkdir "${TEMP_DIR}/micromamba"
          pushd "${TEMP_DIR}/micromamba"
          curl -L -O "https://anaconda.org/conda-forge/micromamba/${{ env.MICROMAMBA_VERSION }}/download/${{ matrix.TARGET_PLATFORM }}/micromamba-${{ env.MICROMAMBA_VERSION }}-0.tar.bz2"
          bsdtar -xf "micromamba-${{ env.MICROMAMBA_VERSION }}-0.tar.bz2"
          echo "MICROMAMBA_FILE=${PWD}/bin/micromamba" >> $GITHUB_ENV
          ls ${PWD}
          ls ${PWD}/bin  
          popd

      - shell: bash -l {0}
        name: Build distribution
        env:
          VERSION: ${{ needs.create_release_job.outputs.VERSION }}
          BLAS_IMPL: ${{ matrix.BLAS_IMPL }}
          CONDA_SOLVER: 'libmamba'
        run: |
          if [ ${{ matrix.TARGET_PLATFORM }} == osx-arm64 ]; then
             constructor -v conda_distribution --platform ${{ matrix.TARGET_PLATFORM }} --conda-exe ${{ env.MICROMAMBA_FILE }}
          else
             constructor -v conda_distribution
          fi

      - shell: bash -l {0}
        name: Set asset name
        env:
          ext: ${{ matrix.INSTALLER_EXTENSION }}
        run: |  
          installer_name=(HyperSpy-*.${{ env.ext }})
          echo "Original installer name: " $installer_name
          # Rename installer name according to BLAS_IMPL
          if [ ${{ matrix.TARGET_PLATFORM }} = osx-arm64 ]; then
             CPU=Silicon
          elif [ ${{ matrix.BLAS_IMPL }} = mkl ]; then
             CPU=Intel
          else
             CPU=AMD
          fi    
          new_installer_name=$(basename $installer_name .${{ env.ext }})-$CPU.${{ env.ext }}   
          mv $installer_name $new_installer_name
          echo "asset_name=$new_installer_name" >> $GITHUB_ENV
          echo "Installer name: "$new_installer_name
          ls

      - name: Get hash
        run: |
          shasum -a 256 ${{ env.asset_name }}

      - name: Install new distribution (Linux)
        if: runner.os == 'linux'
        env:
          install_dir: '${{ github.workspace }}/new_distribution'
        run: |
          echo "install_dir=${{ env.install_dir }}" >> $GITHUB_ENV
          bash ${{ env.asset_name }} -b -p ${{ env.install_dir }}

      - name: Install new distribution (MacOS)
        if: runner.os == 'macos'
        run: |
          echo "install_dir=/Users/runner" >> $GITHUB_ENV
          installer -pkg ${{ env.asset_name }} -target CurrentUserHomeDirectory

      - name: Install new distribution (Windows)
        if: runner.os == 'windows'
        env:
          install_dir: '${{ github.workspace }}\nd'
        shell: bash
        run: |
          echo "install_dir=${{ env.install_dir }}" >> $GITHUB_ENV

      - name: Install new distribution (Windows)
        if: runner.os == 'windows'
        shell: powershell
        run: |
          Start-Process -Wait -FilePath ${{ env.asset_name }} -ArgumentList "/S /AddToPath=0 /RegisterPython=0 /NoScripts=1 /D=${{ env.install_dir }}"

      - name: Upload artifact
        if: startsWith(github.ref, 'refs/tags/') != true
        uses: actions/upload-artifact@v4
        with:
          path: ${{ env.asset_name }}
          name: ${{ env.asset_name }}

      - shell: bash -l {0}
        name: Info new distribution
        run: |
          conda activate "${{ env.install_dir }}"
          conda info
          conda config --show channels
          conda config --show channel_priority
          conda list

      - shell: bash -l {0}
        name: Check latest available hyperspy version
        run: |
          conda activate "${{ env.install_dir }}"     
          python check_hyperspy_latest.py

      - name: Install xvfb
        if: runner.os == 'linux' && always()
        run: |
          sudo apt-get install -y xvfb libxkbcommon-x11-0 libxcb-icccm4 libxcb-image0 libxcb-keysyms1 libxcb-randr0 libxcb-render-util0 libxcb-xinerama0 libxcb-xinput0 libxcb-xfixes0

      - name: Start xvfb
        if: runner.os == 'linux' && always()
        run: |
          sudo /usr/bin/Xvfb ${{ env.DISPLAY }} -screen 0 1280x1024x24 & 

      - shell: bash -l {0}
        name: Install test dependencies
        if: always()
        run: |
          conda activate "${{ env.install_dir }}"
          mamba install ${{ env.TEST_DEPS }}

      - shell: bash -l {0}
        # TODO: revisit at some point, to get it to work on linux (hyperspyui CI works fine...)
        if: runner.os != 'linux' && always()
        name: Test new distribution (HyperSpyUI)
        run: |
          conda activate "${{ env.install_dir }}"
          pytest --pyargs hyperspyui

      - shell: bash -l {0}
        name: Test RosettaSciIO
        if: always()
        run: |
          conda activate "${{ env.install_dir }}"
          # teardown in test_quantumdetector.py doesn't play well with pytest setup
          # revisit when pytest 7.4.4 is unpinned
          pytest --pyargs rsciio --reruns 3 -n 2 -k "not test_quantumdetector.py"

      - shell: bash -l {0}
        name: Test hyperspy
        if: always()
        run: |
          conda activate "${{ env.install_dir }}"
          pytest --pyargs hyperspy --reruns 3 -n 2

      - shell: bash -l {0}
        name: Test hyperspy_gui_ipywidgets
        if: always()
        run: |
          conda activate "${{ env.install_dir }}"
          pytest --pyargs hyperspy_gui_ipywidgets

      - shell: bash -l {0}
        name: Test hyperspy_gui_traitsui
        if: runner.os != 'linux' && always()
        run: |
          conda activate "${{ env.install_dir }}"
          # test_image_contrast_tool not supported with agg backend
          pytest --pyargs hyperspy_gui_traitsui -k "not test_image_contrast_tool"

      - shell: bash -l {0}
        name: Run test exspy
        if: always()
        run: |
          conda activate "${{ env.install_dir }}"
          pytest --pyargs exspy
  
      - shell: bash -l {0}
        name: Run test holospy
        if: always()
        run: |
          conda activate "${{ env.install_dir }}"
          pytest --pyargs holospy

      - shell: bash -l {0}
        name: Run test lumispy
        if: always()
        run: |
          conda activate "${{ env.install_dir }}"
          pytest --pyargs lumispy

      - shell: bash -l {0}
        name: Run test pyxem
        if: always()
        run: |
          conda activate "${{ env.install_dir }}"
          # cause of the TestAddEllipseArrayAsMarkers failure is unknown (seems to be on blas mkl only)
          # test_offest_and_scale failure fixed in https://github.com/pyxem/pyxem/pull/1016
          pytest --pyargs pyxem -k "not TestAddEllipseArrayAsMarkers and not test_offest_and_scale"

      - shell: bash -l {0}
        name: Run test atomap
        if: always()
        run: |
          conda activate "${{ env.install_dir }}"
          # Skip interactive test
          pytest --pyargs atomap -k "not TestSelectAtomsWithGui"

      # - shell: bash -l {0}
      #   name: Run test kikuchipy
      #   if: always()
      #   env:
      #     MPLBACKEND: 'agg'
      #   run: |
      #     conda activate "${{ env.install_dir }}"
      #     pytest --pyargs kikuchipy

      - name: Upload Release Asset
        if: startsWith(github.ref, 'refs/tags/')
        uses: softprops/action-gh-release@9d7c94cfd0a1f3ed45544c887983e9fa900f0564
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          files: ${{ env.asset_name }}
          draft: true


  build_portable:
    name: windows (Portable)
    needs: create_release_job
    runs-on: windows-latest
    env:
      WP_URL: https://github.com/winpython/winpython/releases/download/7.1.20240203final/Winpython64-3.11.8.0.exe
      WP_SHA256: 0fa1bad92e7287632ce6c32c263c694b109b714b5d12e56063b2cd4281a71f7c
      WP_EXE: winpython.exe
      WP_DIR_NAME: WPy64-31180
      #LIB_TO_INSTALL: abtem ase atomap exspy graphviz holospy hdf5plugin hyperspy[all] hyperspyui kikuchipy lumispy matplotlib-scalebar nglview particlespy py4dstem pymatgen pystackreg python-rapidjson pyxem scanning_drift_corr start_jupyter_cm
      LIB_TO_INSTALL: abtem ase atomap dask-image exspy graphviz holospy hdf5plugin hyperspy[all] hyperspyui lumispy matplotlib-scalebar nglview particlespy py4dstem pymatgen pystackreg python-rapidjson pyxem scanning_drift_corr start_jupyter_cm

    steps:
      - uses: actions/checkout@v4
      - name: Download Winpython
        run: |
          Invoke-WebRequest -OutFile ${{ env.WP_EXE }} ${{ env.WP_URL }}
          ls
          $file_hash = (Get-FileHash ${{ env.WP_EXE }} ).Hash
          $file_hash 
          if ($file_hash -ne "${{ env.WP_SHA256 }}") { exit(1) }

      - name: Install Winpython
        run: |
          .\${{ env.WP_EXE }} -y | Out-Null
          ls
          ls ${{ env.WP_DIR_NAME }}

      - name: Install libraries
        shell: cmd
        run: |
          call "${{ env.WP_DIR_NAME }}\scripts\env.bat"
          where python
          where pip
          pip install ${{ env.LIB_TO_INSTALL }}
          # workaround for imageio / pillow incompatiblity?
          pip install --upgrade imageio

      - name: Pip list
        shell: cmd
        run: |
          call "${{ env.WP_DIR_NAME }}\scripts\env.bat"
          pip list

      - shell: bash -l {0}
        name: Set installer name
        env:
          VERSION: ${{ needs.create_release_job.outputs.VERSION }}
        run: |
          installer_name=HyperSpy-bundle-${{ env.VERSION }}-Windows-x86_64-Portable.exe
          echo "asset_name=$installer_name" >> $GITHUB_ENV
          echo $installer_name

      - name: Run pyclean in distribution folder
        shell: cmd
        run: |
          where pip
          pip install pyclean
          where pyclean
          pyclean ${{ env.WP_DIR_NAME }}

      - name: Create installer
        run: |
          7z -mx5 -sfx a ${{ env.asset_name }} ${{ env.WP_DIR_NAME }}
          ls

      - name: Upload artifact
        if: startsWith(github.ref, 'refs/tags/') != true
        uses: actions/upload-artifact@v4
        with:
          path: ${{ env.asset_name }}
          name: ${{ env.asset_name }}

      - name: Install new distribution
        run: |
          Remove-Item -recurse ${{ env.WP_DIR_NAME }}
          ls
          .\${{ env.asset_name }} -y | Out-Null
          echo "After installation"
          ls
          echo ${{ env.asset_name }}

      - name: Check latest available hyperspy version
        shell: cmd
        run: |
          call "${{ env.WP_DIR_NAME }}\scripts\env.bat"
          python check_hyperspy_latest.py

      - name: Install testing libraries
        if: always()
        shell: cmd
        run: |
          call "${{ env.WP_DIR_NAME }}\scripts\env.bat"
          pip install ${{ env.TEST_DEPS }}

      - name: Run test suite (HyperSpyUI)
        if: always()
        shell: cmd
        run: |
          call "${{ env.WP_DIR_NAME }}\scripts\env.bat"
          pytest --pyargs hyperspyui

      - name: Run test hyperspy
        if: always()
        shell: cmd
        run: |
          call "${{ env.WP_DIR_NAME }}\scripts\env.bat"
          pytest --pyargs hyperspy --reruns 3 -n 2

      - name: Run test RosettaSciIO
        if: always()
        shell: cmd
        run: |
          call "${{ env.WP_DIR_NAME }}\scripts\env.bat"
          pytest --pyargs rsciio --reruns 3 -n 2 -k "not test_quantumdetector.py"

      - name: Run test hyperspy_gui_ipywidgets
        if: always()
        shell: cmd
        run: |
          call "${{ env.WP_DIR_NAME }}\scripts\env.bat"
          pytest --pyargs hyperspy_gui_ipywidgets

      - name: Run test hyperspy_gui_traitsui --reruns 3
        if: always()
        shell: cmd
        run: |
          call "${{ env.WP_DIR_NAME }}\scripts\env.bat"
          # test_image_contrast_tool not supported with agg backend
          pytest --pyargs hyperspy_gui_traitsui -k "not test_image_contrast_tool"

      - name: Run test exspy
        if: always()
        shell: cmd
        run: |
          call "${{ env.WP_DIR_NAME }}\scripts\env.bat"
          pytest --pyargs exspy

      - name: Run test holospy
        if: always()
        shell: cmd
        run: |
          call "${{ env.WP_DIR_NAME }}\scripts\env.bat"
          pytest --pyargs holospy

      - name: Run test lumispy
        if: always()
        shell: cmd
        run: |
          call "${{ env.WP_DIR_NAME }}\scripts\env.bat"
          pytest --pyargs lumispy

      - name: Run test pyxem
        if: always()
        shell: cmd
        run: |
          call "${{ env.WP_DIR_NAME }}\scripts\env.bat"
          # cause of the TestAddEllipseArrayAsMarkers failure is unknown (seems to be on blas mkl only)
          pytest --pyargs pyxem -k "not TestAddEllipseArrayAsMarkers"
 
      - name: Run test atomap
        if: always()
        shell: cmd
        run: |
          call "${{ env.WP_DIR_NAME }}\scripts\env.bat"
          # Skip interactive test
          pytest --pyargs atomap -k "not TestSelectAtomsWithGui"

      # - name: Run test kikuchipy
      #   if: always()
      #   shell: cmd
      #   run: |
      #     call "${{ env.WP_DIR_NAME }}\scripts\env.bat"
      #     # Skip these tests until there are fixed
      #     pytest --pyargs kikuchipy -k "not test_load_with_padding"

      - name: Upload Release Asset
        if: startsWith(github.ref, 'refs/tags/')
        uses: softprops/action-gh-release@9d7c94cfd0a1f3ed45544c887983e9fa900f0564
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          files: ${{ env.asset_name }}
          draft: true

  publish_release_job:
    # Set build, build_portable are needed, so that it runs when they are finished 
    needs: [create_release_job, build, build_portable]
    if: startsWith(github.ref, 'refs/tags/')
    runs-on: ubuntu-latest
    steps:
      - name: Finalise release
        # Publish draft release
        uses: eregon/publish-release@01df127f5e9a3c26935118e22e738d95b59d10ce
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          release_id: ${{ needs.create_release_job.outputs.ID }}