Merge pull request #54 from TauferLab/check_metadata_api #1
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Compilation Testing for DYAD | |
| on: | |
| push: | |
| branches: [ main ] | |
| pull_request: | |
| branches: [ main ] | |
| jobs: | |
| compile-flux: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| flux: [ 0.52.0, 0.49.0] | |
| mode: ["FLUX_RPC", "UCX"] | |
| test_mode: ["c", "cpp", "python"] | |
| runs-on: ubuntu-20.04 # Docker-based jobs must run on Ubuntu | |
| env: | |
| FLUX_VERSION: ${{ matrix.flux }} | |
| SPACK_DIR: "/home/runner/work/spack" | |
| DYAD_INSTALL_PREFIX: "/home/runner/work/dyad/install" | |
| DYAD_KVS_NAMESPACE: "test" | |
| DYAD_DTL_MODE: ${{ matrix.mode }} | |
| DYAD_PATH: "/home/runner/work/dyad/temp" | |
| DYAD_TEST_MODE: ${{ matrix.test_mode }} | |
| steps: | |
| - name: Push checkout | |
| if: github.event_name == 'push' | |
| uses: actions/checkout@v3 | |
| - name: PR checkout | |
| if: github.event_name == 'pull_request' | |
| uses: actions/checkout@v3 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| - name: Install system deps | |
| run: | | |
| sudo apt-get install -y --no-install-recommends \ | |
| gcc g++ gfortran \ | |
| autoconf \ | |
| automake \ | |
| libtool \ | |
| libtool-bin \ | |
| openmpi-bin \ | |
| libopenmpi-dev \ | |
| libarchive-dev \ | |
| ncurses-bin \ | |
| hwloc \ | |
| lua5.3 liblua5.3-dev \ | |
| python3 \ | |
| libczmq-dev \ | |
| lua-posix-dev \ | |
| lz4 \ | |
| pkgconf \ | |
| libzmq5 \ | |
| sqlite | |
| sudo pip install jsonschema cffi ply pyyaml | |
| sudo chmod 777 /usr -R | |
| - name: Install Spack | |
| run: | | |
| git clone https://github.com/spack/spack.git ${SPACK_DIR} | |
| - name: Load existing dependencies | |
| run: | | |
| . ${SPACK_DIR}/share/spack/setup-env.sh | |
| mkdir -p /home/runner/.spack | |
| cat > /home/runner/.spack/packages.yaml << 'EOF' | |
| packages: | |
| all: | |
| target: [x86_64] | |
| providers: | |
| mpi: [openmpi] | |
| python: | |
| buildable: False | |
| externals: | |
| - spec: "[email protected]" | |
| prefix: /usr | |
| py-cffi: | |
| buildable: False | |
| externals: | |
| - spec: "[email protected]" | |
| prefix: /usr | |
| py-jsonschema: | |
| buildable: False | |
| externals: | |
| - spec: "[email protected]" | |
| prefix: /usr | |
| py-ply: | |
| buildable: False | |
| externals: | |
| - spec: "[email protected]" | |
| prefix: /usr | |
| py-pyyaml: | |
| buildable: False | |
| externals: | |
| - spec: "[email protected]" | |
| prefix: /usr | |
| czmq: | |
| buildable: False | |
| externals: | |
| - spec: "[email protected]" | |
| prefix: /usr | |
| sqlite: | |
| buildable: False | |
| externals: | |
| - spec: "[email protected]" | |
| prefix: /usr | |
| libzmq: | |
| buildable: False | |
| externals: | |
| - spec: "[email protected]" | |
| prefix: /usr | |
| lua: | |
| buildable: False | |
| externals: | |
| - spec: "[email protected]" | |
| prefix: /usr | |
| lua-luaposix: | |
| buildable: False | |
| externals: | |
| - spec: "[email protected]" | |
| prefix: /usr | |
| lz4: | |
| buildable: False | |
| externals: | |
| - spec: "[email protected]" | |
| prefix: /usr | |
| ncurses: | |
| buildable: False | |
| externals: | |
| - spec: "[email protected]" | |
| prefix: /usr | |
| pkgconf: | |
| buildable: False | |
| externals: | |
| - spec: "[email protected]" | |
| prefix: /usr | |
| hwloc: | |
| buildable: False | |
| externals: | |
| - spec: "[email protected]" | |
| prefix: /usr | |
| libarchive: | |
| buildable: False | |
| externals: | |
| - spec: "[email protected]" | |
| prefix: /usr | |
| autoconf: | |
| buildable: False | |
| externals: | |
| - spec: "[email protected]" | |
| prefix: /usr | |
| automake: | |
| buildable: False | |
| externals: | |
| - spec: "[email protected]" | |
| prefix: /usr | |
| libtool: | |
| buildable: False | |
| externals: | |
| - spec: "[email protected]" | |
| prefix: /usr | |
| m4: | |
| buildable: False | |
| externals: | |
| - spec: "[email protected]" | |
| prefix: /usr | |
| openmpi: | |
| buildable: False | |
| externals: | |
| - spec: "[email protected]" | |
| prefix: /usr | |
| openssl: | |
| buildable: False | |
| externals: | |
| - spec: "[email protected]" | |
| prefix: /usr | |
| pkg-config: | |
| buildable: False | |
| externals: | |
| - spec: "[email protected]" | |
| prefix: /usr | |
| EOF | |
| spack external find | |
| spack spec flux-core@${FLUX_VERSION} | |
| if [[ $DYAD_DTL_MODE == 'UCX' ]]; then | |
| spack spec [email protected] | |
| fi | |
| - name: Install dependencies | |
| run: | | |
| . ${SPACK_DIR}/share/spack/setup-env.sh | |
| spack install -j4 flux-core@${FLUX_VERSION} | |
| if [[ $DYAD_DTL_MODE == 'UCX' ]]; then | |
| spack install -j4 [email protected] | |
| fi | |
| mkdir -p ${DYAD_INSTALL_PREFIX} | |
| spack view --verbose symlink ${DYAD_INSTALL_PREFIX} flux-core@${FLUX_VERSION} | |
| if [[ $DYAD_DTL_MODE == 'UCX' ]]; then | |
| spack view --verbose symlink ${DYAD_INSTALL_PREFIX} [email protected] | |
| fi | |
| - name: Compile DYAD | |
| run: | | |
| echo "Activating spack" | |
| . ${SPACK_DIR}/share/spack/setup-env.sh | |
| echo "Install DYAD" | |
| cd ${GITHUB_WORKSPACE} | |
| ls | |
| pwd | |
| ./autogen.sh | |
| CONFIGURE_FLAGS="" | |
| if [[ $DYAD_DTL_MODE == 'UCX' ]]; then | |
| CONFIGURE_FLAGS=" --enable-ucx " | |
| fi | |
| export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:${DYAD_INSTALL_PREFIX}/lib/pkgconfig | |
| ./configure --enable-dyad-debug ${CONFIGURE_FLAGS} --prefix=${DYAD_INSTALL_PREFIX} \ | |
| CFLAGS="-I${DYAD_INSTALL_PREFIX}/include" \ | |
| CXXFLAGS="-I${DYAD_INSTALL_PREFIX}/include" \ | |
| LDFLAGS="-L${DYAD_INSTALL_PREFIX}/lib" | |
| make install -j | |
| - name: Install PyDYAD | |
| if: ${{ matrix.test_mode == 'python' }} | |
| run: | | |
| cd ${GITHUB_WORKSPACE}/pydyad | |
| python3 -m pip install -e . | |
| cd ${GITHUB_WORKSPACE} | |
| - name: Install Test | |
| if: ${{ matrix.test_mode == 'c' || matrix.test_mode == 'cpp' }} | |
| run: | | |
| . ${SPACK_DIR}/share/spack/setup-env.sh | |
| export CFLAGS="-I${DYAD_INSTALL_PREFIX}/include" | |
| export CXXFLAGS="-I${DYAD_INSTALL_PREFIX}/include" | |
| export LDFLAGS="-L${DYAD_INSTALL_PREFIX}/lib" | |
| cd ${GITHUB_WORKSPACE}/docs/demos/ecp_feb_2023 | |
| make all | |
| - name: Test DYAD with separate FS | |
| run: | | |
| mkdir $DYAD_PATH | |
| . ${SPACK_DIR}/share/spack/setup-env.sh | |
| export PATH=${PATH}:${DYAD_INSTALL_PREFIX}/bin:${DYAD_INSTALL_PREFIX}/sbin | |
| export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${DYAD_INSTALL_PREFIX}/lib | |
| echo "Starting flux brokers" | |
| flux start --test-size=2 /bin/bash ${GITHUB_WORKSPACE}/.github/prod-cons/dyad_prod_cons_test.sh ${DYAD_TEST_MODE} | |