Merge pull request #28 from opendlang/ci_experimentation

Finishing the merge with dmd/ldc and making the CI work (at least

.github/actions/1-setup/action.yml

@@ -0,0 +1,166 @@
+name: Install prerequisites
+inputs:
+  clang_version:
+    required: true
+  llvm_version:
+    required: true
+  arch:
+    required: true
+runs:
+  using: composite
+  steps:
+
+    - name: 'Linux: Install required apt packages'
+      if: runner.os == 'Linux'
+      shell: bash
+      run: |
+        set -eux
+        export DEBIAN_FRONTEND=noninteractive
+        sudo dpkg --add-architecture i386
+        sudo apt-get -q update
+        # Don't use latest gdb v10+ from Ubuntu toolchain PPA with regressions, use official v9
+        sudo apt-get -yq install \
+          git-core cmake g++-multilib \
+          libcurl4 libcurl4:i386 \
+          curl gdb=9.1-0ubuntu1 p7zip-full tzdata unzip zip python3-pip
+
+    - name: 'Linux: Download & extract clang' # into ../clang
+      if: runner.os == 'Linux'
+      shell: bash
+      run: |
+        set -eux
+        cd ..
+        curl -fL --retry 3 --max-time 300 -o clang.tar.xz \
+          https://github.com/llvm/llvm-project/releases/download/llvmorg-${{ inputs.clang_version }}/clang+llvm-${{ inputs.clang_version }}-x86_64-linux-gnu-ubuntu-18.04.tar.xz
+        mkdir clang
+        tar -xf clang.tar.xz --strip 1 -C clang
+        rm clang.tar.xz
+        clang/bin/clang --version
+    - name: 'Windows: Install clang'
+      if: runner.os == 'Windows'
+      shell: bash
+      run: |
+        set -eux
+        cd ..
+        curl -fL --retry 3 --max-time 300 -o clang.exe \
+          https://github.com/llvm/llvm-project/releases/download/llvmorg-${{ inputs.clang_version }}/LLVM-${{ inputs.clang_version }}-win64.exe
+        ./clang.exe //S # double-slash for bash
+        rm clang.exe
+        # C:\Program Files\LLVM\bin should already be in PATH
+        clang-cl --version
+
+    - name: Download & extract LDC-flavoured LLVM # into ../llvm
+      shell: bash
+      run: |
+        set -eux
+        cd ..
+
+        version='${{ inputs.llvm_version }}'
+
+        if [[ "$version" = *.* ]]; then
+          tag="ldc-v$version"
+        else
+          tag=CI
+        fi
+
+        arch='${{ inputs.arch }}'
+
+        # use assertions for untagged builds
+        assertsSuffix="-withAsserts"
+        if [[ '${{ github.ref }}' = refs/tags/* ]]; then
+          assertsSuffix=""
+        fi
+
+        if [[ '${{ runner.os }}' == Windows ]]; then
+          curl -fL --retry 3 --max-time 300 -o llvm.7z \
+            https://github.com/ldc-developers/llvm-project/releases/download/$tag/llvm-$version-windows-$arch$assertsSuffix.7z
+          mkdir llvm
+          cd llvm
+          7z x ../llvm.7z >/dev/null
+          rm ../llvm.7z
+          cd ..
+        else
+          if [[ '${{ runner.os }}' == Linux ]]; then
+            os=linux
+          elif [[ '${{ runner.os }}' == macOS ]]; then
+            os=osx
+          fi
+          curl -fL --retry 3 --max-time 300 -o llvm.tar.xz \
+            https://github.com/ldc-developers/llvm-project/releases/download/$tag/llvm-$version-$os-$arch$assertsSuffix.tar.xz
+          mkdir llvm
+          tar -xf llvm.tar.xz --strip 1 -C llvm
+          rm llvm.tar.xz
+        fi
+
+        llvm/bin/llvm-config --version
+
+    - name: 'Linux: Make lld the default linker'
+      if: runner.os == 'Linux'
+      shell: bash
+      run: |
+        set -eux
+        sudo ln -sf "$(dirname "$PWD")/llvm/bin/ld.lld" /usr/bin/ld
+        ld --version
+
+    - name: Install ninja
+      uses: symmetryinvestments/gha-setup-ninja@v1
+
+    - name: Install D host compiler
+      uses: dlang-community/setup-dlang@v1
+      with:
+        compiler: ldc-latest
+    - name: 'Posix: Clear LD_LIBRARY_PATH env variable' # don't use host druntime/Phobos .so/.dylib etc.
+      if: runner.os != 'Windows'
+      shell: bash
+      run: echo "LD_LIBRARY_PATH=" >> $GITHUB_ENV
+
+    - name: Install lit
+      shell: bash
+      run: |
+        set -euxo pipefail
+        python3 --version
+        python3 -m pip install --user setuptools wheel
+        python3 -m pip install --user lit
+        python3 -c "import lit.main; lit.main.main();" --version . | head -n 1
+
+    # the druntime tests require GNU make
+    - name: 'Windows: Make sure GNU make is installed'
+      if: runner.os == 'Windows'
+      shell: cmd
+      run: make --version
+
+    - name: 'Windows: Download & extract libcurl' # into ../libcurl/ldc2
+      if: runner.os == 'Windows'
+      shell: bash
+      run: |
+        set -eux
+        cd ..
+        curl -fL --retry 3 --max-time 60 -o libcurl.7z \
+          https://github.com/ldc-developers/mingw-w64-libs/releases/download/v8.0.0/libcurl-7.74.0-zlib-static-ipv6-sspi-schannel.7z
+        mkdir libcurl
+        cd libcurl
+        7z x ../libcurl.7z >/dev/null
+        rm ../libcurl.7z
+        mkdir ldc2
+        if [[ '${{ inputs.arch }}' == x64 ]]; then
+          cp dmd2/windows/bin64/libcurl.dll ldc2/
+          cp dmd2/windows/lib64/*.* ldc2/
+        else
+          cp dmd2/windows/bin/libcurl.dll ldc2/
+          cp dmd2/windows/lib32mscoff/*.* ldc2/
+        fi
+        ls -lh ldc2/
+
+    - name: 'Windows: Set LDC_VSDIR env variable' # to somewhat speed-up MSVC auto-detection
+      if: runner.os == 'Windows'
+      shell: bash
+      run: echo "LDC_VSDIR=$(vswhere -latest -property installationPath)" >> $GITHUB_ENV
+
+    - name: 'Windows x86: Make CMake configure 64-bit clang-cl for 32-bit code emission'
+      if: runner.os == 'Windows' && inputs.arch == 'x86'
+      shell: bash
+      run: |
+        set -eux
+        echo "CFLAGS=-m32" >> $GITHUB_ENV
+        echo "CXXFLAGS=-m32" >> $GITHUB_ENV
+        echo "ASMFLAGS=-m32" >> $GITHUB_ENV

.github/actions/2-build-bootstrap/action.yml

@@ -0,0 +1,30 @@
+name: Build bootstrap LDC
+inputs:
+  cmake_flags:
+    required: false
+    default: ''
+  arch:
+    required: false # Windows only
+runs:
+  using: composite
+  steps:
+
+    - name: Check CMake and ninja versions
+      shell: bash
+      run: |
+        set -eux
+        cmake --version
+        ninja --version
+
+    - name: Build bootstrap LDC
+      uses: ./.github/actions/helper-build-ldc
+      with:
+        build_dir: bootstrap-ldc
+        host_dc: ldmd2
+        cmake_flags: >-
+          -DBUILD_SHARED_LIBS=OFF
+          ${{ inputs.cmake_flags }}
+        arch: ${{ inputs.arch }}
+
+    - run: ../bootstrap-ldc/bin/ldc2 --version
+      shell: bash

.github/actions/2a-build-pgo/action.yml

@@ -0,0 +1,32 @@
+name: Build LDC with PGO instrumentation & gather profile from compiling default libs
+inputs:
+  cmake_flags:
+    required: false
+    default: ''
+  arch:
+    required: false # Windows only
+runs:
+  using: composite
+  steps:
+
+    - name: Build LDC with PGO instrumentation & gather profiles from compiling default libs
+      uses: ./.github/actions/helper-build-ldc
+      with:
+        build_dir: pgo-ldc
+        host_dc: ../bootstrap-ldc/bin/ldmd2
+        # tweak -vp-counters-per-site to avoid `LLVM Profile Warning: Unable to track new values: Running out of static counters.`
+        cmake_flags: >-
+          -DBUILD_SHARED_LIBS=OFF
+          "-DDFLAGS_LDC=-fprofile-generate -vp-counters-per-site=1.5"
+          ${{ inputs.cmake_flags }}
+        arch: ${{ inputs.arch }}
+      env:
+        LLVM_PROFILE_FILE: ${{ github.workspace }}/../pgo-ldc/%p.profraw
+
+    - name: Merge PGO profiles # to ../pgo-ldc/merged.profdata
+      shell: bash
+      run: |
+        set -eux
+        cd ../pgo-ldc
+        ../bootstrap-ldc/bin/ldc-profdata merge --output=merged.profdata *.profraw
+        ls -lh *.prof{data,raw}

.github/actions/3-build-cross/action.yml

@@ -0,0 +1,162 @@
+name: Cross-compile LDC
+inputs:
+  arch:
+    required: true
+  os:
+    required: true
+  llvm_version:
+    required: true
+  cmake_flags:
+    required: false
+    default: ''
+  with_pgo:
+    required: false
+    default: false
+  android_ndk_version:
+    required: false
+    default: r21e
+  android_api_level:
+    required: false
+    default: 21
+runs:
+  using: composite
+  steps:
+
+    - name: Download & extract LDC-flavoured LLVM for ${{ inputs.os }}-${{ inputs.arch }} target # into ../llvm-cross
+      shell: bash
+      run: |
+        set -eux
+        cd ..
+        version='${{ inputs.llvm_version }}'
+        if [[ "$version" = *.* ]]; then
+          tag="ldc-v$version"
+        else
+          tag=CI
+        fi
+
+        if [[ '${{ inputs.os }}' == android ]]; then
+          assertsSuffix=""
+        else
+          # Use assertions for untagged builds. Must be the same as for the host LLVM package, because
+          # llvm-config of host package will be used for the cross build configuration.
+          assertsSuffix="-withAsserts"
+          if [[ '${{ github.ref }}' = refs/tags/* ]]; then
+            assertsSuffix=""
+          fi
+        fi
+
+        curl -fL --retry 3 --max-time 300 -o llvm-cross.tar.xz \
+          https://github.com/ldc-developers/llvm-project/releases/download/$tag/llvm-$version-${{ inputs.os }}-${{ inputs.arch }}$assertsSuffix.tar.xz
+        mkdir llvm-cross
+        tar -xf llvm-cross.tar.xz --strip 1 -C llvm-cross
+        rm llvm-cross.tar.xz
+
+    - name: Make non-native llvm-config runnable on host
+      shell: bash
+      run: |
+        set -eux
+        cd ..
+        if [[ '${{ inputs.os }}' == android ]]; then
+          # Android: use a bash script template
+          version="$(llvm/bin/llvm-config --version)" # from native LLVM
+          sed opend/.github/actions/3-build-cross/android-llvm-config.in \
+            -e "s|@LLVM_VERSION@|$version|g" \
+            -e "s|@LLVM_INSTALL_DIR@|$PWD/llvm-cross|g" \
+            -e "s|@LLVM_DEFAULT_TARGET_TRIPLE@|irrelevant-android-triple|g" \
+            -e "s|@LLVM_TARGETS@|AArch64 ARM X86 WebAssembly|g" \
+            > llvm-cross/bin/llvm-config
+          chmod 755 llvm-cross/bin/llvm-config
+        else
+          # copy from native LLVM
+          cp llvm/bin/llvm-config llvm-cross/bin/
+        fi
+
+    # set up DFLAGS to make bootstrap ldmd2 cross-compile/link
+    - name: 'macOS: Set DFLAGS, CROSS_TRIPLE & CROSS_CMAKE_FLAGS'
+      if: inputs.os == 'osx'
+      shell: bash
+      run: |
+        set -eux
+        cd ..
+        arch='${{ inputs.arch }}'
+        triple="$arch-apple-macos"
+        echo "DFLAGS=-mtriple=$triple -L-L$PWD/build-cross-libs/lib -Xcc=-target -Xcc=$triple -Xcc=-isysroot -Xcc=/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk" >> $GITHUB_ENV
+        echo "CROSS_TRIPLE=$triple" >> $GITHUB_ENV
+        echo "CROSS_CMAKE_FLAGS=-DCMAKE_OSX_ARCHITECTURES=$arch" >> $GITHUB_ENV
+
+    - name: 'Android: Download & extract NDK, then set DFLAGS, CROSS_TRIPLE & CROSS_CMAKE_FLAGS'
+      if: inputs.os == 'android'
+      shell: bash
+      run: |
+        set -eux
+        cd ..
+        version='${{ inputs.android_ndk_version }}'
+        curl -fL --retry 3 --max-time 300 -o android-ndk.zip \
+          https://dl.google.com/android/repository/android-ndk-$version-linux-x86_64.zip
+        unzip android-ndk.zip >/dev/null
+        mv "android-ndk-$version" android-ndk
+        rm android-ndk.zip
+
+        # The NDK toolchain file enforces `-g` as base C[XX] flag - remove it to
+        # *significantly* reduce executable sizes
+        toolchainFile="$PWD/android-ndk/build/cmake/android.toolchain.cmake"
+        sed -i 's|^  -g$||' "$toolchainFile"
+
+        arch='${{ inputs.arch }}'
+        apiLevel='${{ inputs.android_api_level }}'
+        cmakeFlags="-DTARGET_SYSTEM='Android;Linux;UNIX'"
+        if [[ "$arch" == armv7a ]]; then
+          triple="$arch-linux-androideabi$apiLevel"
+          cmakeFlags+=' -DANDROID_ABI=armeabi-v7a'
+        elif [[ "$arch" == aarch64 ]]; then
+          triple="$arch-linux-android$apiLevel"
+          cmakeFlags+=' -DANDROID_ABI=arm64-v8a'
+        fi
+        cmakeFlags+=" -DANDROID_NATIVE_API_LEVEL=$apiLevel"
+        cmakeFlags+=" -DANDROID_STL=c++_static"
+        cmakeFlags+=" -DCMAKE_TOOLCHAIN_FILE=$toolchainFile"
+        cmakeFlags+=" -DLDC_LINK_MANUALLY=ON -DD_LINKER_ARGS='-fuse-ld=bfd;-L$PWD/build-cross-libs/lib;-lphobos2-ldc;-ldruntime-ldc'"
+
+        echo "DFLAGS=-mtriple=$triple -fvisibility=hidden -L-L$PWD/build-cross-libs/lib -gcc=$PWD/android-ndk/toolchains/llvm/prebuilt/linux-x86_64/bin/$triple-clang" >> $GITHUB_ENV
+        echo "CROSS_TRIPLE=$triple" >> $GITHUB_ENV
+        echo "CROSS_CMAKE_FLAGS=$cmakeFlags" >> $GITHUB_ENV
+
+    - name: 'macOS: Build mimalloc'
+      if: inputs.os == 'osx'
+      uses: ./.github/actions/helper-mimalloc
+      with:
+        cmake_flags: ${{ env.CROSS_CMAKE_FLAGS }}
+
+    - name: Cross-compile default libraries
+      shell: bash
+      run: |
+        set -euxo pipefail
+        cd ..
+
+        flags='${{ inputs.cmake_flags }}' # may contain double-quotes
+        flags+=" $CROSS_CMAKE_FLAGS"
+        # convert to array, stripping all `-D` prefixes
+        IFS=$'\n' flags=( $(xargs -n1 <<<"$flags" | cut -b3-) )
+
+        echo $PWD
+
+        bootstrap-ldc/bin/ldc-build-runtime --ninja \
+          --buildDir="build-cross-libs" \
+          --dFlags="${DFLAGS// /;}" \
+          --ldcSrcDir="$PWD/opend/ldc" \
+          "${flags[@]}"
+
+    - name: Cross-compile LDC executables
+      uses: ./.github/actions/helper-build-ldc
+      with:
+        build_dir: build-cross
+        host_dc: ../bootstrap-ldc/bin/ldmd2
+        llvm_dir: llvm-cross
+        specify_install_dir: true
+        cmake_flags: >-
+          -DCMAKE_CROSSCOMPILING=True
+          ${{ inputs.os == 'osx' && '-DALTERNATIVE_MALLOC_O="$PWD/../build-mimalloc/CMakeFiles/mimalloc-obj.dir/src/static.c.o"' || '' }}
+          ${{ inputs.cmake_flags }}
+          ${{ inputs.with_pgo == 'true' && '-DDFLAGS_LDC=-fprofile-use=../pgo-ldc/merged.profdata' || '' }}
+          ${{ env.CROSS_CMAKE_FLAGS }}
+        build_targets: ldc2 ldmd2 ldc-build-runtime ldc-build-plugin ldc-profdata ldc-profgen ldc-prune-cache timetrace2txt