From 986855231d039371ce8ee9400cf5639b2e97f242 Mon Sep 17 00:00:00 2001 From: kuqin12 <42554914+kuqin12@users.noreply.github.com> Date: Wed, 31 May 2023 15:10:41 -0700 Subject: [PATCH] Adding support for self-hosted pools (#154) This change added support for self-hosted agent pools for potentially onboarding Windows on ARM systems as there is no plan for Microsoft hosted Windows ARM agents as of today. The main change is around the flag of installing tools or not. The existing routine will expect the incoming flag being a boolean type, which does not work when this parameter is being populated using a variable (string). This change will update the flag to string type as well. After changing, the only file being synced to individual repos is `MuDevOpsWrapper.yml`. And each repo can configure their own build matrix for build target, architecture list, run on selfhost agents or not, etc. This is considered a breaking change, due to: 1. `Steps/PrGate.yml` updated the `extra_install_steps` to be invoked before `stuart_setup` and `stuart_update`. But this change does not knowingly break the existing consumers. 2. `Jobs/PrGate.yml` updated the parameters to accept `parameters.build_matrix`, which needs the consumers to update top level yaml script to comply with the updated template. --------- Co-authored-by: Michael Kubacki --- .sync/Files.yml | 73 ++---------- .sync/azure_pipelines/MuDevOpsWrapper.yml | 90 +++++++++++++++ .../custom/mu_feature_config/Ubuntu-GCC5.yml | 46 -------- .../custom/mu_feature_config/Windows-VS.yml | 44 ------- .../custom/mu_feature_ipmi/Ubuntu-GCC5.yml | 39 ------- .../custom/mu_feature_ipmi/Windows-VS.yml | 37 ------ .../custom/mu_feature_mm_supv/Ubuntu-GCC5.yml | 40 ------- .../custom/mu_feature_mm_supv/Windows-VS.yml | 38 ------ .../custom/mu_oem_sample/Ubuntu-GCC5.yml | 40 ------- .../custom/mu_oem_sample/Windows-VS.yml | 38 ------ .../matrix_dependent/Ubuntu-CLANGPDB.yml | 40 ------- .../matrix_dependent/Ubuntu-GCC5.yml | 41 ------- .../matrix_dependent/Windows-CLANGPDB.yml | 41 ------- .../matrix_dependent/Windows-VS.yml | 37 ------ Jobs/PrGate.yml | 108 ++++++++++++------ Steps/PrGate.yml | 13 ++- 16 files changed, 182 insertions(+), 583 deletions(-) create mode 100644 .sync/azure_pipelines/MuDevOpsWrapper.yml delete mode 100644 .sync/azure_pipelines/custom/mu_feature_config/Ubuntu-GCC5.yml delete mode 100644 .sync/azure_pipelines/custom/mu_feature_config/Windows-VS.yml delete mode 100644 .sync/azure_pipelines/custom/mu_feature_ipmi/Ubuntu-GCC5.yml delete mode 100644 .sync/azure_pipelines/custom/mu_feature_ipmi/Windows-VS.yml delete mode 100644 .sync/azure_pipelines/custom/mu_feature_mm_supv/Ubuntu-GCC5.yml delete mode 100644 .sync/azure_pipelines/custom/mu_feature_mm_supv/Windows-VS.yml delete mode 100644 .sync/azure_pipelines/custom/mu_oem_sample/Ubuntu-GCC5.yml delete mode 100644 .sync/azure_pipelines/custom/mu_oem_sample/Windows-VS.yml delete mode 100644 .sync/azure_pipelines/matrix_dependent/Ubuntu-CLANGPDB.yml delete mode 100644 .sync/azure_pipelines/matrix_dependent/Ubuntu-GCC5.yml delete mode 100644 .sync/azure_pipelines/matrix_dependent/Windows-CLANGPDB.yml delete mode 100644 .sync/azure_pipelines/matrix_dependent/Windows-VS.yml diff --git a/.sync/Files.yml b/.sync/Files.yml index 97a05d15..09bc342c 100644 --- a/.sync/Files.yml +++ b/.sync/Files.yml @@ -27,76 +27,13 @@ # set up to use the YAML file. The file is simply available for pipelines to be connected # if needed. group: -# Azure Pipelines - Custom - microsoft/mu_feature_config - - files: - - source: .sync/azure_pipelines/custom/mu_feature_config/Ubuntu-GCC5.yml - dest: .azurepipelines/Ubuntu-GCC5.yml - template: true - - source: .sync/azure_pipelines/custom/mu_feature_config/Windows-VS.yml - dest: .azurepipelines/Windows-VS.yml - template: true - repos: | - microsoft/mu_feature_config - -# Azure Pipelines - Custom - microsoft/mu_feature_ipmi - - files: - - source: .sync/azure_pipelines/custom/mu_feature_ipmi/Ubuntu-GCC5.yml - dest: .azurepipelines/Ubuntu-GCC5.yml - template: true - - source: .sync/azure_pipelines/custom/mu_feature_ipmi/Windows-VS.yml - dest: .azurepipelines/Windows-VS.yml - template: true - repos: | - microsoft/mu_feature_ipmi - -# Azure Pipelines - Custom - microsoft/mu_feature_mm_supv - - files: - - source: .sync/azure_pipelines/custom/mu_feature_mm_supv/Ubuntu-GCC5.yml - dest: .azurepipelines/Ubuntu-GCC5.yml - template: true - - source: .sync/azure_pipelines/custom/mu_feature_mm_supv/Windows-VS.yml - dest: .azurepipelines/Windows-VS.yml - template: true - repos: | - microsoft/mu_feature_mm_supv - -# Azure Pipelines - Custom - microsoft/mu_oem_sample - - files: - - source: .sync/azure_pipelines/custom/mu_oem_sample/Ubuntu-GCC5.yml - dest: .azurepipelines/Ubuntu-GCC5.yml - template: true - - source: .sync/azure_pipelines/custom/mu_oem_sample/Windows-VS.yml - dest: .azurepipelines/Windows-VS.yml - template: true - repos: | - microsoft/mu_oem_sample - -# Azure Pipelines - Matrix Dependent - All - - files: - - source: .sync/azure_pipelines/matrix_dependent/Ubuntu-CLANGPDB.yml - dest: .azurepipelines/Ubuntu-CLANGPDB.yml - template: true - - source: .sync/azure_pipelines/matrix_dependent/Ubuntu-GCC5.yml - dest: .azurepipelines/Ubuntu-GCC5.yml - template: true - - source: .sync/azure_pipelines/matrix_dependent/Windows-CLANGPDB.yml - dest: .azurepipelines/Windows-CLANGPDB.yml - template: true - - source: .sync/azure_pipelines/matrix_dependent/Windows-VS.yml - dest: .azurepipelines/Windows-VS.yml - template: true - repos: | - microsoft/mu_basecore - # Azure Pipelines - Matrix Dependent - GCC & VS Only - files: - - source: .sync/azure_pipelines/matrix_dependent/Ubuntu-GCC5.yml - dest: .azurepipelines/Ubuntu-GCC5.yml - template: true - - source: .sync/azure_pipelines/matrix_dependent/Windows-VS.yml - dest: .azurepipelines/Windows-VS.yml + - source: .sync/azure_pipelines/MuDevOpsWrapper.yml + dest: .azurepipelines/MuDevOpsWrapper.yml template: true repos: | + microsoft/mu_basecore microsoft/mu_common_intel_min_platform microsoft/mu_feature_dfci microsoft/mu_plus @@ -104,6 +41,10 @@ group: microsoft/mu_silicon_intel_tiano microsoft/mu_tiano_platforms microsoft/mu_tiano_plus + microsoft/mu_feature_config + microsoft/mu_oem_sample + microsoft/mu_feature_ipmi + microsoft/mu_feature_mm_supv # Azure Pipelines - Ignored Repos # - microsoft/mu diff --git a/.sync/azure_pipelines/MuDevOpsWrapper.yml b/.sync/azure_pipelines/MuDevOpsWrapper.yml new file mode 100644 index 00000000..d911ba70 --- /dev/null +++ b/.sync/azure_pipelines/MuDevOpsWrapper.yml @@ -0,0 +1,90 @@ +## @file +# Azure Pipeline build file for a build using mu_devops. +# +# NOTE: This file is automatically synchronized from Mu DevOps. Update the original file there +# instead of the file in this repo. +# +# - Mu DevOps Repo: https://github.com/microsoft/mu_devops +# - File Sync Settings: https://github.com/microsoft/mu_devops/blob/main/.sync/Files.yml +# +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## + +{% import '../../../Version.njk' as sync_version -%} + +resources: + repositories: + - repository: mu_devops + type: github + endpoint: microsoft + name: microsoft/mu_devops + ref: refs/tags/{{ sync_version.mu_devops }} + +parameters: +- name: do_ci_build + displayName: Perform Stuart CI Build + type: boolean + default: true +- name: do_ci_setup + displayName: Perform Stuart CI Setup + type: boolean + default: true +- name: do_non_ci_build + displayName: Perform non-CI Stuart Build + type: boolean + default: false +- name: do_non_ci_setup + displayName: Perform non-CI Stuart Setup + type: boolean + default: false +- name: do_pr_eval + displayName: Perform Stuart PR Evaluation + type: boolean + default: true +- name: container_build + displayName: Flag for whether this repo should do stuart_setup + type: boolean + default: false +- name: os_type + displayName: OS type on the self-hosted agent pools + type: string + values: + - Windows_NT + - Linux + default: Windows_NT +- name: build_matrix + displayName: Build matrix for this repository + type: object +- name: pool_name + displayName: Variable name that hosts pool name to be used for self-hosted agents + type: string + default: pool_name +- name: extra_install_step + displayName: Extra Install Steps + type: stepList + default: + - script: echo No extra steps provided +- name: extra_jobs + displayName: Extra Jobs to be run after build + type: jobList + default: [] + +jobs: +- template: Jobs/PrGate.yml@mu_devops + parameters: + do_ci_build: ${{ parameters.do_ci_build }} + do_ci_setup: ${{ parameters.do_ci_setup }} + do_pr_eval: ${{ parameters.do_pr_eval }} + do_non_ci_setup: ${{ parameters.do_non_ci_setup }} + do_non_ci_build: ${{ parameters.do_non_ci_build }} + build_matrix: ${{ parameters.build_matrix }} + os_type: ${{ parameters.os_type }} + pool_name: ${{ parameters.pool_name }} + extra_install_step: ${{ parameters.extra_install_step }} + tool_chain_tag: $(tool_chain_tag) + vm_image: $(vm_image) + container_build: ${{ parameters.container_build }} + linux_container_image: {{ sync_version.linux_build_container }} + +- ${{ parameters.extra_jobs }} diff --git a/.sync/azure_pipelines/custom/mu_feature_config/Ubuntu-GCC5.yml b/.sync/azure_pipelines/custom/mu_feature_config/Ubuntu-GCC5.yml deleted file mode 100644 index 706ef8bf..00000000 --- a/.sync/azure_pipelines/custom/mu_feature_config/Ubuntu-GCC5.yml +++ /dev/null @@ -1,46 +0,0 @@ -## @file -# Azure Pipeline build file for a build using ubuntu and GCC5. -# -# NOTE: This file is automatically synchronized from Mu DevOps. Update the original file there -# instead of the file in this repo. -# -# - Mu DevOps Repo: https://github.com/microsoft/mu_devops -# - File Sync Settings: https://github.com/microsoft/mu_devops/blob/main/.sync/Files.yml -# -# Copyright (c) Microsoft Corporation. -# SPDX-License-Identifier: BSD-2-Clause-Patent -## - -{% import '../../../Version.njk' as sync_version -%} - -resources: - repositories: - - repository: mu_devops - type: github - endpoint: microsoft - name: microsoft/mu_devops - ref: refs/tags/{{ sync_version.mu_devops }} - -variables: -- group: architectures-arm-64-x86-64 -- group: tool-chain-ubuntu-gcc - -jobs: -- template: Jobs/PrGate.yml@mu_devops - parameters: - arch_list: $(arch_list) - do_ci_build: true - do_ci_setup: true - packages: SetupDataPkg - target_list: DEBUG,RELEASE,NO-TARGET,NOOPT - tool_chain_tag: $(tool_chain_tag) - vm_image: $(vm_image) - container_build: true - linux_container_image: {{ sync_version.linux_build_container }} - -- template: Jobs/Python/RunDevTests.yml@mu_devops - parameters: - code_cov_enabled: true - custom_job_name: ubuntu - root_package_folder: SetupDataPkg/Tools - vm_image: $(vm_image) diff --git a/.sync/azure_pipelines/custom/mu_feature_config/Windows-VS.yml b/.sync/azure_pipelines/custom/mu_feature_config/Windows-VS.yml deleted file mode 100644 index 434c8b50..00000000 --- a/.sync/azure_pipelines/custom/mu_feature_config/Windows-VS.yml +++ /dev/null @@ -1,44 +0,0 @@ -## @file -# Azure Pipeline build file for a build using Windows and Visual Studio. -# -# NOTE: This file is automatically synchronized from Mu DevOps. Update the original file there -# instead of the file in this repo. -# -# - Mu DevOps Repo: https://github.com/microsoft/mu_devops -# - File Sync Settings: https://github.com/microsoft/mu_devops/blob/main/.sync/Files.yml -# -# Copyright (c) Microsoft Corporation. -# SPDX-License-Identifier: BSD-2-Clause-Patent -## - -{% import '../../../Version.njk' as sync_version -%} - -resources: - repositories: - - repository: mu_devops - type: github - endpoint: microsoft - name: microsoft/mu_devops - ref: refs/tags/{{ sync_version.mu_devops }} - -variables: -- group: architectures-x86-64 -- group: tool-chain-windows-visual-studio-latest - -jobs: -- template: Jobs/PrGate.yml@mu_devops - parameters: - arch_list: $(arch_list) - do_ci_build: true - do_ci_setup: true - packages: SetupDataPkg - target_list: DEBUG,RELEASE,NO-TARGET,NOOPT - tool_chain_tag: $(tool_chain_tag) - vm_image: $(vm_image) - -- template: Jobs/Python/RunDevTests.yml@mu_devops - parameters: - code_cov_enabled: true - custom_job_name: windows - root_package_folder: SetupDataPkg/Tools - vm_image: $(vm_image) diff --git a/.sync/azure_pipelines/custom/mu_feature_ipmi/Ubuntu-GCC5.yml b/.sync/azure_pipelines/custom/mu_feature_ipmi/Ubuntu-GCC5.yml deleted file mode 100644 index fb48a136..00000000 --- a/.sync/azure_pipelines/custom/mu_feature_ipmi/Ubuntu-GCC5.yml +++ /dev/null @@ -1,39 +0,0 @@ -## @file -# Azure Pipeline build file for a build using ubuntu and GCC5. -# -# NOTE: This file is automatically synchronized from Mu DevOps. Update the original file there -# instead of the file in this repo. -# -# - Mu DevOps Repo: https://github.com/microsoft/mu_devops -# - File Sync Settings: https://github.com/microsoft/mu_devops/blob/main/.sync/Files.yml -# -# Copyright (c) Microsoft Corporation. -# SPDX-License-Identifier: BSD-2-Clause-Patent -## - -{% import '../../../Version.njk' as sync_version -%} - -resources: - repositories: - - repository: mu_devops - type: github - endpoint: microsoft - name: microsoft/mu_devops - ref: refs/tags/{{ sync_version.mu_devops }} - -variables: -- group: architectures-arm64-x86-64 -- group: tool-chain-ubuntu-gcc - -jobs: -- template: Jobs/PrGate.yml@mu_devops - parameters: - arch_list: $(arch_list) - do_ci_build: true - do_ci_setup: true - packages: IpmiFeaturePkg - target_list: DEBUG,RELEASE,NO-TARGET,NOOPT - tool_chain_tag: $(tool_chain_tag) - vm_image: $(vm_image) - container_build: true - linux_container_image: {{ sync_version.linux_build_container }} diff --git a/.sync/azure_pipelines/custom/mu_feature_ipmi/Windows-VS.yml b/.sync/azure_pipelines/custom/mu_feature_ipmi/Windows-VS.yml deleted file mode 100644 index 599a694a..00000000 --- a/.sync/azure_pipelines/custom/mu_feature_ipmi/Windows-VS.yml +++ /dev/null @@ -1,37 +0,0 @@ -## @file -# Azure Pipeline build file for a build using Windows and Visual Studio. -# -# NOTE: This file is automatically synchronized from Mu DevOps. Update the original file there -# instead of the file in this repo. -# -# - Mu DevOps Repo: https://github.com/microsoft/mu_devops -# - File Sync Settings: https://github.com/microsoft/mu_devops/blob/main/.sync/Files.yml -# -# Copyright (c) Microsoft Corporation. -# SPDX-License-Identifier: BSD-2-Clause-Patent -## - -{% import '../../../Version.njk' as sync_version -%} - -resources: - repositories: - - repository: mu_devops - type: github - endpoint: microsoft - name: microsoft/mu_devops - ref: refs/tags/{{ sync_version.mu_devops }} - -variables: -- group: architectures-arm64-x86-64 -- group: tool-chain-windows-visual-studio-latest - -jobs: -- template: Jobs/PrGate.yml@mu_devops - parameters: - arch_list: $(arch_list) - do_ci_build: true - do_ci_setup: true - packages: IpmiFeaturePkg - target_list: DEBUG,RELEASE,NO-TARGET,NOOPT - tool_chain_tag: $(tool_chain_tag) - vm_image: $(vm_image) diff --git a/.sync/azure_pipelines/custom/mu_feature_mm_supv/Ubuntu-GCC5.yml b/.sync/azure_pipelines/custom/mu_feature_mm_supv/Ubuntu-GCC5.yml deleted file mode 100644 index ff3eb39b..00000000 --- a/.sync/azure_pipelines/custom/mu_feature_mm_supv/Ubuntu-GCC5.yml +++ /dev/null @@ -1,40 +0,0 @@ -## @file -# Azure Pipeline build file for a build using ubuntu and GCC5. -# -# NOTE: This file is automatically synchronized from Mu DevOps. Update the original file there -# instead of the file in this repo. -# -# - Mu DevOps Repo: https://github.com/microsoft/mu_devops -# - File Sync Settings: https://github.com/microsoft/mu_devops/blob/main/.sync/Files.yml -# -# Copyright (c) Microsoft Corporation. -# SPDX-License-Identifier: BSD-2-Clause-Patent -## - -{% import '../../../Version.njk' as sync_version -%} - -resources: - repositories: - - repository: mu_devops - type: github - endpoint: microsoft - name: microsoft/mu_devops - ref: refs/tags/{{ sync_version.mu_devops }} - -variables: -- group: architectures-x86-64 -- group: tool-chain-ubuntu-gcc - -jobs: -- template: Jobs/PrGate.yml@mu_devops - parameters: - arch_list: $(arch_list) - do_ci_build: true - do_ci_setup: true - do_pr_eval: false - packages: MmSupervisorPkg - target_list: DEBUG,RELEASE,NO-TARGET,NOOPT - tool_chain_tag: $(tool_chain_tag) - vm_image: $(vm_image) - container_build: true - linux_container_image: {{ sync_version.linux_build_container }} diff --git a/.sync/azure_pipelines/custom/mu_feature_mm_supv/Windows-VS.yml b/.sync/azure_pipelines/custom/mu_feature_mm_supv/Windows-VS.yml deleted file mode 100644 index 640aec80..00000000 --- a/.sync/azure_pipelines/custom/mu_feature_mm_supv/Windows-VS.yml +++ /dev/null @@ -1,38 +0,0 @@ -## @file -# Azure Pipeline build file for a build using Windows and Visual Studio. -# -# NOTE: This file is automatically synchronized from Mu DevOps. Update the original file there -# instead of the file in this repo. -# -# - Mu DevOps Repo: https://github.com/microsoft/mu_devops -# - File Sync Settings: https://github.com/microsoft/mu_devops/blob/main/.sync/Files.yml -# -# Copyright (c) Microsoft Corporation. -# SPDX-License-Identifier: BSD-2-Clause-Patent -## - -{% import '../../../Version.njk' as sync_version -%} - -resources: - repositories: - - repository: mu_devops - type: github - endpoint: microsoft - name: microsoft/mu_devops - ref: refs/tags/{{ sync_version.mu_devops }} - -variables: -- group: architectures-x86-64 -- group: tool-chain-windows-visual-studio-latest - -jobs: -- template: Jobs/PrGate.yml@mu_devops - parameters: - arch_list: $(arch_list) - do_ci_build: true - do_ci_setup: true - do_pr_eval: false - packages: MmSupervisorPkg - target_list: DEBUG,RELEASE,NO-TARGET,NOOPT - tool_chain_tag: $(tool_chain_tag) - vm_image: $(vm_image) diff --git a/.sync/azure_pipelines/custom/mu_oem_sample/Ubuntu-GCC5.yml b/.sync/azure_pipelines/custom/mu_oem_sample/Ubuntu-GCC5.yml deleted file mode 100644 index 84f60b93..00000000 --- a/.sync/azure_pipelines/custom/mu_oem_sample/Ubuntu-GCC5.yml +++ /dev/null @@ -1,40 +0,0 @@ -## @file -# Azure Pipeline build file for a build using ubuntu and GCC5. -# -# NOTE: This file is automatically synchronized from Mu DevOps. Update the original file there -# instead of the file in this repo. -# -# - Mu DevOps Repo: https://github.com/microsoft/mu_devops -# - File Sync Settings: https://github.com/microsoft/mu_devops/blob/main/.sync/Files.yml -# -# Copyright (c) Microsoft Corporation. -# SPDX-License-Identifier: BSD-2-Clause-Patent -## - -{% import '../../../Version.njk' as sync_version -%} - -resources: - repositories: - - repository: mu_devops - type: github - endpoint: microsoft - name: microsoft/mu_devops - ref: refs/tags/{{ sync_version.mu_devops }} - -variables: -- group: architectures-arm-64-x86-64 -- group: tool-chain-ubuntu-gcc - -jobs: -- template: Jobs/PrGate.yml@mu_devops - parameters: - arch_list: $(arch_list) - do_ci_build: true - do_ci_setup: true - do_non_ci_setup: true - packages: OemPkg - target_list: DEBUG,RELEASE,NO-TARGET,NOOPT - tool_chain_tag: $(tool_chain_tag) - vm_image: $(vm_image) - container_build: true - linux_container_image: {{ sync_version.linux_build_container }} diff --git a/.sync/azure_pipelines/custom/mu_oem_sample/Windows-VS.yml b/.sync/azure_pipelines/custom/mu_oem_sample/Windows-VS.yml deleted file mode 100644 index 2465d35c..00000000 --- a/.sync/azure_pipelines/custom/mu_oem_sample/Windows-VS.yml +++ /dev/null @@ -1,38 +0,0 @@ -## @file -# Azure Pipeline build file for a build using Windows and Visual Studio. -# -# NOTE: This file is automatically synchronized from Mu DevOps. Update the original file there -# instead of the file in this repo. -# -# - Mu DevOps Repo: https://github.com/microsoft/mu_devops -# - File Sync Settings: https://github.com/microsoft/mu_devops/blob/main/.sync/Files.yml -# -# Copyright (c) Microsoft Corporation. -# SPDX-License-Identifier: BSD-2-Clause-Patent -## - -{% import '../../../Version.njk' as sync_version -%} - -resources: - repositories: - - repository: mu_devops - type: github - endpoint: microsoft - name: microsoft/mu_devops - ref: refs/tags/{{ sync_version.mu_devops }} - -variables: -- group: architectures-x86-64 -- group: tool-chain-windows-visual-studio-latest - -jobs: -- template: Jobs/PrGate.yml@mu_devops - parameters: - arch_list: $(arch_list) - do_ci_build: true - do_ci_setup: true - do_non_ci_setup: true - packages: OemPkg - target_list: DEBUG,RELEASE,NO-TARGET,NOOPT - tool_chain_tag: $(tool_chain_tag) - vm_image: $(vm_image) diff --git a/.sync/azure_pipelines/matrix_dependent/Ubuntu-CLANGPDB.yml b/.sync/azure_pipelines/matrix_dependent/Ubuntu-CLANGPDB.yml deleted file mode 100644 index de20ea67..00000000 --- a/.sync/azure_pipelines/matrix_dependent/Ubuntu-CLANGPDB.yml +++ /dev/null @@ -1,40 +0,0 @@ -## @file -# Azure Pipeline build file for a build using Ubuntu and CLANGPDB. -# -# IMPORTANT: This file requires the local repo to contain a .azurepipelines/Matrix-Build-Job-Clang.yml file that -# performs the actual build steps for the configurations and packages supported by the repo. -# -# NOTE: This file is automatically synchronized from Mu DevOps. Update the original file there -# instead of the file in this repo. -# -# - Mu DevOps Repo: https://github.com/microsoft/mu_devops -# - File Sync Settings: https://github.com/microsoft/mu_devops/blob/main/.sync/Files.yml -# -# Copyright (c) Microsoft Corporation. -# SPDX-License-Identifier: BSD-2-Clause-Patent -## - -{% import '../../Version.njk' as sync_version -%} - -resources: - repositories: - - repository: mu_devops - type: github - endpoint: microsoft - name: microsoft/mu_devops - ref: refs/tags/{{ sync_version.mu_devops }} - -variables: -- group: architectures-x86-64 -- group: tool-chain-clangpdb -- group: tool-chain-clangpdb-ubuntu-vm-image - -jobs: -- template: Matrix-Build-Job-Clang.yml - parameters: - arch_list: $(arch_list) - extra_steps: - - script: sudo apt-get install mingw-w64 - displayName: Install Windows Resource Compiler for Linux - tool_chain_tag: $(tool_chain_tag) - vm_image: $(vm_image) diff --git a/.sync/azure_pipelines/matrix_dependent/Ubuntu-GCC5.yml b/.sync/azure_pipelines/matrix_dependent/Ubuntu-GCC5.yml deleted file mode 100644 index 533baec7..00000000 --- a/.sync/azure_pipelines/matrix_dependent/Ubuntu-GCC5.yml +++ /dev/null @@ -1,41 +0,0 @@ -## @file -# Azure Pipeline build file for a build using Ubuntu and GCC. -# -# IMPORTANT: This file requires the local repo to contain a .azurepipelines/Matrix-Build-Job.yml file that -# performs the actual build steps for the configurations and packages supported by the repo. -# -# NOTE: This file is automatically synchronized from Mu DevOps. Update the original file there -# instead of the file in this repo. -# -# - Mu DevOps Repo: https://github.com/microsoft/mu_devops -# - File Sync Settings: https://github.com/microsoft/mu_devops/blob/main/.sync/Files.yml -# -# Copyright (c) Microsoft Corporation. -# SPDX-License-Identifier: BSD-2-Clause-Patent -## - -{% import '../../Version.njk' as sync_version -%} - -resources: - repositories: - - repository: mu_devops - type: github - endpoint: microsoft - name: microsoft/mu_devops - ref: refs/tags/{{ sync_version.mu_devops }} - containers: - - container: linux-gcc - image: {{ sync_version.linux_build_container }} - -variables: -- group: architectures-arm-64-x86-64 -- group: tool-chain-ubuntu-gcc - -jobs: -- template: Matrix-Build-Job.yml - parameters: - arch_list: $(arch_list) - extra_build_args: CODE_COVERAGE=TRUE - tool_chain_tag: 'GCC5' - vm_image: $(vm_image) - container_image: linux-gcc diff --git a/.sync/azure_pipelines/matrix_dependent/Windows-CLANGPDB.yml b/.sync/azure_pipelines/matrix_dependent/Windows-CLANGPDB.yml deleted file mode 100644 index 84f9d74e..00000000 --- a/.sync/azure_pipelines/matrix_dependent/Windows-CLANGPDB.yml +++ /dev/null @@ -1,41 +0,0 @@ -## @file -# Azure Pipeline build file for a build using Windows and CLANGPDB. -# -# IMPORTANT: This file requires the local repo to contain a .azurepipelines/Matrix-Build-Job-Clang.yml file that -# performs the actual build steps for the configurations and packages supported by the repo. -# -# NOTE: This file is automatically synchronized from Mu DevOps. Update the original file there -# instead of the file in this repo. -# -# - Mu DevOps Repo: https://github.com/microsoft/mu_devops -# - File Sync Settings: https://github.com/microsoft/mu_devops/blob/main/.sync/Files.yml -# -# Copyright (c) Microsoft Corporation. -# SPDX-License-Identifier: BSD-2-Clause-Patent -## - -{% import '../../Version.njk' as sync_version -%} - -resources: - repositories: - - repository: mu_devops - type: github - endpoint: microsoft - name: microsoft/mu_devops - ref: refs/tags/{{ sync_version.mu_devops }} - -variables: -- group: architectures-x86-64 -- group: tool-chain-clangpdb -- group: tool-chain-clangpdb-windows-vm-image - -jobs: -- template: Matrix-Build-Job-Clang.yml - parameters: - arch_list: $(arch_list) - extra_install_step: - - powershell: choco install llvm - displayName: Install LLVM and CLANG - condition: and(gt(variables.pkg_count, 0), succeeded()) - tool_chain_tag: $(tool_chain_tag) - vm_image: $(vm_image) diff --git a/.sync/azure_pipelines/matrix_dependent/Windows-VS.yml b/.sync/azure_pipelines/matrix_dependent/Windows-VS.yml deleted file mode 100644 index f3f9a24d..00000000 --- a/.sync/azure_pipelines/matrix_dependent/Windows-VS.yml +++ /dev/null @@ -1,37 +0,0 @@ -## @file -# Azure Pipeline build file for a build using Windows and the latest Visual Studio toolchain supported. -# -# IMPORTANT: This file requires the local repo to contain a .azurepipelines/Matrix-Build-Job.yml file that -# performs the actual build steps for the configurations and packages supported by the repo. -# -# NOTE: This file is automatically synchronized from Mu DevOps. Update the original file there -# instead of the file in this repo. -# -# - Mu DevOps Repo: https://github.com/microsoft/mu_devops -# - File Sync Settings: https://github.com/microsoft/mu_devops/blob/main/.sync/Files.yml -# -# Copyright (c) Microsoft Corporation. -# SPDX-License-Identifier: BSD-2-Clause-Patent -## - -{% import '../../Version.njk' as sync_version -%} - -resources: - repositories: - - repository: mu_devops - type: github - endpoint: microsoft - name: microsoft/mu_devops - ref: refs/tags/{{ sync_version.mu_devops }} - -variables: -- group: architectures-x86-64 -- group: tool-chain-windows-visual-studio-latest - -jobs: -- template: Matrix-Build-Job.yml - parameters: - arch_list: $(arch_list) - extra_build_args: CODE_COVERAGE=TRUE - tool_chain_tag: $(tool_chain_tag) - vm_image: $(vm_image) diff --git a/Jobs/PrGate.yml b/Jobs/PrGate.yml index 7729e3b2..264e0cb1 100644 --- a/Jobs/PrGate.yml +++ b/Jobs/PrGate.yml @@ -7,10 +7,6 @@ ## parameters: -- name: arch_list - displayName: Architectures (e.g. IA32, X64) - type: string - default: '' - name: build_file displayName: Stuart Build File type: string @@ -68,64 +64,110 @@ parameters: displayName: Virtual Machine Image (e.g. windows-latest) type: string default: 'windows-latest' +- name: extra_install_step + displayName: Extra Install Steps + type: stepList + default: [] +- name: pool_name + displayName: Variable name that hosts pool name to be used for self-hosted agents + type: string + default: '' +- name: os_type + displayName: OS type on the self-hosted agent pools + type: string + values: + - Windows_NT + - Linux + default: Windows_NT +- name: build_matrix + type: object + # Each element in the matrix will be used in a separate job + # Required fields: + # Pkgs: Specify what packages in the target repo are to be built for this job + # Targets: Specify which targets are to be built in this job, i.e. DEBUG, + # RELEASE, NO-TARGET, NOOPT. The acceptable values depend on the + # Pkgs to be built. + # ArchList: List of architectures to be supported by the packages (e.g. IA32, X64) + # Optional fields: + # SelfHostAgent: A boolean indicating whether this job should be run on the selfhosted + # "pool_name". If the matrix entry does not specify this element, it + # will be treated as false. + # Example: + # TARGET_MDEMODULE_DEBUG: + # Pkgs: 'MdeModulePkg' + # Targets: 'DEBUG,NOOPT' + # ArchList: 'IA32,X64,AARCH64' + # TARGET_TEST_POLICY_ARM: + # Pkgs: 'UnitTestFrameworkPkg,PolicyServicePkg' + # Targets: 'DEBUG,RELEASE,NO-TARGET,NOOPT' + # ArchList: 'IA32,X64' + # SelfHostAgent: true # Build step jobs: - - template: ../Jobs/CreateBuildMatrix.yml - parameters: - group_package_list: ${{ parameters.packages }} - target_list: ${{ parameters.target_list }} - - - job: Build - dependsOn: CreateBuildMatrix - condition: Succeeded('CreateBuildMatrix') - - # Use matrix to speed up the build process - strategy: - matrix: $[ dependencies.CreateBuildMatrix.outputs['CalculateMatrix.Matrix'] ] +- ${{ each item in parameters.build_matrix }}: + - job: Build_${{ item.Key }} + timeoutInMinutes: 120 + condition: | + and(not(Canceled()), + or(${{ ne(item.Value.SelfHostAgent, true) }}, ne(variables['${{ parameters.pool_name }}'], ''))) workspace: clean: all - pool: - vmImage: ${{ parameters.vm_image }} + ${{ if eq(item.Value.SelfHostAgent, true) }}: + pool: + name: $(${{ parameters.pool_name }}) + demands: + - Agent.OS -equals ${{ parameters.os_type }} + ${{ else }}: + pool: + vmImage: ${{ parameters.vm_image }} # Use a container if one was specified. - ${{ if and(eq(parameters.container_build, true), not(contains(parameters.vm_image, 'windows'))) }}: + ${{ if and(eq(parameters.container_build, true), not(contains(parameters.vm_image, 'windows')), ne(item.Value.SelfHostAgent, true)) }}: container: image: ${{ parameters.linux_container_image }} options: ${{ parameters.linux_container_options }} steps: # Add local path to ensure pip install modules are discoverable. - - ${{ if and(eq(parameters.container_build, true), not(contains(parameters.vm_image, 'windows'))) }}: + - ${{ if and(eq(parameters.container_build, true), not(contains(parameters.vm_image, 'windows')), ne(item.Value.SelfHostAgent, true)) }}: - script: echo "##vso[task.prependpath]/home/vsts_azpcontainer/.local/bin" displayName: Add User Local Bin to Path - ${{ parameters.extra_steps }} - template: ../Steps/PrGate.yml parameters: - artifacts_identifier: '$(package) $(target)' + artifacts_identifier: '${{ item.Key }} ${{ item.Value.Targets }}' build_file: ${{ parameters.build_file }} - build_pkgs: $(package) - build_targets: $(target) - build_archs: ${{ parameters.arch_list }} + build_pkgs: ${{ item.Value.Pkgs }} + build_targets: ${{ item.Value.Targets }} + build_archs: ${{ item.Value.ArchList }} do_ci_build: ${{ parameters.do_ci_build }} do_ci_setup: ${{ parameters.do_ci_setup }} do_non_ci_build: ${{ parameters.do_non_ci_build }} do_non_ci_setup: ${{ parameters.do_non_ci_setup }} do_pr_eval: ${{ parameters.do_pr_eval }} tool_chain_tag: ${{ parameters.tool_chain_tag }} - install_tools: ${{ not(parameters.container_build) }} + install_tools: ${{ and(not(eq(item.Value.SelfHostAgent, true)), not(parameters.container_build)) }} + extra_install_step: ${{ parameters.extra_install_step }} + # This is to handle the matrices that do not specify this. + ${{ if eq(item.Value.SelfHostAgent, true) }}: + self_host_agent: true + ${{ else }}: + self_host_agent: false extra_build_args: CODE_COVERAGE=TRUE - - job: PublishCoverage - dependsOn: Build - workspace: - clean: all +- job: PublishCoverage + dependsOn: + - ${{ each item in parameters.build_matrix }}: + - Build_${{ item.Key }} + workspace: + clean: all - pool: - vmImage: 'windows-latest' + pool: + vmImage: 'windows-latest' - steps: - - template: ../Steps/PublishCodeCoverage.yml + steps: + - template: ../Steps/PublishCodeCoverage.yml diff --git a/Steps/PrGate.yml b/Steps/PrGate.yml index 15c1a6ed..466f2e0f 100644 --- a/Steps/PrGate.yml +++ b/Steps/PrGate.yml @@ -74,6 +74,10 @@ parameters: displayName: Perform self checkout step type: boolean default: true +- name: self_host_agent + displayName: Indicate if this is a selfhost agent + type: boolean + default: false steps: - ${{ if eq(parameters.checkout_self, true) }}: @@ -112,6 +116,12 @@ steps: - ${{ if eq(parameters.install_tools, true) }}: - template: InstallCoverageTools.yml +# Potential Extra steps +- ${{ if eq(parameters.self_host_agent, false) }}: + - ${{ parameters.extra_install_step }} +- ${{ else }}: + - bash: echo "##[warning]A self-hosted agent build requested extra install steps. Those are not supported right now." + # Build repo - ${{ if eq(parameters.do_ci_setup, true) }}: - task: CmdLine@2 @@ -133,9 +143,6 @@ steps: script: stuart_update -c ${{ parameters.build_file }} -p $(pkgs_to_build) -t ${{ parameters.build_targets}} -a ${{ parameters.build_archs}} TOOL_CHAIN_TAG=${{ parameters.tool_chain_tag}} condition: and(gt(variables.pkg_count, 0), succeeded()) -# Potential Extra steps -- ${{ parameters.extra_install_step }} - - ${{ if eq(parameters.do_non_ci_build, true) }}: - task: CmdLine@2 displayName: Build and Test ${{ parameters.build_pkgs }} ${{ parameters.build_archs}}