From e1b6bfa8f096f00031327ad41cad5e10e6ba94f6 Mon Sep 17 00:00:00 2001 From: Matthias Frei Date: Wed, 13 Sep 2023 12:17:32 +0200 Subject: [PATCH 1/7] ci: upload binaries as artifacts Upload binaries built in the build step as build artifacts. Inlcudes some cleanup around the uploading of artifacts and related hooks. Now only the handling of the bazel-remote and go-module-proxy is left in the pre-command/pre-exit hooks. The rest is moved into step-specific metahook-hooks. The log artifacts are no longer combined into a single tar.gz. That has never seemed very convenient to me. Let's see if this works well. --- .buildkite/cleanup-leftovers.sh | 11 +++++ .buildkite/hooks/pre-artifact | 76 --------------------------------- .buildkite/hooks/pre-command | 11 ++--- .buildkite/hooks/pre-exit | 21 +-------- .buildkite/pipeline.yml | 32 ++++++++------ .buildkite/pipeline_lib.sh | 6 ++- 6 files changed, 42 insertions(+), 115 deletions(-) create mode 100755 .buildkite/cleanup-leftovers.sh delete mode 100755 .buildkite/hooks/pre-artifact diff --git a/.buildkite/cleanup-leftovers.sh b/.buildkite/cleanup-leftovers.sh new file mode 100755 index 0000000000..61d982171a --- /dev/null +++ b/.buildkite/cleanup-leftovers.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +echo "~~~ Cleaning up any leftovers" +cntrs="$(docker ps -aq | grep -v -f <(docker ps -q --filter "name=go-module-proxy" --filter "name=bazel-remote-cache"))" +[ -n "$cntrs" ] && { echo "Remove leftover containers..."; docker rm -f $cntrs; } +echo "Remove leftover networks" +docker network prune -f +echo "Remove leftover volumes" +docker volume prune -f + +rm -rf bazel-testlogs logs/* traces gen gen-cache /tmp/test-artifacts diff --git a/.buildkite/hooks/pre-artifact b/.buildkite/hooks/pre-artifact deleted file mode 100755 index 231320d4ed..0000000000 --- a/.buildkite/hooks/pre-artifact +++ /dev/null @@ -1,76 +0,0 @@ -#!/bin/bash - -set -eo pipefail - -# Attempt to do clean docker topology shutdown. -# This lets applications flush the logs to avoid cutting them off. -if [ -f "gen/scion-dc.yml" ]; then - # This hook is global, therefore we need to disable fail on error because - # not all docker-compose tests are following the pattern - # COMPOSE_PROJECT_NAME=scion, COMPOSE_FILE=gen/scion-dc.yml - # and logs in stdout. - set +e - docker-compose -f gen/scion-dc.yml -p scion stop - - docker-compose -f gen/scion-dc.yml -p scion logs --no-color > logs/scion-dc.log - - for s in $(docker-compose -f gen/scion-dc.yml -p scion ps --services); do - cat logs/scion-dc.log | grep $s| cut -f2 -d"|" > logs/${s#"scion_"}.log - done - - docker-compose -f gen/scion-dc.yml -p scion down -v - - # a subset of tests are using testgen and they do a collect log and dc stop - # on their own. Therefore the above code produces empty files that are confusing. - # Given the limitation of buildkite that the hook is global we just cleanup - # the empty files. - find . -type f -empty -delete - - set -e -fi - -# Now we build the artifact name next, for this we first need TARGET and BUILD, -# see below. -# -# For PRs the target is the pull request, otherwise it is the branch. -TARGET="$BUILDKITE_PULL_REQUEST" -if [ "$BUILDKITE_PULL_REQUEST" == "false" ]; then - TARGET="$BUILDKITE_BRANCH" -fi -TARGET="${TARGET//\//_}" -echo "\$TARGET=$TARGET" - -# For nightly builds instead of the build number print nightly and the date. -BUILD="build-${BUILDKITE_BUILD_NUMBER}" -[ -n "$NIGHTLY" ] && BUILD=nightly-"$(date +%s)" -echo "\$BUILD=$BUILD" - -ARTIFACTS="buildkite.${BUILDKITE_ORGANIZATION_SLUG}.${TARGET}.${BUILD}.${BUILDKITE_STEP_KEY:-unset}.${BUILDKITE_JOB_ID}" -mkdir -p "artifacts/$ARTIFACTS" artifacts.out - -function save { - if [ -d "$1" ]; then - echo Found artifacts: "$1" - cp -R "$1" "artifacts/$ARTIFACTS" - fi -} - -# Also store remote cache logs -cache_ctr=$(docker ps -aq -f "name=bazel-remote-cache") -if [ ! -z "$cache_ctr" ]; then - mkdir -p logs/docker - docker logs bazel-remote-cache > logs/docker/bazel-remote-cache.log -fi - -save "bazel-testlogs" -save "outputs" -save "logs" -save "traces" -save "gen" -save "gen-cache" -save "/tmp/test-artifacts" - -tar chaf "artifacts.out/$ARTIFACTS.tar.gz" -C artifacts "$ARTIFACTS" -rm -rf artifacts - -echo "Output tar= artifacts.out/$ARTIFACTS.tar.gz" diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index f43be60aee..0650d36442 100755 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -11,15 +11,12 @@ printenv PATH # Install build tools on first job for this runner. .buildkite/provision-agent.sh -# Clean up left-overs from previous run -PRE_COMMAND_SETUP=true . .buildkite/hooks/pre-exit - set -euo pipefail -echo "--- Increase receive network buffer size" +echo "~~~ Increase receive network buffer size" sudo sysctl -w net.core.rmem_max=1048576 -echo "--- Setting up bazel environment" +echo "~~~ Setting up bazel environment" if [ -z ${BAZEL_REMOTE_S3_ACCESS_KEY_ID+x} ]; then echo "S3 env not set, not starting bazel remote proxy" @@ -41,12 +38,12 @@ else fi echo "test --test_env CI" >> $HOME/.bazelrc -echo "--- Starting bazel remote cache proxy" +echo "~~~ Starting bazel remote cache proxy" # Start bazel remote cache proxy for S3 # Note that S3 keys are injected by buildkite, see # https://buildkite.com/docs/pipelines/secrets#storing-secrets-with-the-elastic-ci-stack-for-aws docker-compose -f .buildkite/hooks/bazel-remote.yml -p bazel_remote up -d -echo "--- Starting go module proxy" +echo "~~~ Starting go module proxy" docker-compose -f .buildkite/hooks/go-module-proxy.yml -p athens up -d diff --git a/.buildkite/hooks/pre-exit b/.buildkite/hooks/pre-exit index 1b85eab6d4..ba1758cc14 100644 --- a/.buildkite/hooks/pre-exit +++ b/.buildkite/hooks/pre-exit @@ -1,8 +1,7 @@ #!/bin/bash - -if [ -f ".buildkite/hooks/bazel-remote.yml" -a -z "$PRE_COMMAND_SETUP" ]; then - echo "--- Uploading bazel-remote and go-module-proxy logs/metrics" +if [ -f ".buildkite/hooks/bazel-remote.yml" ]; then + echo "~~~ Uploading bazel-remote and go-module-proxy logs/metrics" curl http://localhost:8080/metrics > bazel-remote-cache.metrics docker logs bazel-remote-cache &> bazel-remote-cache.log @@ -11,19 +10,3 @@ if [ -f ".buildkite/hooks/bazel-remote.yml" -a -z "$PRE_COMMAND_SETUP" ]; then buildkite-agent artifact upload "bazel-remote-cache.*;go-module-proxy.*" fi - -echo "--- Cleaning up the topology" - -./scion.sh topo_clean - -echo "--- Cleaning up docker containers/networks/volumes" -cntrs="$(docker ps -aq | grep -v -f <(docker ps -q --filter "name=go-module-proxy" --filter "name=bazel-remote-cache"))" -[ -n "$cntrs" ] && { echo "Remove leftover containers..."; docker rm -f $cntrs; } - -echo "Remove leftover networks" -docker network prune -f -echo "Remove leftover volumes" -docker volume prune -f - -echo "--- Cleaning up logs and artifacts" -rm -rf bazel-testlogs logs/* traces gen gen-cache /tmp/test-artifacts diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index aac4913f2f..53cd3e0302 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -6,6 +6,8 @@ steps: - bazel build --verbose_failures --announce_rc //:all - bazel run --verbose_failures //docker:prod //docker:test key: build + artifact_paths: + - "bin/*" retry: &automatic-retry automatic: - exit_status: -1 # Agent was lost @@ -17,7 +19,7 @@ steps: - bazel test --config=race --config=unit_all key: unit_tests artifact_paths: - - "artifacts.out/**/*" + - "bazel-testlogs/**/*" retry: *automatic-retry timeout_in_minutes: 20 - label: "Lint :bash:" @@ -69,14 +71,22 @@ steps: - tools/await-connectivity - ./bin/scion_integration || ( echo "^^^ +++" && false ) - ./bin/end2end_integration || ( echo "^^^ +++" && false ) - plugins: &shutdown-scion-post-command + plugins: &scion-run-hooks - scionproto/metahook#v0.3.0: + pre-command: .buildkite/cleanup-leftovers.sh post-command: | - echo "--- Shutting down SCION topology" + echo "~~~ Shutting down SCION topology" ./scion.sh stop - echo "SCION topology successfully shut down" - artifact_paths: - - "artifacts.out/**/*" + pre-artifacts: | + if [ -f "gen/scion-dc.yml" ]; then + tools/dc collect_logs scion logs/ + fi + pre-exit: .buildkite/cleanup-leftovers.sh + artifact_paths: &scion-run-artifact-paths + - "logs" + - "traces" + - "gen" + - "gen-cache" timeout_in_minutes: 15 key: e2e_integration_tests_v2 retry: *automatic-retry @@ -90,9 +100,8 @@ steps: - tools/await-connectivity - ./bin/end2end_integration || ( echo "^^^ +++" && false ) - ./tools/integration/revocation_test.sh - plugins: *shutdown-scion-post-command - artifact_paths: - - "artifacts.out/**/*" + plugins: *scion-run-hooks + artifact_paths: *scion-run-artifact-paths timeout_in_minutes: 15 key: e2e_revocation_test_v2 retry: *automatic-retry @@ -106,9 +115,8 @@ steps: - tools/await-connectivity - echo "--- run tests" - ./bin/end2end_integration -d || ( echo "^^^ +++" && false ) - plugins: *shutdown-scion-post-command - artifact_paths: - - "artifacts.out/**/*" + plugins: *scion-run-hooks + artifact_paths: *scion-run-artifact-paths timeout_in_minutes: 15 key: docker_integration_e2e_default retry: *automatic-retry diff --git a/.buildkite/pipeline_lib.sh b/.buildkite/pipeline_lib.sh index 7f1de50165..575d1833f1 100644 --- a/.buildkite/pipeline_lib.sh +++ b/.buildkite/pipeline_lib.sh @@ -44,8 +44,12 @@ gen_bazel_test_steps() { echo " command:" echo " - bazel test --test_output=streamed $test $args $cache" echo " key: \"${name////_}\"" + echo " plugins:" + echo " - scionproto/metahook#v0.3.0:" + echo " pre-command: .buildkite/cleanup-leftovers.sh" + echo " pre-exit: .buildkite/cleanup-leftovers.sh" echo " artifact_paths:" - echo " - \"artifacts.out/**/*\"" + echo " - \"bazel-testlogs/**/*\"" echo " timeout_in_minutes: 20" echo " retry:" echo " automatic:" From f0f1823ed420e76ca8f64c7b32ffba94071b8c69 Mon Sep 17 00:00:00 2001 From: Matthias Frei Date: Wed, 13 Sep 2023 13:32:37 +0200 Subject: [PATCH 2/7] fixup: paths, experiment with junit --- .buildkite/cleanup-leftovers.sh | 2 +- .buildkite/pipeline.yml | 19 ++++++++++--------- .buildkite/pipeline_lib.sh | 3 ++- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/.buildkite/cleanup-leftovers.sh b/.buildkite/cleanup-leftovers.sh index 61d982171a..cc1bb05b3e 100755 --- a/.buildkite/cleanup-leftovers.sh +++ b/.buildkite/cleanup-leftovers.sh @@ -8,4 +8,4 @@ docker network prune -f echo "Remove leftover volumes" docker volume prune -f -rm -rf bazel-testlogs logs/* traces gen gen-cache /tmp/test-artifacts +rm -rf bazel-testlogs logs/* traces gen gen-cache /tmp/test-artifacts test-out.tar.gz diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 53cd3e0302..140c1ae76b 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -3,11 +3,12 @@ env: steps: - label: "Build :bazel:" command: - - bazel build --verbose_failures --announce_rc //:all + - bazel build --verbose_failures --announce_rc //:scion //:scion-ci - bazel run --verbose_failures //docker:prod //docker:test key: build artifact_paths: - - "bin/*" + - "bazel-bin/scion.tar" + - "bazel-bin/scion-ci.tar" retry: &automatic-retry automatic: - exit_status: -1 # Agent was lost @@ -18,8 +19,10 @@ steps: command: - bazel test --config=race --config=unit_all key: unit_tests - artifact_paths: - - "bazel-testlogs/**/*" + plugins: + - test-collector#v1.0.0: + files: "bazel-testlogs/**/test.xml" + format: "junit" retry: *automatic-retry timeout_in_minutes: 20 - label: "Lint :bash:" @@ -77,16 +80,14 @@ steps: post-command: | echo "~~~ Shutting down SCION topology" ./scion.sh stop - pre-artifacts: | + pre-artifact: | if [ -f "gen/scion-dc.yml" ]; then tools/dc collect_logs scion logs/ fi + tar -chaf test-out.tar.gz $(ls -d logs traces gen gen-cache) # ls -d to filter missing directories pre-exit: .buildkite/cleanup-leftovers.sh artifact_paths: &scion-run-artifact-paths - - "logs" - - "traces" - - "gen" - - "gen-cache" + - test-out.tar.gz timeout_in_minutes: 15 key: e2e_integration_tests_v2 retry: *automatic-retry diff --git a/.buildkite/pipeline_lib.sh b/.buildkite/pipeline_lib.sh index 575d1833f1..ced606e47a 100644 --- a/.buildkite/pipeline_lib.sh +++ b/.buildkite/pipeline_lib.sh @@ -47,9 +47,10 @@ gen_bazel_test_steps() { echo " plugins:" echo " - scionproto/metahook#v0.3.0:" echo " pre-command: .buildkite/cleanup-leftovers.sh" + echo " pre-artifact: tar -chaf bazel-testlogs.tar.gz bazel-testlogs" echo " pre-exit: .buildkite/cleanup-leftovers.sh" echo " artifact_paths:" - echo " - \"bazel-testlogs/**/*\"" + echo " - \"bazel-testlogs.tar.gz\"" echo " timeout_in_minutes: 20" echo " retry:" echo " automatic:" From 4ccd17b66cadcc614e87608a1b163a7d055128b5 Mon Sep 17 00:00:00 2001 From: Matthias Frei Date: Wed, 13 Sep 2023 14:15:01 +0200 Subject: [PATCH 3/7] fixup: remove test collector, requires setup test analytics --- .buildkite/pipeline.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 140c1ae76b..db6446e432 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -20,9 +20,10 @@ steps: - bazel test --config=race --config=unit_all key: unit_tests plugins: - - test-collector#v1.0.0: - files: "bazel-testlogs/**/test.xml" - format: "junit" + - scionproto/metahook#v0.3.0: + pre-artifact: tar -chaf bazel-testlogs.tar.gz bazel-testlogs + artifact_paths: + - bazel-testlogs.tar.gz retry: *automatic-retry timeout_in_minutes: 20 - label: "Lint :bash:" From 469a138701b96415becb906e35ec3b8e9eac209e Mon Sep 17 00:00:00 2001 From: Matthias Frei Date: Tue, 19 Sep 2023 11:25:14 +0200 Subject: [PATCH 4/7] ci: annotate build step to make it easier to find artifacts --- .buildkite/pipeline.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index db6446e432..2b395acf2d 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -9,6 +9,13 @@ steps: artifact_paths: - "bazel-bin/scion.tar" - "bazel-bin/scion-ci.tar" + plugins: + - scionproto/metahook#v0.3.0: + post-artifact: | + cat << EOF | buildkite-agent annotate + - SCION binaries + - SCION test tools and utilities binaries + EOF retry: &automatic-retry automatic: - exit_status: -1 # Agent was lost From 110adf42a814b157832fa16efce170785a5dea2f Mon Sep 17 00:00:00 2001 From: Matthias Frei Date: Tue, 19 Sep 2023 11:40:28 +0200 Subject: [PATCH 5/7] fixup: style, README badge --- .buildkite/pipeline.yml | 3 ++- README.md | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 2b395acf2d..3cb00fdb4f 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -12,7 +12,8 @@ steps: plugins: - scionproto/metahook#v0.3.0: post-artifact: | - cat << EOF | buildkite-agent annotate + cat << EOF | buildkite-agent annotate --style "info" + #### Build outputs - SCION binaries - SCION test tools and utilities binaries EOF diff --git a/README.md b/README.md index e98ad5d610..7fb110f61e 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ [![Slack chat](https://img.shields.io/badge/chat%20on-slack-blue?logo=slack)](https://scionproto.slack.com) [![ReadTheDocs](https://img.shields.io/badge/doc-reference-blue?version=latest&style=flat&label=docs&logo=read-the-docs&logoColor=white)](https://docs.scion.org/en/latest) [![Documentation](https://img.shields.io/badge/go.dev-reference-007d9c?logo=go&logoColor=white)](https://pkg.go.dev/github.com/scionproto/scion) -[![Build Status](https://badge.buildkite.com/e7ca347d947c23883ad7c3a4d091c2df5ae7feb52b238d29a1.svg?branch=master)](https://buildkite.com/scionproto/scion) +[![Nightly Build](https://badge.buildkite.com/b70b65b38a75eb8724f41a6f1203c9327cfb767f07a0c1934e.svg)](https://buildkite.com/scionproto/scion-nightly/builds/latest) [![Go Report Card](https://goreportcard.com/badge/github.com/scionproto/scion)](https://goreportcard.com/report/github.com/scionproto/scion) [![GitHub issues](https://img.shields.io/github/issues/scionproto/scion/help%20wanted.svg?label=help%20wanted&color=purple)](https://github.com/scionproto/scion/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22) [![GitHub issues](https://img.shields.io/github/issues/scionproto/scion/good%20first%20issue.svg?label=good%20first%20issue&color=purple)](https://github.com/scionproto/scion/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22) From 249b5eb9011c12d3dba427ab3ab9ad3d2e35972f Mon Sep 17 00:00:00 2001 From: Matthias Frei Date: Tue, 19 Sep 2023 12:07:59 +0200 Subject: [PATCH 6/7] doc: mention pre-built binaries in readme --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 7fb110f61e..865d6074e9 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,8 @@ To find out how to work with SCION, please visit our [documentation site](https://docs.scion.org/en/latest/dev/setup.html) for instructions on how to install build dependencies, build and run SCION. +Pre-built binaries for x86-64 Linux are also available from the [latest nightly build](https://buildkite.com/scionproto/scion-nightly/builds/latest). + ## Contributing Interested in contribution to the SCION project? Please visit our From 233f35580e61e92c12c825e51fbead57bc03ae67 Mon Sep 17 00:00:00 2001 From: Matthias Frei Date: Tue, 19 Sep 2023 14:53:58 +0200 Subject: [PATCH 7/7] fixup: doc phrasing --- .buildkite/pipeline.yml | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 3cb00fdb4f..65ed0a2038 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -15,7 +15,7 @@ steps: cat << EOF | buildkite-agent annotate --style "info" #### Build outputs - SCION binaries - - SCION test tools and utilities binaries + - SCION test tools and utilities EOF retry: &automatic-retry automatic: diff --git a/README.md b/README.md index 865d6074e9..0c3016ad61 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ To find out how to work with SCION, please visit our [documentation site](https://docs.scion.org/en/latest/dev/setup.html) for instructions on how to install build dependencies, build and run SCION. -Pre-built binaries for x86-64 Linux are also available from the [latest nightly build](https://buildkite.com/scionproto/scion-nightly/builds/latest). +Pre-built binaries for x86-64 Linux are available from the [latest nightly build](https://buildkite.com/scionproto/scion-nightly/builds/latest). ## Contributing