Merge branch 'master' into FI-1512-move-rosetta-dependencies

.github/workflows-source/ci-main.yml

@@ -259,7 +259,7 @@ jobs:
           rm -rf "/cache/job/${CI_JOB_NAME}/${CI_RUN_ID}"
         env:
           BAZEL_COMMAND: "build"
-          RUN_ON_DIFF_ONLY: ${{ !contains(github.event.pull_request.labels.*.name, 'CI_ALL_BAZEL_TARGETS') }}
+          RUN_ON_DIFF_ONLY: ${{ github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'CI_ALL_BAZEL_TARGETS') }}
           MERGE_BASE_SHA: ${{ github.event.pull_request.base.sha }}
       - name: Upload build-ic.tar
         uses: actions/upload-artifact@v4

.github/workflows/ci-main.yml

@@ -345,7 +345,7 @@ jobs:
           rm -rf "/cache/job/${CI_JOB_NAME}/${CI_RUN_ID}"
         env:
           BAZEL_COMMAND: "build"
-          RUN_ON_DIFF_ONLY: ${{ !contains(github.event.pull_request.labels.*.name, 'CI_ALL_BAZEL_TARGETS') }}
+          RUN_ON_DIFF_ONLY: ${{ github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'CI_ALL_BAZEL_TARGETS') }}
           MERGE_BASE_SHA: ${{ github.event.pull_request.base.sha }}
       - name: Upload build-ic.tar
         uses: actions/upload-artifact@v4

ci/scripts/run-build-ic.sh

@@ -18,9 +18,8 @@ done
 # run build with release on protected branches or if a pull_request is targeting an rc branch
 if [ "${IS_PROTECTED_BRANCH:-}" == "true" ] || [[ "${CI_MERGE_REQUEST_TARGET_BRANCH_NAME:-}" == "rc--"* ]]; then
     ci/container/build-ic.sh -i -c -b
-fi
 # check if the job requested running only on diff, otherwise run full build with no release
-if [[ "${RUN_ON_DIFF_ONLY:-}" == "true" ]]; then
+elif [[ "${RUN_ON_DIFF_ONLY:-}" == "true" ]]; then
     TARGETS=$(ci/bazel-scripts/diff.sh)
     ARGS=(--no-release)
 
@@ -39,6 +38,10 @@ if [[ "${RUN_ON_DIFF_ONLY:-}" == "true" ]]; then
     fi
 
     if [ ${#ARGS[@]} -eq 1 ]; then
+        if [ "${IS_PROTECTED_BRANCH:-}" == "true" ]; then
+            echo "Error: No changes to build on protected branch. Aborting."
+            exit 1
+        fi
         echo "No changes that require building IC-OS, binaries or canisters."
         touch build-ic.tar
         exit 0

ic-os/boundary-guestos/context/Dockerfile

@@ -26,7 +26,7 @@ WORKDIR /tmp
 # Download and verify ic-gateway
 RUN \
     curl -L -O https://github.com/dfinity/ic-gateway/releases/download/v0.1.55/ic-gateway_0.1.55_amd64.deb && \
-    echo "57ef6ed420bca99cc2e5857a862d630d6c1f5ab3  ic-gateway_0.1.55_amd64.deb" | shasum -c
+    echo "b360fa670fef8d097fc814b718ea5333a7de5c52d654c8f850a55bb014b006e1  ic-gateway_0.1.55_amd64.deb" | sha256sum -c
 
 #
 # Second build stage:
@@ -177,15 +177,6 @@ RUN addgroup node_exporter && \
     adduser --system --disabled-password --shell /usr/sbin/nologin --no-create-home socks && \
     adduser socks socks && chmod +s /usr/sbin/danted
 
-# Setup prober
-RUN \
-    addgroup prober && \
-    adduser --system --disabled-password --shell /usr/sbin/nologin --no-create-home prober && \
-    adduser prober prober && \
-    mkdir -p /var/log/prober && \
-    chown prober:prober /var/log/prober && \
-    chmod 755 /var/log/prober
-
 # Install node_exporter
 RUN chown root:root /etc/node_exporter \
                     /usr/local/bin/node_exporter && \

ic-os/boundary-guestos/context/Dockerfile.base

@@ -16,42 +16,18 @@ ARG CROWDSEC_BOUNCER_VERSION=1.0.14
 ARG CROWDSEC_BOUNCER_SHA=c99919c6ecc69904249ff71b32ea791ce2eec32e5a9e8d4c9c6554131a4fb331
 
 # Vector version to install
-ARG VECTOR_VERSION=0.36.0
-ARG VECTOR_SHA=1e7c8d47768a8286c0de517aeb6d7b12e85d91d11c2d282583827c92b83083cf
+ARG VECTOR_VERSION=0.41.1
+ARG VECTOR_SHA=a4fe30ec0e9ef959bac8b6de013a28e22890f6e5a5afd00d42ef7affaf432442
 
 # Node exporter version to install
-ARG NODE_EXPORTER_VERSION=1.8.1
-ARG NODE_EXPORTER_SHA=fbadb376afa7c883f87f70795700a8a200f7fd45412532cc1938a24d41078011
+ARG NODE_EXPORTER_VERSION=1.8.2
+ARG NODE_EXPORTER_SHA=6809dd0b3ec45fd6e992c19071d6b5253aed3ead7bf0686885a51d85c6643c66
 
 #
-# First build stage: download software, build and verify it (such that it
-# does not change under our noses).
+# - Download and cache minimal Ubuntu Server 24.04 LTS Docker image
+# - Install and cache upstream packages from built-in Ubuntu repositories
 #
-FROM ubuntu:20.04 AS download
-
-USER root:root
-
-ENV TZ=UTC
-RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
-RUN apt-get -y update && apt-get -y upgrade && apt-get -y --no-install-recommends install \
-    ca-certificates \
-    curl \
-    pkg-config \
-    libffi-dev \
-    libssl-dev \
-    ssl-cert \
-    rustc \
-    cargo \
-    perl \
-    git \
-    autoconf \
-    make \
-    automake \
-    g++ \
-    gcc \
-    uuid-dev
-
-WORKDIR /tmp
+FROM ubuntu:24.04
 
 ARG CROWDSEC_BOUNCER_VERSION
 ARG CROWDSEC_BOUNCER_SHA
@@ -60,29 +36,6 @@ ARG VECTOR_SHA
 ARG NODE_EXPORTER_VERSION
 ARG NODE_EXPORTER_SHA
 
-# Download and verify vector
-RUN \
-    curl -L -O https://packages.timber.io/vector/${VECTOR_VERSION}/vector_${VECTOR_VERSION}-1_amd64.deb && \
-    echo "${VECTOR_SHA}  vector_${VECTOR_VERSION}-1_amd64.deb" | shasum -c
-
-# Download and verify node_exporter
-RUN \
-    curl -L -O https://github.com/prometheus/node_exporter/releases/download/v${NODE_EXPORTER_VERSION}/node_exporter-${NODE_EXPORTER_VERSION}.linux-amd64.tar.gz && \
-    echo "${NODE_EXPORTER_SHA}  node_exporter-${NODE_EXPORTER_VERSION}.linux-amd64.tar.gz" | shasum -c
-
-# Download and verify crowdsec bouncer
-RUN \
-    curl -L -O https://github.com/blind-oracle/cs-firewall-bouncer/releases/download/v${CROWDSEC_BOUNCER_VERSION}/crowdsec-firewall-bouncer-nftables_${CROWDSEC_BOUNCER_VERSION}_amd64.deb && \
-    echo "${CROWDSEC_BOUNCER_SHA}  crowdsec-firewall-bouncer-nftables_${CROWDSEC_BOUNCER_VERSION}_amd64.deb" | shasum -c
-
-#
-# Second build stage:
-# - Download and cache minimal Ubuntu Server 20.04 LTS Docker image
-# - Install and cache upstream packages from built-in Ubuntu repositories
-# - Copy downloaded archives from first build stage into the target image
-#
-FROM ubuntu:20.04
-
 USER root:root
 
 ENV TZ=UTC
@@ -106,54 +59,60 @@ RUN \
         gettext-base \
         haveged \
         initramfs-tools \
-        inotify-tools \
         iproute2 \
         iputils-ping \
-        isc-dhcp-client \
         less \
-        libffi-dev \
-        liblmdb0 \
-        libssl-dev \
-        linux-image-virtual-hwe-20.04 \
         logrotate \
         lshw \
         lvm2 \
+        nano \
         net-tools \
         netbase \
         nftables \
-        opensc \
-        openssh-client openssh-server \
+        openssh-client \
+        openssh-server \
         parted \
-        pciutils \
-        pcsc-tools pcscd \
-        policycoreutils \
-        python \
-        python3-dev \
         rsync \
         ssl-cert \
         sudo \
-        systemd systemd-sysv systemd-journal-remote \
+        systemd \
+        systemd-sysv \
+        systemd-journal-remote \
+        tcpdump \
+        traceroute \
+        mtr-tiny \
         udev \
         usbutils \
         vim \
         zstd
 
+WORKDIR /tmp
+
+# Download and verify vector
+RUN \
+    curl -L -O https://packages.timber.io/vector/${VECTOR_VERSION}/vector_${VECTOR_VERSION}-1_amd64.deb && \
+    echo "${VECTOR_SHA}  vector_${VECTOR_VERSION}-1_amd64.deb" | sha256sum -c
+
+# Download and verify node_exporter
+RUN \
+    curl -L -O https://github.com/prometheus/node_exporter/releases/download/v${NODE_EXPORTER_VERSION}/node_exporter-${NODE_EXPORTER_VERSION}.linux-amd64.tar.gz && \
+    echo "${NODE_EXPORTER_SHA}  node_exporter-${NODE_EXPORTER_VERSION}.linux-amd64.tar.gz" | sha256sum -c
+
+# Download and verify crowdsec bouncer
+RUN \
+    curl -L -O https://github.com/blind-oracle/cs-firewall-bouncer/releases/download/v${CROWDSEC_BOUNCER_VERSION}/crowdsec-firewall-bouncer-nftables_${CROWDSEC_BOUNCER_VERSION}_amd64.deb && \
+    echo "${CROWDSEC_BOUNCER_SHA}  crowdsec-firewall-bouncer-nftables_${CROWDSEC_BOUNCER_VERSION}_amd64.deb" | sha256sum -c
+
 # Install vector
-ARG VECTOR_VERSION
-COPY --from=download /tmp/vector_${VECTOR_VERSION}-1_amd64.deb /tmp/vector_${VECTOR_VERSION}-1_amd64.deb
 RUN dpkg -i --force-confold /tmp/vector_${VECTOR_VERSION}-1_amd64.deb && \
     rm /tmp/vector_${VECTOR_VERSION}-1_amd64.deb
 
 # Install node_exporter
-ARG NODE_EXPORTER_VERSION
-COPY --from=download /tmp/node_exporter-${NODE_EXPORTER_VERSION}.linux-amd64.tar.gz /tmp/node_exporter-${NODE_EXPORTER_VERSION}.linux-amd64.tar.gz
 RUN cd /tmp/ && \
     mkdir -p /etc/node_exporter && \
     tar --strip-components=1 -C /usr/local/bin/ -zvxf node_exporter-${NODE_EXPORTER_VERSION}.linux-amd64.tar.gz node_exporter-${NODE_EXPORTER_VERSION}.linux-amd64/node_exporter && \
     rm /tmp/node_exporter-${NODE_EXPORTER_VERSION}.linux-amd64.tar.gz
 
 # Install crowdsec bouncer
-ARG CROWDSEC_BOUNCER_VERSION
-COPY --from=download /tmp/crowdsec-firewall-bouncer-nftables_${CROWDSEC_BOUNCER_VERSION}_amd64.deb /tmp/crowdsec-firewall-bouncer-nftables_${CROWDSEC_BOUNCER_VERSION}_amd64.deb
 RUN dpkg -i --force-confold /tmp/crowdsec-firewall-bouncer-nftables_${CROWDSEC_BOUNCER_VERSION}_amd64.deb && \
     rm /tmp/crowdsec-firewall-bouncer-nftables_${CROWDSEC_BOUNCER_VERSION}_amd64.deb

ic-os/boundary-guestos/defs.bzl

@@ -23,9 +23,7 @@ def image_deps(mode):
         "bootfs": {},
         "rootfs": {
             "//publish/binaries:canary-proxy": "/opt/ic/bin/canary-proxy:0755",
-            "//publish/binaries:boundary-node-prober": "/opt/ic/bin/boundary-node-prober:0755",
             "//publish/binaries:certificate-issuer": "/opt/ic/bin/certificate-issuer:0755",
-            "//publish/binaries:ic-balance-exporter": "/opt/ic/bin/ic-balance-exporter:0755",
             "//publish/binaries:systemd-journal-gatewayd-shim": "/opt/ic/bin/systemd-journal-gatewayd-shim:0755",
             "//publish/binaries:ic-boundary": "/opt/ic/bin/ic-boundary:0755",
         },

ic-os/boundary-guestos/docs/Components.adoc

@@ -5,10 +5,6 @@ The Boundary Node has a number of specific components:
 - `ic-gateway` is used to proxy HTTP calls to `ic-boundary` using IC protocol
 - `ic-boundary` forwards the IC requests to replicas
 
-== Prober
-- Creates canisters that are probed
-- Performs dynamic probes and export the results to Prometheus
-
 == Socks proxy
 - A SOCKS proxy will be running on port 1443 and 1080 on the boundary nodes,
  only accessible from IPv6 netblock addresses.

ic-os/boundary-guestos/scripts/build-bootstrap-config-image.sh

@@ -77,9 +77,6 @@ options may be specified:
   --denylist_url url
     Specify the url where to download denylist
 
-  --prober-identity path
-    specify an identity file for the prober
-
   --system-domains
     comma-delimited list of domains serving system canisters (e.g., ic0.dev or ic0.app)
 
@@ -292,9 +289,6 @@ function build_ic_bootstrap_tar() {
             --denylist_url)
                 local DENYLIST_URL="$2"
                 ;;
-            --prober-identity)
-                local PROBER_IDENTITY="$2"
-                ;;
             --system-domains)
                 local SYSTEM_DOMAINS="$2"
                 ;;
@@ -519,12 +513,6 @@ logging_user=${LOGGING_USER:-"undefined"}
 logging_password=${LOGGING_PASSWORD:-"undefined"}
 EOF
 
-    # setup the prober identity
-    if [[ -n "${PROBER_IDENTITY:-}" ]]; then
-        echo "Using prober identity ${PROBER_IDENTITY}"
-        cp "${PROBER_IDENTITY}" "${BOOTSTRAP_TMPDIR}/prober_identity.pem"
-    fi
-
     # setup the certificates
     if [[ -n "${CERT_DIR:-}" && -f "${CERT_DIR}/fullchain.pem" && -f "${CERT_DIR}/privkey.pem" && -f "${CERT_DIR}/chain.pem" ]]; then
         echo "Using certificates ${CERT_DIR}/fullchain.pem ${CERT_DIR}/privkey.pem ${CERT_DIR}/chain.pem"

ic-os/boundary-guestos/scripts/build-deployment.sh

@@ -43,7 +43,6 @@ Arguments:
        --nns_urls=                      specify a file that lists on each line a nns url of the form `http://[ip]:port` this file will override nns urls derived from input json file
        --replicas-ipv6=                 specify a file that lists on each line an ipv6 firewall rule to allow replicas of the form `ipv6-addr/prefix-length` (# comments and trailing whitespace will be stripped)
        --denylist=                      a deny list of canisters
-       --prober-identity=               specify an identity file for the prober
        --geolite2-country-db=           specify path to GeoLite2 Country Database
        --cert-issuer-creds              specify a credentials file for certificate-issuer
        --cert-issuer-identity           specify an identity file for certificate-issuer
@@ -102,9 +101,6 @@ for argument in "${@}"; do
         --denylist=*)
             DENY_LIST="${argument#*=}"
             ;;
-        --prober-identity=*)
-            PROBER_IDENTITY="${argument#*=}"
-            ;;
         --geolite2-country-db=*)
             GEOLITE2_COUNTRY_DB="${argument#*=}"
             ;;
@@ -206,20 +202,18 @@ VALUES=$(echo ${CONFIG} \
     .ipv6_gateway,
     .ipv4_gateway,
     .ipv4_address,
-    .prober,
     .hostname,
     .subnet_type,
     .subnet_idx,
     .node_idx,
     .type
 ] | join("\u0001")')
-while IFS=$'\1' read -r ipv6_address ipv6_gateway ipv4_gateway ipv4_address prober hostname subnet_type subnet_idx node_idx type; do
+while IFS=$'\1' read -r ipv6_address ipv6_gateway ipv4_gateway ipv4_address hostname subnet_type subnet_idx node_idx type; do
     eval "declare -A __RAW_NODE_$NODES=(
         ['ipv6_address']=$ipv6_address
         ['ipv6_gateway']=$ipv6_gateway
-	['ipv4_gateway']=$ipv4_gateway
+	    ['ipv4_gateway']=$ipv4_gateway
         ['ipv4_address']=$ipv4_address
-        ['prober']=$prober
         ['hostname']=$hostname
         ['subnet_type']=$subnet_type
         ['subnet_idx']=$subnet_idx
@@ -374,28 +368,6 @@ function generate_network_config() {
     done
 }
 
-function generate_prober_config() {
-    for n in $NODES; do
-        declare -n NODE=$n
-        if [[ "${NODE["type"]}" == "boundary" ]]; then
-            local hostname=${NODE["hostname"]}
-            local subnet_idx=${NODE["subnet_idx"]}
-            local node_idx=${NODE["node_idx"]}
-            local prober=${NODE["prober"]}
-
-            NODE_PREFIX=${DEPLOYMENT}.$subnet_idx.$node_idx
-
-            mkdir -p "${CONFIG_DIR}/${NODE_PREFIX}"
-
-            # copy prober identity if enabled
-            if [[ -f "${PROBER_IDENTITY:-}" && "${prober:-}" == "true" ]]; then
-                echo "Using prober identity ${PROBER_IDENTITY}"
-                cp "${PROBER_IDENTITY}" "${CONFIG_DIR}/${NODE_PREFIX}/prober_identity.pem"
-            fi
-        fi
-    done
-}
-
 function copy_ssh_keys() {
     for n in $NODES; do
         declare -n NODE=$n
@@ -678,7 +650,6 @@ function main() {
     create_tarball_structure
     generate_boundary_node_config
     generate_network_config
-    generate_prober_config
     copy_ssh_keys
     copy_certs
     copy_deny_list

ic-os/components/boundary-guestos.bzl

@@ -20,9 +20,6 @@ component_files = {
     Label("boundary-guestos/etc/initramfs-tools/scripts/init-bottom/set-machine-id"): "/etc/initramfs-tools/scripts/init-bottom/set-machine-id",
     Label("boundary-guestos/etc/initramfs-tools/scripts/init-premount/verity-root"): "/etc/initramfs-tools/scripts/init-premount/verity-root",
     Label("boundary-guestos/etc/nftables.conf"): "/etc/nftables.conf",
-    Label("boundary-guestos/etc/prober/ic_public_key.pem"): "/etc/prober/ic_public_key.pem",
-    Label("boundary-guestos/etc/prober-wallets-exporter/wallets.json"): "/etc/prober-wallets-exporter/wallets.json",
-    Label("boundary-guestos/etc/prober/wallets.json"): "/etc/prober/wallets.json",
     Label("boundary-guestos/etc/resolv.conf"): "/etc/resolv.conf",
     Label("boundary-guestos/etc/security/limits.conf"): "/etc/security/limits.conf",
     Label("boundary-guestos/etc/stunnel/stunnel.conf"): "/etc/stunnel/stunnel.conf",
@@ -47,8 +44,6 @@ component_files = {
     Label("boundary-guestos/etc/systemd/system/logrotate.timer"): "/etc/systemd/system/logrotate.timer",
     Label("boundary-guestos/etc/systemd/system/nftables.service.d/override.conf"): "/etc/systemd/system/nftables.service.d/override.conf",
     Label("boundary-guestos/etc/systemd/system/node_exporter.service"): "/etc/systemd/system/node_exporter.service",
-    Label("boundary-guestos/etc/systemd/system/prober.service"): "/etc/systemd/system/prober.service",
-    Label("boundary-guestos/etc/systemd/system/prober-wallets-exporter.service"): "/etc/systemd/system/prober-wallets-exporter.service",
     Label("boundary-guestos/etc/systemd/system/relabel-machine-id.service"): "/etc/systemd/system/relabel-machine-id.service",
     Label("boundary-guestos/etc/systemd/system/retry-ipv6-config.service"): "/etc/systemd/system/retry-ipv6-config.service",
     Label("boundary-guestos/etc/systemd/system/save-machine-id.service"): "/etc/systemd/system/save-machine-id.service",
@@ -62,7 +57,6 @@ component_files = {
     Label("boundary-guestos/etc/systemd/system/setup-ic-gateway.service"): "/etc/systemd/system/setup-ic-gateway.service",
     Label("boundary-guestos/etc/systemd/system/setup-lvs.service"): "/etc/systemd/system/setup-lvs.service",
     Label("boundary-guestos/etc/systemd/system/setup-nftables.service"): "/etc/systemd/system/setup-nftables.service",
-    Label("boundary-guestos/etc/systemd/system/setup-prober.service"): "/etc/systemd/system/setup-prober.service",
     Label("boundary-guestos/etc/systemd/system/setup-ssh-account-keys.service"): "/etc/systemd/system/setup-ssh-account-keys.service",
     Label("boundary-guestos/etc/systemd/system/setup-ssh-keys.service"): "/etc/systemd/system/setup-ssh-keys.service",
     Label("boundary-guestos/etc/systemd/system/setup-var-log.service"): "/etc/systemd/system/setup-var-log.service",
@@ -94,7 +88,6 @@ component_files = {
     Label("boundary-guestos/opt/ic/bin/setup-ic-gateway.sh"): "/opt/ic/bin/setup-ic-gateway.sh",
     Label("boundary-guestos/opt/ic/bin/setup-lvs.sh"): "/opt/ic/bin/setup-lvs.sh",
     Label("boundary-guestos/opt/ic/bin/setup-nftables.sh"): "/opt/ic/bin/setup-nftables.sh",
-    Label("boundary-guestos/opt/ic/bin/setup-prober.sh"): "/opt/ic/bin/setup-prober.sh",
     Label("boundary-guestos/opt/ic/bin/setup-ssh-account-keys.sh"): "/opt/ic/bin/setup-ssh-account-keys.sh",
     Label("boundary-guestos/opt/ic/bin/setup-ssh-keys.sh"): "/opt/ic/bin/setup-ssh-keys.sh",
     Label("boundary-guestos/opt/ic/bin/setup-var-encryption.sh"): "/opt/ic/bin/setup-var-encryption.sh",