feat(BOUN-1169): Replace nginx/icx-proxy/certsyncer with ic-gateway […

…RUN_ALL_BAZEL_TARGETS] (#342) This is an intermediate step to decentralization. The goal is to evaluate `ic-gateway` on the current centralized Boundary Nodes setup before switching to a separate `ic-gateway` + API BNs. It replaces: * `nginx` * `certificate-syncer` * `icx-proxy` --------- Co-authored-by: Igor Novgorodov <[email protected]> Co-authored-by: IDX GitLab Automation <[email protected]>
dfinity · Sep 23, 2024 · 8ed97f1 · 8ed97f1
1 parent 292444b
commit 8ed97f1
Show file tree

Hide file tree

Showing 57 changed files with 509 additions and 2,321 deletions.
diff --git a/ic-os/README.adoc b/ic-os/README.adoc
@@ -7,7 +7,7 @@ IC-OS is an umbrella term for all the operating systems within the IC, including
 * SetupOS: Responsible for booting a new replica node and installing HostOS and GuestOS.
 * HostOS: The operating system that runs on the host machine. Its main responsibility is to launch and run the GuestOS in a virtual machine. In terms of its capabilities, it is intentionally limited by design to not perform any trusted capabilities.
 * GuestOS: The operating system that runs inside a virtual machine on the HostOS. The core IC protocol is executed within the GuestOS.
-* Boundary-GuestOS: The operating system that runs on boundary nodes. It contains all the services necessary to fulfill the two main tasks of the boundary nodes: (1) route ICP API requests to a healthy replica of the right subnet (`ic-boundary`); and (2) translate HTTP requests into ICP API requests to allow direct access to dapps from the browser (`icx-proxy`).
+* Boundary-GuestOS: The operating system that runs on boundary nodes. It contains all the services necessary to fulfill the two main tasks of the boundary nodes: (1) route ICP API requests to a healthy replica of the right subnet (`ic-boundary`); and (2) translate HTTP requests into ICP API requests to allow direct access to dapps from the browser.
 
 == Managing IC-OS files
 

diff --git a/ic-os/boundary-guestos/context/Dockerfile b/ic-os/boundary-guestos/context/Dockerfile
@@ -23,6 +23,11 @@ USER root:root
 
 WORKDIR /tmp
 
+# Download and verify ic-gateway
+RUN \
+    curl -L -O https://github.com/dfinity/ic-gateway/releases/download/v0.1.53/ic-gateway_0.1.53_amd64.deb && \
+    echo "a08db301f91301649973367cc10fd3b117b45dfe  ic-gateway_0.1.53_amd64.deb" | shasum -c
+
 #
 # Second build stage:
 # - Construct base images with the differences between dev and prod (IC-OS root filesystem)
@@ -51,6 +56,10 @@ FROM image-${BUILD_TYPE}
 
 USER root:root
 
+COPY --from=download /tmp/ic-gateway_0.1.53_amd64.deb /tmp/ic-gateway_0.1.53_amd64.deb
+RUN dpkg -i --force-confold /tmp/ic-gateway_0.1.53_amd64.deb && \
+    rm /tmp/ic-gateway_0.1.53_amd64.deb
+
 RUN mkdir -p /boot/config \
              /boot/efi \
              /boot/grub
@@ -116,6 +125,9 @@ RUN for SERVICE in /etc/systemd/system/*; do \
     systemd-resolved \
     systemd-journal-gatewayd
 
+# TODO remove when nginx is removed from base image
+RUN systemctl disable nginx
+
 # Add user/group entries specified here: /usr/lib/sysusers.d/systemd.conf
 # E.g., systemd-timesync/coredump
 RUN systemd-sysusers && \
@@ -187,14 +199,6 @@ RUN chown root:root /etc/node_exporter \
 # will be cached when only the binaries change.
 COPY opt /opt
 
-RUN rm -rf /etc/nginx/sites-enabled/nginx.conf
-
-RUN mkdir -p /var/www/html && chown www-data:www-data /var/www/html
-
-# Install other files (e.g. the uninstall-script)
-COPY var/www/html /var/www/html
-RUN chmod 0644 /var/www/html/*
-
 # Clear all files that may lead to indeterministic build.
 RUN apt-get clean && \
     rm -rf \
@@ -214,13 +218,5 @@ RUN find /opt -type d -exec chmod 0755 {} \+ && \
     find /opt -type f -exec chmod 0644 {} \+ && \
     chmod 0755 /opt/ic/bin/*
 
-# Take care of nginx files
-RUN chmod 0755 /etc/nginx/* && \
-    chmod 0644 /etc/nginx/nginx.conf && \
-    rm -rf /etc/nginx/conf.d/nginx-global.conf && \
-    rm -rf /etc/nginx/conf.d/default.conf && \
-    rm -rf /etc/nginx/sites-enabled/default && \
-    rm -rf /etc/nginx/conf.d/default
-
 # Increase the default limit on the number of open files for all systemd services
 RUN sed -i '/^#DefaultLimitNOFILE=/c\DefaultLimitNOFILE=1048576:1048576' /etc/systemd/system.conf
diff --git a/ic-os/boundary-guestos/defs.bzl b/ic-os/boundary-guestos/defs.bzl
@@ -25,9 +25,7 @@ def image_deps(mode):
             "//publish/binaries:canary-proxy": "/opt/ic/bin/canary-proxy:0755",
             "//publish/binaries:boundary-node-prober": "/opt/ic/bin/boundary-node-prober:0755",
             "//publish/binaries:certificate-issuer": "/opt/ic/bin/certificate-issuer:0755",
-            "//publish/binaries:certificate-syncer": "/opt/ic/bin/certificate-syncer:0755",
             "//publish/binaries:ic-balance-exporter": "/opt/ic/bin/ic-balance-exporter:0755",
-            "//publish/binaries:icx-proxy": "/opt/ic/bin/icx-proxy:0755",
             "//publish/binaries:systemd-journal-gatewayd-shim": "/opt/ic/bin/systemd-journal-gatewayd-shim:0755",
             "//publish/binaries:ic-boundary": "/opt/ic/bin/ic-boundary:0755",
         },

diff --git a/ic-os/boundary-guestos/docs/Boot.adoc b/ic-os/boundary-guestos/docs/Boot.adoc
@@ -30,9 +30,7 @@ service are started in the IC-OS boot sequence:
 
 - Start node exporter
 
-- Start nginx
-
-- Start icx_proxy
+- Start ic-gateway
 
 - Start danted socks-proxy
 
@@ -126,15 +124,9 @@ Service: `node_exporter.service`. Depends on `setup-node_exporter-keys.service`.
 
 Starts the `node_exporter` service to make machine metrics accessible externally.
 
-==  Start nginx
-
-Service: `nginx.service`
-
-Default starter script that comes with the package. Relevant information is here: `nginx.service.d/override.conf` and in `/etc/nginx`
-
-==  Start icx_proxy
+==  Start ic-gateway
 
-Service: icx-proxy.service
+Service: ic-gateway.service
 
 - TODO BOUN-179
 

diff --git a/ic-os/boundary-guestos/docs/Components.adoc b/ic-os/boundary-guestos/docs/Components.adoc
@@ -2,13 +2,8 @@
 
 The Boundary Node has a number of specific components:
 
-== Nginx and `ic_router.js`
-Nginx is used as reverse proxy that terminates TLS, forwards requests to `ic-boundary` and `icx-proxy`, and provides a caching layer for static assets.
-
-== `icx-proxy`
-Transforms HTTP-to-canister request as API calls.
-More details can be found link:https://github.com/dfinity/icx-proxy[here].
-
+- `ic-gateway` is used to proxy HTTP calls to `ic-boundary` using IC protocol
+- `ic-boundary` forwards the IC requests to replicas
 
 == Prober
 - Creates canisters that are probed

diff --git a/ic-os/boundary-guestos/scripts/build-bootstrap-config-image.sh b/ic-os/boundary-guestos/scripts/build-bootstrap-config-image.sh
@@ -154,10 +154,6 @@ options may be specified:
     time between peeks by the certificate issuer to fetch a new task from the
     certificate orchestrator.
 
-  --certificate_syncer_polling_interval_sec
-    time between polling the certificate issuer for custom domain updates (i.e.,
-    newly registered, modified, or removed custom domains).
-
   --ic_registry_local_store
     path to a local registry store to be used instead of the one provided by the
     registry replicator.
@@ -365,9 +361,6 @@ function build_ic_bootstrap_tar() {
             --certificate_issuer_peek_sleep_sec)
                 CERTIFICATE_ISSUER_PEEK_SLEEP_SEC="$2"
                 ;;
-            --certificate_syncer_polling_interval_sec)
-                CERTIFICATE_SYNCER_POLLING_INTERVAL_SEC="$2"
-                ;;
             --ic_registry_local_store)
                 IC_REGISTRY_LOCAL_STORE="$2"
                 ;;
@@ -561,12 +554,6 @@ ${CERTIFICATE_ISSUER_PEEK_SLEEP_SEC:+certificate_issuer_peek_sleep_sec=${CERTIFI
 EOF
     fi
 
-    if [[ ! -z "${CERTIFICATE_SYNCER_POLLING_INTERVAL_SEC:-}" ]]; then
-        cat >"${BOOTSTRAP_TMPDIR}/certificate_syncer.conf" <<EOF
-certificate_syncer_polling_interval_sec=${CERTIFICATE_SYNCER_POLLING_INTERVAL_SEC}
-EOF
-    fi
-
     # use the registry local store
     if [[ -n "${IC_REGISTRY_LOCAL_STORE:-}" ]]; then
         echo "Using the registry local store at ${IC_REGISTRY_LOCAL_STORE}"

diff --git a/ic-os/components/boundary-guestos.bzl b/ic-os/components/boundary-guestos.bzl
@@ -3,7 +3,6 @@ Enumerate every component file dependency for Boundary-GuestOS
 """
 
 component_files = {
-    Label("boundary-guestos/etc/certificate-syncer/domain.tmpl"): "/etc/certificate-syncer/domain.tmpl",
     Label("boundary-guestos/etc/chrony/chrony.conf"): "/etc/chrony/chrony.conf",
     Label("boundary-guestos/etc/crowdsec/bouncers/crowdsec-firewall-bouncer.yaml"): "/etc/crowdsec/bouncers/crowdsec-firewall-bouncer.yaml",
     Label("boundary-guestos/etc/crypttab"): "/etc/crypttab",
@@ -20,24 +19,7 @@ component_files = {
     Label("boundary-guestos/etc/initramfs-tools/modules"): "/etc/initramfs-tools/modules",
     Label("boundary-guestos/etc/initramfs-tools/scripts/init-bottom/set-machine-id"): "/etc/initramfs-tools/scripts/init-bottom/set-machine-id",
     Label("boundary-guestos/etc/initramfs-tools/scripts/init-premount/verity-root"): "/etc/initramfs-tools/scripts/init-premount/verity-root",
-    Label("boundary-guestos/etc/logrotate.d/nginx"): "/etc/logrotate.d/nginx",
     Label("boundary-guestos/etc/nftables.conf"): "/etc/nftables.conf",
-    Label("boundary-guestos/etc/nginx/conf.d/000-nginx-global.conf"): "/etc/nginx/conf.d/000-nginx-global.conf",
-    Label("boundary-guestos/etc/nginx/conf.d/001-rosetta-nginx.conf"): "/etc/nginx/conf.d/001-rosetta-nginx.conf",
-    Label("boundary-guestos/etc/nginx/conf.d/002-mainnet-nginx.conf"): "/etc/nginx/conf.d/002-mainnet-nginx.conf",
-    Label("boundary-guestos/etc/nginx/ffdhe4096.pem"): "/etc/nginx/ffdhe4096.pem",
-    Label("boundary-guestos/etc/nginx/ic_public_key.pem"): "/etc/nginx/ic_public_key.pem",
-    Label("boundary-guestos/etc/nginx/includes/cache.conf"): "/etc/nginx/includes/cache.conf",
-    Label("boundary-guestos/etc/nginx/includes/error_pages.conf"): "/etc/nginx/includes/error_pages.conf",
-    Label("boundary-guestos/etc/nginx/includes/method_get.conf"): "/etc/nginx/includes/method_get.conf",
-    Label("boundary-guestos/etc/nginx/includes/method_post.conf"): "/etc/nginx/includes/method_post.conf",
-    Label("boundary-guestos/etc/nginx/includes/options.conf"): "/etc/nginx/includes/options.conf",
-    Label("boundary-guestos/etc/nginx/includes/proxy_headers.conf"): "/etc/nginx/includes/proxy_headers.conf",
-    Label("boundary-guestos/etc/nginx/includes/request_id.conf"): "/etc/nginx/includes/request_id.conf",
-    Label("boundary-guestos/etc/nginx/includes/response_headers.conf"): "/etc/nginx/includes/response_headers.conf",
-    Label("boundary-guestos/etc/nginx/includes/slice.conf"): "/etc/nginx/includes/slice.conf",
-    Label("boundary-guestos/etc/nginx/includes/whitelist_rosetta.conf"): "/etc/nginx/includes/whitelist_rosetta.conf",
-    Label("boundary-guestos/etc/nginx/nginx.conf"): "/etc/nginx/nginx.conf",
     Label("boundary-guestos/etc/prober/ic_public_key.pem"): "/etc/prober/ic_public_key.pem",
     Label("boundary-guestos/etc/prober-wallets-exporter/wallets.json"): "/etc/prober-wallets-exporter/wallets.json",
     Label("boundary-guestos/etc/prober/wallets.json"): "/etc/prober/wallets.json",
@@ -51,7 +33,6 @@ component_files = {
     Label("boundary-guestos/etc/systemd/system/bootstrap-ic-node.service"): "/etc/systemd/system/bootstrap-ic-node.service",
     Label("boundary-guestos/etc/systemd/system/canary-proxy.service"): "/etc/systemd/system/canary-proxy.service",
     Label("boundary-guestos/etc/systemd/system/certificate-issuer.service"): "/etc/systemd/system/certificate-issuer.service",
-    Label("boundary-guestos/etc/systemd/system/certificate-syncer.service"): "/etc/systemd/system/certificate-syncer.service",
     Label("boundary-guestos/etc/systemd/system/chrony-var.service"): "/etc/systemd/system/chrony-var.service",
     Label("boundary-guestos/etc/systemd/system/crowdsec-firewall-bouncer.service"): "/etc/systemd/system/crowdsec-firewall-bouncer.service",
     Label("boundary-guestos/etc/systemd/system/danted.service"): "/etc/systemd/system/danted.service",
@@ -62,10 +43,9 @@ component_files = {
     Label("boundary-guestos/etc/systemd/system-generators/mount-generator"): "/etc/systemd/system-generators/mount-generator",
     Label("boundary-guestos/etc/systemd/system-generators/systemd-gpt-auto-generator"): "/etc/systemd/system-generators/systemd-gpt-auto-generator",
     Label("boundary-guestos/etc/systemd/system/ic-boundary.service"): "/etc/systemd/system/ic-boundary.service",
-    Label("boundary-guestos/etc/systemd/system/icx-proxy.service"): "/etc/systemd/system/icx-proxy.service",
+    Label("boundary-guestos/etc/systemd/system/ic-gateway.service"): "/etc/systemd/system/ic-gateway.service",
     Label("boundary-guestos/etc/systemd/system/logrotate.timer"): "/etc/systemd/system/logrotate.timer",
     Label("boundary-guestos/etc/systemd/system/nftables.service.d/override.conf"): "/etc/systemd/system/nftables.service.d/override.conf",
-    Label("boundary-guestos/etc/systemd/system/nginx.service.d/override.conf"): "/etc/systemd/system/nginx.service.d/override.conf",
     Label("boundary-guestos/etc/systemd/system/node_exporter.service"): "/etc/systemd/system/node_exporter.service",
     Label("boundary-guestos/etc/systemd/system/prober.service"): "/etc/systemd/system/prober.service",
     Label("boundary-guestos/etc/systemd/system/prober-wallets-exporter.service"): "/etc/systemd/system/prober-wallets-exporter.service",
@@ -74,16 +54,14 @@ component_files = {
     Label("boundary-guestos/etc/systemd/system/save-machine-id.service"): "/etc/systemd/system/save-machine-id.service",
     Label("boundary-guestos/etc/systemd/system/setup-canary-proxy.service"): "/etc/systemd/system/setup-canary-proxy.service",
     Label("boundary-guestos/etc/systemd/system/setup-certificate-issuer.service"): "/etc/systemd/system/setup-certificate-issuer.service",
-    Label("boundary-guestos/etc/systemd/system/setup-certificate-syncer.service"): "/etc/systemd/system/setup-certificate-syncer.service",
     Label("boundary-guestos/etc/systemd/system/setup-conf-encryption.service"): "/etc/systemd/system/setup-conf-encryption.service",
     Label("boundary-guestos/etc/systemd/system/setup-crowdsec.service"): "/etc/systemd/system/setup-crowdsec.service",
     Label("boundary-guestos/etc/systemd/system/setup-encryption.service"): "/etc/systemd/system/setup-encryption.service",
     Label("boundary-guestos/etc/systemd/system/setup-hostname.service"): "/etc/systemd/system/setup-hostname.service",
     Label("boundary-guestos/etc/systemd/system/setup-ic-boundary.service"): "/etc/systemd/system/setup-ic-boundary.service",
-    Label("boundary-guestos/etc/systemd/system/setup-icx-proxy.service"): "/etc/systemd/system/setup-icx-proxy.service",
+    Label("boundary-guestos/etc/systemd/system/setup-ic-gateway.service"): "/etc/systemd/system/setup-ic-gateway.service",
     Label("boundary-guestos/etc/systemd/system/setup-lvs.service"): "/etc/systemd/system/setup-lvs.service",
     Label("boundary-guestos/etc/systemd/system/setup-nftables.service"): "/etc/systemd/system/setup-nftables.service",
-    Label("boundary-guestos/etc/systemd/system/setup-nginx.service"): "/etc/systemd/system/setup-nginx.service",
     Label("boundary-guestos/etc/systemd/system/setup-prober.service"): "/etc/systemd/system/setup-prober.service",
     Label("boundary-guestos/etc/systemd/system/setup-ssh-account-keys.service"): "/etc/systemd/system/setup-ssh-account-keys.service",
     Label("boundary-guestos/etc/systemd/system/setup-ssh-keys.service"): "/etc/systemd/system/setup-ssh-keys.service",
@@ -108,16 +86,14 @@ component_files = {
     Label("boundary-guestos/opt/ic/bin/save-machine-id.sh"): "/opt/ic/bin/save-machine-id.sh",
     Label("boundary-guestos/opt/ic/bin/setup-canary-proxy.sh"): "/opt/ic/bin/setup-canary-proxy.sh",
     Label("boundary-guestos/opt/ic/bin/setup-certificate-issuer.sh"): "/opt/ic/bin/setup-certificate-issuer.sh",
-    Label("boundary-guestos/opt/ic/bin/setup-certificate-syncer.sh"): "/opt/ic/bin/setup-certificate-syncer.sh",
     Label("boundary-guestos/opt/ic/bin/setup-conf-encryption.sh"): "/opt/ic/bin/setup-conf-encryption.sh",
     Label("boundary-guestos/opt/ic/bin/setup-crowdsec.sh"): "/opt/ic/bin/setup-crowdsec.sh",
     Label("boundary-guestos/opt/ic/bin/setup-encryption.sh"): "/opt/ic/bin/setup-encryption.sh",
     Label("boundary-guestos/opt/ic/bin/setup-hostname.sh"): "/opt/ic/bin/setup-hostname.sh",
     Label("boundary-guestos/opt/ic/bin/setup-ic-boundary.sh"): "/opt/ic/bin/setup-ic-boundary.sh",
-    Label("boundary-guestos/opt/ic/bin/setup-icx-proxy.sh"): "/opt/ic/bin/setup-icx-proxy.sh",
+    Label("boundary-guestos/opt/ic/bin/setup-ic-gateway.sh"): "/opt/ic/bin/setup-ic-gateway.sh",
     Label("boundary-guestos/opt/ic/bin/setup-lvs.sh"): "/opt/ic/bin/setup-lvs.sh",
     Label("boundary-guestos/opt/ic/bin/setup-nftables.sh"): "/opt/ic/bin/setup-nftables.sh",
-    Label("boundary-guestos/opt/ic/bin/setup-nginx.sh"): "/opt/ic/bin/setup-nginx.sh",
     Label("boundary-guestos/opt/ic/bin/setup-prober.sh"): "/opt/ic/bin/setup-prober.sh",
     Label("boundary-guestos/opt/ic/bin/setup-ssh-account-keys.sh"): "/opt/ic/bin/setup-ssh-account-keys.sh",
     Label("boundary-guestos/opt/ic/bin/setup-ssh-keys.sh"): "/opt/ic/bin/setup-ssh-keys.sh",
@@ -126,6 +102,4 @@ component_files = {
     Label("boundary-guestos/opt/ic/bin/setup-vector.sh"): "/opt/ic/bin/setup-vector.sh",
     Label("boundary-guestos/opt/ic/bin/setup-version-metric.sh"): "/opt/ic/bin/setup-version-metric.sh",
     Label("boundary-guestos/opt/ic/share/ark.pem"): "/opt/ic/share/ark.pem",
-    Label("boundary-guestos/var/www/html/451.html"): "/var/www/html/451.html",
-    Label("boundary-guestos/var/www/html/uninstall-script.js"): "/var/www/html/uninstall-script.js",
 }
diff --git a/ic-os/components/boundary-guestos/etc/certificate-syncer/domain.tmpl b/ic-os/components/boundary-guestos/etc/certificate-syncer/domain.tmpl
diff --git a/ic-os/components/boundary-guestos/etc/logrotate.d/nginx b/ic-os/components/boundary-guestos/etc/logrotate.d/nginx
diff --git a/ic-os/components/boundary-guestos/etc/nftables.conf b/ic-os/components/boundary-guestos/etc/nftables.conf
@@ -24,12 +24,10 @@ define icmp_v6_types_accept = {
 define ipv6_monitoring_tcp_ports = {
   9100, # node-exporter
   9313, # prober
-  9314, # icx-proxy
+  9314, # ic-gateway
   9315, # prober-wallets-exporter
-  9316, # vector-nginx
   9317, # vector
   9321, # certificate-issuer
-  9322, # certificate-syncer
   9323, # danted (socks proxy)
   9324, # ic-boundary
 }