From 8ed97f18bad8bf93ff98fc266e26ab503b44f00c Mon Sep 17 00:00:00 2001 From: r-birkner <103420898+r-birkner@users.noreply.github.com> Date: Mon, 23 Sep 2024 20:31:44 +0200 Subject: [PATCH] feat(BOUN-1169): Replace nginx/icx-proxy/certsyncer with ic-gateway [RUN_ALL_BAZEL_TARGETS] (#342) This is an intermediate step to decentralization. The goal is to evaluate `ic-gateway` on the current centralized Boundary Nodes setup before switching to a separate `ic-gateway` + API BNs. It replaces: * `nginx` * `certificate-syncer` * `icx-proxy` --------- Co-authored-by: Igor Novgorodov Co-authored-by: IDX GitLab Automation --- ic-os/README.adoc | 2 +- ic-os/boundary-guestos/context/Dockerfile | 28 +- ic-os/boundary-guestos/defs.bzl | 2 - ic-os/boundary-guestos/docs/Boot.adoc | 14 +- ic-os/boundary-guestos/docs/Components.adoc | 9 +- .../scripts/build-bootstrap-config-image.sh | 13 - ic-os/components/boundary-guestos.bzl | 32 +- .../etc/certificate-syncer/domain.tmpl | 77 --- .../boundary-guestos/etc/logrotate.d/nginx | 29 - .../boundary-guestos/etc/nftables.conf | 4 +- .../etc/nginx/conf.d/000-nginx-global.conf | 83 --- .../etc/nginx/conf.d/001-rosetta-nginx.conf | 57 -- .../etc/nginx/conf.d/002-mainnet-nginx.conf | 365 ------------ .../boundary-guestos/etc/nginx/ffdhe4096.pem | 13 - .../etc/nginx/ic_public_key.pem | 5 - .../etc/nginx/includes/cache.conf | 3 - .../etc/nginx/includes/error_pages.conf | 8 - .../etc/nginx/includes/method_get.conf | 7 - .../etc/nginx/includes/method_post.conf | 7 - .../etc/nginx/includes/options.conf | 12 - .../etc/nginx/includes/proxy_headers.conf | 31 - .../etc/nginx/includes/request_id.conf | 2 - .../etc/nginx/includes/response_headers.conf | 6 - .../etc/nginx/includes/slice.conf | 3 - .../etc/nginx/includes/whitelist_rosetta.conf | 70 --- .../boundary-guestos/etc/nginx/nginx.conf | 176 ------ .../systemd/system/certificate-syncer.service | 26 - .../etc/systemd/system/ic-boundary.service | 5 +- .../etc/systemd/system/ic-gateway.service | 16 + .../etc/systemd/system/icx-proxy.service | 41 -- .../system/nginx.service.d/override.conf | 8 - .../system/setup-certificate-syncer.service | 11 - ...proxy.service => setup-ic-gateway.service} | 4 +- .../etc/systemd/system/setup-nginx.service | 12 - .../etc/systemd/system/vector.service | 1 - .../etc/tmpfiles.d/ic-node.conf | 5 +- .../boundary-guestos/etc/vector/vector.yaml | 66 -- .../opt/ic/bin/bootstrap-ic-node.sh | 1 - .../opt/ic/bin/setup-certificate-syncer.sh | 52 -- .../opt/ic/bin/setup-ic-boundary.sh | 12 - .../opt/ic/bin/setup-ic-gateway.sh | 206 +++++++ .../opt/ic/bin/setup-icx-proxy.sh | 146 ----- .../opt/ic/bin/setup-nginx.sh | 140 ----- .../boundary-guestos/var/www/html/451.html | 282 --------- .../var/www/html/uninstall-script.js | 13 - rs/tests/boundary_nodes/BUILD.bazel | 8 +- .../bn_integration_on_playnet_test.rs | 9 +- .../boundary_nodes/bn_integration_test.rs | 8 +- ...mainnet_bn_ic_gateway_performance_test.rs} | 6 +- rs/tests/driver/src/driver/boundary_node.rs | 6 - rs/tests/driver/src/driver/prometheus_vm.rs | 19 +- .../nns/ic_mainnet_nns_recovery/src/lib.rs | 2 +- .../boundary_nodes_integration.rs | 563 +++++++----------- .../src/boundary_nodes/performance_test.rs | 14 +- .../certificate_orchestrator.rs | 46 +- .../src/custom_domains_integration/setup.rs | 42 +- .../spec_compliance/spec_compliance.rs | 2 +- 57 files changed, 509 insertions(+), 2321 deletions(-) delete mode 100644 ic-os/components/boundary-guestos/etc/certificate-syncer/domain.tmpl delete mode 100644 ic-os/components/boundary-guestos/etc/logrotate.d/nginx delete mode 100644 ic-os/components/boundary-guestos/etc/nginx/conf.d/000-nginx-global.conf delete mode 100644 ic-os/components/boundary-guestos/etc/nginx/conf.d/001-rosetta-nginx.conf delete mode 100644 ic-os/components/boundary-guestos/etc/nginx/conf.d/002-mainnet-nginx.conf delete mode 100644 ic-os/components/boundary-guestos/etc/nginx/ffdhe4096.pem delete mode 100644 ic-os/components/boundary-guestos/etc/nginx/ic_public_key.pem delete mode 100644 ic-os/components/boundary-guestos/etc/nginx/includes/cache.conf delete mode 100644 ic-os/components/boundary-guestos/etc/nginx/includes/error_pages.conf delete mode 100644 ic-os/components/boundary-guestos/etc/nginx/includes/method_get.conf delete mode 100644 ic-os/components/boundary-guestos/etc/nginx/includes/method_post.conf delete mode 100644 ic-os/components/boundary-guestos/etc/nginx/includes/options.conf delete mode 100644 ic-os/components/boundary-guestos/etc/nginx/includes/proxy_headers.conf delete mode 100644 ic-os/components/boundary-guestos/etc/nginx/includes/request_id.conf delete mode 100644 ic-os/components/boundary-guestos/etc/nginx/includes/response_headers.conf delete mode 100644 ic-os/components/boundary-guestos/etc/nginx/includes/slice.conf delete mode 100644 ic-os/components/boundary-guestos/etc/nginx/includes/whitelist_rosetta.conf delete mode 100644 ic-os/components/boundary-guestos/etc/nginx/nginx.conf delete mode 100644 ic-os/components/boundary-guestos/etc/systemd/system/certificate-syncer.service create mode 100644 ic-os/components/boundary-guestos/etc/systemd/system/ic-gateway.service delete mode 100644 ic-os/components/boundary-guestos/etc/systemd/system/icx-proxy.service delete mode 100644 ic-os/components/boundary-guestos/etc/systemd/system/nginx.service.d/override.conf delete mode 100644 ic-os/components/boundary-guestos/etc/systemd/system/setup-certificate-syncer.service rename ic-os/components/boundary-guestos/etc/systemd/system/{setup-icx-proxy.service => setup-ic-gateway.service} (60%) delete mode 100644 ic-os/components/boundary-guestos/etc/systemd/system/setup-nginx.service delete mode 100644 ic-os/components/boundary-guestos/opt/ic/bin/setup-certificate-syncer.sh create mode 100755 ic-os/components/boundary-guestos/opt/ic/bin/setup-ic-gateway.sh delete mode 100755 ic-os/components/boundary-guestos/opt/ic/bin/setup-icx-proxy.sh delete mode 100755 ic-os/components/boundary-guestos/opt/ic/bin/setup-nginx.sh delete mode 100644 ic-os/components/boundary-guestos/var/www/html/451.html delete mode 100644 ic-os/components/boundary-guestos/var/www/html/uninstall-script.js rename rs/tests/boundary_nodes/{mainnet_bn_icx_proxy_performance_test.rs => mainnet_bn_ic_gateway_performance_test.rs} (74%) diff --git a/ic-os/README.adoc b/ic-os/README.adoc index eefb29374fc..357e1d43378 100644 --- a/ic-os/README.adoc +++ b/ic-os/README.adoc @@ -7,7 +7,7 @@ IC-OS is an umbrella term for all the operating systems within the IC, including * SetupOS: Responsible for booting a new replica node and installing HostOS and GuestOS. * HostOS: The operating system that runs on the host machine. Its main responsibility is to launch and run the GuestOS in a virtual machine. In terms of its capabilities, it is intentionally limited by design to not perform any trusted capabilities. * GuestOS: The operating system that runs inside a virtual machine on the HostOS. The core IC protocol is executed within the GuestOS. -* Boundary-GuestOS: The operating system that runs on boundary nodes. It contains all the services necessary to fulfill the two main tasks of the boundary nodes: (1) route ICP API requests to a healthy replica of the right subnet (`ic-boundary`); and (2) translate HTTP requests into ICP API requests to allow direct access to dapps from the browser (`icx-proxy`). +* Boundary-GuestOS: The operating system that runs on boundary nodes. It contains all the services necessary to fulfill the two main tasks of the boundary nodes: (1) route ICP API requests to a healthy replica of the right subnet (`ic-boundary`); and (2) translate HTTP requests into ICP API requests to allow direct access to dapps from the browser. == Managing IC-OS files diff --git a/ic-os/boundary-guestos/context/Dockerfile b/ic-os/boundary-guestos/context/Dockerfile index b7c194eea85..677e27d6198 100644 --- a/ic-os/boundary-guestos/context/Dockerfile +++ b/ic-os/boundary-guestos/context/Dockerfile @@ -23,6 +23,11 @@ USER root:root WORKDIR /tmp +# Download and verify ic-gateway +RUN \ + curl -L -O https://github.com/dfinity/ic-gateway/releases/download/v0.1.53/ic-gateway_0.1.53_amd64.deb && \ + echo "a08db301f91301649973367cc10fd3b117b45dfe ic-gateway_0.1.53_amd64.deb" | shasum -c + # # Second build stage: # - Construct base images with the differences between dev and prod (IC-OS root filesystem) @@ -51,6 +56,10 @@ FROM image-${BUILD_TYPE} USER root:root +COPY --from=download /tmp/ic-gateway_0.1.53_amd64.deb /tmp/ic-gateway_0.1.53_amd64.deb +RUN dpkg -i --force-confold /tmp/ic-gateway_0.1.53_amd64.deb && \ + rm /tmp/ic-gateway_0.1.53_amd64.deb + RUN mkdir -p /boot/config \ /boot/efi \ /boot/grub @@ -116,6 +125,9 @@ RUN for SERVICE in /etc/systemd/system/*; do \ systemd-resolved \ systemd-journal-gatewayd +# TODO remove when nginx is removed from base image +RUN systemctl disable nginx + # Add user/group entries specified here: /usr/lib/sysusers.d/systemd.conf # E.g., systemd-timesync/coredump RUN systemd-sysusers && \ @@ -187,14 +199,6 @@ RUN chown root:root /etc/node_exporter \ # will be cached when only the binaries change. COPY opt /opt -RUN rm -rf /etc/nginx/sites-enabled/nginx.conf - -RUN mkdir -p /var/www/html && chown www-data:www-data /var/www/html - -# Install other files (e.g. the uninstall-script) -COPY var/www/html /var/www/html -RUN chmod 0644 /var/www/html/* - # Clear all files that may lead to indeterministic build. RUN apt-get clean && \ rm -rf \ @@ -214,13 +218,5 @@ RUN find /opt -type d -exec chmod 0755 {} \+ && \ find /opt -type f -exec chmod 0644 {} \+ && \ chmod 0755 /opt/ic/bin/* -# Take care of nginx files -RUN chmod 0755 /etc/nginx/* && \ - chmod 0644 /etc/nginx/nginx.conf && \ - rm -rf /etc/nginx/conf.d/nginx-global.conf && \ - rm -rf /etc/nginx/conf.d/default.conf && \ - rm -rf /etc/nginx/sites-enabled/default && \ - rm -rf /etc/nginx/conf.d/default - # Increase the default limit on the number of open files for all systemd services RUN sed -i '/^#DefaultLimitNOFILE=/c\DefaultLimitNOFILE=1048576:1048576' /etc/systemd/system.conf diff --git a/ic-os/boundary-guestos/defs.bzl b/ic-os/boundary-guestos/defs.bzl index a0610fc6d6f..8b89f8d5fd8 100644 --- a/ic-os/boundary-guestos/defs.bzl +++ b/ic-os/boundary-guestos/defs.bzl @@ -25,9 +25,7 @@ def image_deps(mode): "//publish/binaries:canary-proxy": "/opt/ic/bin/canary-proxy:0755", "//publish/binaries:boundary-node-prober": "/opt/ic/bin/boundary-node-prober:0755", "//publish/binaries:certificate-issuer": "/opt/ic/bin/certificate-issuer:0755", - "//publish/binaries:certificate-syncer": "/opt/ic/bin/certificate-syncer:0755", "//publish/binaries:ic-balance-exporter": "/opt/ic/bin/ic-balance-exporter:0755", - "//publish/binaries:icx-proxy": "/opt/ic/bin/icx-proxy:0755", "//publish/binaries:systemd-journal-gatewayd-shim": "/opt/ic/bin/systemd-journal-gatewayd-shim:0755", "//publish/binaries:ic-boundary": "/opt/ic/bin/ic-boundary:0755", }, diff --git a/ic-os/boundary-guestos/docs/Boot.adoc b/ic-os/boundary-guestos/docs/Boot.adoc index dbf6125e8c9..7080e087e8e 100644 --- a/ic-os/boundary-guestos/docs/Boot.adoc +++ b/ic-os/boundary-guestos/docs/Boot.adoc @@ -30,9 +30,7 @@ service are started in the IC-OS boot sequence: - Start node exporter -- Start nginx - -- Start icx_proxy +- Start ic-gateway - Start danted socks-proxy @@ -126,15 +124,9 @@ Service: `node_exporter.service`. Depends on `setup-node_exporter-keys.service`. Starts the `node_exporter` service to make machine metrics accessible externally. -== Start nginx - -Service: `nginx.service` - -Default starter script that comes with the package. Relevant information is here: `nginx.service.d/override.conf` and in `/etc/nginx` - -== Start icx_proxy +== Start ic-gateway -Service: icx-proxy.service +Service: ic-gateway.service - TODO BOUN-179 diff --git a/ic-os/boundary-guestos/docs/Components.adoc b/ic-os/boundary-guestos/docs/Components.adoc index a1ec9ec2420..276d24f9ec0 100644 --- a/ic-os/boundary-guestos/docs/Components.adoc +++ b/ic-os/boundary-guestos/docs/Components.adoc @@ -2,13 +2,8 @@ The Boundary Node has a number of specific components: -== Nginx and `ic_router.js` -Nginx is used as reverse proxy that terminates TLS, forwards requests to `ic-boundary` and `icx-proxy`, and provides a caching layer for static assets. - -== `icx-proxy` -Transforms HTTP-to-canister request as API calls. -More details can be found link:https://github.com/dfinity/icx-proxy[here]. - +- `ic-gateway` is used to proxy HTTP calls to `ic-boundary` using IC protocol +- `ic-boundary` forwards the IC requests to replicas == Prober - Creates canisters that are probed diff --git a/ic-os/boundary-guestos/scripts/build-bootstrap-config-image.sh b/ic-os/boundary-guestos/scripts/build-bootstrap-config-image.sh index e428624c89b..dd420e5a04e 100755 --- a/ic-os/boundary-guestos/scripts/build-bootstrap-config-image.sh +++ b/ic-os/boundary-guestos/scripts/build-bootstrap-config-image.sh @@ -154,10 +154,6 @@ options may be specified: time between peeks by the certificate issuer to fetch a new task from the certificate orchestrator. - --certificate_syncer_polling_interval_sec - time between polling the certificate issuer for custom domain updates (i.e., - newly registered, modified, or removed custom domains). - --ic_registry_local_store path to a local registry store to be used instead of the one provided by the registry replicator. @@ -365,9 +361,6 @@ function build_ic_bootstrap_tar() { --certificate_issuer_peek_sleep_sec) CERTIFICATE_ISSUER_PEEK_SLEEP_SEC="$2" ;; - --certificate_syncer_polling_interval_sec) - CERTIFICATE_SYNCER_POLLING_INTERVAL_SEC="$2" - ;; --ic_registry_local_store) IC_REGISTRY_LOCAL_STORE="$2" ;; @@ -561,12 +554,6 @@ ${CERTIFICATE_ISSUER_PEEK_SLEEP_SEC:+certificate_issuer_peek_sleep_sec=${CERTIFI EOF fi - if [[ ! -z "${CERTIFICATE_SYNCER_POLLING_INTERVAL_SEC:-}" ]]; then - cat >"${BOOTSTRAP_TMPDIR}/certificate_syncer.conf" < 444535f9-378a-3dfa-1b86-04bc9e05a303 -map $request_id $request_uuid { - "~^(\w{8})(\w{4})(\w{4})(\w{4})(.*)$" "$1-$2-$3-$4-$5"; -} - -# Observability -include "includes/request_id.conf"; - -### Caching - -# Cache for static responses (e.g `/`) -proxy_ignore_headers X-Accel-Expires Expires Cache-Control Vary Set-Cookie; -proxy_cache_path /var/cache/nginx/static levels=1:2 keys_zone=cache_static:256m max_size=4000m use_temp_path=off; -proxy_cache_lock on; -proxy_cache_lock_age 10s; -proxy_cache_key $host$request_uri$slice_range; -proxy_cache_valid 200 206 301 302 10s; -proxy_cache_valid 404 5s; - -### Rate Limiting -limit_req_status 429; - -# for custom-domain registrations -limit_req_zone global zone=rgs_global:32k rate=10r/s; -limit_req_zone $binary_remote_addr zone=rgs_per_ip:1m rate=1r/s; - -root /var/www/html; - -# Any direct HTTPS access without correct domain name will default to returning a 404. -server { - listen 443 ssl reuseport backlog=4096; - listen [::]:443 ssl ipv6only=on reuseport backlog=4096; # setting ipv6only=on once turns it on for all instances on that port - - server_name _; - - return 404; -} - -# Any direct HTTP traffic will be redirected to HTTPS via 301. -server { - listen 80 reuseport backlog=4096; - listen [::]:80 ipv6only=on reuseport backlog=4096; # setting ipv6only=on once turns it on for all instances on that port - - server_name _; - - return 301 https://$host$request_uri; -} diff --git a/ic-os/components/boundary-guestos/etc/nginx/conf.d/001-rosetta-nginx.conf b/ic-os/components/boundary-guestos/etc/nginx/conf.d/001-rosetta-nginx.conf deleted file mode 100644 index 75f088620a8..00000000000 --- a/ic-os/components/boundary-guestos/etc/nginx/conf.d/001-rosetta-nginx.conf +++ /dev/null @@ -1,57 +0,0 @@ -server { - listen 443 ssl; - listen [::]:443 ssl; - - server_name .rosetta.dfinity.network; - include "/run/ic-node/etc/nginx/conf.d/server_rosetta_domain.conf"; - - include "includes/whitelist_rosetta.conf"; - - location ~ (/api/v2/status|/health) { - # Observability - include "includes/request_id.conf"; - - # Prereqs - include "includes/method_get.conf"; - - # Proxy - proxy_pass "http://ic_boundary"; - include "includes/proxy_headers.conf"; - } - - location ~ /api/v2/canister/[0-9a-zA-Z\-]+/query { - # Observability - include "includes/request_id.conf"; - - # Prereqs - include "includes/method_post.conf"; - - # Proxy - proxy_pass "http://ic_boundary"; - include "includes/proxy_headers.conf"; - } - - location ~ /api/(v2|v3)/canister/[0-9a-zA-Z\-]+/call { - # Observability - include "includes/request_id.conf"; - - # Prereqs - include "includes/method_post.conf"; - - # Proxy - proxy_pass "http://ic_boundary"; - include "includes/proxy_headers.conf"; - } - - location ~ /api/v2/canister/[0-9a-zA-Z\-]+/read_state { - # Observability - include "includes/request_id.conf"; - - # Prereqs - include "includes/method_post.conf"; - - # Proxy - proxy_pass "http://ic_boundary"; - include "includes/proxy_headers.conf"; - } -} diff --git a/ic-os/components/boundary-guestos/etc/nginx/conf.d/002-mainnet-nginx.conf b/ic-os/components/boundary-guestos/etc/nginx/conf.d/002-mainnet-nginx.conf deleted file mode 100644 index b828a4ab032..00000000000 --- a/ic-os/components/boundary-guestos/etc/nginx/conf.d/002-mainnet-nginx.conf +++ /dev/null @@ -1,365 +0,0 @@ -# Server blocks for custom domains -include "/var/opt/nginx/domains.conf"; - -server { - listen 443 ssl; - listen [::]:443 ssl; - - server_name .boundary.dfinity.network; - server_name .mainnet.dfinity.network; - - location / { - # Observability - include "includes/request_id.conf"; - - return 308 $scheme://$primary_application_domain$request_uri; - } -} - -# API Domains ONLY -server { - listen 443 ssl; - listen [::]:443 ssl; - - include "/run/ic-node/etc/nginx/conf.d/api_domain.conf"; - include "includes/error_pages.conf"; - - location = / { - # Observability - include "includes/request_id.conf"; - - include "includes/method_get.conf"; - return 302 https://dashboard.internetcomputer.org/; - } - - location ~ (/api/v2/status|/health) { - # Observability - include "includes/request_id.conf"; - - # Prereqs - include "includes/method_get.conf"; - - # CORS - set $cors_allow_methods "HEAD, GET"; - include "includes/response_headers.conf"; - include "includes/options.conf"; - - # Proxy - proxy_pass "http://ic_boundary"; - include "includes/proxy_headers.conf"; - } - - location ~ /api/v2/canister/[0-9a-zA-Z\-]+/query { - # Observability - include "includes/request_id.conf"; - - # Prereqs - include "includes/method_post.conf"; - - # CORS - set $cors_allow_methods "HEAD, POST"; - include "includes/response_headers.conf"; - include "includes/options.conf"; - - # Proxy - proxy_pass "http://ic_boundary"; - include "includes/proxy_headers.conf"; - } - - location ~ /api/(v2|v3)/canister/[0-9a-zA-Z\-]+/call { - # Observability - include "includes/request_id.conf"; - - # Prereqs - include "includes/method_post.conf"; - - # CORS - set $cors_allow_methods "HEAD, POST"; - include "includes/response_headers.conf"; - include "includes/options.conf"; - - # Proxy - proxy_pass "http://ic_boundary"; - include "includes/proxy_headers.conf"; - } - - location ~ /api/v2/(canister|subnet)/[0-9a-zA-Z\-]+/read_state { - # Observability - include "includes/request_id.conf"; - - # Prereqs - include "includes/method_post.conf"; - - # CORS - set $cors_allow_methods "HEAD, POST"; - include "includes/response_headers.conf"; - include "includes/options.conf"; - - # Proxy - proxy_pass "http://ic_boundary"; - include "includes/proxy_headers.conf"; - } -} - -# System and Application Domains ONLY -server { - listen 443 ssl; - listen [::]:443 ssl; - - include "/run/ic-node/etc/nginx/conf.d/server_domain.conf"; - include "includes/error_pages.conf"; - - location = / { - # Observability - include "includes/request_id.conf"; - - include "includes/method_get.conf"; - return 302 https://dashboard.internetcomputer.org/; - } - - location ~ (/api/v2/status|/health) { - # Observability - include "includes/request_id.conf"; - - # Prereqs - include "includes/method_get.conf"; - - # CORS - set $cors_allow_methods "HEAD, GET"; - include "includes/response_headers.conf"; - include "includes/options.conf"; - - # Proxy - proxy_pass "http://ic_boundary"; - include "includes/proxy_headers.conf"; - } - - location ~ /api/v2/canister/[0-9a-zA-Z\-]+/query { - # Observability - include "includes/request_id.conf"; - - # Prereqs - include "includes/method_post.conf"; - - # CORS - set $cors_allow_methods "HEAD, POST"; - include "includes/response_headers.conf"; - include "includes/options.conf"; - - # Proxy - proxy_pass "http://ic_boundary"; - include "includes/proxy_headers.conf"; - } - - location ~ /api/(v2|v3)/canister/[0-9a-zA-Z\-]+/call { - # Observability - include "includes/request_id.conf"; - - # Prereqs - include "includes/method_post.conf"; - - # CORS - set $cors_allow_methods "HEAD, POST"; - include "includes/response_headers.conf"; - include "includes/options.conf"; - - # Proxy - proxy_pass "http://ic_boundary"; - include "includes/proxy_headers.conf"; - } - - location ~ /api/v2/(canister|subnet)/[0-9a-zA-Z\-]+/read_state { - # Observability - include "includes/request_id.conf"; - - # Prereqs - include "includes/method_post.conf"; - - # CORS - set $cors_allow_methods "HEAD, POST"; - include "includes/response_headers.conf"; - include "includes/options.conf"; - - # Proxy - proxy_pass "http://ic_boundary"; - include "includes/proxy_headers.conf"; - } - - # Custom Domains - location ~ /registrations\/?$ { - # Observability - include "includes/request_id.conf"; - - # Prereqs - include "includes/method_post.conf"; - - # Limits - limit_req zone=rgs_global nodelay; - limit_req zone=rgs_per_ip nodelay; - - # CORS - set $cors_allow_methods "HEAD, POST"; - include "includes/response_headers.conf"; - include "includes/options.conf"; - - # Proxy - proxy_pass "http://cert_issuer"; - include "includes/proxy_headers.conf"; - } - - location ~ /registrations/[0-9a-zA-Z]+$ { - # Observability - include "includes/request_id.conf"; - - # Limits - limit_req zone=rgs_global nodelay; - limit_req zone=rgs_per_ip nodelay; - - # CORS - set $cors_allow_methods "HEAD, GET, PUT, DELETE"; - include "includes/response_headers.conf"; - include "includes/options.conf"; - - # Proxy - proxy_pass "http://cert_issuer"; - include "includes/proxy_headers.conf"; - } -} - -server { - listen 443 ssl; - listen [::]:443 ssl; - - include "/run/ic-node/etc/nginx/conf.d/server_raw_domain.conf"; - include "includes/error_pages.conf"; - - location = / { - # Observability - include "includes/request_id.conf"; - - include "includes/method_get.conf"; - return 302 https://dashboard.internetcomputer.org/; - } - - location = /api/v2/status { - # Observability - include "includes/request_id.conf"; - - include "includes/method_get.conf"; - return 307 $scheme://$primary_application_domain$request_uri; - } - - location ~ /api/v2/canister/[0-9a-zA-Z\-]+/query { - # Observability - include "includes/request_id.conf"; - - include "includes/method_post.conf"; - return 307 $scheme://$primary_application_domain$request_uri; - } - - location ~ /api/(v2|v3)/canister/[0-9a-zA-Z\-]+/call { - # Observability - include "includes/request_id.conf"; - - include "includes/method_post.conf"; - return 307 $scheme://$primary_application_domain$request_uri; - } - - location ~ /api/v2/(canister|subnet)/[0-9a-zA-Z\-]+/read_state { - # Observability - include "includes/request_id.conf"; - - include "includes/method_post.conf"; - return 307 $scheme://$primary_application_domain$request_uri; - } -} - -server { - listen 443 ssl; - listen [::]:443 ssl; - - include "/run/ic-node/etc/nginx/conf.d/server_domain_escaped.conf"; - include "includes/error_pages.conf"; - - ######################################################## - # TODO as a follow up for BOUN-65 - # Eventually we need to remove the redirects - ######################################################## - - location = /api/v2/status { - # Observability - include "includes/request_id.conf"; - - include "includes/method_get.conf"; - return 307 $scheme://$primary_api_domain$request_uri; - } - - location ~ /api/v2/canister/[0-9a-zA-Z\-]+/query { - # Observability - include "includes/request_id.conf"; - - include "includes/method_post.conf"; - return 307 $scheme://$primary_api_domain$request_uri; - } - - location ~ /api/(v2|v3)/canister/[0-9a-zA-Z\-]+/call { - # Observability - include "includes/request_id.conf"; - - include "includes/method_post.conf"; - return 307 $scheme://$primary_api_domain$request_uri; - } - - location ~ /api/v2/(canister|subnet)/[0-9a-zA-Z\-]+/read_state { - # Observability - include "includes/request_id.conf"; - - include "includes/method_post.conf"; - return 307 $scheme://$primary_api_domain$request_uri; - } - - location / { - # Observability - include "includes/request_id.conf"; - - # CORS - set $cors_allow_methods "HEAD, GET, OPTIONS"; - include "includes/response_headers.conf"; - include "includes/options.conf"; - - # Cache - include "includes/cache.conf"; - - proxy_pass http://icx_proxy; - include "includes/proxy_headers.conf"; - - proxy_set_header x-icx-require-certification "1"; - - add_header "X-Cache-Status" $upstream_cache_status; - } -} - -server { - listen 443 ssl; - listen [::]:443 ssl; - - include "/run/ic-node/etc/nginx/conf.d/server_raw_domain_escaped.conf"; - include "includes/error_pages.conf"; - - location / { - # Observability - include "includes/request_id.conf"; - - # CORS - set $cors_allow_methods "HEAD, GET, POST, OPTIONS"; - include "includes/response_headers.conf"; - include "includes/options.conf"; - - # Cache - include "includes/cache.conf"; - include "includes/slice.conf"; - - proxy_pass http://icx_proxy; - include "includes/proxy_headers.conf"; - } -} diff --git a/ic-os/components/boundary-guestos/etc/nginx/ffdhe4096.pem b/ic-os/components/boundary-guestos/etc/nginx/ffdhe4096.pem deleted file mode 100644 index 3cf0fcbc011..00000000000 --- a/ic-os/components/boundary-guestos/etc/nginx/ffdhe4096.pem +++ /dev/null @@ -1,13 +0,0 @@ ------BEGIN DH PARAMETERS----- -MIICCAKCAgEA//////////+t+FRYortKmq/cViAnPTzx2LnFg84tNpWp4TZBFGQz -+8yTnc4kmz75fS/jY2MMddj2gbICrsRhetPfHtXV/WVhJDP1H18GbtCFY2VVPe0a -87VXE15/V8k1mE8McODmi3fipona8+/och3xWKE2rec1MKzKT0g6eXq8CrGCsyT7 -YdEIqUuyyOP7uWrat2DX9GgdT0Kj3jlN9K5W7edjcrsZCwenyO4KbXCeAvzhzffi -7MA0BM0oNC9hkXL+nOmFg/+OTxIy7vKBg8P+OxtMb61zO7X8vC7CIAXFjvGDfRaD -ssbzSibBsu/6iGtCOGEfz9zeNVs7ZRkDW7w09N75nAI4YbRvydbmyQd62R0mkff3 -7lmMsPrBhtkcrv4TCYUTknC0EwyTvEN5RPT9RFLi103TZPLiHnH1S/9croKrnJ32 -nuhtK8UiNjoNq8Uhl5sN6todv5pC1cRITgq80Gv6U93vPBsg7j/VnXwl5B0rZp4e -8W5vUsMWTfT7eTDp5OWIV7asfV9C1p9tGHdjzx1VA0AEh/VbpX4xzHpxNciG77Qx -iu1qHgEtnmgyqQdgCpGBMMRtx3j5ca0AOAkpmaMzy4t6Gh25PXFAADwqTs6p+Y0K -zAqCkc3OyX3Pjsm1Wn+IpGtNtahR9EGC4caKAH5eZV9q//////////8CAQI= ------END DH PARAMETERS----- diff --git a/ic-os/components/boundary-guestos/etc/nginx/ic_public_key.pem b/ic-os/components/boundary-guestos/etc/nginx/ic_public_key.pem deleted file mode 100644 index 21ccf24074f..00000000000 --- a/ic-os/components/boundary-guestos/etc/nginx/ic_public_key.pem +++ /dev/null @@ -1,5 +0,0 @@ ------BEGIN PUBLIC KEY----- -MIGCMB0GDSsGAQQBgtx8BQMBAgEGDCsGAQQBgtx8BQMCAQNhAIFMDm7HH6tYOwi9 -gTc8JVw8NxsuhIY8mKTx4It0I10U+12cDNVG2WhfkToMCyzFNBWDv0tDkuRn25bW -W5u0y3FxEvhHLg1aTRRQX/10hLASkQkcX4e5iINGP5gJGguqrg== ------END PUBLIC KEY----- diff --git a/ic-os/components/boundary-guestos/etc/nginx/includes/cache.conf b/ic-os/components/boundary-guestos/etc/nginx/includes/cache.conf deleted file mode 100644 index 6759a8765f7..00000000000 --- a/ic-os/components/boundary-guestos/etc/nginx/includes/cache.conf +++ /dev/null @@ -1,3 +0,0 @@ -# Enable caching -proxy_buffering on; -proxy_cache cache_static; diff --git a/ic-os/components/boundary-guestos/etc/nginx/includes/error_pages.conf b/ic-os/components/boundary-guestos/etc/nginx/includes/error_pages.conf deleted file mode 100644 index 07d94d3ca2e..00000000000 --- a/ic-os/components/boundary-guestos/etc/nginx/includes/error_pages.conf +++ /dev/null @@ -1,8 +0,0 @@ -proxy_intercept_errors on; - -error_page 451 /451.html; - -location /451.html { - internal; - root /var/www/html; -} diff --git a/ic-os/components/boundary-guestos/etc/nginx/includes/method_get.conf b/ic-os/components/boundary-guestos/etc/nginx/includes/method_get.conf deleted file mode 100644 index 89003a26ace..00000000000 --- a/ic-os/components/boundary-guestos/etc/nginx/includes/method_get.conf +++ /dev/null @@ -1,7 +0,0 @@ -if ($request_method !~ "HEAD|GET|OPTIONS") { - # Observability - include "includes/request_id.conf"; - - add_header "Allow" "HEAD, GET, OPTIONS" always; - return 405; -} diff --git a/ic-os/components/boundary-guestos/etc/nginx/includes/method_post.conf b/ic-os/components/boundary-guestos/etc/nginx/includes/method_post.conf deleted file mode 100644 index 82db4fc058a..00000000000 --- a/ic-os/components/boundary-guestos/etc/nginx/includes/method_post.conf +++ /dev/null @@ -1,7 +0,0 @@ -if ($request_method !~ "POST|OPTIONS") { - # Observability - include "includes/request_id.conf"; - - add_header "Allow" "POST, OPTIONS" always; - return 405; -} diff --git a/ic-os/components/boundary-guestos/etc/nginx/includes/options.conf b/ic-os/components/boundary-guestos/etc/nginx/includes/options.conf deleted file mode 100644 index 65b443fe63a..00000000000 --- a/ic-os/components/boundary-guestos/etc/nginx/includes/options.conf +++ /dev/null @@ -1,12 +0,0 @@ -if ($request_method = "OPTIONS") { - add_header Content-Type "text/plain; charset=utf-8" always; - add_header Content-Length "0" always; - - # Observability - include "includes/request_id.conf"; - - # required because any `add_header` within an `if` will remove previously set `add_header` - include "includes/response_headers.conf"; - - return 204; -} diff --git a/ic-os/components/boundary-guestos/etc/nginx/includes/proxy_headers.conf b/ic-os/components/boundary-guestos/etc/nginx/includes/proxy_headers.conf deleted file mode 100644 index a538ec5eb8c..00000000000 --- a/ic-os/components/boundary-guestos/etc/nginx/includes/proxy_headers.conf +++ /dev/null @@ -1,31 +0,0 @@ -# Basic headers -proxy_set_header Host $host; -proxy_set_header X-Real-IP $remote_addr; -proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; -proxy_set_header X-Forwarded-Proto $scheme; -proxy_set_header Connection ""; - -# Request-ID -proxy_set_header X-Request-ID $request_uuid; - -# Remove CORS-related headers -proxy_hide_header Access-Control-Allow-Origin; -proxy_hide_header Access-Control-Allow-Methods; -proxy_hide_header Access-Control-Allow-Credentials; -proxy_hide_header Access-Control-Allow-Headers; -proxy_hide_header Access-Control-Expose-Headers; -proxy_hide_header Access-Control-Max-Age; - -# Headers used for logging -proxy_hide_header x-ic-error-cause; -proxy_hide_header x-ic-subnet-id; -proxy_hide_header x-ic-canister-id-cbor; -proxy_hide_header x-ic-cache-status; -proxy_hide_header x-ic-cache-bypass-reason; -proxy_hide_header x-ic-country-code; -proxy_hide_header x-ic-node-id; -proxy_hide_header x-ic-request-type; -proxy_hide_header x-ic-subnet-type; -proxy_hide_header x-ic-sender; -proxy_hide_header x-ic-retries; -proxy_hide_header x-ic-method-name; diff --git a/ic-os/components/boundary-guestos/etc/nginx/includes/request_id.conf b/ic-os/components/boundary-guestos/etc/nginx/includes/request_id.conf deleted file mode 100644 index 02f89ebf40b..00000000000 --- a/ic-os/components/boundary-guestos/etc/nginx/includes/request_id.conf +++ /dev/null @@ -1,2 +0,0 @@ -# Response -add_header X-Request-ID $request_uuid always; diff --git a/ic-os/components/boundary-guestos/etc/nginx/includes/response_headers.conf b/ic-os/components/boundary-guestos/etc/nginx/includes/response_headers.conf deleted file mode 100644 index b4fc5799aa8..00000000000 --- a/ic-os/components/boundary-guestos/etc/nginx/includes/response_headers.conf +++ /dev/null @@ -1,6 +0,0 @@ -# CORS -add_header "Access-Control-Allow-Origin" "*" always; -add_header "Access-Control-Allow-Methods" "$cors_allow_methods" always; -add_header "Access-Control-Allow-Headers" "DNT,User-Agent,X-Requested-With,If-None-Match,If-Modified-Since,Cache-Control,Content-Type,Range,Cookie,X-Ic-Canister-Id" always; -add_header "Access-Control-Expose-Headers" "Accept-Ranges,Content-Length,Content-Range,X-Request-Id,X-Ic-Canister-Id" always; -add_header "Access-Control-Max-Age" "600" always; diff --git a/ic-os/components/boundary-guestos/etc/nginx/includes/slice.conf b/ic-os/components/boundary-guestos/etc/nginx/includes/slice.conf deleted file mode 100644 index b7256ed5703..00000000000 --- a/ic-os/components/boundary-guestos/etc/nginx/includes/slice.conf +++ /dev/null @@ -1,3 +0,0 @@ -# Enable slicing -slice 1m; -proxy_set_header Range $slice_range; diff --git a/ic-os/components/boundary-guestos/etc/nginx/includes/whitelist_rosetta.conf b/ic-os/components/boundary-guestos/etc/nginx/includes/whitelist_rosetta.conf deleted file mode 100644 index ae97bdee6cd..00000000000 --- a/ic-os/components/boundary-guestos/etc/nginx/includes/whitelist_rosetta.conf +++ /dev/null @@ -1,70 +0,0 @@ -# Internal Traffic (Pritunl VPN) -allow 2001:4d78:40d::/48; # FR1-old -allow 2602:fb2b:110::/48; # FR1 -allow 2a00:fb01:400::/56; # ZH1 -allow 2607:fb58:9005::/48; # SF1-old -allow 2602:fb2b:100::/48; # SF1 -allow 2607:f6f0:3004::/48; # CH1-old -allow 2602:fb2b:120::/48; # CH1 -allow 2a0b:21c0:4003:2::/64; # LN1 -allow 2a0b:21c0:4006:100::/56; # LN1-add -allow fda6:8d22:43e1::/48; # kubernetes internal - -# Third Party Clients -allow 212.71.124.194; -allow 149.97.209.182; -allow 54.85.1.24; -allow 34.224.245.49; -allow 54.205.248.136; -allow 52.0.183.139; -allow 34.198.224.121; -allow 50.19.243.132; -allow 184.73.195.123; -allow 52.23.35.172; -allow 34.202.32.58; -allow 50.16.21.213; -allow 3.227.72.188; -allow 3.214.85.212; -allow 87.239.207.244; -allow 172.111.167.93; -allow 54.250.134.234; -allow 18.176.155.169; -allow 3.112.138.57; -allow 54.238.51.31; -allow 13.114.212.150; -allow 52.197.6.219; -allow 18.179.76.71; -allow 13.230.19.97; -allow 52.194.107.16; -allow 54.248.13.68; -allow 18.176.159.241; -allow 47.52.252.183; -allow 35.74.17.158; -allow 54.199.26.21; -allow 161.117.238.176; -allow 47.75.13.236; -allow 13.112.52.91; -allow 34.142.37.197; -allow 65.109.53.116; -allow 47.244.167.76; -allow 34.195.53.91; -allow 18.235.35.236; -allow 52.4.132.28; -allow 52.30.132.101; -allow 54.216.66.11; -allow 46.137.182.253; -allow 35.79.207.60; -allow 18.176.159.242; -allow 54.199.153.43; -allow 44.197.125.86; -allow 54.159.132.13; -allow 18.235.42.46; -allow 44.195.76.177; -allow 52.2.138.32; -allow 44.199.174.45; -allow 35.168.176.14; -allow 13.42.94.126; -allow 18.169.224.211; -allow 18.170.10.91; - -deny all; diff --git a/ic-os/components/boundary-guestos/etc/nginx/nginx.conf b/ic-os/components/boundary-guestos/etc/nginx/nginx.conf deleted file mode 100644 index 91b54de0b50..00000000000 --- a/ic-os/components/boundary-guestos/etc/nginx/nginx.conf +++ /dev/null @@ -1,176 +0,0 @@ -user www-data; -worker_processes auto; -pid /run/nginx.pid; - -events { - worker_connections 8192; -} - -# During an nginx reload, all the existing workers finish up with their clients -# and shutdown gracefully. If their clients don't finish up, they never shut -# down. We've chosen a 10 minute timeout to limit the impact on long downloads -# that happen to occur during an nginx reload. -worker_shutdown_timeout 10m; - -http { - sendfile on; - tcp_nopush on; - tcp_nodelay on; - - # Keepalive settings - ## Idle connection timeout - keepalive_timeout 30; # default 75s - - ## Close keepalive connections more frequently to free per-connection resources - keepalive_time 10m; # default 1h - keepalive_requests 500; # default 1000 - - # Client timeouts - ## Full header reading timeout - client_header_timeout 15s; # default 60s - ## Timeout between two read operations from the client, not full body transmit time - client_body_timeout 15s; # default 60s - ## Timeout between two write operations to the client, not full body transmit time - send_timeout 15s; # default 60s - ## More aggressively close timed out connections, don't linger in FIN_WAIT1 - reset_timedout_connection on; - - # Do not emit server version in headers - server_tokens off; - - # Disable body checking since it's streamed directly to backend and limited there - client_max_body_size 0; - - # Required for keepalive connections - proxy_http_version 1.1; - - # Increase timeout a bit - proxy_read_timeout 120; - - # Keepalives - proxy_socket_keepalive on; - - # Disable request & response buffering since we don't need to cache anything here - proxy_request_buffering off; - proxy_buffering off; - - # Enlarge buffers, they should be big enough to fit a full response header. - # IC response headers can be quite big due to `ic-certificate` and similar - proxy_buffer_size 32k; - proxy_buffers 4 64k; - proxy_busy_buffers_size 64k; - proxy_connect_timeout 20s; # default 60s - - # HTTP2 - http2 on; - http2_max_concurrent_streams 100; - - # Increase some hash sizes - server_names_hash_bucket_size 128; - - include "/etc/nginx/mime.types"; - default_type application/octet-stream; - - ## - # TLS Settings - ## - - # Dropping SSLv3, ref: POODLE - # Dropping TLSv1/v1.1, deprecated: https://datatracker.ietf.org/doc/rfc8996/ - ssl_protocols TLSv1.2 TLSv1.3; - - # Switching this off is recommended when no old TLS version is used and/or weak ciphers are disabled - # This allows client to choose the cipher that it has better support for (e.g. AES hardware acceleration) - # See https://wiki.mozilla.org/Security/Server_Side_TLS#Recommended_configurations (Cipher preference: client chooses) - ssl_prefer_server_ciphers off; - - ssl_certificate /run/ic-node/etc/nginx/certs/fullchain.pem; - ssl_trusted_certificate /run/ic-node/etc/nginx/certs/chain.pem; - ssl_certificate_key /run/ic-node/etc/nginx/keys/privkey.pem; - - # Predefined DHE params from RFC 7919 https://github.com/internetstandards/dhe_groups/blob/master/ffdhe4096.pem - ssl_dhparam /etc/nginx/ffdhe4096.pem; - - # Increase session timeout to 1 hour (Cloudflare uses even 18 hours) - # See https://blog.cloudflare.com/tls-session-resumption-full-speed-and-secure/ - # 256MB should be good for ~1M sessions - ssl_session_timeout 1h; - ssl_session_cache shared:SSL:256m; - - # Session tickets seem to weaken Perfect Forward Secrecy. - # See: - # - https://www.imperialviolet.org/2013/06/27/botchingpfs.html - # - https://blog.compass-security.com/2017/06/about-tls-perfect-forward-secrecy-and-session-resumption/ - ssl_session_tickets off; - - # prime256v1 (NIST P-256) curve is considered less safe - # See http://safecurves.cr.yp.to/ - ssl_ecdh_curve secp521r1:secp384r1; - - # ssllabs.com Cipher Strength - see https://wiki.mozilla.org/Security/Server_Side_TLS#Recommended_configurations - # TLSv1.3 ciphers are enabled with it by default and don't need to be specified here - ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384; - - # Lowered from default 16k to get faster TTFB - ssl_buffer_size 4k; - - # OCSP stapling - ssl_stapling on; - ssl_stapling_verify on; - resolver 1.1.1.1 1.0.0.1 [2606:4700:4700::1111] [2606:4700:4700::1001] valid=300s; # Cloudflare - resolver_timeout 5s; - - ## - # Logging Settings - ## - - log_format access escape=json '{' - '"body_bytes_sent":' '"$body_bytes_sent",' - '"bytes_sent":' '"$bytes_sent",' - '"cache_bypass_reason":' '"$upstream_http_x_ic_cache_bypass_reason",' - '"cache_status":' '"$upstream_http_x_ic_cache_status",' - '"cache_status_nginx":' '"$upstream_cache_status",' - '"content_length":' '"$content_length",' - '"content_type":' '"$content_type",' - '"error_cause":' '"$upstream_http_x_ic_error_cause",' - '"geo_country_code":' '"$upstream_http_x_ic_country_code",' - '"hostname":' '"$hostname",' - '"http_host":' '"$http_host",' - '"http_origin":' '"$http_origin",' - '"http_referer":' '"$http_referer",' - '"http_user_agent":' '"$http_user_agent",' - '"https":' '"$https",' - '"ic_canister_id":' '"$upstream_http_x_ic_canister_id",' - '"ic_canister_id_cbor":' '"$upstream_http_x_ic_canister_id_cbor",' - '"ic_method_name":' '"$upstream_http_x_ic_method_name",' - '"ic_node_id":' '"$upstream_http_x_ic_node_id",' - '"ic_request_type":' '"$upstream_http_x_ic_request_type",' - '"ic_sender":' '"$upstream_http_x_ic_sender",' - '"ic_subnet_id":' '"$upstream_http_x_ic_subnet_id",' - '"msec":' '"$msec",' - '"query_string":' '"$query_string",' - '"remote_addr":' '"$remote_addr",' - '"retries":' '"$upstream_http_x_ic_retries",' - '"request_id":' '"$request_uuid",' - '"request_length":' '"$request_length",' - '"request_method":' '"$request_method",' - '"request_time":' '"$request_time",' - '"request_time_headers":' '"$upstream_header_time",' - '"request_uri":' '"$request_uri",' - '"server_protocol":' '"$server_protocol",' - '"ssl_cipher":' '"$ssl_cipher",' - '"ssl_protocol":' '"$ssl_protocol",' - '"status":' '"$status",' - '"status_upstream":' '"$upstream_status",' - '"upstream":' '"$upstream_addr"' - '}'; - - access_log syslog:server=unix:/var/lib/vector/nginx.socket,tag=access,nohostname access; - error_log /dev/null emerg; - - ## - # Virtual Host Configs - ## - - include /etc/nginx/conf.d/*.conf; -} diff --git a/ic-os/components/boundary-guestos/etc/systemd/system/certificate-syncer.service b/ic-os/components/boundary-guestos/etc/systemd/system/certificate-syncer.service deleted file mode 100644 index dcbff4ef347..00000000000 --- a/ic-os/components/boundary-guestos/etc/systemd/system/certificate-syncer.service +++ /dev/null @@ -1,26 +0,0 @@ -[Unit] -Description=Certificate Syncer -After=network-online.target -Wants=network-online.target -After=setup-certificate-syncer.service -BindsTo=setup-certificate-syncer.service - -[Service] -LogRateLimitIntervalSec=1ms -LogRateLimitBurst=1000 -User=root -Group=root -Restart=always -EnvironmentFile=/run/ic-node/etc/default/certificate-syncer -ExecStart=/bin/bash -c ' \ - /opt/ic/bin/certificate-syncer \ - --certificates-exporter-uri "http://localhost:3000/certificates" \ - --local-certificates-path "/var/opt/nginx/certs" \ - --local-configuration-path "/var/opt/nginx/domains.conf" \ - --configuration-template-path "/etc/certificate-syncer/domain.tmpl" \ - --metrics-addr "[::]:9322" \ - ${POLLING_INTERVAL_SEC:+ --polling-interval-sec "${POLLING_INTERVAL_SEC}"} \ -' - -[Install] -WantedBy=multi-user.target diff --git a/ic-os/components/boundary-guestos/etc/systemd/system/ic-boundary.service b/ic-os/components/boundary-guestos/etc/systemd/system/ic-boundary.service index df345f2fdbd..4ba60cc5d1c 100644 --- a/ic-os/components/boundary-guestos/etc/systemd/system/ic-boundary.service +++ b/ic-os/components/boundary-guestos/etc/systemd/system/ic-boundary.service @@ -17,18 +17,15 @@ ExecStart=/bin/bash -c ' \ --local-store-path /var/opt/registry/store \ --nns-pub-key-pem /run/ic-node/etc/default/nns_public_key.pem \ --nns-urls "${NNS_URL}" \ - --http-unix-socket /run/ic-node/ic-boundary.socket \ + --http-port 9000 \ --metrics-addr "[::]:9324" \ --log-stdout \ --log-failed-requests-only \ --nftables-system-replicas-path /run/ic-node/etc/nftables/system_replicas.ruleset \ - --geoip-db /run/ic-node/etc/ic-boundary/GeoLite2-Country.mmdb \ --disable-latency-routing \ --retry-update-call \ --rate-limit-per-second-per-subnet "1000" \ --http-client-count "8" \ - ${MAX_CONCURRENCY:+ --max-concurrency "${MAX_CONCURRENCY}"} \ - ${SHED_EWMA_PARAM:+ --shed-ewma-param "${SHED_EWMA_PARAM}"} \ ${CACHE_SIZE:+ --cache-size-bytes "${CACHE_SIZE}"} \ ${CACHE_ITEM_MAX_SIZE:+ --cache-max-item-size-bytes "${CACHE_ITEM_MAX_SIZE}"} \ ${CACHE_TTL:+ --cache-ttl-seconds "${CACHE_TTL}"} \ diff --git a/ic-os/components/boundary-guestos/etc/systemd/system/ic-gateway.service b/ic-os/components/boundary-guestos/etc/systemd/system/ic-gateway.service new file mode 100644 index 00000000000..103e7523d2b --- /dev/null +++ b/ic-os/components/boundary-guestos/etc/systemd/system/ic-gateway.service @@ -0,0 +1,16 @@ +[Unit] +Description=IC-Gateway +After=network-online.target +Wants=network-online.target +After=setup-ic-gateway.service +BindsTo=setup-ic-gateway.service + +[Service] +User=root +Group=root +Restart=always +EnvironmentFile=/run/ic-node/etc/ic-gateway/env +ExecStart=/usr/sbin/ic-gateway + +[Install] +WantedBy=multi-user.target diff --git a/ic-os/components/boundary-guestos/etc/systemd/system/icx-proxy.service b/ic-os/components/boundary-guestos/etc/systemd/system/icx-proxy.service deleted file mode 100644 index c975ce207b6..00000000000 --- a/ic-os/components/boundary-guestos/etc/systemd/system/icx-proxy.service +++ /dev/null @@ -1,41 +0,0 @@ -[Unit] -Description=ICX Proxy Service -After=network-online.target -Wants=network-online.target -After=setup-icx-proxy.service -BindsTo=setup-icx-proxy.service - -[Service] -User=root -Group=root -LimitNOFILE=524288 -EnvironmentFile=/run/ic-node/etc/icx-proxy/env -ExecStart=/bin/bash -c ' \ - /opt/ic/bin/icx-proxy \ - --unix-socket /run/ic-node/icx-proxy.socket \ - --canister-alias "personhood:g3wsl-eqaaa-aaaan-aaaaa-cai" \ - --canister-alias "identity:rdmx6-jaaaa-aaaaa-aaadq-cai" \ - --canister-alias "nns:qoctq-giaaa-aaaaa-aaaea-cai" \ - --canister-alias "dscvr:h5aet-waaaa-aaaab-qaamq-cai" \ - --metrics-addr "[::]:9314" \ - --root-key "/run/ic-node/etc/icx-proxy/root_key.der" \ - --replica-unix-socket /run/ic-node/ic-boundary.socket \ - --pre-isolation-canisters /run/ic-node/etc/icx-proxy/pre_isolation_canisters.txt \ - --geoip-db /run/ic-node/etc/icx-proxy/GeoLite2-Country.mmdb \ - ${DENYLIST_URL:+ --denylist-url "${DENYLIST_URL}"} \ - --denylist-initial /run/ic-node/etc/icx-proxy/denylist.json \ - --allowlist /run/ic-node/etc/icx-proxy/allowlist.txt \ - --quiet \ - ${DOMAINS} \ - ${DOMAINS_SYSTEM} \ - ${DOMAINS_APPLICATION} \ -' - -Restart=always -RestartSec=10 -KillSignal=SIGINT -StartLimitBurst=5 -StartLimitInterval=0 - -[Install] -WantedBy=multi-user.target diff --git a/ic-os/components/boundary-guestos/etc/systemd/system/nginx.service.d/override.conf b/ic-os/components/boundary-guestos/etc/systemd/system/nginx.service.d/override.conf deleted file mode 100644 index 8fa84064696..00000000000 --- a/ic-os/components/boundary-guestos/etc/systemd/system/nginx.service.d/override.conf +++ /dev/null @@ -1,8 +0,0 @@ -[Unit] -After=setup-nginx.service -BindsTo=setup-nginx.service - -[Service] -LimitNOFILE=524288 -LogRateLimitIntervalSec=1ms -LogRateLimitBurst=1000 diff --git a/ic-os/components/boundary-guestos/etc/systemd/system/setup-certificate-syncer.service b/ic-os/components/boundary-guestos/etc/systemd/system/setup-certificate-syncer.service deleted file mode 100644 index 52b09d3c12f..00000000000 --- a/ic-os/components/boundary-guestos/etc/systemd/system/setup-certificate-syncer.service +++ /dev/null @@ -1,11 +0,0 @@ -[Unit] -Description=Setup Certificate Syncer -DefaultDependencies=no -After=bootstrap-ic-node.service -Requires=bootstrap-ic-node.service - -[Service] -Type=oneshot -RemainAfterExit=true -ExecCondition=/opt/ic/bin/setup-certificate-syncer.sh -ExecStart=/usr/bin/true diff --git a/ic-os/components/boundary-guestos/etc/systemd/system/setup-icx-proxy.service b/ic-os/components/boundary-guestos/etc/systemd/system/setup-ic-gateway.service similarity index 60% rename from ic-os/components/boundary-guestos/etc/systemd/system/setup-icx-proxy.service rename to ic-os/components/boundary-guestos/etc/systemd/system/setup-ic-gateway.service index 3758ff86308..ee35f04691c 100644 --- a/ic-os/components/boundary-guestos/etc/systemd/system/setup-icx-proxy.service +++ b/ic-os/components/boundary-guestos/etc/systemd/system/setup-ic-gateway.service @@ -1,5 +1,5 @@ [Unit] -Description=Setup environment files for icx-proxy +Description=Setup environment for ic-gateway DefaultDependencies=no After=bootstrap-ic-node.service Requires=bootstrap-ic-node.service @@ -7,4 +7,4 @@ Requires=bootstrap-ic-node.service [Service] Type=oneshot RemainAfterExit=true -ExecStart=/opt/ic/bin/setup-icx-proxy.sh +ExecStart=/opt/ic/bin/setup-ic-gateway.sh diff --git a/ic-os/components/boundary-guestos/etc/systemd/system/setup-nginx.service b/ic-os/components/boundary-guestos/etc/systemd/system/setup-nginx.service deleted file mode 100644 index 1dc14c04b89..00000000000 --- a/ic-os/components/boundary-guestos/etc/systemd/system/setup-nginx.service +++ /dev/null @@ -1,12 +0,0 @@ -[Unit] -Description=Setup variable files (e.g. domain) for nginx -DefaultDependencies=no -After=bootstrap-ic-node.service -Requires=bootstrap-ic-node.service -Wants=network-online.target -After=network-online.target - -[Service] -Type=oneshot -RemainAfterExit=true -ExecStart=/opt/ic/bin/setup-nginx.sh diff --git a/ic-os/components/boundary-guestos/etc/systemd/system/vector.service b/ic-os/components/boundary-guestos/etc/systemd/system/vector.service index ec41778fecd..6ea513a8288 100644 --- a/ic-os/components/boundary-guestos/etc/systemd/system/vector.service +++ b/ic-os/components/boundary-guestos/etc/systemd/system/vector.service @@ -3,7 +3,6 @@ Description=Vector After=network-online.target Wants=network-online.target After=setup-vector.service -Before=nginx.service BindsTo=setup-vector.service [Service] diff --git a/ic-os/components/boundary-guestos/etc/tmpfiles.d/ic-node.conf b/ic-os/components/boundary-guestos/etc/tmpfiles.d/ic-node.conf index e653aef3e9d..bf47cce768c 100644 --- a/ic-os/components/boundary-guestos/etc/tmpfiles.d/ic-node.conf +++ b/ic-os/components/boundary-guestos/etc/tmpfiles.d/ic-node.conf @@ -12,9 +12,6 @@ d /run/ic-node/etc/node_exporter 0755 root root - d /run/ic-node/config 0755 root root - d /run/node_exporter/collector_textfile 0755 root root - d /run/ic-node/etc/socks 0755 root root - -d /run/ic-node/etc/nginx 0755 root root - -d /run/ic-node/etc/nginx/conf.d 0755 root root - -d /run/ic-node/etc/nginx/certs 0755 root root - -d /run/ic-node/etc/nginx/keys 0755 root root - +d /run/ic-node/etc/ic-gateway 0755 root root - d /run/ic-node/etc/nftables 0755 root root - d /run/ic-node/etc/default 0755 root root - diff --git a/ic-os/components/boundary-guestos/etc/vector/vector.yaml b/ic-os/components/boundary-guestos/etc/vector/vector.yaml index 359a163c7e9..6d8781f038e 100644 --- a/ic-os/components/boundary-guestos/etc/vector/vector.yaml +++ b/ic-os/components/boundary-guestos/etc/vector/vector.yaml @@ -5,36 +5,14 @@ sources: vector_metrics: type: internal_metrics - nginx_socket: - type: socket - - mode: unix_datagram - socket_file_mode: 666 - path: /var/lib/vector/nginx.socket - - decoding: - codec: syslog - journal: type: journald include_units: - certificate-issuer - - certificate-syncer - ic-boundary - danted transforms: - nginx_access_parsed: - type: remap - inputs: - - nginx_socket - - source: | - . = parse_json!(.message) - - # Inject the environment - .env = "${ENV:?ENV must be provided}" - danted: type: filter inputs: @@ -101,24 +79,6 @@ transforms: .service = "certificate-issuer" .timestamp = parse_timestamp!(.timestamp, "%+") - certificate_syncer: - type: filter - inputs: - - journal - - condition: ._SYSTEMD_UNIT == "certificate-syncer.service" - - certificate_syncer_normalized: - type: remap - inputs: - - certificate_syncer - - source: | - . = parse_json!(.message) - - .service = "certificate-syncer" - .timestamp = parse_timestamp!(.timestamp, "%+") - ic_boundary: type: filter inputs: @@ -168,36 +128,10 @@ sinks: suppress_timestamp: true flush_period_secs: 7200 - vector_aws: - type: http - inputs: - - nginx_access_parsed - - uri: "${LOGGING_URL:?LOGGING_URL must be provided}" - auth: - strategy: basic - user: "${LOGGING_USER:?LOGGING_USER must be provided}" - password: "${LOGGING_PASSWORD:?LOGGING_PASSWORD must be provided}" - - compression: zstd - encoding: - codec: native - - buffer: - type: memory - # nginx event json is usually < 2kb, even less if parsed, so 100k shouldn't take much memory - max_events: 100000 - - batch: - max_bytes: 10485760 # 10MB - max_events: 10000 # 10k - timeout_secs: 5 - console: type: console inputs: - certificate_issuer_normalized - - certificate_syncer_normalized - ic_boundary_filtered encoding: diff --git a/ic-os/components/boundary-guestos/opt/ic/bin/bootstrap-ic-node.sh b/ic-os/components/boundary-guestos/opt/ic/bin/bootstrap-ic-node.sh index 8cbd01e3e4f..6e742e8dffb 100755 --- a/ic-os/components/boundary-guestos/opt/ic/bin/bootstrap-ic-node.sh +++ b/ic-os/components/boundary-guestos/opt/ic/bin/bootstrap-ic-node.sh @@ -64,7 +64,6 @@ function process_bootstrap() { certificate_issuer_enc_key.pem certificate_issuer_identity.pem certificate_issuer.conf - certificate_syncer.conf denylist.json ic_boundary.conf canister-ratelimit.yml diff --git a/ic-os/components/boundary-guestos/opt/ic/bin/setup-certificate-syncer.sh b/ic-os/components/boundary-guestos/opt/ic/bin/setup-certificate-syncer.sh deleted file mode 100644 index 4c7604a4c9d..00000000000 --- a/ic-os/components/boundary-guestos/opt/ic/bin/setup-certificate-syncer.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/bash - -set -euox pipefail -source '/opt/ic/bin/helpers.shlib' -source '/opt/ic/bin/exec_condition.shlib' - -readonly IDENTITY_PEM="${BOOT_DIR}/certificate_issuer_identity.pem" - -readonly RUN_DIR='/run/ic-node/etc/default' -readonly ENV_FILE="${RUN_DIR}/certificate-syncer" -readonly CONFIG_FILE="${BOOT_DIR}/certificate_syncer.conf" - -# Read the config variables. The files must be of the form -# "key=value" for each line with a specific set of keys permissible (see -# code below). -function read_variables() { - if [[ ! -d "${BOOT_DIR}" ]]; then - err "missing node configuration directory: ${BOOT_DIR}" - exit 1 - fi - - if [ -f "${CONFIG_FILE}" ]; then - # Read limited set of keys. Be extra-careful quoting values as it could - # otherwise lead to executing arbitrary shell code! - while IFS="=" read -r key value; do - case "${key}" in - "certificate_syncer_polling_interval_sec") POLLING_INTERVAL_SEC="${value}" ;; - esac - done <"${CONFIG_FILE}" - fi -} - -function generate_config() { - mkdir -p $(dirname "${ENV_FILE}") - - cat >"${ENV_FILE}" <"${DENY_LIST_DST}" + else + cp "${DENY_LIST_SRC}" "${DENY_LIST_DST}" + fi +} + +function generate_config() { + # Setup network key + get_nns_der >"${ROOT_KEY}" + + local DOMAINS_APP=$(join_by , ${APPLICATION_DOMAINS[@]}) + local DOMAINS_SYSTEM=$(join_by , ${SYSTEM_DOMAINS[@]}) + local DOMAINS_API=$(join_by , ${API_DOMAINS[@]}) + + # Allow denylist canister + cat >"${RUN_DIR}/allowlist.txt" <"${ENV_FILE}" <>"${ENV_FILE}" + fi + + if [ ! -z "${LOGGING_URL}" ]; then + echo "LOG_VECTOR_URL=\"${LOGGING_URL}\"" >>"${ENV_FILE}" + echo "LOG_VECTOR_USER=\"${LOGGING_USER}\"" >>"${ENV_FILE}" + echo "LOG_VECTOR_PASS=\"${LOGGING_PASSWORD}\"" >>"${ENV_FILE}" + fi + + if [ ! -z "${MAX_CONCURRENCY}" ]; then + echo "LOAD_MAX_CONCURRENCY=\"${MAX_CONCURRENCY}\"" >>"${ENV_FILE}" + fi + + if [ ! -z "${SHED_EWMA_PARAM}" ]; then + echo "LOAD_SHED_EWMA_PARAM=\"${SHED_EWMA_PARAM}\"" >>"${ENV_FILE}" + fi +} + +function setup_pre_isolation_canisters() { + local -r SRC_CANISTERS_PATH="${BOOT_DIR}/pre_isolation_canisters.txt" + local -r DST_CANISTERS_PATH="${RUN_DIR}/pre_isolation_canisters.txt" + + if [[ ! -f "${SRC_CANISTERS_PATH}" ]]; then + touch "${DST_CANISTERS_PATH}" + else + cp "${SRC_CANISTERS_PATH}" "${DST_CANISTERS_PATH}" + fi +} + +function setup_certs() { + mkdir -p "${RUN_DIR}/certs" + + if [ -f "${BOOT_DIR}/certs/fullchain.pem" ]; then + cp "${BOOT_DIR}/certs/fullchain.pem" "${RUN_DIR}/certs/cert.pem" + else + cp "/etc/ssl/certs/ssl-cert-snakeoil.pem" "${RUN_DIR}/certs/cert.pem" + fi + + if [ -f "${BOOT_DIR}/certs/privkey.pem" ]; then + cp "${BOOT_DIR}/certs/privkey.pem" "${RUN_DIR}/certs/cert.key" + else + cp "/etc/ssl/private/ssl-cert-snakeoil.key" "${RUN_DIR}/certs/cert.key" + fi +} + +function main() { + mkdir -p "${RUN_DIR}" + + read_variables + setup_pre_isolation_canisters + setup_geoip_db + copy_deny_list + setup_certs + generate_config +} + +main "$@" diff --git a/ic-os/components/boundary-guestos/opt/ic/bin/setup-icx-proxy.sh b/ic-os/components/boundary-guestos/opt/ic/bin/setup-icx-proxy.sh deleted file mode 100755 index 66056366655..00000000000 --- a/ic-os/components/boundary-guestos/opt/ic/bin/setup-icx-proxy.sh +++ /dev/null @@ -1,146 +0,0 @@ -#!/bin/bash - -set -euox pipefail -source '/opt/ic/bin/helpers.shlib' - -readonly BN_CONFIG="${BOOT_DIR}/bn_vars.conf" - -readonly RUN_DIR='/run/ic-node/etc/icx-proxy' -readonly ENV_FILE="${RUN_DIR}/env" -readonly ROOT_KEY="${RUN_DIR}/root_key.der" - -SYSTEM_DOMAINS=() -APPLICATION_DOMAINS=() - -# Read the config variables. The files must be of the form -# "key=value" for each line with a specific set of keys permissible (see -# code below). -function read_variables() { - if [[ ! -d "${BOOT_DIR}" ]]; then - err "missing node configuration directory: ${BOOT_DIR}" - exit 1 - fi - if [ ! -f "${BN_CONFIG}" ]; then - err "missing domain configuration: ${BN_CONFIG}" - exit 1 - fi - - # Read limited set of keys. Be extra-careful quoting values as it could - # otherwise lead to executing arbitrary shell code! - while IFS="=" read -r key value; do - case "${key}" in - "system_domains") SYSTEM_DOMAINS+=("${value}") ;; - "application_domains") APPLICATION_DOMAINS+=("${value}") ;; - "denylist_url") DENYLIST_URL="${value}" ;; - esac - done <"${BN_CONFIG}" - - if [[ "${#SYSTEM_DOMAINS[@]}" -eq 0 ]]; then - err "SYSTEM_DOMAINS variable not set. icx-proxy won't be configured." - exit 1 - fi - - if [[ "${#APPLICATION_DOMAINS[@]}" -eq 0 ]]; then - err "APPLICATION_DOMAINS variable not set. icx-proxy won't be configured." - exit 1 - fi - - check_nns_pem -} - -function setup_geolite2_dbs() { - local -r BOOT_DBS="${BOOT_DIR}/geolite2_dbs" - local -r EMPTY_DBS='/etc/geoip' - - if [[ ! -d "${BOOT_DBS}" ]]; then - err "missing geolite2 dbs dir '${BOOT_DBS}', defaulting to empty dbs '${EMPTY_DBS}'" - local -r DBS_SRC="${EMPTY_DBS}" - else - local -r DBS_SRC="${BOOT_DBS}" - fi - - cp "${DBS_SRC}/GeoLite2-Country.mmdb" "${RUN_DIR}" -} - -function copy_deny_list() { - local -r DENY_LIST_SRC="${BOOT_DIR}/denylist.json" - local -r DENY_LIST_DST="${RUN_DIR}/denylist.json" - - if [[ -f "${DENY_LIST_DST}" ]]; then - echo "${DENY_LIST_DST} already present, skipping" - return - fi - - if [[ ! -f "${DENY_LIST_SRC}" ]]; then - echo '{"canisters":{}}' >"${DENY_LIST_DST}" - else - cp "${DENY_LIST_SRC}" "${DENY_LIST_DST}" - fi -} - -function generate_icx_proxy_config() { - local -r DOMAINS=( - "${SYSTEM_DOMAINS[@]}" - "${APPLICATION_DOMAINS[@]}" - ) - - local -A UNIQUE_DOMAINS - - for DOMAIN in "${DOMAINS[@]}"; do - UNIQUE_DOMAINS[$DOMAIN]=0 - done - - for DOMAIN in "${!UNIQUE_DOMAINS[@]}"; do - ARG_DOMAINS+=("--domain ${DOMAIN}") - done - - for DOMAIN in "${SYSTEM_DOMAINS[@]}"; do - local DOMAIN_ESCAPED=${DOMAIN//\./\\.} - ARG_DOMAINS_SYSTEM+=("--domain-system-regex '^([^.]+\.)?(raw\.)?${DOMAIN_ESCAPED}$'") - done - - for DOMAIN in "${APPLICATION_DOMAINS[@]}"; do - local DOMAIN_ESCAPED=${DOMAIN//\./\\.} - ARG_DOMAINS_APPLICATION+=("--domain-app-regex '^([^.]+\.)?(raw\.)?${DOMAIN_ESCAPED}$'") - done - - mkdir -p "${RUN_DIR}" - - # Setup network key - get_nns_der >"${ROOT_KEY}" - - cat >"${ENV_FILE}" <"${RUN_DIR}/allowlist.txt" <"${DOMAIN_DIR}/set_primary_application_domain.conf" - echo "map nop \$primary_api_domain { default ${API_DOMAINS[0]}; }" >"${DOMAIN_DIR}/set_primary_api_domain.conf" - - local -r DOMAINS=( - "${SYSTEM_DOMAINS[@]}" - "${APPLICATION_DOMAINS[@]}" - ) - - local -A UNIQUE_DOMAINS - - for DOMAIN in "${DOMAINS[@]}"; do - UNIQUE_DOMAINS[$DOMAIN]=0 - done - - # server names - for DOMAIN in "${!UNIQUE_DOMAINS[@]}"; do - local DOMAIN_ESCAPED=${DOMAIN//\./\\.} - - echo "server_name .rosetta-exchanges.${DOMAIN};" >>"${DOMAIN_DIR}/server_rosetta_domain.conf" - echo "server_name ~^([^.]+\.${DOMAIN_ESCAPED})$;" >>"${DOMAIN_DIR}/server_domain_escaped.conf" - echo "server_name ~^([^.]+\.raw\.${DOMAIN_ESCAPED})$;" >>"${DOMAIN_DIR}/server_raw_domain_escaped.conf" - echo "server_name ${DOMAIN};" >>"${DOMAIN_DIR}/server_domain.conf" - echo "server_name raw.${DOMAIN};" >>"${DOMAIN_DIR}/server_raw_domain.conf" - done - - # api domains - for DOMAIN in "${API_DOMAINS[@]}"; do - echo "server_name ${DOMAIN};" >>"${DOMAIN_DIR}/api_domain.conf" - done -} - -function setup_custom_domains() { - local -r SERVER_BLOCKS='/var/opt/nginx/domains.conf' - mkdir -p "$(dirname ${SERVER_BLOCKS})" - - if [[ ! -f "${SERVER_BLOCKS}" ]]; then - touch "${SERVER_BLOCKS}" - fi -} - -function main() { - read_variables - copy_certs - setup_domains - setup_custom_domains -} - -main "$@" diff --git a/ic-os/components/boundary-guestos/var/www/html/451.html b/ic-os/components/boundary-guestos/var/www/html/451.html deleted file mode 100644 index 3019c0799d4..00000000000 --- a/ic-os/components/boundary-guestos/var/www/html/451.html +++ /dev/null @@ -1,282 +0,0 @@ - - - - Internet Computer - Canister unavailable for legal reasons - - - - - - -
- -
-

This app is powered by

- -
- - -
-

451 (Unavailable for policy reasons)

-

- - The page you are looking for is currently being blocked. -

- - The owner of the Internet Computer boundary node that is - routing this page has placed its canister smart contract - URL under review due to possible - Code of Conduct violations. - -

-
- - - - -
- - diff --git a/ic-os/components/boundary-guestos/var/www/html/uninstall-script.js b/ic-os/components/boundary-guestos/var/www/html/uninstall-script.js deleted file mode 100644 index b65caae44af..00000000000 --- a/ic-os/components/boundary-guestos/var/www/html/uninstall-script.js +++ /dev/null @@ -1,13 +0,0 @@ -// install immediately -self.addEventListener('install', () => self.skipWaiting()); -self.addEventListener('activate', () => { - // uninstall itself & then reload page - self.registration - .unregister() - .then(function () { - return self.clients.matchAll(); - }) - .then(function (clients) { - clients.forEach((client) => client.navigate(client.url)); - }); -}); diff --git a/rs/tests/boundary_nodes/BUILD.bazel b/rs/tests/boundary_nodes/BUILD.bazel index be6fb1c6751..e75180d7369 100644 --- a/rs/tests/boundary_nodes/BUILD.bazel +++ b/rs/tests/boundary_nodes/BUILD.bazel @@ -1,5 +1,5 @@ load("//rs/tests:common.bzl", "BOUNDARY_NODE_GUESTOS_RUNTIME_DEPS", "COUNTER_CANISTER_RUNTIME_DEPS", "CUSTOM_DOMAINS_RUNTIME_DEPS", "DEPENDENCIES", "GRAFANA_RUNTIME_DEPS", "GUESTOS_RUNTIME_DEPS", "MACRO_DEPENDENCIES", "UNIVERSAL_VM_RUNTIME_DEPS") -load("//rs/tests:system_tests.bzl", "system_test_nns") +load("//rs/tests:system_tests.bzl", "system_test", "system_test_nns") package(default_visibility = ["//rs:system-tests-pkg"]) @@ -91,7 +91,7 @@ system_test_nns( deps = DEPENDENCIES + ["//rs/tests"], ) -system_test_nns( +system_test( name = "mainnet_bn_performance_test", colocated_test_driver_vm_required_host_features = ["performance"], colocated_test_driver_vm_resources = { @@ -111,8 +111,8 @@ system_test_nns( deps = DEPENDENCIES + ["//rs/tests"], ) -system_test_nns( - name = "mainnet_bn_icx_proxy_performance_test", +system_test( + name = "mainnet_bn_ic_gateway_performance_test", colocated_test_driver_vm_required_host_features = ["performance"], colocated_test_driver_vm_resources = { "vcpus": 64, diff --git a/rs/tests/boundary_nodes/bn_integration_on_playnet_test.rs b/rs/tests/boundary_nodes/bn_integration_on_playnet_test.rs index 52265846804..a591cb3fdf9 100644 --- a/rs/tests/boundary_nodes/bn_integration_on_playnet_test.rs +++ b/rs/tests/boundary_nodes/bn_integration_on_playnet_test.rs @@ -7,9 +7,8 @@ use ic_system_test_driver::systest; use ic_tests::boundary_nodes::boundary_nodes_integration::{ canister_allowlist_test, canister_test, denylist_test, direct_to_replica_options_test, direct_to_replica_rosetta_test, direct_to_replica_test, headers_test, http_canister_test, - http_endpoint_test, icx_proxy_test, nginx_valid_config_test, prefix_canister_id_test, - proxy_http_canister_test, reboot_test, redirect_http_to_https_test, redirect_to_dashboard_test, - redirect_to_non_raw_test, seo_test, + http_endpoint_test, ic_gateway_test, prefix_canister_id_test, proxy_http_canister_test, + reboot_test, redirect_http_to_https_test, redirect_to_dashboard_test, seo_test, }; use ic_tests::boundary_nodes::setup::setup_ic_with_bn; use ic_tests::boundary_nodes::{constants::BOUNDARY_NODE_NAME, helpers::BoundaryNodeHttpsConfig}; @@ -30,12 +29,10 @@ fn main() -> Result<()> { .add_test(systest!(http_canister_test)) .add_test(systest!(prefix_canister_id_test)) .add_test(systest!(proxy_http_canister_test)) - .add_test(systest!(nginx_valid_config_test)) .add_test(systest!(redirect_http_to_https_test)) .add_test(systest!(redirect_to_dashboard_test)) - .add_test(systest!(redirect_to_non_raw_test)) .add_test(systest!(http_endpoint_test)) - .add_test(systest!(icx_proxy_test)) + .add_test(systest!(ic_gateway_test)) .add_test(systest!(direct_to_replica_test)) .add_test(systest!(direct_to_replica_rosetta_test)) .add_test(systest!(direct_to_replica_options_test)) diff --git a/rs/tests/boundary_nodes/bn_integration_test.rs b/rs/tests/boundary_nodes/bn_integration_test.rs index 1d1c5770316..7dfa41bbd6b 100644 --- a/rs/tests/boundary_nodes/bn_integration_test.rs +++ b/rs/tests/boundary_nodes/bn_integration_test.rs @@ -11,9 +11,9 @@ use ic_tests::boundary_nodes::{ asset_canister_test, canister_allowlist_test, canister_routing_test, canister_test, denylist_test, direct_to_replica_options_test, direct_to_replica_rosetta_test, direct_to_replica_test, headers_test, http_canister_test, http_endpoint_test, - icx_proxy_test, nginx_valid_config_test, prefix_canister_id_test, proxy_http_canister_test, + ic_gateway_test, prefix_canister_id_test, proxy_http_canister_test, read_state_via_subnet_path_test, reboot_test, redirect_http_to_https_test, - redirect_to_dashboard_test, redirect_to_non_raw_test, seo_test, + redirect_to_dashboard_test, seo_test, }, constants::BOUNDARY_NODE_NAME, helpers::BoundaryNodeHttpsConfig, @@ -37,12 +37,10 @@ fn main() -> Result<()> { .add_test(systest!(http_canister_test)) .add_test(systest!(prefix_canister_id_test)) .add_test(systest!(proxy_http_canister_test)) - .add_test(systest!(nginx_valid_config_test)) .add_test(systest!(redirect_http_to_https_test)) .add_test(systest!(redirect_to_dashboard_test)) - .add_test(systest!(redirect_to_non_raw_test)) .add_test(systest!(http_endpoint_test)) - .add_test(systest!(icx_proxy_test)) + .add_test(systest!(ic_gateway_test)) .add_test(systest!(direct_to_replica_test)) .add_test(systest!(direct_to_replica_rosetta_test)) .add_test(systest!(direct_to_replica_options_test)) diff --git a/rs/tests/boundary_nodes/mainnet_bn_icx_proxy_performance_test.rs b/rs/tests/boundary_nodes/mainnet_bn_ic_gateway_performance_test.rs similarity index 74% rename from rs/tests/boundary_nodes/mainnet_bn_icx_proxy_performance_test.rs rename to rs/tests/boundary_nodes/mainnet_bn_ic_gateway_performance_test.rs index 655dca8de40..e51b4e63a09 100644 --- a/rs/tests/boundary_nodes/mainnet_bn_icx_proxy_performance_test.rs +++ b/rs/tests/boundary_nodes/mainnet_bn_ic_gateway_performance_test.rs @@ -1,12 +1,14 @@ use anyhow::Result; use ic_system_test_driver::{driver::group::SystemTestGroup, systest}; -use ic_tests::boundary_nodes::performance_test::{empty_setup, mainnet_query_calls_icx_proxy_test}; +use ic_tests::boundary_nodes::performance_test::{ + empty_setup, mainnet_query_calls_ic_gateway_test, +}; use std::{env, net::Ipv6Addr, time::Duration}; fn main() -> Result<()> { let ipv6 = env::var("BOUNDARY_NODE_IPV6").expect("environment variable is not provided"); let bn_ipv6 = ipv6.parse::().expect("invalid ipv6"); - let test = move |env| mainnet_query_calls_icx_proxy_test(env, bn_ipv6); + let test = move |env| mainnet_query_calls_ic_gateway_test(env, bn_ipv6); SystemTestGroup::new() .with_setup(empty_setup) .add_test(systest!(test)) diff --git a/rs/tests/driver/src/driver/boundary_node.rs b/rs/tests/driver/src/driver/boundary_node.rs index 41d69a9ab8e..9c639dae589 100644 --- a/rs/tests/driver/src/driver/boundary_node.rs +++ b/rs/tests/driver/src/driver/boundary_node.rs @@ -78,7 +78,6 @@ pub struct BoundaryNodeCustomDomainsConfig { pub task_delay_sec: Option, pub task_error_delay_sec: Option, pub peek_sleep_sec: Option, - pub polling_interval_sec: Option, } /// A builder for the initial configuration of an IC boundary node. @@ -647,11 +646,6 @@ fn create_config_disk_image( cmd.arg("--certificate_issuer_peek_sleep_sec") .arg(peek_sleep_sec.to_string()); } - - if let Some(polling_interval_sec) = cfg.polling_interval_sec { - cmd.arg("--certificate_syncer_polling_interval_sec") - .arg(polling_interval_sec.to_string()); - } } let key = "PATH"; diff --git a/rs/tests/driver/src/driver/prometheus_vm.rs b/rs/tests/driver/src/driver/prometheus_vm.rs index 13dee1e4b66..33c25676f04 100644 --- a/rs/tests/driver/src/driver/prometheus_vm.rs +++ b/rs/tests/driver/src/driver/prometheus_vm.rs @@ -60,7 +60,6 @@ const REPLICA_METRICS_PORT: u16 = 9090; const ORCHESTRATOR_METRICS_PORT: u16 = 9091; const NODE_EXPORTER_METRICS_PORT: u16 = 9100; const IC_BOUNDARY_METRICS_PORT: u16 = 9324; -const BOUNDARY_NODE_NGINX_PORT: u16 = 9316; const PROMETHEUS_DOMAIN_NAME: &str = "prometheus"; const GRAFANA_DOMAIN_NAME: &str = "grafana"; @@ -78,7 +77,6 @@ const NODE_EXPORTER_PROMETHEUS_TARGET: &str = "node_exporter.json"; const LEDGER_CANISTER_PROMETHEUS_TARGET: &str = "ledger_canister.json"; const BN_PROMETHEUS_TARGET: &str = "boundary_nodes.json"; const BN_EXPORTER_PROMETHEUS_TARGET: &str = "boundary_nodes_exporter.json"; -const BN_NGINX_PROMETHEUS_TARGET: &str = "boundary_nodes_nginx.json"; pub struct PrometheusVm { universal_vm: UniversalVm, @@ -298,7 +296,6 @@ impl HasPrometheus for TestEnv { NODE_EXPORTER_PROMETHEUS_TARGET, BN_PROMETHEUS_TARGET, BN_EXPORTER_PROMETHEUS_TARGET, - BN_NGINX_PROMETHEUS_TARGET, ]; if farm_url_for_ledger_canister.is_some() { target_json_files.push(LEDGER_CANISTER_PROMETHEUS_TARGET); @@ -394,8 +391,6 @@ fn write_prometheus_config_dir(config_dir: PathBuf, scrape_interval: Duration) - Path::new(PROMETHEUS_SCRAPING_TARGETS_DIR).join(BN_PROMETHEUS_TARGET); let boundary_nodes_exporter_scraping_targets_path = Path::new(PROMETHEUS_SCRAPING_TARGETS_DIR).join(BN_EXPORTER_PROMETHEUS_TARGET); - let boundary_nodes_nginx_scraping_targets_path = - Path::new(PROMETHEUS_SCRAPING_TARGETS_DIR).join(BN_NGINX_PROMETHEUS_TARGET); let replica_scraping_targets_path = Path::new(PROMETHEUS_SCRAPING_TARGETS_DIR).join(REPLICA_PROMETHEUS_TARGET); let orchestrator_scraping_targets_path = @@ -416,10 +411,6 @@ fn write_prometheus_config_dir(config_dir: PathBuf, scrape_interval: Duration) - "job_name": "boundary_nodes_exporter", "file_sd_configs": [{"files": [boundary_nodes_exporter_scraping_targets_path]}], }, - { - "job_name": "boundary_nodes_nginx", - "file_sd_configs": [{"files": [boundary_nodes_nginx_scraping_targets_path]}], - }, {"job_name": "replica", "file_sd_configs": [{"files": [replica_scraping_targets_path]}]}, {"job_name": "orchestrator", "file_sd_configs": [{"files": [orchestrator_scraping_targets_path]}]}, { @@ -452,7 +443,7 @@ fn sync_prometheus_config_dir_with_boundary_nodes( ) -> Result<()> { let mut boundary_nodes_p8s_static_configs: Vec = Vec::new(); let mut boundary_nodes_exporter_p8s_static_configs: Vec = Vec::new(); - let mut boundary_nodes_nginx_p8s_static_configs: Vec = Vec::new(); + let bns: Vec<(String, Ipv6Addr)> = env .get_deployed_boundary_nodes() .into_iter() @@ -478,10 +469,6 @@ fn sync_prometheus_config_dir_with_boundary_nodes( targets: vec![format!("[{:?}]:{:?}", ipv6, NODE_EXPORTER_METRICS_PORT)], labels: labels.clone(), }); - boundary_nodes_nginx_p8s_static_configs.push(PrometheusStaticConfig { - targets: vec![format!("[{:?}]:{:?}", ipv6, BOUNDARY_NODE_NGINX_PORT)], - labels: labels.clone(), - }); } for (name, p8s_static_configs) in &[ (BN_PROMETHEUS_TARGET, boundary_nodes_p8s_static_configs), @@ -489,10 +476,6 @@ fn sync_prometheus_config_dir_with_boundary_nodes( BN_EXPORTER_PROMETHEUS_TARGET, boundary_nodes_exporter_p8s_static_configs, ), - ( - BN_NGINX_PROMETHEUS_TARGET, - boundary_nodes_nginx_p8s_static_configs, - ), ] { ::serde_json::to_writer( &File::create(prometheus_config_dir.join(name))?, diff --git a/rs/tests/nns/ic_mainnet_nns_recovery/src/lib.rs b/rs/tests/nns/ic_mainnet_nns_recovery/src/lib.rs index 609c3568eb5..d15849708f6 100644 --- a/rs/tests/nns/ic_mainnet_nns_recovery/src/lib.rs +++ b/rs/tests/nns/ic_mainnet_nns_recovery/src/lib.rs @@ -501,7 +501,7 @@ fn setup_boundary_node( sudo systemctl daemon-reload sudo systemctl restart ic-boundary "#)).unwrap_or_else(|e| { - panic!("Could not reconfigure nginx on {BOUNDARY_NODE_NAME} to only route to the recovered NNS because {e:?}",) + panic!("Could not reconfigure ic-boundary on {BOUNDARY_NODE_NAME} to only route to the recovered NNS because {e:?}",) }); info!(logger, "Waiting until {BOUNDARY_NODE_NAME} is healthy ..."); diff --git a/rs/tests/src/boundary_nodes/boundary_nodes_integration.rs b/rs/tests/src/boundary_nodes/boundary_nodes_integration.rs index 26fec2fee15..32d8848b33a 100644 --- a/rs/tests/src/boundary_nodes/boundary_nodes_integration.rs +++ b/rs/tests/src/boundary_nodes/boundary_nodes_integration.rs @@ -33,13 +33,14 @@ use ic_system_test_driver::{ RetrieveIpv4Addr, SshSession, READY_WAIT_TIMEOUT, RETRY_BACKOFF, }, }, + retry_with_msg_async, util::{ agent_observes_canister_module, agent_using_call_v2_endpoint, assert_create_agent, block_on, }, }; use std::{env, iter, net::SocketAddrV6, time::Duration}; -use anyhow::{anyhow, bail, Error}; +use anyhow::{anyhow, bail, Context, Error}; use futures::stream::FuturesUnordered; use ic_agent::{ agent::http_transport::{ @@ -49,12 +50,18 @@ use ic_agent::{ export::Principal, Agent, }; +use reqwest::{redirect::Policy, ClientBuilder, Method}; use serde::Deserialize; use slog::{error, info, Logger}; -use tokio::runtime::Runtime; +use tokio::{runtime::Runtime, time::sleep}; + const CANISTER_RETRY_TIMEOUT: Duration = Duration::from_secs(30); const CANISTER_RETRY_BACKOFF: Duration = Duration::from_secs(2); +fn runtime() -> Runtime { + Runtime::new().expect("Could not create tokio runtime") +} + async fn install_canister(env: TestEnv, logger: Logger, path: &str) -> Result { let install_node = env .topology_snapshot() @@ -81,7 +88,7 @@ async fn install_canister(env: TestEnv, logger: Logger, path: &str) -> Result&1") - .unwrap(); - - info!(logger, "nginx test result = '{}'", cmd_output.trim()); - - if !cmd_output.trim().contains("test is successful") { - panic!("nginx config failed validation"); - } -} - /* tag::catalog[] Title:: Boundary nodes denylist blocking test @@ -1211,7 +1179,7 @@ pub fn denylist_test(env: TestEnv) { .get_snapshot() .unwrap(); - let rt = tokio::runtime::Runtime::new().expect("Could not create tokio runtime."); + let rt = runtime(); rt.block_on(async move { info!(&logger, "creating replica agent"); let agent = assert_create_agent(install_node.as_ref().unwrap().0.as_str()).await; @@ -1224,7 +1192,7 @@ pub fn denylist_test(env: TestEnv) { .expect("Could not create http_counter canister"); info!(&logger, "Waiting for canisters to finish installing..."); - ic_system_test_driver::retry_with_msg_async!( + retry_with_msg_async!( format!( "agent of {} observes canister module {}", install_node.as_ref().unwrap().0, @@ -1245,8 +1213,8 @@ pub fn denylist_test(env: TestEnv) { info!(&logger, "created canister={canister_id}"); - // Update the denylist and restart icx-proxy - let denylist_command = format!(r#"echo "{{\"canisters\":{{\"{}\": {{}}}}}}" | sudo tee /run/ic-node/etc/icx-proxy/denylist.json && sudo service icx-proxy restart"#, canister_id); + // Update the denylist and restart ic-gateway + let denylist_command = format!(r#"echo "{{\"canisters\":{{\"{}\": {{}}}}}}" | sudo tee /run/ic-node/etc/ic-gateway/denylist.json && sudo service ic-gateway restart"#, canister_id); info!( logger, "update denylist {BOUNDARY_NODE_NAME} with {denylist_command}" @@ -1256,9 +1224,9 @@ pub fn denylist_test(env: TestEnv) { } // Wait a bit for the restart to complete - tokio::time::sleep(Duration::from_secs(3)).await; + sleep(Duration::from_secs(3)).await; - let client_builder = reqwest::ClientBuilder::new(); + let client_builder = ClientBuilder::new(); let (client_builder, host) = if let Some(playnet) = boundary_node.get_playnet() { (client_builder, playnet) } else { @@ -1273,7 +1241,7 @@ pub fn denylist_test(env: TestEnv) { // Probe the blocked canister, we should get a 451 let url = &format!("https://{canister_id}.raw.{host}/"); - ic_system_test_driver::retry_with_msg_async!( + retry_with_msg_async!( format!("GET {} (expecting 451)", url), &logger, READY_WAIT_TIMEOUT, @@ -1285,7 +1253,7 @@ pub fn denylist_test(env: TestEnv) { .await? .status(); - if res != reqwest::StatusCode::UNAVAILABLE_FOR_LEGAL_REASONS { + if res != StatusCode::UNAVAILABLE_FOR_LEGAL_REASONS { bail!("expected 451, got {res}"); } @@ -1322,7 +1290,7 @@ pub fn canister_allowlist_test(env: TestEnv) { .get_snapshot() .unwrap(); - let rt = tokio::runtime::Runtime::new().expect("Could not create tokio runtime."); + let rt = runtime(); rt.block_on(async move { info!(&logger, "creating replica agent"); let agent = assert_create_agent(install_node.as_ref().unwrap().0.as_str()).await; @@ -1335,7 +1303,7 @@ pub fn canister_allowlist_test(env: TestEnv) { .expect("Could not create http_counter canister"); info!(&logger, "Waiting for canisters to finish installing..."); - ic_system_test_driver::retry_with_msg_async!( + retry_with_msg_async!( format!( "agent of {} observes canister module {}", install_node.as_ref().unwrap().0, @@ -1356,7 +1324,7 @@ pub fn canister_allowlist_test(env: TestEnv) { info!(&logger, "created canister={canister_id}"); - let client_builder = reqwest::ClientBuilder::new(); + let client_builder = ClientBuilder::new(); let (client_builder, host) = if let Some(playnet) = boundary_node.get_playnet() { (client_builder, playnet) } else { @@ -1371,7 +1339,7 @@ pub fn canister_allowlist_test(env: TestEnv) { // Check canister is available let url = &format!("https://{canister_id}.raw.{host}/"); - ic_system_test_driver::retry_with_msg_async!( + retry_with_msg_async!( format!("GET {}", url), &logger, READY_WAIT_TIMEOUT, @@ -1384,7 +1352,7 @@ pub fn canister_allowlist_test(env: TestEnv) { .expect("Could not perform get request.") .status(); - if res != reqwest::StatusCode::OK { + if res != StatusCode::OK { bail!("expected OK, got {}", res); } @@ -1392,8 +1360,8 @@ pub fn canister_allowlist_test(env: TestEnv) { } ).await.unwrap(); - // Update the denylist and restart icx-proxy - let denylist_command = format!(r#"echo "{{\"canisters\":{{\"{}\": {{}}}}}}" | sudo tee /run/ic-node/etc/icx-proxy/denylist.json && sudo service icx-proxy restart"#, canister_id); + // Update the denylist and restart ic-gateway + let denylist_command = format!(r#"echo "{{\"canisters\":{{\"{}\": {{}}}}}}" | sudo tee /run/ic-node/etc/ic-gateway/denylist.json && sudo service ic-gateway restart"#, canister_id); info!( logger, "update denylist {BOUNDARY_NODE_NAME} with {denylist_command}" @@ -1403,10 +1371,10 @@ pub fn canister_allowlist_test(env: TestEnv) { } // Wait a bit for the restart to complete - tokio::time::sleep(Duration::from_secs(3)).await; + sleep(Duration::from_secs(3)).await; // Check canister is restricted - ic_system_test_driver::retry_with_msg_async!( + retry_with_msg_async!( format!("GET {} (expecting 451)", url), &logger, READY_WAIT_TIMEOUT, @@ -1419,7 +1387,7 @@ pub fn canister_allowlist_test(env: TestEnv) { .expect("Could not perform get request.") .status(); - if res != reqwest::StatusCode::UNAVAILABLE_FOR_LEGAL_REASONS { + if res != StatusCode::UNAVAILABLE_FOR_LEGAL_REASONS { bail!("expected 451, got {}", res); } @@ -1427,8 +1395,8 @@ pub fn canister_allowlist_test(env: TestEnv) { } ).await.unwrap(); - // Update the allowlist and restart icx-proxy - let allowlist_command = format!(r#"echo "{}" | sudo tee /run/ic-node/etc/icx-proxy/allowlist.txt && sudo service icx-proxy restart"#, canister_id); + // Update the allowlist and restart ic-gateway + let allowlist_command = format!(r#"echo "{}" | sudo tee /run/ic-node/etc/ic-gateway/allowlist.txt && sudo service ic-gateway restart"#, canister_id); info!( logger, "update allowlist {BOUNDARY_NODE_NAME} with {allowlist_command}" @@ -1438,10 +1406,10 @@ pub fn canister_allowlist_test(env: TestEnv) { } // Wait a bit for the restart to complete - tokio::time::sleep(Duration::from_secs(3)).await; + sleep(Duration::from_secs(3)).await; // Check canister is available - ic_system_test_driver::retry_with_msg_async!( + retry_with_msg_async!( format!("GET {}", url), &logger, READY_WAIT_TIMEOUT, @@ -1454,7 +1422,7 @@ pub fn canister_allowlist_test(env: TestEnv) { .expect("Could not perform get request.") .status(); - if res != reqwest::StatusCode::OK { + if res != StatusCode::OK { bail!("expected OK, got {}", res); } @@ -1473,7 +1441,7 @@ pub fn redirect_http_to_https_test(env: TestEnv) { .get_snapshot() .unwrap(); - let client_builder = reqwest::ClientBuilder::new().redirect(reqwest::redirect::Policy::none()); + let client_builder = ClientBuilder::new().redirect(Policy::none()); let (client_builder, host_orig) = if let Some(playnet) = boundary_node.get_playnet() { (client_builder, playnet) } else { @@ -1487,7 +1455,7 @@ pub fn redirect_http_to_https_test(env: TestEnv) { }; let client = client_builder.build().unwrap(); - let rt = tokio::runtime::Runtime::new().expect("Could not create tokio runtime."); + let rt = runtime(); let futs = FuturesUnordered::new(); @@ -1500,7 +1468,7 @@ pub fn redirect_http_to_https_test(env: TestEnv) { async move { let res = client.get(format!("http://{host}/")).send().await?; - if res.status() != reqwest::StatusCode::MOVED_PERMANENTLY { + if res.status() != StatusCode::PERMANENT_REDIRECT { bail!("{name} failed: {}", res.status()) } @@ -1522,7 +1490,7 @@ pub fn redirect_http_to_https_test(env: TestEnv) { async move { let res = client.get(format!("http://raw.{host}/")).send().await?; - if res.status() != reqwest::StatusCode::MOVED_PERMANENTLY { + if res.status() != StatusCode::PERMANENT_REDIRECT { bail!("{name} failed: {}", res.status()) } @@ -1570,9 +1538,9 @@ pub fn redirect_to_dashboard_test(env: TestEnv) { .get_snapshot() .unwrap(); - let client_builder = reqwest::ClientBuilder::new() + let client_builder = ClientBuilder::new() .danger_accept_invalid_certs(boundary_node.uses_snake_oil_certs()) - .redirect(reqwest::redirect::Policy::none()); + .redirect(Policy::none()); let (client_builder, host_orig) = if let Some(playnet) = boundary_node.get_playnet() { (client_builder, playnet) } else { @@ -1585,7 +1553,7 @@ pub fn redirect_to_dashboard_test(env: TestEnv) { }; let client = client_builder.build().unwrap(); - let rt = tokio::runtime::Runtime::new().expect("Could not create tokio runtime."); + let rt = runtime(); let futs = FuturesUnordered::new(); @@ -1598,7 +1566,7 @@ pub fn redirect_to_dashboard_test(env: TestEnv) { async move { let res = client.get(format!("https://{host}/")).send().await?; - if res.status() != reqwest::StatusCode::FOUND { + if res.status() != StatusCode::TEMPORARY_REDIRECT { bail!("{name} failed: {}", res.status()) } @@ -1620,7 +1588,7 @@ pub fn redirect_to_dashboard_test(env: TestEnv) { async move { let res = client.get(format!("https://raw.{host}/")).send().await?; - if res.status() != reqwest::StatusCode::FOUND { + if res.status() != StatusCode::TEMPORARY_REDIRECT { bail!("{name} failed: {}", res.status()) } @@ -1659,162 +1627,7 @@ pub fn redirect_to_dashboard_test(env: TestEnv) { .expect("test suite failed"); } -pub fn redirect_to_non_raw_test(env: TestEnv) { - let logger = env.logger(); - - let boundary_node = env - .get_deployed_boundary_node(BOUNDARY_NODE_NAME) - .unwrap() - .get_snapshot() - .unwrap(); - - let client_builder = reqwest::ClientBuilder::new().redirect(reqwest::redirect::Policy::none()); - let (client_builder, host_orig) = if let Some(playnet) = boundary_node.get_playnet() { - (client_builder, playnet) - } else { - let host = "ic0.app"; - let bn_addr = SocketAddrV6::new(boundary_node.ipv6(), 443, 0, 0); - let client_builder = client_builder - .danger_accept_invalid_certs(true) - .resolve(&format!("raw.{host}"), bn_addr.into()); - (client_builder, host.to_string()) - }; - let client = client_builder.build().unwrap(); - - let rt = tokio::runtime::Runtime::new().expect("Could not create tokio runtime."); - - let futs = FuturesUnordered::new(); - - let host = host_orig.clone(); - futs.push(rt.spawn({ - let client = client.clone(); - let name = "redirect status to non-raw domain"; - info!(&logger, "Starting subtest {}", name); - - async move { - let res = client - .get(format!("https://raw.{host}/api/v2/status")) - .send() - .await?; - - if res.status() != reqwest::StatusCode::TEMPORARY_REDIRECT { - bail!("{name} failed: {}", res.status()) - } - - let location_hdr = res.headers().get("Location").unwrap().to_str().unwrap(); - if location_hdr != format!("https://{host}/api/v2/status") { - bail!("{name} failed: wrong location header: {}", location_hdr) - } - - Ok(()) - } - })); - - let host = host_orig.clone(); - futs.push(rt.spawn({ - let client = client.clone(); - let name = "redirect query to non-raw domain"; - info!(&logger, "Starting subtest {}", name); - - async move { - let res = client - .post(format!("https://raw.{host}/api/v2/canister/CID/query")) - .body("body") - .send() - .await?; - - if res.status() != reqwest::StatusCode::TEMPORARY_REDIRECT { - bail!("{name} failed: {}", res.status()) - } - - let location_hdr = res.headers().get("Location").unwrap().to_str().unwrap(); - if location_hdr != format!("https://{host}/api/v2/canister/CID/query") { - bail!("{name} failed: wrong location header: {}", location_hdr) - } - - Ok(()) - } - })); - - let host = host_orig.clone(); - futs.push(rt.spawn({ - let client = client.clone(); - let name = "redirect call to non-raw domain"; - info!(&logger, "Starting subtest {}", name); - - async move { - let res = client - .post(format!("https://raw.{host}/api/v2/canister/CID/call")) - .body("body") - .send() - .await?; - - if res.status() != reqwest::StatusCode::TEMPORARY_REDIRECT { - bail!("{name} failed: {}", res.status()) - } - - let location_hdr = res.headers().get("Location").unwrap().to_str().unwrap(); - if location_hdr != format!("https://{host}/api/v2/canister/CID/call") { - bail!("{name} failed: wrong location header: {}", location_hdr) - } - - Ok(()) - } - })); - - let host = host_orig; - futs.push(rt.spawn({ - let client = client; - let name = "redirect read_state to non-raw domain"; - info!(&logger, "Starting subtest {}", name); - - async move { - let res = client - .post(format!("https://raw.{host}/api/v2/canister/CID/read_state")) - .body("body") - .send() - .await?; - - if res.status() != reqwest::StatusCode::TEMPORARY_REDIRECT { - bail!("{name} failed: {}", res.status()) - } - - let location_hdr = res.headers().get("Location").unwrap().to_str().unwrap(); - if location_hdr != format!("https://{host}/api/v2/canister/CID/read_state") { - bail!("{name} failed: wrong location header: {}", location_hdr) - } - - Ok(()) - } - })); - - rt.block_on(async move { - let mut cnt_err = 0; - info!(&logger, "Waiting for subtests"); - - for fut in futs { - match fut.await { - Ok(Err(err)) => { - error!(logger, "test failed: {}", err); - cnt_err += 1; - } - Err(err) => { - error!(logger, "test panicked: {}", err); - cnt_err += 1; - } - _ => {} - } - } - - match cnt_err { - 0 => Ok(()), - _ => bail!("failed with {cnt_err} errors"), - } - }) - .expect("test suite failed"); -} - -// this tests the HTTP endpoint of the boundary node (anything that goes to icx-proxy) +// this tests the HTTP endpoint of the boundary node pub fn http_endpoint_test(env: TestEnv) { let logger_orig = env.logger(); @@ -1824,11 +1637,11 @@ pub fn http_endpoint_test(env: TestEnv) { .get_snapshot() .unwrap(); - let rt = tokio::runtime::Runtime::new().expect("Could not create tokio runtime."); + let rt = runtime(); let asset_canister_orig = rt.block_on(env.deploy_asset_canister()).unwrap(); - let client_builder = reqwest::ClientBuilder::new().redirect(reqwest::redirect::Policy::none()); + let client_builder = ClientBuilder::new().redirect(Policy::none()); let (client_builder, host_orig) = if let Some(playnet) = boundary_node.get_playnet() { (client_builder, playnet) } else { @@ -1844,7 +1657,7 @@ pub fn http_endpoint_test(env: TestEnv) { let futs = FuturesUnordered::new(); - // fetching standard assets (html page, JS script) through icx-proxy + // fetching standard assets (html page) let host = host_orig.clone(); let logger = logger_orig.clone(); let asset_canister = asset_canister_orig.clone(); @@ -1871,7 +1684,7 @@ pub fn http_endpoint_test(env: TestEnv) { .send() .await?; - if res.status() != reqwest::StatusCode::OK { + if res.status() != StatusCode::OK { bail!("{name} failed: {}", res.status()) } @@ -1879,7 +1692,7 @@ pub fn http_endpoint_test(env: TestEnv) { let body = String::from_utf8_lossy(&body); if !body.contains("Hello World!") { - bail!("{name} failed: expected icx-response but got {body}") + bail!("{name} failed: expected response but got {body}") } let hello_world_js = vec![ @@ -1905,15 +1718,25 @@ pub fn http_endpoint_test(env: TestEnv) { .send() .await?; - if res.status() != reqwest::StatusCode::OK { + if res.status() != StatusCode::OK { bail!("{name} failed: {}", res.status()) } + if let Some(v) = res.headers().get("x-ic-canister-id") { + let hdr = v.to_str().unwrap(); + let id = asset_canister.canister_id.to_string(); + if hdr != id { + bail!("{name} failed: header x-ic-canister-id is incorrect ({hdr} != {id})",); + } + } else { + bail!("{name} failed: header x-ic-canister-id not found"); + } + let body = res.bytes().await?.to_vec(); let body = String::from_utf8_lossy(&body); if !body.contains(r#"console.log("Hello World!")"#) { - bail!("{name} failed: expected icx-response but got {body}") + bail!("{name} failed: expected response but got {body}") } Ok(()) @@ -1953,7 +1776,7 @@ pub fn http_endpoint_test(env: TestEnv) { .send() .await?; - if res.status() != reqwest::StatusCode::OK { + if res.status() != StatusCode::OK { bail!("{name} failed: {}", res.status()) } @@ -1961,7 +1784,7 @@ pub fn http_endpoint_test(env: TestEnv) { let body = String::from_utf8_lossy(&body); if !body.contains("Do re mi, A B C, 1 2 3") { - bail!("{name} failed: expected icx-response but got {body}") + bail!("{name} failed: expected response but got {body}") } Ok(()) @@ -1995,7 +1818,7 @@ pub fn http_endpoint_test(env: TestEnv) { .expect("test suite failed"); } -pub fn icx_proxy_test(env: TestEnv) { +pub fn ic_gateway_test(env: TestEnv) { let logger = env.logger(); let boundary_node = env @@ -2004,7 +1827,7 @@ pub fn icx_proxy_test(env: TestEnv) { .get_snapshot() .unwrap(); - let rt = tokio::runtime::Runtime::new().expect("Could not create tokio runtime."); + let rt = runtime(); let canister_id = rt .block_on(install_canister( @@ -2014,7 +1837,7 @@ pub fn icx_proxy_test(env: TestEnv) { )) .unwrap(); - let client_builder = reqwest::ClientBuilder::new().redirect(reqwest::redirect::Policy::none()); + let client_builder = ClientBuilder::new().redirect(Policy::none()); let (client_builder, host_orig) = if let Some(playnet) = boundary_node.get_playnet() { (client_builder, playnet) } else { @@ -2033,7 +1856,7 @@ pub fn icx_proxy_test(env: TestEnv) { let host = host_orig.clone(); futs.push(rt.spawn({ let client = client.clone(); - let name = "get sent to icx-proxy via /_/raw/"; + let name = "get sent to ic-gateway via /_/raw/"; info!(&logger, "Starting subtest {}", name); async move { @@ -2042,15 +1865,15 @@ pub fn icx_proxy_test(env: TestEnv) { .send() .await?; - if res.status() != reqwest::StatusCode::INTERNAL_SERVER_ERROR { + if res.status() != StatusCode::INTERNAL_SERVER_ERROR { bail!("{name} failed: {}", res.status()) } let body = res.bytes().await?.to_vec(); let body = String::from_utf8_lossy(&body); - if !body.contains("Body does not pass verification") { - bail!("{name} failed: expected 'Body does not pass verification' but got {body}") + if !body.contains("Response verification failed") { + bail!("{name} failed: expected 'Response verification failed' but got {body}") } Ok(()) @@ -2060,7 +1883,7 @@ pub fn icx_proxy_test(env: TestEnv) { let host = host_orig; futs.push(rt.spawn({ let client = client; - let name = "get sent to icx-proxy via raw domain"; + let name = "get sent to ic-gateway via raw domain"; info!(&logger, "Starting subtest {}", name); async move { @@ -2069,7 +1892,7 @@ pub fn icx_proxy_test(env: TestEnv) { .send() .await?; - if res.status() != reqwest::StatusCode::OK { + if res.status() != StatusCode::OK { bail!("{name} failed: {}", res.status()) } @@ -2077,7 +1900,7 @@ pub fn icx_proxy_test(env: TestEnv) { let body = String::from_utf8_lossy(&body); if !body.contains("Counter is 0") { - bail!("{name} failed: expected icx-response but got {body}") + bail!("{name} failed: expected response but got {body}") } Ok(()) @@ -2119,7 +1942,7 @@ pub fn direct_to_replica_test(env: TestEnv) { .get_snapshot() .expect("failed to get BN snapshot"); - let client_builder = reqwest::ClientBuilder::new().redirect(reqwest::redirect::Policy::none()); + let client_builder = ClientBuilder::new().redirect(Policy::none()); let (client_builder, host_orig) = if let Some(playnet) = boundary_node.get_playnet() { (client_builder, playnet) } else { @@ -2151,7 +1974,7 @@ pub fn direct_to_replica_test(env: TestEnv) { .send() .await?; - if res.status() != reqwest::StatusCode::OK { + if res.status() != StatusCode::OK { bail!("{name} failed: {}", res.status()) } @@ -2197,7 +2020,7 @@ pub fn direct_to_replica_test(env: TestEnv) { .map_err(|err| anyhow!(format!("failed to create canister: {}", err)))?; info!(&logger, "Waiting for canisters to finish installing..."); - ic_system_test_driver::retry_with_msg_async!( + retry_with_msg_async!( format!( "agent of {} observes canister module {}", install_url.to_string(), @@ -2258,7 +2081,7 @@ pub fn direct_to_replica_test(env: TestEnv) { .map_err(|err| anyhow!(format!("failed to create canister: {}", err)))?; info!(&logger, "Waiting for canisters to finish installing..."); - ic_system_test_driver::retry_with_msg_async!( + retry_with_msg_async!( format!( "agent of {} observes canister module {}", install_url.to_string(), @@ -2336,7 +2159,7 @@ pub fn direct_to_replica_options_test(env: TestEnv) { .get_snapshot() .expect("failed to get BN snapshot"); - let client_builder = reqwest::ClientBuilder::new().redirect(reqwest::redirect::Policy::none()); + let client_builder = ClientBuilder::new().redirect(Policy::none()); let (client_builder, host_orig) = if let Some(playnet) = boundary_node.get_playnet() { (client_builder, playnet) } else { @@ -2370,7 +2193,7 @@ pub fn direct_to_replica_options_test(env: TestEnv) { .map_err(|err| anyhow!(format!("failed to create canister: {}", err)))?; info!(&logger, "Waiting for canisters to finish installing..."); - ic_system_test_driver::retry_with_msg_async!( + retry_with_msg_async!( format!( "agent of {} observes canister module {}", install_url.to_string(), @@ -2399,29 +2222,39 @@ pub fn direct_to_replica_options_test(env: TestEnv) { struct TestCase { name: String, path: String, + method: Method, + expect: StatusCode, allowed_methods: String, } let test_cases = [ TestCase { name: "status OPTIONS".into(), + method: Method::GET, + expect: StatusCode::OK, path: "/api/v2/status".into(), allowed_methods: "HEAD, GET".into(), }, TestCase { name: "query OPTIONS".into(), + method: Method::POST, + expect: StatusCode::BAD_REQUEST, path: format!("/api/v2/canister/{cid}/query"), - allowed_methods: "HEAD, POST".into(), + allowed_methods: "POST".into(), }, TestCase { name: "call OPTIONS".into(), + method: Method::POST, + expect: StatusCode::BAD_REQUEST, path: format!("/api/v2/canister/{cid}/call"), - allowed_methods: "HEAD, POST".into(), + allowed_methods: "POST".into(), }, TestCase { - name: "read_status OPTIONS".into(), + name: "read_state OPTIONS".into(), + method: Method::POST, + expect: StatusCode::BAD_REQUEST, path: format!("/api/v2/canister/{cid}/read_state"), - allowed_methods: "HEAD, POST".into(), + allowed_methods: "POST".into(), }, ]; @@ -2431,6 +2264,8 @@ pub fn direct_to_replica_options_test(env: TestEnv) { let TestCase { name, + method, + expect, path, allowed_methods, } = tc; @@ -2441,20 +2276,26 @@ pub fn direct_to_replica_options_test(env: TestEnv) { let mut url = reqwest::Url::parse(&format!("https://{host}"))?; url.set_path(&path); - - let req = reqwest::Request::new(reqwest::Method::OPTIONS, url); - + let req = reqwest::Request::new(Method::OPTIONS, url); let res = client.execute(req).await?; - if res.status() != reqwest::StatusCode::NO_CONTENT { + // Both 200 and 204 are valid OPTIONS codes + if ![StatusCode::NO_CONTENT, StatusCode::OK].contains(&res.status()) { bail!("{name} failed: {}", res.status()) } + // Normalize & sort header values so that they can be compared regardless of their order + fn normalize(hdr: &str) -> String { + let mut hdr = hdr.split(',').map(|x| x.trim().to_ascii_lowercase()).collect::>(); + hdr.sort(); + hdr.join(",") + } + + // Check pre-flight CORS headers for (k, v) in [ ("Access-Control-Allow-Origin", "*"), ("Access-Control-Allow-Methods", &allowed_methods), ("Access-Control-Allow-Headers", "DNT,User-Agent,X-Requested-With,If-None-Match,If-Modified-Since,Cache-Control,Content-Type,Range,Cookie,X-Ic-Canister-Id"), - ("Access-Control-Expose-Headers", "Accept-Ranges,Content-Length,Content-Range,X-Request-Id,X-Ic-Canister-Id"), ("Access-Control-Max-Age", "600"), ] { let hdr = res @@ -2462,8 +2303,38 @@ pub fn direct_to_replica_options_test(env: TestEnv) { .get(k) .ok_or_else(|| anyhow!("missing {k} header"))?.to_str()?; - if hdr != v { - bail!("wrong {k} header: {hdr}, expected {v}") + let hdr = normalize(hdr); + let expect = normalize(v); + + if hdr != expect { + bail!("wrong {k} header: {hdr} expected {expect}") + } + } + + // Check non-pre-flight CORS headers + let mut url = reqwest::Url::parse(&format!("https://{host}"))?; + url.set_path(&path); + let req = reqwest::Request::new(method, url); + let res = client.execute(req).await?; + + if res.status() != expect { + bail!("{name} failed: expected {expect}, got {}", res.status()) + } + + for (k, v) in [ + ("Access-Control-Allow-Origin", "*"), + ("Access-Control-Expose-Headers", "Accept-Ranges,Content-Length,Content-Range,X-Request-Id,X-Ic-Canister-Id"), + ] { + let hdr = res + .headers() + .get(k) + .ok_or_else(|| anyhow!("missing {k} header"))?.to_str()?; + + let hdr = normalize(hdr); + let expect = normalize(v); + + if hdr != expect { + bail!("wrong {k} header: {hdr} expected {expect}") } } @@ -2508,9 +2379,9 @@ pub fn direct_to_replica_rosetta_test(env: TestEnv) { let bn_addr = SocketAddrV6::new(boundary_node.ipv6(), 443, 0, 0); - let client = reqwest::ClientBuilder::new() + let client = ClientBuilder::new() .danger_accept_invalid_certs(true) - .redirect(reqwest::redirect::Policy::none()) + .redirect(Policy::none()) .resolve("rosetta.dfinity.network", bn_addr.into()) .build() .expect("failed to build http client"); @@ -2533,7 +2404,7 @@ pub fn direct_to_replica_rosetta_test(env: TestEnv) { .send() .await?; - if res.status() != reqwest::StatusCode::OK { + if res.status() != StatusCode::OK { bail!("{name} failed: {}", res.status()) } @@ -2578,7 +2449,7 @@ pub fn direct_to_replica_rosetta_test(env: TestEnv) { .map_err(|err| anyhow!(format!("failed to create canister: {}", err)))?; info!(&logger, "Waiting for canisters to finish installing..."); - ic_system_test_driver::retry_with_msg_async!( + retry_with_msg_async!( format!( "agent of {} observes canister module {}", install_url.to_string(), @@ -2639,7 +2510,7 @@ pub fn direct_to_replica_rosetta_test(env: TestEnv) { .map_err(|err| anyhow!(format!("failed to create canister: {}", err)))?; info!(&logger, "Waiting for canisters to finish installing..."); - ic_system_test_driver::retry_with_msg_async!( + retry_with_msg_async!( format!( "agent of {} observes canister module {}", install_url.to_string(), @@ -2717,12 +2588,12 @@ pub fn seo_test(env: TestEnv) { .get_snapshot() .unwrap(); - let rt = tokio::runtime::Runtime::new().expect("Could not create tokio runtime."); + let rt = runtime(); // create an asset canister for the test let asset_canister_orig = rt.block_on(env.deploy_asset_canister()).unwrap(); - let client_builder = reqwest::ClientBuilder::new().redirect(reqwest::redirect::Policy::none()); + let client_builder = ClientBuilder::new().redirect(Policy::none()); let (client_builder, host_orig) = if let Some(playnet) = boundary_node.get_playnet() { (client_builder, playnet) } else { @@ -2743,7 +2614,7 @@ pub fn seo_test(env: TestEnv) { let asset_canister = asset_canister_orig.clone(); futs.push(rt.spawn({ - let name = "get sent to icx-proxy if you're a bot"; + let name = "get sent to ic-gateway if you're a bot"; info!(&logger, "Starting subtest {}", name); async move { @@ -2768,7 +2639,7 @@ pub fn seo_test(env: TestEnv) { .send() .await?; - if res.status() != reqwest::StatusCode::OK { + if res.status() != StatusCode::OK { bail!("{name} failed: {}", res.status()) } @@ -2776,7 +2647,7 @@ pub fn seo_test(env: TestEnv) { let body = String::from_utf8_lossy(&body); if !body.contains("Hello World!") { - bail!("{name} failed: expected icx-response but got {body}") + bail!("{name} failed: expected response but got {body}") } Ok(()) diff --git a/rs/tests/src/boundary_nodes/performance_test.rs b/rs/tests/src/boundary_nodes/performance_test.rs index 2207fb5af57..8e6727bd5d3 100644 --- a/rs/tests/src/boundary_nodes/performance_test.rs +++ b/rs/tests/src/boundary_nodes/performance_test.rs @@ -145,8 +145,6 @@ pub fn setup(bn_https_config: BoundaryNodeHttpsConfig, env: TestEnv) { } // Execute update calls (without polling) with an increasing req/s rate, against a counter canister via the boundary node agent. -// At the moment 300 req/s is the maximum defined by the rate limiter in 000-nginx-global.conf - pub fn update_calls_test(env: TestEnv) { let rps_min = 50; let rps_max = 450; @@ -211,8 +209,6 @@ pub fn update_calls_test(env: TestEnv) { } // Execute query calls with an increasing req/s rate, against a counter canister via the boundary node agent. -// In order to observe rates>1 req/s on the replica, caching should be disabled in 002-mainnet-nginx.conf - pub fn query_calls_test(env: TestEnv) { let rps_min = 500; let rps_max = 6500; @@ -352,7 +348,7 @@ pub fn mainnet_query_calls_test(env: TestEnv, bn_ipv6: Ipv6Addr) { } } -pub fn mainnet_query_calls_icx_proxy_test(env: TestEnv, bn_ipv6: Ipv6Addr) { +pub fn mainnet_query_calls_ic_gateway_test(env: TestEnv, bn_ipv6: Ipv6Addr) { const ROOT_HOST: &str = "icp0.io"; const MAINNET_STREAMING_CANISTER_ID: &str = "4evdk-jqaaa-aaaan-qel6q-cai"; const MAINNET_COUNTER_CANISTER_ID: &str = "3muos-6yaaa-aaaaa-qaaua-cai"; @@ -365,9 +361,9 @@ pub fn mainnet_query_calls_icx_proxy_test(env: TestEnv, bn_ipv6: Ipv6Addr) { const NUM_AGENTS: usize = 100; - // The amount of traffic that will be directed to ICX Proxy, the remaining traffic will be direct canister query calls. - const ICX_PROXY_TRAFFIC_PERCENTAGE: f64 = 20.0; - // ICX Proxy traffic will be distributed among these requests according to their weights. + // The amount of traffic that will be HTTP, the remaining traffic will be direct canister query calls. + const HTTP_TRAFFIC_PERCENTAGE: f64 = 20.0; + // HTTP traffic will be distributed among these requests according to their weights. let weighted_http_requests = [ (format!("https://{streaming_canister_host}/1mb.json"), 25), (format!("https://{streaming_canister_host}/2mb.json"), 30), @@ -450,7 +446,7 @@ pub fn mainnet_query_calls_icx_proxy_test(env: TestEnv, bn_ipv6: Ipv6Addr) { CallMode::Query, ); - if prob < ICX_PROXY_TRAFFIC_PERCENTAGE { + if prob < HTTP_TRAFFIC_PERCENTAGE { let start_time = Instant::now(); let result = http_client diff --git a/rs/tests/src/custom_domains_integration/certificate_orchestrator.rs b/rs/tests/src/custom_domains_integration/certificate_orchestrator.rs index 97971b60ae8..6aa6daa721e 100644 --- a/rs/tests/src/custom_domains_integration/certificate_orchestrator.rs +++ b/rs/tests/src/custom_domains_integration/certificate_orchestrator.rs @@ -1,12 +1,12 @@ /* tag::catalog[] Title:: Custom domains end-to-end test -Goal:: Verify that custom domains is working end-to-end (certificate orchestrator, certificate issuer, certificate syncer) +Goal:: Verify that custom domains is working end-to-end (certificate orchestrator, certificate issuer) Runbook: . Set up the certificate orhcestrator on the IC. . Start a UVM exposing a nameserver, a mock Cloudflare API, a mock certificate authority -. Start a boundary node running the certificate issuer and syncer +. Start a boundary node running the certificate issuer . Register some custom domains Success:: All custom domain services on the boundary node come up healthy and process the registration requests successfully @@ -16,11 +16,10 @@ Coverage:: End-to-end registration processing end::catalog[] */ use crate::custom_domains_integration::setup::{ - access_domain, create_bn_http_client, get_certificate_syncer_state, get_registration_status, - get_service_errors, remove_dns_records, remove_registration, setup_asset_canister, - setup_dns_records, submit_registration_request, update_dns_records, update_registration, - GetRequestState, RegistrationRequestState, RemoveRequestState, UpdateRequestState, - BOUNDARY_NODE_VM_ID, + access_domain, create_bn_http_client, get_registration_status, get_service_errors, + remove_dns_records, remove_registration, setup_asset_canister, setup_dns_records, + submit_registration_request, update_dns_records, update_registration, GetRequestState, + RegistrationRequestState, RemoveRequestState, UpdateRequestState, BOUNDARY_NODE_VM_ID, }; use ic_system_test_driver::{ driver::boundary_node::BoundaryNodeVm, driver::test_env::TestEnv, util::block_on, @@ -93,7 +92,7 @@ pub fn test_end_to_end_registration(env: TestEnv) { } } - // wait for the certificate syncer to update the nginx config + // wait a bit tokio::time::sleep(Duration::from_secs(2)).await; // check that the custom domain is being served by the BN by checking the content of the canister @@ -103,13 +102,6 @@ pub fn test_end_to_end_registration(env: TestEnv) { "Site content of the custom domain is not correct" ); - // check that the syncer sees the custom domain and maps it to the right canister - assert_eq!( - get_certificate_syncer_state(&boundary_node, domain_name), - asset_canister_id.to_string(), - "Certificate syncer does not know about the custom domain or maps it to the wrong canister" - ); - // need to wait a second to prevent being rate-limited tokio::time::sleep(Duration::from_secs(1)).await; @@ -147,7 +139,7 @@ pub fn test_end_to_end_registration(env: TestEnv) { UpdateRequestState::Rejected(reason) => panic!("Failed to update the custom domain: {reason}"), }; - // wait for the certificate syncer to update the nginx config + // wait a bit tokio::time::sleep(Duration::from_secs(2)).await; // check that the custom domain is being served by the BN by checking the content of the canister @@ -157,13 +149,6 @@ pub fn test_end_to_end_registration(env: TestEnv) { "Site content of the custom domain is not correct" ); - // check that the syncer sees the custom domain and maps it to the right canister - assert_eq!( - get_certificate_syncer_state(&boundary_node, domain_name), - new_asset_canister_id.to_string(), - "Certificate syncer does not know about the custom domain or maps it to the wrong canister" - ); - info!( logger, "Custom domain has been successfully updated" @@ -195,25 +180,12 @@ pub fn test_end_to_end_registration(env: TestEnv) { GetRequestState::Rejected(reason) => panic!("Failed to delete the custom domain: {reason}"), }; - // make sure the certificate syncer removed the domain - assert_eq!( - get_certificate_syncer_state(&boundary_node, domain_name), - "", - "Certificate syncer still has an entry for the deleted custom domain" - ); - info!( logger, "Custom domain has been successfully removed" ); - // check that there are no issues with the syncer (failed certification) - assert_eq!( - get_service_errors(&boundary_node, "certificate-syncer"), - "-- No entries --", - "There were errors in the syncer" - ); - + // check that there are no issues with the issuer (failed certification) assert_eq!( get_service_errors(&boundary_node, "certificate-issuer"), "-- No entries --", diff --git a/rs/tests/src/custom_domains_integration/setup.rs b/rs/tests/src/custom_domains_integration/setup.rs index 74e358ec5d4..8412d5d7e69 100644 --- a/rs/tests/src/custom_domains_integration/setup.rs +++ b/rs/tests/src/custom_domains_integration/setup.rs @@ -79,10 +79,10 @@ pub fn setup(env: TestEnv) { .await .context("failed to generate pebble certificate")?; - // Certificates (Nginx) - let nginx_pair = generate_leaf_certificate_pair(&remote_docker_host, &ca_pair, "ic0.app") + // Certificates + let cert_pair = generate_leaf_certificate_pair(&remote_docker_host, &ca_pair, "ic0.app") .await - .context("failed to generate nginx certificate")?; + .context("failed to generate BN certificate")?; // CoreDNS + Cloudflare API Work Directory let work_dir = exec_ssh_mktemp(&remote_docker_host, MkTempMode::Dir)?; @@ -161,11 +161,11 @@ pub fn setup(env: TestEnv) { ) .await?; - // Install self-signed certificates (pebble and nginx) + // Install self-signed certificates configure_boundary_node_trust_certificate(&boundary_node, &ca_pair.certificate).await?; - // Update Nginx TLS Certificates - update_nginx_tls_certificate(&boundary_node, &nginx_pair).await?; + // Update TLS Certificates + update_tls_certificate(&boundary_node, &cert_pair).await?; // Update /etc/hosts on Boundary Node update_etc_hosts(&boundary_node, "ic0.app", &boundary_node.ipv6().to_string()).await?; @@ -510,7 +510,6 @@ async fn setup_boundary_node( task_delay_sec: Some(5), task_error_delay_sec: Some(10), peek_sleep_sec: Some(5), - polling_interval_sec: Some(1), }; // Start Boundary Node @@ -724,10 +723,7 @@ async fn update_etc_hosts(vm: &dyn SshSession, name: &str, ip: &str) -> Result<( Ok(()) } -async fn update_nginx_tls_certificate( - vm: &dyn SshSession, - pair: &CertificatePair, -) -> Result<(), Error> { +async fn update_tls_certificate(vm: &dyn SshSession, pair: &CertificatePair) -> Result<(), Error> { let CertificatePair { key, certificate: cert, @@ -736,23 +732,18 @@ async fn update_nginx_tls_certificate( vm.block_on_bash_script(&indoc::formatdoc! {r#" set -euo pipefail - echo "--> Installing nginx private key" - sudo bash -c 'cat > /run/ic-node/etc/nginx/keys/privkey.pem < Installing private key" + sudo bash -c 'cat > /run/ic-node/etc/ic-gateway/certs/cert.key < Installing nginx certificate chain" - sudo bash -c 'cat > /run/ic-node/etc/nginx/certs/chain.pem < Installing certificate chain" + sudo bash -c 'cat > /run/ic-node/etc/ic-gateway/certs/cert.pem < Installing nginx full certificate chain" - sudo bash -c 'cat > /run/ic-node/etc/nginx/certs/fullchain.pem < Restarting Nginx" - sudo nginx -s reload + echo "--> Restarting ic-gateway" + sudo systemctl restart ic-gateway "#})?; Ok(()) @@ -1294,13 +1285,6 @@ pub async fn get_registration_status( } } -pub fn get_certificate_syncer_state(vm: &dyn SshSession, domain_name: &str) -> String { - let cmd = format!( - r#"cat /var/opt/nginx/domain_canister_mappings.js | grep -o '"{domain_name}":"[^"]*' | sed 's/"{domain_name}":"//'"# - ); - vm.block_on_bash_script(&cmd).unwrap().trim().to_string() -} - fn get_service_status(vm: &dyn SshSession, service: &str) -> String { vm.block_on_bash_script(&format!("systemctl is-active {service} 2>&1")) .unwrap() diff --git a/rs/tests/testing_verification/spec_compliance/spec_compliance.rs b/rs/tests/testing_verification/spec_compliance/spec_compliance.rs index bf54ca77149..4f1f653cf40 100644 --- a/rs/tests/testing_verification/spec_compliance/spec_compliance.rs +++ b/rs/tests/testing_verification/spec_compliance/spec_compliance.rs @@ -78,7 +78,7 @@ pub fn config_impl(env: TestEnv, deploy_bn_and_nns_canisters: bool, http_request .allocate_vm(&env) .expect("Allocation of BoundaryNode failed.") .for_ic(&env, "") - .use_ipv6_certs() + .use_real_certs_and_dns() .start(&env) .expect("failed to setup BoundaryNode VM"); }