feat: publish https outcalls adapter with http enabled for dfx

ic-os/guestos/defs.bzl

@@ -34,7 +34,7 @@ def image_deps(mode, malicious = False):
             "//publish/binaries:guestos_tool": "/opt/ic/bin/guestos_tool:0755",
             "//publish/binaries:ic-btc-adapter": "/opt/ic/bin/ic-btc-adapter:0755",
             "//publish/binaries:ic-consensus-pool-util": "/opt/ic/bin/ic-consensus-pool-util:0755",
-            "//publish/binaries:ic-https-outcalls-adapter": "/opt/ic/bin/ic-https-outcalls-adapter:0755",
+            "//rs/https_outcalls/adapter:ic-https-outcalls-adapter": "/opt/ic/bin/ic-https-outcalls-adapter:0755",  # `//publish/binaries:ic-https-outcalls-adapter` is for testing and must NOT be used here
             "//publish/binaries:ic-crypto-csp": "/opt/ic/bin/ic-crypto-csp:0755",
             "//publish/binaries:ic-regedit": "/opt/ic/bin/ic-regedit:0755",
             "//publish/binaries:ic-recovery": "/opt/ic/bin/ic-recovery:0755",

publish/binaries/BUILD.bazel

@@ -42,7 +42,7 @@ BINARIES = {
     "ic-boundary": "//rs/boundary_node/ic_boundary:ic-boundary",
     "ic-boundary-tls": "//rs/boundary_node/ic_boundary:ic-boundary-tls",
     "ic-starter": "//rs/starter:ic-starter",
-    "ic-https-outcalls-adapter": "//rs/https_outcalls/adapter:ic-https-outcalls-adapter",
+    "ic-https-outcalls-adapter": "//rs/https_outcalls/adapter:ic-outcalls-adapter-with-http",
     "ic-consensus-pool-util": "//rs/artifact_pool:ic-consensus-pool-util",
     "ic-crypto-csp": "//rs/crypto:ic-crypto-csp",
     "ic-nns-init": "//rs/nns/init:ic-nns-init",
@@ -82,6 +82,7 @@ TESTONLY_BINARIES = [
     "ic-backup",
     "ic-boundary",
     "ic-boundary-tls",
+    "ic-https-outcalls-adapter",
     "ic-nns-init",
     "ic-prep",
     "ic-recovery",

rs/https_outcalls/adapter/BUILD.bazel

@@ -48,15 +48,40 @@ rust_library(
     deps = DEPENDENCIES,
 )
 
+# Same target as above but allows the adapter to make HTTP calls.
+rust_library(
+    name = "adapter_with_http",
+    testonly = True,
+    srcs = glob(["src/**"]),
+    aliases = ALIASES,
+    crate_features = ["http"],
+    crate_name = "ic_https_outcalls_adapter",
+    proc_macro_deps = MACRO_DEPENDENCIES,
+    version = "0.1.0",
+    deps = DEPENDENCIES,
+)
+
 rust_binary(
     name = "ic-https-outcalls-adapter",
     srcs = ["src/main.rs"],
     aliases = ALIASES,
     proc_macro_deps = MACRO_DEPENDENCIES,
-    visibility = ["//publish:__subpackages__"],
+    visibility = ["//ic-os/guestos:__subpackages__"],
     deps = DEPENDENCIES + [":adapter"],
 )
 
+# Same target as above but allows the adapter to make HTTP calls.
+# This target is used for local testing (e.g. DFX)
+rust_binary(
+    name = "ic-outcalls-adapter-with-http",
+    testonly = True,
+    srcs = ["src/main.rs"],
+    aliases = ALIASES,
+    proc_macro_deps = MACRO_DEPENDENCIES,
+    visibility = ["//publish:__subpackages__"],
+    deps = DEPENDENCIES + [":adapter_with_http"],
+)
+
 rust_test(
     name = "adapter_test",
     aliases = ALIASES,
@@ -73,3 +98,13 @@ rust_test_suite(
     tags = ["requires-network"],
     deps = [":adapter"] + DEPENDENCIES + DEV_DEPENDENCIES,
 )
+
+rust_test_suite(
+    name = "adapter_integration_with_http",
+    srcs = glob(["tests/**/*.rs"]),
+    aliases = ALIASES,
+    crate_features = ["http"],
+    proc_macro_deps = MACRO_DEPENDENCIES + MACRO_DEV_DEPENDENCIES,
+    tags = ["requires-network"],
+    deps = [":adapter_with_http"] + DEPENDENCIES + DEV_DEPENDENCIES,
+)

rs/https_outcalls/adapter/src/lib.rs

@@ -75,16 +75,18 @@ impl AdapterServer {
             }
         };
 
-        let https_client = Client::builder()
+        let builder = Client::builder()
             .use_rustls_tls()
-            .https_only(true)
             .http1_only()
             .redirect(Policy::none())
             .referer(false)
             .default_headers(HeaderMap::new())
-            .connect_timeout(timeout)
-            .build()
-            .expect("Failed to create HTTPS client");
+            .connect_timeout(timeout);
+
+        #[cfg(not(feature = "http"))]
+        let builder = builder.https_only(true);
+
+        let https_client = builder.build().expect("Failed to create HTTPS client");
 
         let canister_http = CanisterHttp::new(https_client, socks_client, logger, metrics);
 

rs/https_outcalls/adapter/src/metrics.rs

@@ -5,6 +5,7 @@ use prometheus::{IntCounter, IntCounterVec};
 pub(crate) const LABEL_BODY_RECEIVE_SIZE: &str = "body_receive_size";
 pub(crate) const LABEL_BODY_RECEIVE_TIMEOUT: &str = "body_receive_timeout";
 pub(crate) const LABEL_HEADER_RECEIVE_SIZE: &str = "header_receive_size";
+#[cfg(not(feature = "http"))]
 pub(crate) const LABEL_HTTP_SCHEME: &str = "http_scheme";
 pub(crate) const LABEL_HTTP_METHOD: &str = "http_method";
 pub(crate) const LABEL_RESPONSE_HEADERS: &str = "response_headers";

rs/https_outcalls/adapter/src/rpc_server.rs

@@ -1,7 +1,7 @@
 use crate::metrics::{
     AdapterMetrics, LABEL_BODY_RECEIVE_SIZE, LABEL_BODY_RECEIVE_TIMEOUT, LABEL_CONNECT,
-    LABEL_DOWNLOAD, LABEL_HEADER_RECEIVE_SIZE, LABEL_HTTP_METHOD, LABEL_HTTP_SCHEME,
-    LABEL_REQUEST_HEADERS, LABEL_RESPONSE_HEADERS, LABEL_UPLOAD, LABEL_URL_PARSE,
+    LABEL_DOWNLOAD, LABEL_HEADER_RECEIVE_SIZE, LABEL_HTTP_METHOD, LABEL_REQUEST_HEADERS,
+    LABEL_RESPONSE_HEADERS, LABEL_UPLOAD, LABEL_URL_PARSE,
 };
 use core::convert::TryFrom;
 use futures::StreamExt;
@@ -86,7 +86,9 @@ impl CanisterHttpService for CanisterHttp {
             )
         })?;
 
+        #[cfg(not(feature = "http"))]
         if url.scheme() != "https" {
+            use crate::metrics::LABEL_HTTP_SCHEME;
             debug!(
                 self.logger,
                 "Got request with no or http scheme specified. {}", url

rs/https_outcalls/adapter/tests/server_test.rs

@@ -21,10 +21,14 @@ mod test {
     use unix::UnixListenerDrop;
     use uuid::Uuid;
     use warp::{
+        filters::BoxedFilter,
         http::{header::HeaderValue, Response, StatusCode},
         Filter,
     };
 
+    #[cfg(feature = "http")]
+    use std::net::IpAddr;
+
     // Selfsigned localhost cert
     const CERT: &str = "
 -----BEGIN CERTIFICATE-----
@@ -74,7 +78,7 @@ MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgob29X4H4m2XOkSZE
         dir
     }
 
-    fn start_server(cert_dir: &TempDir) -> String {
+    fn warp_server() -> BoxedFilter<(impl warp::Reply,)> {
         let basic_post = warp::post()
             .and(warp::path("post"))
             .and(warp::body::json())
@@ -83,7 +87,6 @@ MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgob29X4H4m2XOkSZE
         let basic_get = warp::get()
             .and(warp::path("get"))
             .map(|| warp::reply::json(&"Hello"));
-
         let invalid_header = warp::get().and(warp::path("invalid")).map(|| unsafe {
             Response::builder()
                 .header(
@@ -108,14 +111,17 @@ MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgob29X4H4m2XOkSZE
 
         let basic_head = warp::head().and(warp::path("head")).map(warp::reply::reply);
 
-        let routes = basic_post
+        basic_post
             .or(basic_get)
             .or(basic_head)
             .or(get_response_size)
             .or(get_delay)
-            .or(invalid_header);
+            .or(invalid_header)
+            .boxed()
+    }
 
-        let (addr, fut) = warp::serve(routes)
+    fn start_server(cert_dir: &TempDir) -> String {
+        let (addr, fut) = warp::serve(warp_server())
             .tls()
             .cert_path(cert_dir.path().join("cert.crt"))
             .key_path(cert_dir.path().join("key.pem"))
@@ -125,6 +131,14 @@ MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgob29X4H4m2XOkSZE
         format!("localhost:{}", addr.port())
     }
 
+    #[cfg(feature = "http")]
+    fn start_http_server(ip: IpAddr) -> String {
+        let (addr, fut) = warp::serve(warp_server()).bind_ephemeral((ip, 0));
+
+        tokio::spawn(fut);
+        format!("{}:{}", ip, addr.port())
+    }
+
     #[tokio::test]
     async fn test_canister_http_server() {
         let server_config = Config {
@@ -146,6 +160,7 @@ MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgob29X4H4m2XOkSZE
         assert_eq!(http_response.status, StatusCode::OK.as_u16() as u32);
     }
 
+    #[cfg(not(feature = "http"))]
     #[tokio::test]
     async fn test_canister_http_http_protocol() {
         // Check that error is returned if a `http` url is specified.
@@ -175,6 +190,30 @@ MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgob29X4H4m2XOkSZE
             .contains(&"Url need to specify https scheme".to_string()));
     }
 
+    #[cfg(feature = "http")]
+    #[tokio::test]
+    async fn test_canister_http_http_protocol_allowed() {
+        // Check that error is returned if a `http` url is specified.
+        let server_config = Config {
+            ..Default::default()
+        };
+
+        let url = start_http_server("127.0.0.1".parse().unwrap());
+        let mut client = spawn_grpc_server(server_config);
+
+        let request = tonic::Request::new(CanisterHttpSendRequest {
+            url: format!("http://{}/get", &url),
+            headers: Vec::new(),
+            method: HttpMethod::Get as i32,
+            body: "hello".to_string().as_bytes().to_vec(),
+            max_response_size_bytes: 512,
+            socks_proxy_allowed: false,
+        });
+        let response = client.canister_http_send(request).await;
+        let http_response = response.unwrap().into_inner();
+        assert_eq!(http_response.status, StatusCode::OK.as_u16() as u32);
+    }
+
     #[tokio::test]
     async fn test_canister_http_server_post() {
         let server_config = Config {