feat: upload br encoding to asset canister (#3791)

CHANGELOG.md

@@ -94,6 +94,15 @@ Adds support for the `log_visibility` canister setting, which configures which u
 Valid options are `controllers` and `public`. The setting can be used with the `--log-visibility` flag in `dfx canister create`
 and `dfx canister update-settings`, or in `dfx.json` under `canisters[].initialization_values.log_visibility`.
 
+## Asset canister synchronization
+
+### feat: support `brotli` encoding
+
+Asset synchronization now not only supports `identity` and `gzip`, but also `brotli` encoding.
+The default encodings are still
+- `identity` and `gzip` for MIME types `.txt`, `.html` and `.js`
+- `identity` for anything else
+
 ## Dependencies
 
 ### Frontend canister

e2e/tests-dfx/assetscanister.bash

@@ -835,6 +835,41 @@ check_permission_failure() {
   diff encoded-compressed-2 src/e2e_project_frontend/assets/notreally.js
 }
 
+@test "can use brotli compression" {
+  install_asset assetscanister
+  for i in $(seq 1 400); do
+    echo "some easily duplicate text $i" >>src/e2e_project_frontend/assets/notreally.js
+  done
+
+  echo '[{
+      "match": "*.js",
+      "encodings": ["identity", "br"]
+    }
+  ]' > src/e2e_project_frontend/assets/.ic-assets.json5
+
+  dfx_start
+  assert_command dfx deploy
+  dfx canister call --query e2e_project_frontend list '(record{})'
+
+  ID=$(dfx canister id e2e_project_frontend)
+  PORT=$(get_webserver_port)
+
+  assert_command curl -v --output not-compressed http://localhost:"$PORT"/notreally.js?canisterId="$ID"
+  assert_not_match "content-encoding:"
+  diff not-compressed src/e2e_project_frontend/assets/notreally.js
+
+  assert_command curl -v --output encoded-compressed-1.br -H "Accept-Encoding: br" http://localhost:"$PORT"/notreally.js?canisterId="$ID"
+  assert_match "content-encoding: br"
+  brotli --decompress encoded-compressed-1.br
+  diff encoded-compressed-1 src/e2e_project_frontend/assets/notreally.js
+
+  # should split up accept-encoding lines with more than one encoding
+  assert_command curl -v --output encoded-compressed-2.br -H "Accept-Encoding: br, deflate, gzip" http://localhost:"$PORT"/notreally.js?canisterId="$ID"
+  assert_match "content-encoding: br"
+  brotli --decompress encoded-compressed-2.br
+  diff encoded-compressed-2 src/e2e_project_frontend/assets/notreally.js
+}
+
 @test "leaves in place files that were already installed" {
   install_asset assetscanister
   dd if=/dev/urandom of=src/e2e_project_frontend/assets/asset1.bin bs=400000 count=1

src/canisters/frontend/ic-asset/Cargo.toml

@@ -13,6 +13,7 @@ keywords = ["internet-computer", "assets", "icp", "dfinity"]
 
 [dependencies]
 backoff.workspace = true
+brotli = "6.0.0"
 candid = { workspace = true }
 derivative = "2.2.0"
 dfx-core = { path = "../../../dfx-core" }

src/canisters/frontend/ic-asset/src/asset/content.rs

@@ -1,4 +1,5 @@
 use crate::asset::content_encoder::ContentEncoder;
+use brotli::CompressorWriter;
 use dfx_core::error::fs::FsError;
 use flate2::write::GzEncoder;
 use flate2::Compression;
@@ -28,6 +29,7 @@ impl Content {
     pub fn encode(&self, encoder: &ContentEncoder) -> Result<Content, std::io::Error> {
         match encoder {
             ContentEncoder::Gzip => self.to_gzip(),
+            ContentEncoder::Brotli => self.to_brotli(),
             ContentEncoder::Identity => Ok(self.clone()),
         }
     }
@@ -42,6 +44,19 @@ impl Content {
         })
     }
 
+    pub fn to_brotli(&self) -> Result<Content, std::io::Error> {
+        let mut compressed_data = Vec::new();
+        {
+            let mut compressor = CompressorWriter::new(&mut compressed_data, 4096, 11, 22);
+            compressor.write_all(&self.data)?;
+            compressor.flush()?;
+        }
+        Ok(Content {
+            data: compressed_data,
+            media_type: self.media_type.clone(),
+        })
+    }
+
     pub fn sha256(&self) -> Vec<u8> {
         Sha256::digest(&self.data).to_vec()
     }

src/canisters/frontend/ic-asset/src/asset/content_encoder.rs

@@ -4,13 +4,16 @@ use serde::{Deserialize, Serialize};
 #[serde(rename_all = "lowercase")]
 pub enum ContentEncoder {
     Gzip,
+    #[serde(alias = "br")]
+    Brotli,
     Identity,
 }
 
 impl std::fmt::Display for ContentEncoder {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match &self {
             ContentEncoder::Gzip => f.write_str("gzip"),
+            ContentEncoder::Brotli => f.write_str("br"),
             ContentEncoder::Identity => f.write_str("identity"),
         }
     }

src/canisters/frontend/ic-asset/src/evidence/mod.rs

@@ -1,5 +1,5 @@
 use crate::asset::content::Content;
-use crate::asset::content_encoder::ContentEncoder::Gzip;
+use crate::asset::content_encoder::ContentEncoder::{Brotli, Gzip};
 use crate::batch_upload::operations::assemble_batch_operations;
 use crate::batch_upload::operations::AssetDeletionReason::Obsolete;
 use crate::batch_upload::plumbing::{make_project_assets, ProjectAsset};
@@ -123,14 +123,15 @@ fn hash_set_asset_content(
 
     let content = {
         let identity = Content::load(&ad.source).map_err(LoadContentFailed)?;
-        if args.content_encoding == "identity" {
-            identity
-        } else if args.content_encoding == "gzip" {
-            identity
+        match args.content_encoding.as_str() {
+            "identity" => identity,
+            "br" | "brotli" => identity
+                .encode(&Brotli)
+                .map_err(|e| EncodeContentFailed(ad.key.clone(), Brotli, e))?,
+            "gzip" => identity
                 .encode(&Gzip)
-                .map_err(|e| EncodeContentFailed(ad.key.clone(), Gzip, e))?
-        } else {
-            unreachable!("unhandled content encoder");
+                .map_err(|e| EncodeContentFailed(ad.key.clone(), Gzip, e))?,
+            _ => unreachable!("unhandled content encoder"),
         }
     };
     hasher.update(&content.data);