From 5ee2a1cfe190b90d2e7d24291914400308892684 Mon Sep 17 00:00:00 2001 From: Severin Siffert Date: Wed, 19 Jun 2024 10:16:46 +0200 Subject: [PATCH] feat: upload br encoding to asset canister (#3791) --- CHANGELOG.md | 9 +++++ Cargo.lock | 37 +++++++++++++++++++ e2e/tests-dfx/assetscanister.bash | 35 ++++++++++++++++++ src/canisters/frontend/ic-asset/Cargo.toml | 1 + .../frontend/ic-asset/src/asset/content.rs | 15 ++++++++ .../ic-asset/src/asset/content_encoder.rs | 3 ++ .../frontend/ic-asset/src/evidence/mod.rs | 17 +++++---- 7 files changed, 109 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 596bbe9b8b..22a511b2a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -94,6 +94,15 @@ Adds support for the `log_visibility` canister setting, which configures which u Valid options are `controllers` and `public`. The setting can be used with the `--log-visibility` flag in `dfx canister create` and `dfx canister update-settings`, or in `dfx.json` under `canisters[].initialization_values.log_visibility`. +## Asset canister synchronization + +### feat: support `brotli` encoding + +Asset synchronization now not only supports `identity` and `gzip`, but also `brotli` encoding. +The default encodings are still +- `identity` and `gzip` for MIME types `.txt`, `.html` and `.js` +- `identity` for anything else + ## Dependencies ### Frontend canister diff --git a/Cargo.lock b/Cargo.lock index 9752ec4db5..5434dfa232 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -189,6 +189,21 @@ dependencies = [ "memchr", ] +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" +dependencies = [ + "alloc-no-stdlib", +] + [[package]] name = "allocator-api2" version = "0.2.18" @@ -640,6 +655,27 @@ dependencies = [ "syn_derive", ] +[[package]] +name = "brotli" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74f7971dbd9326d58187408ab83117d8ac1bb9c17b085fdacd1cf2f598719b6b" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "4.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a45bd2e4095a8b518033b128020dd4a55aab1c0a381ba4404a472630f4bc362" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + [[package]] name = "bs58" version = "0.4.0" @@ -2691,6 +2727,7 @@ name = "ic-asset" version = "0.20.0" dependencies = [ "backoff", + "brotli", "candid", "derivative", "dfx-core", diff --git a/e2e/tests-dfx/assetscanister.bash b/e2e/tests-dfx/assetscanister.bash index f75f3e676d..b15da9ce22 100644 --- a/e2e/tests-dfx/assetscanister.bash +++ b/e2e/tests-dfx/assetscanister.bash @@ -835,6 +835,41 @@ check_permission_failure() { diff encoded-compressed-2 src/e2e_project_frontend/assets/notreally.js } +@test "can use brotli compression" { + install_asset assetscanister + for i in $(seq 1 400); do + echo "some easily duplicate text $i" >>src/e2e_project_frontend/assets/notreally.js + done + + echo '[{ + "match": "*.js", + "encodings": ["identity", "br"] + } + ]' > src/e2e_project_frontend/assets/.ic-assets.json5 + + dfx_start + assert_command dfx deploy + dfx canister call --query e2e_project_frontend list '(record{})' + + ID=$(dfx canister id e2e_project_frontend) + PORT=$(get_webserver_port) + + assert_command curl -v --output not-compressed http://localhost:"$PORT"/notreally.js?canisterId="$ID" + assert_not_match "content-encoding:" + diff not-compressed src/e2e_project_frontend/assets/notreally.js + + assert_command curl -v --output encoded-compressed-1.br -H "Accept-Encoding: br" http://localhost:"$PORT"/notreally.js?canisterId="$ID" + assert_match "content-encoding: br" + brotli --decompress encoded-compressed-1.br + diff encoded-compressed-1 src/e2e_project_frontend/assets/notreally.js + + # should split up accept-encoding lines with more than one encoding + assert_command curl -v --output encoded-compressed-2.br -H "Accept-Encoding: br, deflate, gzip" http://localhost:"$PORT"/notreally.js?canisterId="$ID" + assert_match "content-encoding: br" + brotli --decompress encoded-compressed-2.br + diff encoded-compressed-2 src/e2e_project_frontend/assets/notreally.js +} + @test "leaves in place files that were already installed" { install_asset assetscanister dd if=/dev/urandom of=src/e2e_project_frontend/assets/asset1.bin bs=400000 count=1 diff --git a/src/canisters/frontend/ic-asset/Cargo.toml b/src/canisters/frontend/ic-asset/Cargo.toml index 3139c6edda..9ff2183901 100644 --- a/src/canisters/frontend/ic-asset/Cargo.toml +++ b/src/canisters/frontend/ic-asset/Cargo.toml @@ -13,6 +13,7 @@ keywords = ["internet-computer", "assets", "icp", "dfinity"] [dependencies] backoff.workspace = true +brotli = "6.0.0" candid = { workspace = true } derivative = "2.2.0" dfx-core = { path = "../../../dfx-core" } diff --git a/src/canisters/frontend/ic-asset/src/asset/content.rs b/src/canisters/frontend/ic-asset/src/asset/content.rs index 0567eb9fed..bcc7e7e218 100644 --- a/src/canisters/frontend/ic-asset/src/asset/content.rs +++ b/src/canisters/frontend/ic-asset/src/asset/content.rs @@ -1,4 +1,5 @@ use crate::asset::content_encoder::ContentEncoder; +use brotli::CompressorWriter; use dfx_core::error::fs::FsError; use flate2::write::GzEncoder; use flate2::Compression; @@ -28,6 +29,7 @@ impl Content { pub fn encode(&self, encoder: &ContentEncoder) -> Result { match encoder { ContentEncoder::Gzip => self.to_gzip(), + ContentEncoder::Brotli => self.to_brotli(), ContentEncoder::Identity => Ok(self.clone()), } } @@ -42,6 +44,19 @@ impl Content { }) } + pub fn to_brotli(&self) -> Result { + let mut compressed_data = Vec::new(); + { + let mut compressor = CompressorWriter::new(&mut compressed_data, 4096, 11, 22); + compressor.write_all(&self.data)?; + compressor.flush()?; + } + Ok(Content { + data: compressed_data, + media_type: self.media_type.clone(), + }) + } + pub fn sha256(&self) -> Vec { Sha256::digest(&self.data).to_vec() } diff --git a/src/canisters/frontend/ic-asset/src/asset/content_encoder.rs b/src/canisters/frontend/ic-asset/src/asset/content_encoder.rs index b38f84ac8f..d8390f7aa4 100644 --- a/src/canisters/frontend/ic-asset/src/asset/content_encoder.rs +++ b/src/canisters/frontend/ic-asset/src/asset/content_encoder.rs @@ -4,6 +4,8 @@ use serde::{Deserialize, Serialize}; #[serde(rename_all = "lowercase")] pub enum ContentEncoder { Gzip, + #[serde(alias = "br")] + Brotli, Identity, } @@ -11,6 +13,7 @@ impl std::fmt::Display for ContentEncoder { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match &self { ContentEncoder::Gzip => f.write_str("gzip"), + ContentEncoder::Brotli => f.write_str("br"), ContentEncoder::Identity => f.write_str("identity"), } } diff --git a/src/canisters/frontend/ic-asset/src/evidence/mod.rs b/src/canisters/frontend/ic-asset/src/evidence/mod.rs index e6f12cd792..2a64a032e2 100644 --- a/src/canisters/frontend/ic-asset/src/evidence/mod.rs +++ b/src/canisters/frontend/ic-asset/src/evidence/mod.rs @@ -1,5 +1,5 @@ use crate::asset::content::Content; -use crate::asset::content_encoder::ContentEncoder::Gzip; +use crate::asset::content_encoder::ContentEncoder::{Brotli, Gzip}; use crate::batch_upload::operations::assemble_batch_operations; use crate::batch_upload::operations::AssetDeletionReason::Obsolete; use crate::batch_upload::plumbing::{make_project_assets, ProjectAsset}; @@ -123,14 +123,15 @@ fn hash_set_asset_content( let content = { let identity = Content::load(&ad.source).map_err(LoadContentFailed)?; - if args.content_encoding == "identity" { - identity - } else if args.content_encoding == "gzip" { - identity + match args.content_encoding.as_str() { + "identity" => identity, + "br" | "brotli" => identity + .encode(&Brotli) + .map_err(|e| EncodeContentFailed(ad.key.clone(), Brotli, e))?, + "gzip" => identity .encode(&Gzip) - .map_err(|e| EncodeContentFailed(ad.key.clone(), Gzip, e))? - } else { - unreachable!("unhandled content encoder"); + .map_err(|e| EncodeContentFailed(ad.key.clone(), Gzip, e))?, + _ => unreachable!("unhandled content encoder"), } }; hasher.update(&content.data);