From 4cc2b892e0ea9e32bf2e7940964f5422672b5833 Mon Sep 17 00:00:00 2001 From: Martin Raszyk Date: Mon, 30 Sep 2024 12:09:01 +0200 Subject: [PATCH] feat(control-panel): support deploying large station WASM --- .../generated/control-panel/control_panel.did | 11 +++ .../control-panel/control_panel.did.d.ts | 6 ++ .../control-panel/control_panel.did.js | 8 ++ core/control-panel/api/spec.did | 11 +++ core/control-panel/api/src/canister.rs | 2 + core/control-panel/impl/src/core/config.rs | 13 ++- core/control-panel/impl/src/core/memory.rs | 6 +- core/control-panel/impl/src/core/mod.rs | 2 +- .../impl/src/services/canister.rs | 3 + .../control-panel/impl/src/services/deploy.rs | 25 +++--- libs/orbit-essentials/src/types.rs | 2 +- tests/integration/src/control_panel_tests.rs | 3 +- tests/integration/src/system_upgrade_tests.rs | 79 +---------------- tests/integration/src/utils.rs | 86 +++++++++++++++++-- 14 files changed, 157 insertions(+), 100 deletions(-) diff --git a/apps/wallet/src/generated/control-panel/control_panel.did b/apps/wallet/src/generated/control-panel/control_panel.did index 6fd377537..4d2a048c3 100644 --- a/apps/wallet/src/generated/control-panel/control_panel.did +++ b/apps/wallet/src/generated/control-panel/control_panel.did @@ -245,12 +245,23 @@ type CanDeployStationResult = variant { Err : ApiError; }; +type WasmModuleExtraChunks = record { + // The asset canister from which the chunks are to be retrieved. + store_canister : principal; + // The list of chunk hashes in the order they should be appended to the wasm module. + chunk_hashes_list : vec blob; + // The hash of the assembled wasm module. + wasm_module_hash : blob; +}; + // The canister modules required for the control panel. type UploadCanisterModulesInput = record { // The upgrader wasm module to use for the station canister. upgrader_wasm_module : opt blob; // The station wasm module to use. station_wasm_module : opt blob; + // Optional extra chunks of the station canister wasm module. + station_wasm_module_extra_chunks : opt opt WasmModuleExtraChunks; }; // The result of uploading canister modules. diff --git a/apps/wallet/src/generated/control-panel/control_panel.did.d.ts b/apps/wallet/src/generated/control-panel/control_panel.did.d.ts index 21ee568d4..68c710b89 100644 --- a/apps/wallet/src/generated/control-panel/control_panel.did.d.ts +++ b/apps/wallet/src/generated/control-panel/control_panel.did.d.ts @@ -178,6 +178,7 @@ export interface UpdateWaitingListInput { export type UpdateWaitingListResult = { 'Ok' : null } | { 'Err' : ApiError }; export interface UploadCanisterModulesInput { + 'station_wasm_module_extra_chunks' : [] | [[] | [WasmModuleExtraChunks]], 'station_wasm_module' : [] | [Uint8Array | number[]], 'upgrader_wasm_module' : [] | [Uint8Array | number[]], } @@ -199,6 +200,11 @@ export type UserSubscriptionStatus = { 'Unsubscribed' : null } | { 'Approved' : null } | { 'Denylisted' : null } | { 'Pending' : null }; +export interface WasmModuleExtraChunks { + 'wasm_module_hash' : Uint8Array | number[], + 'chunk_hashes_list' : Array, + 'store_canister' : Principal, +} export interface WasmModuleRegistryEntryDependency { 'name' : string, 'version' : string, diff --git a/apps/wallet/src/generated/control-panel/control_panel.did.js b/apps/wallet/src/generated/control-panel/control_panel.did.js index 6d27b6f55..9a5519d2f 100644 --- a/apps/wallet/src/generated/control-panel/control_panel.did.js +++ b/apps/wallet/src/generated/control-panel/control_panel.did.js @@ -245,7 +245,15 @@ export const idlFactory = ({ IDL }) => { 'Ok' : IDL.Null, 'Err' : ApiError, }); + const WasmModuleExtraChunks = IDL.Record({ + 'wasm_module_hash' : IDL.Vec(IDL.Nat8), + 'chunk_hashes_list' : IDL.Vec(IDL.Vec(IDL.Nat8)), + 'store_canister' : IDL.Principal, + }); const UploadCanisterModulesInput = IDL.Record({ + 'station_wasm_module_extra_chunks' : IDL.Opt( + IDL.Opt(WasmModuleExtraChunks) + ), 'station_wasm_module' : IDL.Opt(IDL.Vec(IDL.Nat8)), 'upgrader_wasm_module' : IDL.Opt(IDL.Vec(IDL.Nat8)), }); diff --git a/core/control-panel/api/spec.did b/core/control-panel/api/spec.did index 6fd377537..4d2a048c3 100644 --- a/core/control-panel/api/spec.did +++ b/core/control-panel/api/spec.did @@ -245,12 +245,23 @@ type CanDeployStationResult = variant { Err : ApiError; }; +type WasmModuleExtraChunks = record { + // The asset canister from which the chunks are to be retrieved. + store_canister : principal; + // The list of chunk hashes in the order they should be appended to the wasm module. + chunk_hashes_list : vec blob; + // The hash of the assembled wasm module. + wasm_module_hash : blob; +}; + // The canister modules required for the control panel. type UploadCanisterModulesInput = record { // The upgrader wasm module to use for the station canister. upgrader_wasm_module : opt blob; // The station wasm module to use. station_wasm_module : opt blob; + // Optional extra chunks of the station canister wasm module. + station_wasm_module_extra_chunks : opt opt WasmModuleExtraChunks; }; // The result of uploading canister modules. diff --git a/core/control-panel/api/src/canister.rs b/core/control-panel/api/src/canister.rs index f03b9352c..37da9e719 100644 --- a/core/control-panel/api/src/canister.rs +++ b/core/control-panel/api/src/canister.rs @@ -1,4 +1,5 @@ use candid::{CandidType, Deserialize}; +use orbit_essentials::types::WasmModuleExtraChunks; #[derive(CandidType, serde::Serialize, Deserialize, Clone, Debug, Eq, PartialEq)] pub struct UploadCanisterModulesInput { @@ -6,4 +7,5 @@ pub struct UploadCanisterModulesInput { pub upgrader_wasm_module: Option>, #[serde(deserialize_with = "orbit_essentials::deserialize::deserialize_option_blob")] pub station_wasm_module: Option>, + pub station_wasm_module_extra_chunks: Option>, } diff --git a/core/control-panel/impl/src/core/config.rs b/core/control-panel/impl/src/core/config.rs index 0a5c8d089..33486041a 100644 --- a/core/control-panel/impl/src/core/config.rs +++ b/core/control-panel/impl/src/core/config.rs @@ -3,7 +3,7 @@ use crate::core::ic_cdk::api::time; use crate::SYSTEM_VERSION; use ic_stable_structures::{storable::Bound, Storable}; use orbit_essentials::storable; -use orbit_essentials::types::Timestamp; +use orbit_essentials::types::{Timestamp, WasmModuleExtraChunks}; use std::borrow::Cow; #[storable] @@ -15,6 +15,9 @@ pub struct CanisterConfig { /// The station canister wasm module that will be used to deploy new stations. pub station_wasm_module: Vec, + /// Optional extra chunks of the station canister wasm module. + pub station_wasm_module_extra_chunks: Option, + /// Last time the canister was upgraded or initialized. pub last_upgrade_timestamp: Timestamp, @@ -27,6 +30,7 @@ impl Default for CanisterConfig { Self { upgrader_wasm_module: vec![], station_wasm_module: vec![], + station_wasm_module_extra_chunks: None, last_upgrade_timestamp: time(), version: None, } @@ -34,10 +38,15 @@ impl Default for CanisterConfig { } impl CanisterConfig { - pub fn new(upgrader_wasm_module: Vec, station_wasm_module: Vec) -> Self { + pub fn new( + upgrader_wasm_module: Vec, + station_wasm_module: Vec, + station_wasm_module_extra_chunks: Option, + ) -> Self { Self { upgrader_wasm_module, station_wasm_module, + station_wasm_module_extra_chunks, last_upgrade_timestamp: time(), version: Some(SYSTEM_VERSION.to_string()), } diff --git a/core/control-panel/impl/src/core/memory.rs b/core/control-panel/impl/src/core/memory.rs index 08a6e1434..85c5857cb 100644 --- a/core/control-panel/impl/src/core/memory.rs +++ b/core/control-panel/impl/src/core/memory.rs @@ -70,16 +70,16 @@ mod tests { #[test] fn test_canister_config() { - let config = CanisterConfig::new(Vec::new(), Vec::new()); + let config = CanisterConfig::new(Vec::new(), Vec::new(), None); write_canister_config(config.clone()); assert_eq!(canister_config(), Some(config)); } #[test] fn test_update_canister_config() { - let config = CanisterConfig::new(Vec::new(), Vec::new()); + let config = CanisterConfig::new(Vec::new(), Vec::new(), None); write_canister_config(config.clone()); - let new_config = CanisterConfig::new(vec![1], vec![2]); + let new_config = CanisterConfig::new(vec![1], vec![2], None); write_canister_config(new_config.clone()); assert_eq!(canister_config(), Some(new_config)); } diff --git a/core/control-panel/impl/src/core/mod.rs b/core/control-panel/impl/src/core/mod.rs index 3e6a209cc..c81fee665 100644 --- a/core/control-panel/impl/src/core/mod.rs +++ b/core/control-panel/impl/src/core/mod.rs @@ -49,7 +49,7 @@ pub mod test_utils { } pub fn init_canister_config() { - let config = CanisterConfig::new(Vec::new(), Vec::new()); + let config = CanisterConfig::new(Vec::new(), Vec::new(), None); write_canister_config(config); } } diff --git a/core/control-panel/impl/src/services/canister.rs b/core/control-panel/impl/src/services/canister.rs index 1fc350b8d..b9c8ed7c8 100644 --- a/core/control-panel/impl/src/services/canister.rs +++ b/core/control-panel/impl/src/services/canister.rs @@ -57,6 +57,9 @@ impl CanisterService { if let Some(station_wasm_module) = input.station_wasm_module { config.station_wasm_module = station_wasm_module; } + if let Some(station_wasm_module_extra_chunks) = input.station_wasm_module_extra_chunks { + config.station_wasm_module_extra_chunks = station_wasm_module_extra_chunks; + } write_canister_config(config); Ok(()) diff --git a/core/control-panel/impl/src/services/deploy.rs b/core/control-panel/impl/src/services/deploy.rs index 09dc2542c..5c18d4143 100644 --- a/core/control-panel/impl/src/services/deploy.rs +++ b/core/control-panel/impl/src/services/deploy.rs @@ -11,6 +11,7 @@ use ic_cdk::api::id as self_canister_id; use ic_cdk::api::management_canister::main::{self as mgmt}; use lazy_static::lazy_static; use orbit_essentials::api::ServiceResult; +use orbit_essentials::install_chunked_code::install_chunked_code; use std::sync::Arc; lazy_static! { @@ -47,8 +48,9 @@ impl DeployService { let config = canister_config().ok_or(DeployError::Failed { reason: "Canister config not initialized.".to_string(), })?; - let station_wasm_module = config.station_wasm_module; let upgrader_wasm_module = config.upgrader_wasm_module; + let station_wasm_module = config.station_wasm_module; + let station_wasm_module_extra_chunks = config.station_wasm_module_extra_chunks; let can_deploy_station_response = user.can_deploy_station(); match can_deploy_station_response { @@ -97,11 +99,8 @@ impl DeployService { .collect::>(); // installs the station canister with the associated upgrader wasm module - mgmt::install_code(mgmt::InstallCodeArgument { - mode: mgmt::CanisterInstallMode::Install, - canister_id: station_canister.canister_id, - wasm_module: station_wasm_module, - arg: Encode!(&station_api::SystemInstall::Init(station_api::SystemInit { + let station_install_arg = + Encode!(&station_api::SystemInstall::Init(station_api::SystemInit { name: input.name.clone(), admins, upgrader: station_api::SystemUpgraderInput::WasmModule(upgrader_wasm_module), @@ -111,12 +110,16 @@ impl DeployService { })) .map_err(|err| DeployError::Failed { reason: err.to_string(), - })?, - }) + })?; + install_chunked_code( + station_canister.canister_id, + mgmt::CanisterInstallMode::Install, + station_wasm_module, + station_wasm_module_extra_chunks, + station_install_arg, + ) .await - .map_err(|(_, err)| DeployError::Failed { - reason: err.to_string(), - })?; + .map_err(|err| DeployError::Failed { reason: err })?; self.user_service .add_deployed_station(&user.id, station_canister.canister_id, ctx) diff --git a/libs/orbit-essentials/src/types.rs b/libs/orbit-essentials/src/types.rs index 0e224d4c2..9bb1eb416 100644 --- a/libs/orbit-essentials/src/types.rs +++ b/libs/orbit-essentials/src/types.rs @@ -1,7 +1,7 @@ use candid::{CandidType, Deserialize, Principal}; use serde::Serialize; -#[derive(CandidType, Deserialize, Serialize, Clone, Debug)] +#[derive(CandidType, Deserialize, Serialize, Clone, Debug, Eq, PartialEq)] pub struct WasmModuleExtraChunks { pub store_canister: Principal, #[serde(deserialize_with = "crate::deserialize::deserialize_vec_blob")] diff --git a/tests/integration/src/control_panel_tests.rs b/tests/integration/src/control_panel_tests.rs index d404d0be5..135e310b6 100644 --- a/tests/integration/src/control_panel_tests.rs +++ b/tests/integration/src/control_panel_tests.rs @@ -512,8 +512,9 @@ fn upload_canister_modules_authorization() { upload_canister_modules(&env, canister_ids.control_panel, controller); let upload_canister_modules_args = UploadCanisterModulesInput { - station_wasm_module: None, upgrader_wasm_module: None, + station_wasm_module: None, + station_wasm_module_extra_chunks: None, }; let res: (ApiResult<()>,) = update_candid_as( &env, diff --git a/tests/integration/src/system_upgrade_tests.rs b/tests/integration/src/system_upgrade_tests.rs index a94a4ba84..778aa6852 100644 --- a/tests/integration/src/system_upgrade_tests.rs +++ b/tests/integration/src/system_upgrade_tests.rs @@ -1,14 +1,12 @@ -use crate::setup::{create_canister, get_canister_wasm, setup_new_env, WALLET_ADMIN_USER}; +use crate::setup::{get_canister_wasm, setup_new_env, WALLET_ADMIN_USER}; use crate::utils::{ execute_request, execute_request_with_extra_ticks, get_core_canister_health_status, - get_system_info, + get_system_info, upload_canister_chunks_to_asset_canister, }; use crate::{CanisterIds, TestEnv}; -use candid::{CandidType, Encode, Principal}; +use candid::{Encode, Principal}; use orbit_essentials::api::ApiResult; -use orbit_essentials::types::WasmModuleExtraChunks; use pocket_ic::{update_candid_as, PocketIc}; -use sha2::{Digest, Sha256}; use station_api::{ HealthStatus, NotifyFailedStationUpgradeInput, RequestOperationInput, RequestStatusDTO, SystemInstall, SystemUpgrade, SystemUpgradeOperationInput, SystemUpgradeTargetDTO, @@ -17,77 +15,6 @@ use upgrader_api::InitArg; const EXTRA_TICKS: u64 = 50; -#[derive(CandidType)] -struct StoreArg { - pub key: String, - pub content: Vec, - pub content_type: String, - pub content_encoding: String, - pub sha256: Option>, -} - -fn hash(data: Vec) -> Vec { - let mut hasher = Sha256::new(); - hasher.update(data); - hasher.finalize().to_vec() -} - -fn upload_canister_chunks_to_asset_canister( - env: &PocketIc, - canister_name: &str, - chunk_len: usize, -) -> (Vec, WasmModuleExtraChunks) { - // create and install the asset canister - let asset_canister_id = create_canister(env, Principal::anonymous()); - env.install_canister( - asset_canister_id, - get_canister_wasm("assetstorage"), - Encode!(&()).unwrap(), - None, - ); - - // get canister wasm - let canister_wasm = get_canister_wasm(canister_name).to_vec(); - let mut hasher = Sha256::new(); - hasher.update(&canister_wasm); - let canister_wasm_hash = hasher.finalize().to_vec(); - - // chunk canister - let mut chunks = canister_wasm.chunks(chunk_len); - let base_chunk: &[u8] = chunks.next().unwrap(); - assert!(!base_chunk.is_empty()); - let chunks: Vec<&[u8]> = chunks.collect(); - assert!(chunks.len() >= 2); - - // upload chunks to asset canister - for chunk in &chunks { - let chunk_hash = hash(chunk.to_vec()); - let store_arg = StoreArg { - key: hex::encode(chunk_hash.clone()), - content: chunk.to_vec(), - content_type: "application/octet-stream".to_string(), - content_encoding: "identity".to_string(), - sha256: Some(chunk_hash), - }; - update_candid_as::<_, ((),)>( - env, - asset_canister_id, - Principal::anonymous(), - "store", - (store_arg,), - ) - .unwrap(); - } - - let module_extra_chunks = WasmModuleExtraChunks { - store_canister: asset_canister_id, - chunk_hashes_list: chunks.iter().map(|c| hash(c.to_vec())).collect(), - wasm_module_hash: canister_wasm_hash, - }; - - (base_chunk.to_vec(), module_extra_chunks) -} - fn do_successful_station_upgrade( env: &PocketIc, canister_ids: &CanisterIds, diff --git a/tests/integration/src/utils.rs b/tests/integration/src/utils.rs index 102cf62d2..a2222ac2a 100644 --- a/tests/integration/src/utils.rs +++ b/tests/integration/src/utils.rs @@ -1,12 +1,14 @@ -use crate::setup::{get_canister_wasm, WALLET_ADMIN_USER}; -use candid::Principal; +use crate::setup::{create_canister, get_canister_wasm, WALLET_ADMIN_USER}; +use candid::{CandidType, Encode, Principal}; use control_panel_api::UploadCanisterModulesInput; use flate2::{write::GzEncoder, Compression}; use ic_cdk::api::management_canister::main::CanisterStatusResponse; use orbit_essentials::api::ApiResult; use orbit_essentials::cdk::api::management_canister::main::CanisterId; +use orbit_essentials::types::WasmModuleExtraChunks; use pocket_ic::{query_candid_as, update_candid_as, CallError, PocketIc, UserError, WasmResult}; use sha2::Digest; +use sha2::Sha256; use station_api::{ AccountDTO, AddAccountOperationInput, AddUserOperationInput, AllowDTO, ApiErrorDTO, CreateRequestInput, CreateRequestResponse, GetPermissionResponse, GetRequestInput, @@ -711,8 +713,9 @@ pub fn upload_canister_modules(env: &PocketIc, control_panel_id: Principal, cont // upload upgrader let upgrader_wasm = get_canister_wasm("upgrader").to_vec(); let upload_canister_modules_args = UploadCanisterModulesInput { - station_wasm_module: None, upgrader_wasm_module: Some(upgrader_wasm.to_owned()), + station_wasm_module: None, + station_wasm_module_extra_chunks: None, }; let res: (ApiResult<()>,) = update_candid_as( env, @@ -725,10 +728,12 @@ pub fn upload_canister_modules(env: &PocketIc, control_panel_id: Principal, cont res.0.unwrap(); // upload station - let station_wasm = get_canister_wasm("station").to_vec(); + let (base_chunk, module_extra_chunks) = + upload_canister_chunks_to_asset_canister(env, "station", 200_000); let upload_canister_modules_args = UploadCanisterModulesInput { - station_wasm_module: Some(station_wasm.to_owned()), upgrader_wasm_module: None, + station_wasm_module: Some(base_chunk), + station_wasm_module_extra_chunks: Some(Some(module_extra_chunks)), }; let res: (ApiResult<()>,) = update_candid_as( env, @@ -745,3 +750,74 @@ pub fn bump_time_to_avoid_ratelimit(env: &PocketIc) { // the rate limiter aggregation window is 300s and resolution is 10s env.advance_time(Duration::from_secs(300 + 10)); } + +#[derive(CandidType)] +struct StoreArg { + pub key: String, + pub content: Vec, + pub content_type: String, + pub content_encoding: String, + pub sha256: Option>, +} + +fn hash(data: Vec) -> Vec { + let mut hasher = Sha256::new(); + hasher.update(data); + hasher.finalize().to_vec() +} + +pub fn upload_canister_chunks_to_asset_canister( + env: &PocketIc, + canister_name: &str, + chunk_len: usize, +) -> (Vec, WasmModuleExtraChunks) { + // create and install the asset canister + let asset_canister_id = create_canister(env, Principal::anonymous()); + env.install_canister( + asset_canister_id, + get_canister_wasm("assetstorage"), + Encode!(&()).unwrap(), + None, + ); + + // get canister wasm + let canister_wasm = get_canister_wasm(canister_name).to_vec(); + let mut hasher = Sha256::new(); + hasher.update(&canister_wasm); + let canister_wasm_hash = hasher.finalize().to_vec(); + + // chunk canister + let mut chunks = canister_wasm.chunks(chunk_len); + let base_chunk: &[u8] = chunks.next().unwrap(); + assert!(!base_chunk.is_empty()); + let chunks: Vec<&[u8]> = chunks.collect(); + assert!(chunks.len() >= 2); + + // upload chunks to asset canister + for chunk in &chunks { + let chunk_hash = hash(chunk.to_vec()); + let store_arg = StoreArg { + key: hex::encode(chunk_hash.clone()), + content: chunk.to_vec(), + content_type: "application/octet-stream".to_string(), + content_encoding: "identity".to_string(), + sha256: Some(chunk_hash), + }; + update_candid_as::<_, ((),)>( + env, + asset_canister_id, + Principal::anonymous(), + "store", + (store_arg,), + ) + .unwrap(); + } + + let module_extra_chunks = WasmModuleExtraChunks { + store_canister: asset_canister_id, + chunk_hashes_list: chunks.iter().map(|c| hash(c.to_vec())).collect(), + wasm_module_hash: canister_wasm_hash, + }; + + (base_chunk.to_vec(), module_extra_chunks) +}