Merge branch 'master' into vincent/SDK-1636

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 # UNRELEASED
 
+### feat: Also report Motoko stable compatibility warnings
+
+Report upgrade compatibility warnings for Motoko, such as deleted stable variables, in addition to compatibility errors.
+
 ### feat: Support for Motoko's enhanced orthogonal persistence.
 
 Support Motoko's enhanced orthogonal persistence by automatically setting the canister upgrade option `wasm_memory_persistence` based on the Wasm metadata.
@@ -10,6 +14,17 @@ Support Motoko's enhanced orthogonal persistence by automatically setting the ca
 
 `dfx start --pocketic` no longer requires `--clean`, and can persist replica state between runs.
 
+### fix: Scripts always run with current directory set to the project root
+
+Build scripts and other scripts now always run with the working directory
+set to the project root (directory containing dfx.json).
+
+This applies to the following:
+ - build scripts
+ - extension run
+ - tech stack value computation
+ - packtool (vessel, mops etc)
+
 # 0.23.0
 
 ### feat: Add canister snapshots

docs/dfx-json-schema.json

@@ -288,7 +288,7 @@
           "properties": {
             "build": {
               "title": "Build Commands",
-              "description": "Commands that are executed in order to produce this canister's Wasm module. Expected to produce the Wasm in the path specified by the 'wasm' field. No build commands are allowed if the `wasm` field is a URL.",
+              "description": "Commands that are executed in order to produce this canister's Wasm module. Expected to produce the Wasm in the path specified by the 'wasm' field. No build commands are allowed if the `wasm` field is a URL. These commands are executed in the root of the project.",
               "default": [],
               "allOf": [
                 {
@@ -467,7 +467,7 @@
         },
         "post_install": {
           "title": "Post-Install Commands",
-          "description": "One or more commands to run post canister installation.",
+          "description": "One or more commands to run post canister installation. These commands are executed in the root of the project.",
           "default": [],
           "allOf": [
             {
@@ -698,7 +698,7 @@
           ]
         },
         "packtool": {
-          "description": "Main command to run the packtool.",
+          "description": "Main command to run the packtool. This command is executed in the root of the project.",
           "type": [
             "string",
             "null"

e2e/assets/custom_canister/build.sh

@@ -1,2 +1,3 @@
 #!/usr/bin/env bash
+echo "working directory of build script: '$(pwd)'"
 echo CUSTOM_CANISTER2_BUILD_DONE

e2e/assets/metadata/tech_stack/dfx.json

@@ -114,6 +114,17 @@
         "k": {
             "type": "motoko",
             "main": "main.mo"
+        },
+        "m": {
+            "type": "motoko",
+            "main": "main.mo",
+            "tech_stack": {
+                "other": {
+                    "command": {
+                        "cwd": "$(pwd)"
+                    }
+                }
+            }
         }
     }
 }

e2e/assets/post_install/postinstall.sh

@@ -1,2 +1,3 @@
 #!/usr/bin/env bash
+echo "working directory of post-install script: '$(pwd)'"
 echo hello-script

e2e/assets/rust_complex/Cargo.toml

@@ -4,5 +4,5 @@ edition = 2021
 resolver = "2"
 
 [workspace.dependencies]
-ic-cdk = "0.15"
+ic-cdk = "0.16"
 candid = "0.10"

e2e/assets/rust_deps/src/rust_deps/Cargo.toml

@@ -11,7 +11,7 @@ crate-type = ["cdylib"]
 
 [dependencies]
 candid = "0.10"
-ic-cdk = "0.12"
+ic-cdk = "0.16"
 
 [build-dependencies]
 ic-cdk-bindgen = "0.1"

e2e/assets/upgrade/v5.mo

@@ -0,0 +1,11 @@
+actor {
+  stable var newState : Int = 0;
+  public func inc() : async Int {
+    newState += 1;
+    return newState;
+  };
+  public func f() : async ?Int {
+    return ?42;
+  };
+  public query func read() : async Int { return newState; };
+}

e2e/tests-dfx/build.bash

@@ -279,6 +279,21 @@ teardown() {
   assert_match CUSTOM_CANISTER2_BUILD_DONE
 }
 
+@test "custom canister build script runs in project root" {
+  install_asset custom_canister
+  install_asset wasm/identity
+
+  dfx_start
+  dfx canister create custom2
+
+  cd src/e2e_project_backend
+  pwd
+
+  assert_command dfx build custom2
+  assert_match CUSTOM_CANISTER2_BUILD_DONE
+  assert_match "working directory of build script: '.*/working-dir/e2e_project'"
+}
+
 @test "build succeeds with network parameter" {
   dfx_start
   dfx canister create --all --network local

e2e/tests-dfx/extension.bash

@@ -675,6 +675,48 @@ EOF
   assert_eq "pamparam the param is 123"
 }
 
+@test "extension run uses project root" {
+  CACHE_DIR=$(dfx cache show)
+  mkdir -p "$CACHE_DIR"/extensions/test_extension
+
+  cat > "$CACHE_DIR"/extensions/test_extension/test_extension << "EOF"
+#!/usr/bin/env bash
+
+echo "the current directory is '$(pwd)'"
+
+EOF
+
+  chmod +x "$CACHE_DIR"/extensions/test_extension/test_extension
+
+  cat > "$CACHE_DIR"/extensions/test_extension/extension.json <<EOF
+{
+  "name": "test_extension",
+  "version": "0.1.0",
+  "homepage": "https://github.com/dfinity/dfx-extensions",
+  "authors": "DFINITY",
+  "summary": "Test extension for e2e purposes.",
+  "categories": [],
+  "keywords": [],
+  "subcommands": {
+  "abc": {
+  "about": "something something",
+  "args": {
+  }
+  }
+  }
+}
+EOF
+
+  mkdir -p project || exit
+  cd project || exit
+  echo "{}" >dfx.json
+  mkdir -p subdir || exit
+  cd subdir || exit
+
+  assert_command dfx test_extension abc
+  assert_match "the current directory is '.*/working-dir/project'"
+}
+
 @test "run with multiple values for the same parameter" {
   CACHE_DIR=$(dfx cache show)
   mkdir -p "$CACHE_DIR"/extensions/test_extension

e2e/tests-dfx/install.bash

@@ -108,6 +108,20 @@ teardown() {
   assert_match 'hello-script'
 }
 
+@test "post-install tasks run in project root" {
+  install_asset post_install
+  dfx_start
+
+  assert_command dfx canister create --all
+  assert_command dfx build
+
+  cd src/e2e_project_backend
+
+  assert_command dfx canister install postinstall_script
+  assert_match 'hello-script'
+  assert_match "working directory of post-install script: '.*/working-dir/e2e_project'"
+}
+
 @test "post-install tasks receive environment variables" {
   install_asset post_install
   dfx_start

e2e/tests-dfx/metadata.bash

@@ -300,3 +300,21 @@ ic-stable-structures"
   assert_command jq -r '.tech_stack.cdk."ic-cdk" | keys[]' e2e_project_backend.json
   assert_eq "version"
 }
+
+# shellcheck disable=SC2154
+@test "tech stack value generation uses project root as working directory" {
+  dfx_new
+  install_asset metadata/tech_stack
+
+  dfx_start
+
+
+  # m exposes other->command->working-directory
+
+  cd src/e2e_project_backend || exit
+  assert_command dfx deploy m
+  assert_command dfx canister metadata m dfx
+  echo "$stdout" > m.json
+  assert_command jq -r '.tech_stack.other.command.cwd' m.json
+  assert_match ".*/working-dir/e2e_project$"
+}

e2e/tests-dfx/new.bash

@@ -35,6 +35,22 @@ teardown() {
   assert_command_fail dfx new 'a:b'
 }
 
+@test "dfx new --help shows possible backend template names" {
+  assert_command dfx new --help
+  assert_match "\[possible values.*motoko.*\]"
+  assert_match "\[possible values.*rust.*\]"
+  assert_match "\[possible values.*kybra.*\]"
+  assert_match "\[possible values.*azle.*\]"
+}
+
+@test "dfx new --type <bad type> shows possible values" {
+  assert_command_fail dfx new --type bad_type
+  assert_match "\[possible values.*motoko.*\]"
+  assert_match "\[possible values.*rust.*\]"
+  assert_match "\[possible values.*kybra.*\]"
+  assert_match "\[possible values.*azle.*\]"
+}
+
 @test "dfx new readmes contain appropriate links" {
   assert_command dfx new --type rust e2e_rust --no-frontend
   assert_command grep "https://docs.rs/ic-cdk" e2e_rust/README.md

e2e/tests-dfx/packtool.bash

@@ -50,6 +50,25 @@ teardown() {
   assert_eq '("php: No comment.")'
 }
 
+@test "project calls packtool in project root" {
+  install_asset packtool
+  # shellcheck disable=SC1091
+  source configure_packtool.bash
+
+  dfx_start
+
+  cd vessel || exit
+  dfx canister create --all
+  dfx build
+  dfx canister install e2e_project_backend
+
+  assert_command dfx canister call e2e_project_backend rate '("rust")'
+  assert_eq '("rust: So hot right now.")'
+
+  assert_command dfx canister call e2e_project_backend rate '("php")'
+  assert_eq '("php: No comment.")'
+}
+
 @test "failure to invoke the package tool reports the command line and reason" {
   install_asset packtool
   jq '.defaults.build.packtool="./no-such-command that command cannot be invoked"' dfx.json | sponge dfx.json

e2e/tests-dfx/upgrade_check.bash

@@ -33,7 +33,7 @@ teardown() {
   jq '.canisters.hello_backend.main="v2_bad.mo"' dfx.json | sponge dfx.json
   echo yes | (
   assert_command dfx deploy
-  assert_match "Stable interface compatibility check failed"
+  assert_match "Stable interface compatibility check issued an ERROR"
   )
   assert_command dfx canister call hello_backend read '()'
   assert_match "(0 : nat)"
@@ -78,3 +78,17 @@ teardown() {
   assert_command dfx canister call hello_backend f '()'
   assert_match "(opt \"\")"
 }
+
+@test "warning when dropping stable variable" {
+  install_asset upgrade
+  dfx_start
+  dfx deploy
+  dfx canister call hello_backend inc '()'
+  jq '.canisters.hello_backend.main="v5.mo"' dfx.json | sponge dfx.json
+  echo yes | (
+  assert_command dfx deploy
+  assert_match "Stable interface compatibility check issued an ERROR"
+  )
+  assert_command dfx canister call hello_backend read '()'
+  assert_match "(0 : int)"
+}

src/dfx-core/Cargo.toml

@@ -26,6 +26,7 @@ humantime-serde = "1.1.1"
 ic-agent.workspace = true
 ic-utils.workspace = true
 ic-identity-hsm.workspace = true
+itertools.workspace = true
 k256 = { version = "0.11.4", features = ["pem"] }
 keyring.workspace = true
 lazy_static.workspace = true

src/dfx-core/src/config/mod.rs

@@ -1,3 +1,4 @@
 pub mod cache;
 pub mod directories;
 pub mod model;
+pub mod project_templates;

src/dfx-core/src/config/model/dfinity.rs

@@ -271,6 +271,7 @@ pub struct ConfigCanistersCanister {
 
     /// # Post-Install Commands
     /// One or more commands to run post canister installation.
+    /// These commands are executed in the root of the project.
     #[serde(default)]
     pub post_install: SerdeVec<String>,
 
@@ -379,6 +380,7 @@ pub enum CanisterTypeProperties {
         /// Commands that are executed in order to produce this canister's Wasm module.
         /// Expected to produce the Wasm in the path specified by the 'wasm' field.
         /// No build commands are allowed if the `wasm` field is a URL.
+        /// These commands are executed in the root of the project.
         #[schemars(default)]
         build: SerdeVec<String>,
     },
@@ -617,6 +619,7 @@ impl Default for ConfigDefaultsBootstrap {
 #[derive(Clone, Debug, Default, Serialize, Deserialize, JsonSchema)]
 pub struct ConfigDefaultsBuild {
     /// Main command to run the packtool.
+    /// This command is executed in the root of the project.
     pub packtool: Option<String>,
 
     /// Arguments for packtool.