Merge branch 'master' into chore-update-replica-3d6a76efba59d6f03026d…

…6b7c1c9a1dfce96ee93-master

CHANGELOG.md

@@ -2,12 +2,31 @@
 
 # UNRELEASED
 
-## Dependencies
+### feat: Also report Motoko stable compatibility warnings
+
+Report upgrade compatibility warnings for Motoko, such as deleted stable variables, in addition to compatibility errors.
+
+### feat: Support for Motoko's enhanced orthogonal persistence.
+
+Support Motoko's enhanced orthogonal persistence by automatically setting the canister upgrade option `wasm_memory_persistence` based on the Wasm metadata.
 
 ### feat: PocketIC state
 
 `dfx start --pocketic` no longer requires `--clean`, and can persist replica state between runs.
 
+### fix: Scripts always run with current directory set to the project root
+
+Build scripts and other scripts now always run with the working directory
+set to the project root (directory containing dfx.json).
+
+This applies to the following:
+ - build scripts
+ - extension run
+ - tech stack value computation
+ - packtool (vessel, mops etc)
+
+## Dependencies
+
 ### Replica
 
 Updated replica to elected commit 3d6a76efba59d6f03026d6b7c1c9a1dfce96ee93.

Cargo.toml

@@ -21,11 +21,11 @@ license = "Apache-2.0"
 [workspace.dependencies]
 candid = "0.10.4"
 candid_parser = "0.1.4"
-ic-agent = { git = "https://github.com/dfinity/agent-rs.git", rev = "60a64dd2d5368180f03a75df80f475d6e2caf21a" }
+ic-agent = { git = "https://github.com/dfinity/agent-rs.git", rev = "6e11a350112f9b907c4d590d8217f340e153d898" }
 ic-asset = { path = "src/canisters/frontend/ic-asset" }
 ic-cdk = "0.13.5"
-ic-identity-hsm = { git = "https://github.com/dfinity/agent-rs.git", rev = "60a64dd2d5368180f03a75df80f475d6e2caf21a" }
-ic-utils = { git = "https://github.com/dfinity/agent-rs.git", rev = "60a64dd2d5368180f03a75df80f475d6e2caf21a" }
+ic-identity-hsm = { git = "https://github.com/dfinity/agent-rs.git", rev = "6e11a350112f9b907c4d590d8217f340e153d898" }
+ic-utils = { git = "https://github.com/dfinity/agent-rs.git", rev = "6e11a350112f9b907c4d590d8217f340e153d898" }
 
 aes-gcm = "0.10.3"
 anyhow = "1.0.56"

docs/dfx-json-schema.json

@@ -288,7 +288,7 @@
           "properties": {
             "build": {
               "title": "Build Commands",
-              "description": "Commands that are executed in order to produce this canister's Wasm module. Expected to produce the Wasm in the path specified by the 'wasm' field. No build commands are allowed if the `wasm` field is a URL.",
+              "description": "Commands that are executed in order to produce this canister's Wasm module. Expected to produce the Wasm in the path specified by the 'wasm' field. No build commands are allowed if the `wasm` field is a URL. These commands are executed in the root of the project.",
               "default": [],
               "allOf": [
                 {
@@ -467,7 +467,7 @@
         },
         "post_install": {
           "title": "Post-Install Commands",
-          "description": "One or more commands to run post canister installation.",
+          "description": "One or more commands to run post canister installation. These commands are executed in the root of the project.",
           "default": [],
           "allOf": [
             {
@@ -698,7 +698,7 @@
           ]
         },
         "packtool": {
-          "description": "Main command to run the packtool.",
+          "description": "Main command to run the packtool. This command is executed in the root of the project.",
           "type": [
             "string",
             "null"

e2e/assets/custom_canister/build.sh

@@ -1,2 +1,3 @@
 #!/usr/bin/env bash
+echo "working directory of build script: '$(pwd)'"
 echo CUSTOM_CANISTER2_BUILD_DONE

e2e/assets/metadata/tech_stack/dfx.json

@@ -114,6 +114,17 @@
         "k": {
             "type": "motoko",
             "main": "main.mo"
+        },
+        "m": {
+            "type": "motoko",
+            "main": "main.mo",
+            "tech_stack": {
+                "other": {
+                    "command": {
+                        "cwd": "$(pwd)"
+                    }
+                }
+            }
         }
     }
 }

e2e/assets/post_install/postinstall.sh

@@ -1,2 +1,3 @@
 #!/usr/bin/env bash
+echo "working directory of post-install script: '$(pwd)'"
 echo hello-script

e2e/assets/rust_complex/Cargo.toml

@@ -4,5 +4,5 @@ edition = 2021
 resolver = "2"
 
 [workspace.dependencies]
-ic-cdk = "0.15"
+ic-cdk = "0.16"
 candid = "0.10"

e2e/assets/rust_deps/src/rust_deps/Cargo.toml

@@ -11,7 +11,7 @@ crate-type = ["cdylib"]
 
 [dependencies]
 candid = "0.10"
-ic-cdk = "0.12"
+ic-cdk = "0.16"
 
 [build-dependencies]
 ic-cdk-bindgen = "0.1"

e2e/assets/upgrade/v5.mo

@@ -0,0 +1,11 @@
+actor {
+  stable var newState : Int = 0;
+  public func inc() : async Int {
+    newState += 1;
+    return newState;
+  };
+  public func f() : async ?Int {
+    return ?42;
+  };
+  public query func read() : async Int { return newState; };
+}

e2e/assets/wasm_memory_persistence/README.md

@@ -0,0 +1,3 @@
+`classical-actor.wasm` is built from `actor.mo` with `moc -o classical-actor.wasm actor.mo`.
+`enhanced-actor.wasm` is built from `actor.mo` with `moc -o enhanced-actor.wasm --enhanced-orthogonal-persistence actor.mo`.
+`actor.did` is obtained by `moc --idl actor.mo`.

e2e/assets/wasm_memory_persistence/actor.did

@@ -0,0 +1,3 @@
+service : {
+  getVersion: () -> (nat) query;
+}

e2e/assets/wasm_memory_persistence/actor.mo

@@ -0,0 +1,12 @@
+import Prim "mo:prim";
+
+actor {
+    stable var version = 0;
+
+    version += 1;
+    Prim.debugPrint("Deployed actor version " # debug_show (version));
+
+    public query func getVersion() : async Nat {
+        return version;
+    };
+};

e2e/assets/wasm_memory_persistence/dfx.json

@@ -0,0 +1,9 @@
+{
+  "canisters": {
+    "test": {
+      "type": "custom",
+      "candid": "actor.did",
+      "wasm": "classical-actor.wasm"
+    }
+  }
+}

e2e/tests-dfx/build.bash

@@ -279,6 +279,21 @@ teardown() {
   assert_match CUSTOM_CANISTER2_BUILD_DONE
 }
 
+@test "custom canister build script runs in project root" {
+  install_asset custom_canister
+  install_asset wasm/identity
+
+  dfx_start
+  dfx canister create custom2
+
+  cd src/e2e_project_backend
+  pwd
+
+  assert_command dfx build custom2
+  assert_match CUSTOM_CANISTER2_BUILD_DONE
+  assert_match "working directory of build script: '.*/working-dir/e2e_project'"
+}
+
 @test "build succeeds with network parameter" {
   dfx_start
   dfx canister create --all --network local

e2e/tests-dfx/extension.bash

@@ -675,6 +675,48 @@ EOF
   assert_eq "pamparam the param is 123"
 }
 
+@test "extension run uses project root" {
+  CACHE_DIR=$(dfx cache show)
+  mkdir -p "$CACHE_DIR"/extensions/test_extension
+
+  cat > "$CACHE_DIR"/extensions/test_extension/test_extension << "EOF"
+#!/usr/bin/env bash
+
+echo "the current directory is '$(pwd)'"
+
+EOF
+
+  chmod +x "$CACHE_DIR"/extensions/test_extension/test_extension
+
+  cat > "$CACHE_DIR"/extensions/test_extension/extension.json <<EOF
+{
+  "name": "test_extension",
+  "version": "0.1.0",
+  "homepage": "https://github.com/dfinity/dfx-extensions",
+  "authors": "DFINITY",
+  "summary": "Test extension for e2e purposes.",
+  "categories": [],
+  "keywords": [],
+  "subcommands": {
+  "abc": {
+  "about": "something something",
+  "args": {
+  }
+  }
+  }
+}
+EOF
+
+  mkdir -p project || exit
+  cd project || exit
+  echo "{}" >dfx.json
+  mkdir -p subdir || exit
+  cd subdir || exit
+
+  assert_command dfx test_extension abc
+  assert_match "the current directory is '.*/working-dir/project'"
+}
+
 @test "run with multiple values for the same parameter" {
   CACHE_DIR=$(dfx cache show)
   mkdir -p "$CACHE_DIR"/extensions/test_extension

e2e/tests-dfx/install.bash

@@ -108,6 +108,20 @@ teardown() {
   assert_match 'hello-script'
 }
 
+@test "post-install tasks run in project root" {
+  install_asset post_install
+  dfx_start
+
+  assert_command dfx canister create --all
+  assert_command dfx build
+
+  cd src/e2e_project_backend
+
+  assert_command dfx canister install postinstall_script
+  assert_match 'hello-script'
+  assert_match "working directory of post-install script: '.*/working-dir/e2e_project'"
+}
+
 @test "post-install tasks receive environment variables" {
   install_asset post_install
   dfx_start

e2e/tests-dfx/metadata.bash

@@ -300,3 +300,21 @@ ic-stable-structures"
   assert_command jq -r '.tech_stack.cdk."ic-cdk" | keys[]' e2e_project_backend.json
   assert_eq "version"
 }
+
+# shellcheck disable=SC2154
+@test "tech stack value generation uses project root as working directory" {
+  dfx_new
+  install_asset metadata/tech_stack
+
+  dfx_start
+
+
+  # m exposes other->command->working-directory
+
+  cd src/e2e_project_backend || exit
+  assert_command dfx deploy m
+  assert_command dfx canister metadata m dfx
+  echo "$stdout" > m.json
+  assert_command jq -r '.tech_stack.other.command.cwd' m.json
+  assert_match ".*/working-dir/e2e_project$"
+}

e2e/tests-dfx/packtool.bash

@@ -50,6 +50,25 @@ teardown() {
   assert_eq '("php: No comment.")'
 }
 
+@test "project calls packtool in project root" {
+  install_asset packtool
+  # shellcheck disable=SC1091
+  source configure_packtool.bash
+
+  dfx_start
+
+  cd vessel || exit
+  dfx canister create --all
+  dfx build
+  dfx canister install e2e_project_backend
+
+  assert_command dfx canister call e2e_project_backend rate '("rust")'
+  assert_eq '("rust: So hot right now.")'
+
+  assert_command dfx canister call e2e_project_backend rate '("php")'
+  assert_eq '("php: No comment.")'
+}
+
 @test "failure to invoke the package tool reports the command line and reason" {
   install_asset packtool
   jq '.defaults.build.packtool="./no-such-command that command cannot be invoked"' dfx.json | sponge dfx.json

e2e/tests-dfx/sign_send.bash

@@ -32,13 +32,12 @@ teardown() {
   assert_eq "Error: [message.json] already exists, please specify a different output file name."
 
   assert_command dfx canister sign --update hello_backend inc --file message-inc.json
-  assert_eq "Update message generated at [message-inc.json]
-Signed request_status append to update message in [message-inc.json]"
+  assert_eq "Update and request_status message generated at [message-inc.json]"
 
   sleep 10
   echo y | assert_command dfx canister send message-inc.json
   assert_command dfx canister send message-inc.json --status
-  assert_match "To see the content of response, copy-paste the encoded string into cbor.me."
+  assert_match "Response:"
 }
 
 @test "sign outside of a dfx project" {

e2e/tests-dfx/upgrade_check.bash

@@ -33,7 +33,7 @@ teardown() {
   jq '.canisters.hello_backend.main="v2_bad.mo"' dfx.json | sponge dfx.json
   echo yes | (
   assert_command dfx deploy
-  assert_match "Stable interface compatibility check failed"
+  assert_match "Stable interface compatibility check issued an ERROR"
   )
   assert_command dfx canister call hello_backend read '()'
   assert_match "(0 : nat)"
@@ -78,3 +78,17 @@ teardown() {
   assert_command dfx canister call hello_backend f '()'
   assert_match "(opt \"\")"
 }
+
+@test "warning when dropping stable variable" {
+  install_asset upgrade
+  dfx_start
+  dfx deploy
+  dfx canister call hello_backend inc '()'
+  jq '.canisters.hello_backend.main="v5.mo"' dfx.json | sponge dfx.json
+  echo yes | (
+  assert_command dfx deploy
+  assert_match "Stable interface compatibility check issued an ERROR"
+  )
+  assert_command dfx canister call hello_backend read '()'
+  assert_match "(0 : int)"
+}