EOP: Rust Upgrade for RTS

default.nix

@@ -20,44 +20,89 @@ let ic-ref-run =
       cp ${ic-hs-pkgs.ic-hs}/bin/ic-ref-run $out/bin
   ''; in
 
-let
-  nixos-unstable = import nixpkgs.sources.nixpkgs-unstable {};
-in
-
 let haskellPackages = nixpkgs.haskellPackages.override {
       overrides = import nix/haskell-packages.nix nixpkgs subpath;
     }; in
+let llvm_sources = nixpkgs.fetchFromGitHub {
+      owner = "llvm";
+      repo = "llvm-project";
+      rev = "llvmorg-18.1.8";
+      sha256 = "sha256-iiZKMRo/WxJaBXct9GdAcAT3cz9d9pnAcO1mmR6oPNE=";
+    }; in
+let patched-wasm-ld = stdenv.mkDerivation {
+    name = "wasm-ld";
+    src = llvm_sources;
+    nativeBuildInputs = with nixpkgs; [ cmake ninja ];
+    buildInputs = with nixpkgs; [ llvmPackages_18.libllvm libxml2 ];
+
+    patchPhase = ''
+      patch lld/wasm/Relocations.cpp  << EOF
+@@ -104,9 +104,15 @@ void scanRelocations(InputChunk *chunk) {
+     case R_WASM_TABLE_INDEX_SLEB64:
+     case R_WASM_TABLE_INDEX_REL_SLEB:
+     case R_WASM_TABLE_INDEX_REL_SLEB64:
+-      if (requiresGOTAccess(sym))
+-        break;
++      if (!sym->isDefined()) {
++        error(toString(file) + ": relocation " + relocTypeToString(reloc.Type) +
++              " cannot be used against an undefined symbol \`" + toString(*sym) +
++              "\`");
++      }
+       out.elemSec->addEntry(cast<FunctionSymbol>(sym));
++      if (requiresGOTAccess(sym)) {
++        addGOTEntry(sym);
++      }
+       break;
+     case R_WASM_GLOBAL_INDEX_LEB:
+     case R_WASM_GLOBAL_INDEX_I32:
+EOF
+    cd lld
+    '';
+
+    outputs = [ "out" ];
+  }; in
+# Selectively build llvm binary tools.
+# Exclude wasm-ld, as this is patched separately.
+let llvm_bintools = stdenv.mkDerivation {
+    name = "llvm_bintools";
+    src = llvm_sources;
+    nativeBuildInputs = with nixpkgs; [ cmake ninja python3 ];
+    buildInputs = with nixpkgs; [ llvmPackages_18.libllvm libxml2 ];
+
+    preConfigure = ''
+      cd llvm
+    '';
+
+    outputs = [ "out" ];
+  }; in
 let
   rtsBuildInputs = with nixpkgs; [
-    # pulls in clang (wrapped) and clang-13 (unwrapped)
-    llvmPackages_13.clang
-    # pulls in wasm-ld
-    llvmPackages_13.lld
-    llvmPackages_13.bintools
+    llvmPackages_18.clang
+    llvm_bintools
+    patched-wasm-ld
     rustc-nightly
     cargo-nightly
     wasmtime
     rust-bindgen
     python3
-    nixos-unstable.emscripten
   ] ++ pkgs.lib.optional pkgs.stdenv.isDarwin [
     libiconv
   ];
 
   llvmEnv = ''
     # When compiling to wasm, we want to have more control over the flags,
     # so we do not use the nix-provided wrapper in clang
-    export WASM_CLANG="clang-13"
+    export WASM_CLANG="clang-18"
     export WASM_LD=wasm-ld
     # because we use the unwrapped clang, we have to pass in some flags/paths
     # that otherwise the wrapped clang would take care for us
-    export WASM_CLANG_LIB="${nixpkgs.llvmPackages_13.clang-unwrapped.lib}"
+    export WASM_CLANG_LIB="${nixpkgs.llvmPackages_18.clang-unwrapped.lib}"
 
     # When compiling natively, we want to use `clang` (which is a nixpkgs
     # provided wrapper that sets various include paths etc).
     # But for some reason it does not handle building for Wasm well, so
-    # there we use plain clang-13. There is no stdlib there anyways.
-    export CLANG="${nixpkgs.clang_13}/bin/clang"
+    # there we use plain clang-18. There is no stdlib there anyways.
+    export CLANG="${nixpkgs.clang_18}/bin/clang"
   '';
 in
 
@@ -172,7 +217,7 @@ rec {
       vendorRustStdDeps = "${cargoVendorTools}/bin/vendor-rust-std-deps";
 
       # SHA256 of Rust std deps
-      rustStdDepsHash = "sha256-A3WPIx+weu4wIYV7cweGkRxYGAPt7srxBAtMEyPOkhI=";
+      rustStdDepsHash = "sha256-U4BTr1CzFuOMdyLuhw5ry3/u8bkRiPmnMr4pLo3IdOQ=";
 
       # Vendor directory for Rust std deps
       rustStdDeps = nixpkgs.stdenvNoCC.mkDerivation {
@@ -337,7 +382,7 @@ rec {
 
     # extra deps for test/ld
     ldTestDeps =
-      with nixpkgs; [ llvmPackages_13.bintools llvmPackages_13.clang ];
+      with nixpkgs; [ patched-wasm-ld llvmPackages_18.clang ];
 
     testDerivation = args:
       stdenv.mkDerivation (testDerivationArgs // args);

nix/default.nix

@@ -91,7 +91,7 @@ let
 
         # Rust nightly
         (self: super: let
-          rust-channel = self.moz_overlay.rustChannelOf { date = "2023-04-21"; channel = "nightly"; };
+          rust-channel = self.moz_overlay.rustChannelOf { date = "2024-07-28"; channel = "nightly"; };
         in rec {
           rustc-nightly = rust-channel.rust.override {
             targets = [
@@ -108,9 +108,9 @@ let
           };
         })
 
-        # Rust 1.69
+        # Rust stable
         (self: super: let
-          rust-channel = self.moz_overlay.rustChannelOf { date = "2023-04-20"; channel = "stable"; };
+          rust-channel = self.moz_overlay.rustChannelOf { version = "1.78.0"; channel = "stable"; };
         in {
           rustPlatform_moz_stable = self.makeRustPlatform {
             rustc = rust-channel.rust;

nix/drun.nix

@@ -1,6 +1,6 @@
 pkgs:
 { drun =
-    pkgs.rustPlatform.buildRustPackage {
+    pkgs.rustPlatform_moz_stable.buildRustPackage {
       name = "drun";
 
       src = pkgs.sources.ic;
@@ -19,12 +19,10 @@ pkgs:
         lockFile = "${pkgs.sources.ic}/Cargo.lock";
         outputHashes = {
           "build-info-0.0.27" = "sha256-SkwWwDNrTsntkNiCv6rsyTFGazhpRDnKtVzPpYLKF9U=";
-          "derive_more-0.99.8-alpha.0" = "sha256-tEsfYC9oCAsDjinCsUDgRg3q6ruvayuA1lRmsEP9cys=";
-          "ic-btc-interface-0.1.0" = "sha256-JoVg1t62C2FIe0la1oQzidybLj1CyAQy80gkRh/MTn0=";
-          "ic-btc-test-utils-0.1.0" = "sha256-VecEMFjoeiRi0VgJ9CeDoOzdyJbJNiZ5MBmiV1+b7As=";
-          "icrc1-test-env-0.1.1" = "sha256-yWJF+KM8l65Nr0pwR9QeltkqbHDzOLNPVnLhf1mRukQ=";
+          "cloudflare-0.11.0" = "sha256-bJYiypmDI4KEy/VWt/7UcOv+g2CZLb9qUA9c1xlLxhM=";
+          "ic-agent-0.36.0" = "sha256-vDONIVjz0cwVgiszVRIjTKcqRUMHdVwTURflAMqmzHM=";
+          "icrc1-test-env-0.1.1" = "sha256-2PB7e64Owin/Eji3k8UoeWs+pfDfOOTaAyXjvjOZ/4g=";
           "jsonrpc-0.12.1" = "sha256-3FtdZlt2PqVDkE5iKWYIp1eiIELsaYlUPRSP2Xp8ejM=";
-          "libssh2-sys-0.2.23" = "sha256-9Hb7CnPF+lxrVO1NAhS7EXcPVWZutJXr6UWxpptzk4U=";
           "lmdb-rkv-0.14.99" = "sha256-5WcUzapkrc/s3wCBNCuUDhtbp17n67rTbm2rx0qtITg=";
         };
       };
@@ -43,9 +41,64 @@ pkgs:
 
      for file in lib_sources {
 EOF
-
         cd -
 
+        # Disable DTS for `drun`
+        patch rs/config/src/subnet_config.rs << EOF
+@@ -290,9 +290,9 @@ impl SchedulerConfig {
+     }
+
+     pub fn system_subnet() -> Self {
+-        let max_instructions_per_message_without_dts = NumInstructions::from(50 * B);
++        let max_instructions_per_message_without_dts =
++            MAX_INSTRUCTIONS_PER_MESSAGE_WITHOUT_DTS * SYSTEM_SUBNET_FACTOR;
+         let max_instructions_per_install_code = NumInstructions::from(1_000 * B);
+-        let max_instructions_per_slice = NumInstructions::from(10 * B);
+         Self {
+             scheduler_cores: NUMBER_OF_EXECUTION_THREADS,
+             max_paused_executions: MAX_PAUSED_EXECUTIONS,
+@@ -300,20 +300,19 @@ impl SchedulerConfig {
+             // TODO(RUN-993): Enable heap delta rate limiting for system subnets.
+             // Setting initial reserve to capacity effectively disables the rate limiting.
+             heap_delta_initial_reserve: SUBNET_HEAP_DELTA_CAPACITY,
+-            // Round limit is set to allow on average 2B instructions.
+-            // See also comment about \`MAX_INSTRUCTIONS_PER_ROUND\`.
+-            max_instructions_per_round: max_instructions_per_message_without_dts
+-                .max(max_instructions_per_slice)
+-                + NumInstructions::from(2 * B),
++            max_instructions_per_round: MAX_INSTRUCTIONS_PER_ROUND * SYSTEM_SUBNET_FACTOR,
++            // Effectively disable DTS on system subnets.
+             max_instructions_per_message: max_instructions_per_message_without_dts,
+             max_instructions_per_message_without_dts,
+-            max_instructions_per_slice,
++            // Effectively disable DTS on system subnets.
++            max_instructions_per_slice: max_instructions_per_message_without_dts,
+             instruction_overhead_per_execution: INSTRUCTION_OVERHEAD_PER_EXECUTION,
+             instruction_overhead_per_canister: INSTRUCTION_OVERHEAD_PER_CANISTER,
+             instruction_overhead_per_canister_for_finalization:
+                 INSTRUCTION_OVERHEAD_PER_CANISTER_FOR_FINALIZATION,
+             max_instructions_per_install_code,
+-            max_instructions_per_install_code_slice: max_instructions_per_slice,
++            // Effectively disable DTS on system subnets.
++            max_instructions_per_install_code_slice: max_instructions_per_install_code,
+             max_heap_delta_per_iteration: MAX_HEAP_DELTA_PER_ITERATION * SYSTEM_SUBNET_FACTOR,
+             max_message_duration_before_warn_in_seconds:
+                 MAX_MESSAGE_DURATION_BEFORE_WARN_IN_SECONDS,
+EOF
+
+        # static linking of libunwind fails under nix Linux
+        patch rs/monitoring/backtrace/build.rs << EOF
+@@ -1,8 +1,2 @@
+ fn main() {
+-    if std::env::var("TARGET").unwrap() == "x86_64-unknown-linux-gnu" {
+-        println!("cargo:rustc-link-lib=static=unwind");
+-        println!("cargo:rustc-link-lib=static=unwind-ptrace");
+-        println!("cargo:rustc-link-lib=static=unwind-x86_64");
+-        println!("cargo:rustc-link-lib=dylib=lzma");
+-    }
+ }
+EOF
+
         mkdir -p .cargo
         cat > .cargo/config.toml << EOF
 [target.x86_64-apple-darwin]
@@ -63,17 +116,17 @@ EOF
 
       buildInputs = with pkgs; [
         openssl
-        llvm_13
-        llvmPackages_13.libclang
+        llvm_18
+        llvmPackages_18.libclang
         lmdb
         libunwind
         libiconv
       ] ++ pkgs.lib.optional pkgs.stdenv.isDarwin
         pkgs.darwin.apple_sdk.frameworks.Security;
 
       # needed for bindgen
-      LIBCLANG_PATH = "${pkgs.llvmPackages_13.libclang.lib}/lib";
-      CLANG_PATH = "${pkgs.llvmPackages_13.clang}/bin/clang";
+      LIBCLANG_PATH = "${pkgs.llvmPackages_18.libclang.lib}/lib";
+      CLANG_PATH = "${pkgs.llvmPackages_18.clang}/bin/clang";
 
       # needed for ic-protobuf
       PROTOC="${pkgs.protobuf}/bin/protoc";

nix/sources.json

@@ -21,15 +21,15 @@
         "version": "3.2.25"
     },
     "ic": {
-        "branch": "pull/143/head",
+        "branch": "luc/latest-ic-wasm64-test",
         "description": "Internet Computer blockchain source: the client/replica software run by nodes",
         "homepage": "",
-        "owner": "dfinity",
+        "owner": "luc-blaeser",
         "repo": "ic",
-        "rev": "31f02c1b08aef1c6ee4c1b6bb9d25c21c5a4ff41",
-        "sha256": "04k278900kv082axrgv6vga0n8sw6c3lcc9298ih9gjyx8hanfb1",
+        "rev": "7921f5f3dc0d9fb774e3222f8ff6b1c00a086f1a",
+        "sha256": "1ykawbpaqnf1y508vh81m30p813ykmnbffxc3p0hw0p0k1ynq6zz",
         "type": "tarball",
-        "url": "https://github.com/dfinity/ic/archive/31f02c1b08aef1c6ee4c1b6bb9d25c21c5a4ff41.tar.gz",
+        "url": "https://github.com/luc-blaeser/ic/archive/7921f5f3dc0d9fb774e3222f8ff6b1c00a086f1a.tar.gz",
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
     },
     "ic-hs": {
@@ -70,15 +70,15 @@
         "version": "v1.2.0"
     },
     "motoko-base": {
-        "branch": "next-moc",
+        "branch": "pull/589/head",
         "description": "The Motoko base library",
         "homepage": "",
         "owner": "dfinity",
         "repo": "motoko-base",
-        "rev": "dfabfc3201950263a0057f5d1a78d2ede5c87478",
-        "sha256": "1wd9rcmhg82h08l3gz3gmn2xpgalvf679w7yz2g35yzbg2b5g2fd",
+        "rev": "f33777b2e7b28871f6e6b86863d22b7dca254a8b",
+        "sha256": "1q7hwby01kppjfwddyynn60xfvbpmw13qg628anghdxkgysc6x2y",
         "type": "tarball",
-        "url": "https://github.com/dfinity/motoko-base/archive/dfabfc3201950263a0057f5d1a78d2ede5c87478.tar.gz",
+        "url": "https://github.com/dfinity/motoko-base/archive/f33777b2e7b28871f6e6b86863d22b7dca254a8b.tar.gz",
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
     },
     "motoko-matchers": {
@@ -108,16 +108,15 @@
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
     },
     "nixpkgs": {
-        "branch": "release-23.11",
-        "builtin": true,
-        "description": "Nixpkgs/NixOS branches that track the Nixpkgs/NixOS channels",
-        "homepage": null,
+        "branch": "release-24.05",
+        "description": "Nix Packages collection & NixOS",
+        "homepage": "",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "6a5b92486ae7826c07fbfad302f569ceb187b0eb",
-        "sha256": "058kf03v7yh1c4ns96af6jq3ymadv71s7ajv9s05ipl9bnkjfrhm",
+        "rev": "404f983f8c1da3ed403c95df9bd18d7c3b23399d",
+        "sha256": "025alb35qf1iwmz45615ahp400q54582kqzc0238a51ai2w88pxc",
         "type": "tarball",
-        "url": "https://github.com/NixOS/nixpkgs/archive/6a5b92486ae7826c07fbfad302f569ceb187b0eb.tar.gz",
+        "url": "https://github.com/NixOS/nixpkgs/archive/404f983f8c1da3ed403c95df9bd18d7c3b23399d.tar.gz",
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
     },
     "nixpkgs-mozilla": {
@@ -133,18 +132,6 @@
         "url": "https://github.com/mozilla/nixpkgs-mozilla/archive/1ca9ee7192f973fd67b0988bdd77b8c11ae245a6.tar.gz",
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
     },
-    "nixpkgs-unstable": {
-        "branch": "master",
-        "description": "Nix Packages collection & NixOS",
-        "homepage": "",
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "9d2864289e65d0dc841123ae86bc7b3f77b6d12d",
-        "sha256": "0f3bcphzrjm1hs81mhpfx1hqxfnbvhhv5hv82wi1h3f735i279ja",
-        "type": "tarball",
-        "url": "https://github.com/NixOS/nixpkgs/archive/9d2864289e65d0dc841123ae86bc7b3f77b6d12d.tar.gz",
-        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
-    },
     "ocaml-vlq": {
         "branch": "v0.2.0",
         "builtin": false,