__motoko_runtime_information() as Privileged Query for Runtime Stat…

…istics. (#4635)

# Privileged System-Level Query for Runtime Statistics

Exposing a privileged system-level query function `__motoko_runtime_information()` that reports the current runtime statistics of the canister, such as the compiler version, runtime system version, the GC in use, the heap size, the total number of allocated objects, the total amount of reclaimed memory and more. This is useful because several statistics of the reported information cannot be inspected on the IC replica dashboard as they are internal to the Motoko runtime system. This query is **only authorized to the canister controllers and self-calls of the canister**.

### Function Signature
    ```
    __motoko_runtime_information : () -> {
        compilerVersion : Text;
        rtsVersion : Text;
        garbageCollector : Text;
        memorySize : Nat;
        heapSize : Nat;
        totalAllocation : Nat;
        reclaimed : Nat;
        maxLiveSize : Nat;
        stableMemorySize : Nat;
        logicalStableMemorySize : Nat;
        maxStackSize : Nat;
        callbackTableCount : Nat;
        callbackTableSize : Nat;
    }
    ```

Changelog.md

@@ -1,5 +1,37 @@
 # Motoko compiler changelog
 
+## Upcoming
+
+* motoko (`moc`)
+  * debugging: `__motoko_runtime_information()` as privileged query for runtime statistics.
+
+    Exposing a privileged system-level query function `__motoko_runtime_information()` 
+    that reports the current runtime statistics of the canister, such as the heap size, 
+    the total number of allocated objects, the total amount of reclaimed memory and more.
+    This is useful because several statistics of the reported information cannot be 
+    inspected on the IC replica dashboard as they are internal to the Motoko runtime system. 
+    This query is only authorized to the canister controllers and self-calls of the canister.
+
+    ```
+    __motoko_runtime_information : () -> {
+        compilerVersion : Text;
+        rtsVersion : Text;
+        garbageCollector : Text;
+        sanityChecks : Nat;
+        memorySize : Nat;
+        heapSize : Nat;
+        totalAllocation : Nat;
+        reclaimed : Nat;
+        maxLiveSize : Nat;
+        stableMemorySize : Nat;
+        logicalStableMemorySize : Nat;
+        maxStackSize : Nat;
+        callbackTableCount : Nat;
+        callbackTableSize : Nat;
+    }
+    ```
+
+
 ## 0.12.0 (2024-07-26)
 
 * motoko (`moc`)

src/ir_def/construct.ml

@@ -115,6 +115,17 @@ let primE prim es =
     | OtherPrim "blob_size"
     | OtherPrim "text_len" -> T.nat
     | OtherPrim "is_controller" -> T.bool
+    | OtherPrim "rts_version" -> T.text
+    | OtherPrim "rts_memory_size" -> T.nat
+    | OtherPrim "rts_heap_size" -> T.nat
+    | OtherPrim "rts_total_allocation" -> T.nat
+    | OtherPrim "rts_reclaimed" -> T.nat
+    | OtherPrim "rts_max_live_size" -> T.nat
+    | OtherPrim "rts_stable_memory_size" -> T.nat
+    | OtherPrim "rts_logical_stable_memory_size" -> T.nat
+    | OtherPrim "rts_max_stack_size" -> T.nat
+    | OtherPrim "rts_callback_table_count" -> T.nat
+    | OtherPrim "rts_callback_table_size" -> T.nat
     | _ -> assert false (* implement more as needed *)
   in
   let eff = List.(map eff es |> fold_left max_eff T.Triv) in

src/lowering/desugar.ml

@@ -464,6 +464,67 @@ and export_footprint self_id expr =
   )],
   [{ it = I.{ name = lab; var = v }; at = no_region; note = typ }])
 
+and export_runtime_information self_id =
+  let open T in
+  let {lab;typ;_} = motoko_runtime_information_fld in
+  let v = "$"^lab in
+  let scope_con1 = Cons.fresh "T1" (Abs ([], scope_bound)) in
+  let scope_con2 = Cons.fresh "T2" (Abs ([], Any)) in
+  let bind1  = typ_arg scope_con1 Scope scope_bound in
+  let bind2 = typ_arg scope_con2 Scope scope_bound in
+  let gc_strategy = 
+    let open Mo_config in
+    let strategy = match !Flags.gc_strategy with
+    | Flags.MarkCompact -> "compacting"
+    | Flags.Copying -> "copying"
+    | Flags.Generational -> "generational"
+    | Flags.Incremental -> "incremental" in
+    if !Flags.force_gc then (Printf.sprintf "%s force" strategy) else strategy
+  in
+  let prim_call function_name = primE (I.OtherPrim function_name) [] in
+  let information = [
+    ("compilerVersion", textE (Lib.Option.get Source_id.release Source_id.id), T.text);
+    ("garbageCollector", textE gc_strategy, T.text);
+    ("rtsVersion", prim_call "rts_version", T.text);
+    ("sanityChecks", boolE !Mo_config.Flags.sanity, T.bool);
+    ("memorySize", prim_call "rts_memory_size", T.nat);
+    ("heapSize", prim_call "rts_heap_size", T.nat);
+    ("totalAllocation", prim_call "rts_total_allocation", T.nat);
+    ("reclaimed", prim_call "rts_reclaimed", T.nat);
+    ("maxLiveSize", prim_call "rts_max_live_size", T.nat);
+    ("stableMemorySize", prim_call "rts_stable_memory_size", T.nat);
+    ("logicalStableMemorySize", prim_call "rts_logical_stable_memory_size", T.nat);
+    ("maxStackSize", prim_call "rts_max_stack_size", T.nat);
+    ("callbackTableCount", prim_call "rts_callback_table_count", T.nat);
+    ("callbackTableSize", prim_call "rts_callback_table_size", T.nat)
+  ] in
+  let fields = List.map (fun (name, _, typ) -> fresh_var name typ) information in
+  (* Use an object return type to allow adding more data in future. *)
+  let ret_typ = motoko_runtime_information_type in
+  let caller = fresh_var "caller" caller in
+  ([ letD (var v typ) (
+       funcE v (Shared Query) Promises [bind1] [] [ret_typ] (
+           (asyncE T.Fut bind2
+              (blockE ([
+                  letD caller (primE I.ICCallerPrim []);
+                  expD (ifE (orE 
+                      (primE (I.RelPrim (principal, Operator.EqOp)) [varE caller; selfRefE principal])
+                      (primE (I.OtherPrim "is_controller") [varE caller]))
+                    (unitE()) 
+                    (primE (Ir.OtherPrim "trap")
+                      [textE "Unauthorized call of __motoko_runtime_information"]))
+                  ] @
+                  (List.map2 (fun field (_, load_info, _) -> 
+                    letD field load_info
+                  ) fields information))
+                (newObjE T.Object
+                  (List.map2 (fun field (name, _, typ) -> 
+                      { it = Ir.{name; var = id_of_var field}; at = no_region; note = typ }) 
+                    fields information
+                  ) ret_typ))
+              (Con (scope_con1, []))))
+  )],
+  [{ it = I.{ name = lab; var = v }; at = no_region; note = typ }])
 
 and build_actor at ts self_id es obj_typ =
   let candid = build_candid ts obj_typ in
@@ -520,7 +581,8 @@ and build_actor at ts self_id es obj_typ =
                ) fields vs)
             ty)) in
   let footprint_d, footprint_f = export_footprint self_id (with_stable_vars (fun e -> e)) in
-  I.(ActorE (footprint_d @ ds', footprint_f @ fs,
+  let runtime_info_d, runtime_info_f = export_runtime_information self_id in
+  I.(ActorE (footprint_d @ runtime_info_d @ ds', footprint_f @ runtime_info_f @ fs,
      { meta;
        preupgrade = with_stable_vars (fun e -> primE (I.ICStableWrite ty) [e]);
        postupgrade =

src/mo_types/type.ml

@@ -1358,6 +1358,32 @@ let motoko_gc_trigger_fld =
     src = empty_src;
   }
 
+let motoko_runtime_information_type =
+  Obj(Object, [
+    (* Fields must be sorted by label *)
+    {lab = "callbackTableCount"; typ = nat; src = empty_src};
+    {lab = "callbackTableSize"; typ = nat; src = empty_src};
+    {lab = "compilerVersion"; typ = text; src = empty_src};
+    {lab = "garbageCollector"; typ = text; src = empty_src};
+    {lab = "heapSize"; typ = nat; src = empty_src};
+    {lab = "logicalStableMemorySize"; typ = nat; src = empty_src};
+    {lab = "maxLiveSize"; typ = nat; src = empty_src};
+    {lab = "maxStackSize"; typ = nat; src = empty_src};
+    {lab = "memorySize"; typ = nat; src = empty_src};
+    {lab = "reclaimed"; typ = nat; src = empty_src};
+    {lab = "rtsVersion"; typ = text; src = empty_src};
+    {lab = "sanityChecks"; typ = bool; src = empty_src};
+    {lab = "stableMemorySize"; typ = nat; src = empty_src};
+    {lab = "totalAllocation"; typ = nat; src = empty_src};
+  ])
+
+let motoko_runtime_information_fld =
+  { lab = "__motoko_runtime_information";
+    typ = Func(Shared Query, Promises, [scope_bind], [], 
+      [ motoko_runtime_information_type ]);
+    src = empty_src;
+  }
+
 let well_known_actor_fields = [
     motoko_async_helper_fld;
     motoko_stable_var_info_fld;

src/mo_types/type.mli

@@ -263,12 +263,14 @@ val match_stab_sig : field list -> field list -> bool
 
 val string_of_stab_sig : field list -> string
 
+val motoko_runtime_information_type : typ
 
 (* Well-known fields *)
 
 val motoko_async_helper_fld : field
 val motoko_stable_var_info_fld : field
 val motoko_gc_trigger_fld : field
+val motoko_runtime_information_fld : field
 
 val well_known_actor_fields : field list
 val decode_msg_typ : field list -> typ

test/bench/ok/bignum.drun-run-opt.ok

@@ -2,5 +2,5 @@ ingress Completed: Reply: 0x4449444c016c01b3c4b1f204680100010a000000000000000001
 ingress Completed: Reply: 0x4449444c0000
 debug.print: {cycles = 2_512_723; size = +59_652}
 ingress Completed: Reply: 0x4449444c0000
-debug.print: {cycles = 107_695_051; size = +1_817_872}
+debug.print: {cycles = 107_695_281; size = +1_817_872}
 ingress Completed: Reply: 0x4449444c0000

test/bench/ok/heap-32.drun-run-opt.ok

@@ -1,5 +1,5 @@
 ingress Completed: Reply: 0x4449444c016c01b3c4b1f204680100010a00000000000000000101
 ingress Completed: Reply: 0x4449444c0000
 debug.print: (50_227, +29_863_068, 708_174_952)
-debug.print: (50_070, +32_992_212, 766_613_680)
+debug.print: (50_070, +32_992_212, 766_613_520)
 ingress Completed: Reply: 0x4449444c0000

test/bench/ok/heap-32.drun-run.ok

@@ -1,5 +1,5 @@
 ingress Completed: Reply: 0x4449444c016c01b3c4b1f204680100010a00000000000000000101
 ingress Completed: Reply: 0x4449444c0000
 debug.print: (50_227, +29_863_068, 769_000_085)
-debug.print: (50_070, +32_992_212, 830_427_376)
+debug.print: (50_070, +32_992_212, 830_427_128)
 ingress Completed: Reply: 0x4449444c0000

test/bench/ok/palindrome.drun-run-opt.ok

@@ -5,5 +5,5 @@ debug.print: (false, +1_188, 10_488)
 debug.print: (false, +1_188, 11_373)
 debug.print: (true, +868, 11_589)
 debug.print: (false, +868, 10_160)
-debug.print: (false, +868, 11_540)
+debug.print: (false, +868, 11_942)
 ingress Completed: Reply: 0x4449444c0000

test/bench/ok/palindrome.drun-run.ok

@@ -5,5 +5,5 @@ debug.print: (false, +1_188, 11_718)
 debug.print: (false, +1_188, 12_648)
 debug.print: (true, +868, 12_777)
 debug.print: (false, +868, 11_240)
-debug.print: (false, +868, 12_725)
+debug.print: (false, +868, 13_155)
 ingress Completed: Reply: 0x4449444c0000

test/run-drun/ok/runtime-info-acl.drun-run.ok

@@ -0,0 +1,6 @@
+ingress Completed: Reply: 0x4449444c016c01b3c4b1f204680100010a00000000000000000101
+ingress Completed: Reply: 0x4449444c0000
+debug.print: Self information okay
+debug.print: Controllee information okay
+debug.print: IC0503: Canister rwlgt-iiaaa-aaaaa-aaaaa-cai trapped explicitly: Unauthorized call of __motoko_runtime_information
+ingress Completed: Reply: 0x4449444c0000

test/run-drun/ok/runtime-info.drun-run.ok

@@ -0,0 +1,10 @@
+ingress Completed: Reply: 0x4449444c016c01b3c4b1f204680100010a00000000000000000101
+ingress Completed: Reply: 0x4449444c0000
+debug.print: Ignore Diff: (ignored)
+ingress Completed: Reply: 0x4449444c0000
+ingress Completed: Reply: 0x4449444c0000
+debug.print: Ignore Diff: (ignored)
+ingress Completed: Reply: 0x4449444c0000
+ingress Completed: Reply: 0x4449444c0000
+debug.print: Ignore Diff: (ignored)
+ingress Completed: Reply: 0x4449444c0000

test/run-drun/runtime-info-acl.mo

@@ -0,0 +1,44 @@
+import Prim "mo:⛔";
+import Lib "runtime-info-acl/C";
+import Info "runtime-info/info"
+// Test `__motoko_runtime_information` can only be invoked from self or controller.
+actor Self {
+  public func go() : async () {
+    // Test call from Self
+    do {
+      try {
+        ignore await Info.introspect(Self).__motoko_runtime_information();
+        Prim.debugPrint("Self information okay");
+      }
+      catch _e {
+        assert false;
+      }
+    };
+
+    let c = await (Lib.C(Info.introspect(Self)));
+
+    // Test call from controller (Self is a controller of c).
+    do {
+      try {
+        ignore await Info.introspect(c).__motoko_runtime_information();
+        Prim.debugPrint("Controllee information okay");
+      }
+      catch _e {
+        assert false;
+      }
+    };
+
+    // Test callback from non-controller (c is not a controller of Self).
+    do {
+      try {
+        await c.callback();
+        assert false;
+      }
+      catch e {
+        Prim.debugPrint(Prim.errorMessage(e));
+      }
+    }
+  }
+}
+
+//CALL ingress go "DIDL\x00\x00"

test/run-drun/runtime-info-acl/C.mo

@@ -0,0 +1,11 @@
+actor class C(
+   internal : actor {
+      __motoko_runtime_information : () -> async {};
+   }
+) {
+
+   // attempt to call runtime info for the actor passed as reference
+   public shared func callback() : async () {
+      ignore await internal.__motoko_runtime_information(); // should fail unless controller or selfaxs
+   };
+};

test/run-drun/runtime-info.mo

@@ -0,0 +1,47 @@
+import Prim "mo:prim";
+import Info "runtime-info/info";
+actor Self {
+    var array : [var Nat] = [var];
+    let region = Prim.regionNew();
+    ignore Prim.regionGrow(region, 1);
+
+    public func increaseHeap() : async () {
+        array := Prim.Array_init<Nat>(array.size() + 1024 * 1024, 0);
+    };
+
+    func validGC(strategy : Text) : Bool {
+        for (name in ["copying", "compacting", "generational", "incremental"].vals()) {
+            if (strategy == name # " force") {
+                return true;
+            };
+        };
+        return false;
+    };
+
+    public func checkInformation() : async () {
+        let information = await Info.introspect(Self).__motoko_runtime_information();
+        Prim.debugPrint("Ignore Diff: " # debug_show (information));
+        // Runtime information differs between GCs and sanity check options.
+        assert (validGC(information.garbageCollector));
+        assert (information.heapSize > array.size() * 4); // or 8 with 64-bit
+        assert (information.heapSize < information.memorySize);
+        assert (information.maxStackSize > 1024);
+        assert (information.maxStackSize % 1024 == 0);
+        assert (information.totalAllocation > 0);
+        assert (information.stableMemorySize > 0);
+        assert (information.logicalStableMemorySize == information.stableMemorySize);
+        assert (information.callbackTableCount <= information.callbackTableSize);
+    };
+};
+
+// Not calling query __motoko_runtime_information "DIDL\x00\x00" as it differs
+// between the compiler options and changes with every compiler/RTS adjustment.
+
+//SKIP run
+//SKIP run-low
+//SKIP run-ir
+//CALL ingress checkInformation "DIDL\x00\x00"
+//CALL ingress increaseHeap "DIDL\x00\x00"
+//CALL ingress checkInformation "DIDL\x00\x00"
+//CALL ingress increaseHeap "DIDL\x00\x00"
+//CALL ingress checkInformation "DIDL\x00\x00"

test/run-drun/runtime-info/info.mo

@@ -0,0 +1,28 @@
+import Prim "mo:prim";
+
+module {
+    public type RuntimeInformation = {
+        compilerVersion : Text;
+        rtsVersion : Text;
+        garbageCollector : Text;
+        sanityChecks : Bool;
+        memorySize : Nat;
+        heapSize : Nat;
+        totalAllocation : Nat;
+        reclaimed : Nat;
+        maxLiveSize : Nat;
+        stableMemorySize : Nat;
+        logicalStableMemorySize : Nat;
+        maxStackSize : Nat;
+        callbackTableCount : Nat;
+        callbackTableSize : Nat;
+    };
+
+    public type ActorIntrospection = actor {
+        __motoko_runtime_information : () -> async RuntimeInformation;
+    };
+
+    public func introspect(a : actor {}) : ActorIntrospection {
+        (actor (debug_show (Prim.principalOfActor(a))) : ActorIntrospection);
+    };
+};