feat: build and run F3 sidecar in Forest process via FFI

.github/workflows/forest.yml

@@ -47,6 +47,9 @@ jobs:
         uses: mozilla-actions/[email protected]
         timeout-minutes: '${{ fromJSON(env.CACHE_TIMEOUT_MINUTES) }}'
         continue-on-error: true
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: "go.work"
       - name: Cargo Check
         run: cargo check
   build-ubuntu:
@@ -71,6 +74,9 @@ jobs:
         uses: mozilla-actions/[email protected]
         timeout-minutes: '${{ fromJSON(env.CACHE_TIMEOUT_MINUTES) }}'
         continue-on-error: true
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: "go.work"
       - name: Cargo Install
         env:
           # To minimize compile times: https://nnethercote.github.io/perf-book/build-configuration.html#minimizing-compile-times
@@ -95,6 +101,8 @@ jobs:
         run: |
           sudo make install-deps
       - run: cargo publish --dry-run
+        env:
+          FOREST_F3_SIDECAR_FFI_BUILD_OPT_OUT: 1
   forest-cli-check:
     needs:
       - build-ubuntu

Cargo.toml

@@ -153,6 +153,7 @@ reqwest = { version = "0.12", default-features = false, features = [
 rlimit = "0.10"
 rlp = "0.5"
 rs-car-ipfs = "0.3"
+rust2go = { version = "0.3" }
 schemars = { version = "0.8", features = ["chrono", "uuid1"] }
 scopeguard = "1"
 semver = "1"
@@ -228,6 +229,9 @@ regex-automata = "0.4"
 syn = { version = "2", default-features = false, features = ["full", "parsing", "visit", "printing", "extra-traits"] }
 tokio-test = "0.4"
 
+[build-dependencies]
+rust2go = { version = "0.3", features = ["build"] }
+
 # This needs to be set as default. Otherwise, a regular build or test will produce
 # gargantuan artifacts (around 70G for all tests). For a debugging session, you can
 # temporarily comment it out.

Makefile

@@ -82,6 +82,8 @@ lint-clippy:
 	cargo clippy --all-targets --no-default-features --features slim --quiet --no-deps -- --deny=warnings
 	cargo clippy --all-targets --no-default-features --quiet --no-deps -- --deny=warnings
 	cargo clippy --benches --features benchmark-private --quiet --no-deps -- --deny=warnings
+	# check docs.rs build
+	DOCS_RS=1 cargo clippy --all-targets --quiet --no-deps -- --deny=warnings
 
 DOCKERFILES=$(wildcard Dockerfile*)
 lint-docker: $(DOCKERFILES)

build.rs

@@ -0,0 +1,35 @@
+// Copyright 2019-2024 ChainSafe Systems
+// SPDX-License-Identifier: Apache-2.0, MIT
+
+fn main() {
+    // whitelist the cfg for cargo clippy
+    println!("cargo::rustc-check-cfg=cfg(f3sidecar)");
+
+    // Do not build f3-sidecar on docs.rs publishing
+    // No proper version of Go compiler is available.
+    if !is_docs_rs() && is_sidecar_ffi_enabled() {
+        println!("cargo:rustc-cfg=f3sidecar");
+        rust2go::Builder::default()
+            .with_go_src("./f3-sidecar")
+            .with_regen_arg(rust2go::RegenArgs {
+                src: "./src/f3/go_ffi.rs".into(),
+                dst: "./f3-sidecar/ffi_gen.go".into(),
+                without_main: true,
+                ..Default::default()
+            })
+            .build();
+    }
+}
+
+// See <https://docs.rs/about/builds#detecting-docsrs>
+fn is_docs_rs() -> bool {
+    std::env::var("DOCS_RS").is_ok()
+}
+
+fn is_sidecar_ffi_enabled() -> bool {
+    // Opt-out building the F3 sidecar staticlib
+    match std::env::var("FOREST_F3_SIDECAR_FFI_BUILD_OPT_OUT") {
+        Ok(value) => !matches!(value.to_lowercase().as_str(), "1" | "true"),
+        _ => true,
+    }
+}

f3-sidecar/README.md

@@ -47,3 +47,14 @@ flowchart TD
     A --> |storage backend| C[level db]
     A --> |dynamic manifest backend| D[manifest p2p server]
 ```
+
+### To build and run F3 sidecar within Forest via FFI
+
+By default, the Go F3-sidecar is built and linked into Forest binary unless
+environment variable `FOREST_F3_SIDECAR_FFI_BUILD_OPT_OUT=1` is set.
+
+F3 sidecar is not started by default, set `FOREST_F3_SIDECAR_FFI_ENABLED=1` to
+opt in.
+
+Set dynamic manifest server via `FOREST_F3_MANIFEST_SERVER`, e.g.
+`FOREST_F3_MANIFEST_SERVER=12D3KooWENMwUF9YxvQxar7uBWJtZkA6amvK4xWmKXfSiHUo2Qq7`

f3-sidecar/ffi_gen.go

@@ -0,0 +1,210 @@
+package main
+
+/*
+// Generated by rust2go. Please DO NOT edit this C part manually.
+
+#include <stdarg.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdlib.h>
+
+typedef struct EmptyReqRef {
+
+} EmptyReqRef;
+
+typedef struct ListRef {
+  const void *ptr;
+  uintptr_t len;
+} ListRef;
+
+typedef struct StringRef {
+  const uint8_t *ptr;
+  uintptr_t len;
+} StringRef;
+
+// hack from: https://stackoverflow.com/a/69904977
+__attribute__((weak))
+inline void GoF3Node_run_cb(const void *f_ptr, bool resp, const void *slot) {
+((void (*)(bool, const void*))f_ptr)(resp, slot);
+}
+*/
+import "C"
+import (
+	"runtime"
+	"unsafe"
+)
+
+var GoF3NodeImpl GoF3Node
+
+type GoF3Node interface {
+	run(rpc_endpoint string, f3_rpc_endpoint string, finality int64, db string, manifest_server string) bool
+}
+
+//export CGoF3Node_run
+func CGoF3Node_run(rpc_endpoint C.StringRef, f3_rpc_endpoint C.StringRef, finality C.int64_t, db C.StringRef, manifest_server C.StringRef, slot *C.void, cb *C.void) {
+	resp := GoF3NodeImpl.run(newString(rpc_endpoint), newString(f3_rpc_endpoint), newC_int64_t(finality), newString(db), newString(manifest_server))
+	resp_ref, buffer := cvt_ref(cntC_bool, refC_bool)(&resp)
+	C.GoF3Node_run_cb(unsafe.Pointer(cb), resp_ref, unsafe.Pointer(slot))
+	runtime.KeepAlive(resp)
+	runtime.KeepAlive(buffer)
+}
+
+func newString(s_ref C.StringRef) string {
+	return unsafe.String((*byte)(unsafe.Pointer(s_ref.ptr)), s_ref.len)
+}
+func refString(s *string, _ *[]byte) C.StringRef {
+	return C.StringRef{
+		ptr: (*C.uint8_t)(unsafe.StringData(*s)),
+		len: C.uintptr_t(len(*s)),
+	}
+}
+
+func cntString(_ *string, _ *uint) [0]C.StringRef { return [0]C.StringRef{} }
+func new_list_mapper[T1, T2 any](f func(T1) T2) func(C.ListRef) []T2 {
+	return func(x C.ListRef) []T2 {
+		input := unsafe.Slice((*T1)(unsafe.Pointer(x.ptr)), x.len)
+		output := make([]T2, len(input))
+		for i, v := range input {
+			output[i] = f(v)
+		}
+		return output
+	}
+}
+func new_list_mapper_primitive[T1, T2 any](_ func(T1) T2) func(C.ListRef) []T2 {
+	return func(x C.ListRef) []T2 {
+		return unsafe.Slice((*T2)(unsafe.Pointer(x.ptr)), x.len)
+	}
+}
+
+// only handle non-primitive type T
+func cnt_list_mapper[T, R any](f func(s *T, cnt *uint) [0]R) func(s *[]T, cnt *uint) [0]C.ListRef {
+	return func(s *[]T, cnt *uint) [0]C.ListRef {
+		for _, v := range *s {
+			f(&v, cnt)
+		}
+		*cnt += uint(len(*s)) * size_of[R]()
+		return [0]C.ListRef{}
+	}
+}
+
+// only handle primitive type T
+func cnt_list_mapper_primitive[T, R any](_ func(s *T, cnt *uint) [0]R) func(s *[]T, cnt *uint) [0]C.ListRef {
+	return func(s *[]T, cnt *uint) [0]C.ListRef { return [0]C.ListRef{} }
+}
+
+// only handle non-primitive type T
+func ref_list_mapper[T, R any](f func(s *T, buffer *[]byte) R) func(s *[]T, buffer *[]byte) C.ListRef {
+	return func(s *[]T, buffer *[]byte) C.ListRef {
+		if len(*buffer) == 0 {
+			return C.ListRef{
+				ptr: unsafe.Pointer(nil),
+				len: C.uintptr_t(len(*s)),
+			}
+		}
+		ret := C.ListRef{
+			ptr: unsafe.Pointer(&(*buffer)[0]),
+			len: C.uintptr_t(len(*s)),
+		}
+		children_bytes := int(size_of[R]()) * len(*s)
+		children := (*buffer)[:children_bytes]
+		*buffer = (*buffer)[children_bytes:]
+		for _, v := range *s {
+			child := f(&v, buffer)
+			len := unsafe.Sizeof(child)
+			copy(children, unsafe.Slice((*byte)(unsafe.Pointer(&child)), len))
+			children = children[len:]
+		}
+		return ret
+	}
+}
+
+// only handle primitive type T
+func ref_list_mapper_primitive[T, R any](_ func(s *T, buffer *[]byte) R) func(s *[]T, buffer *[]byte) C.ListRef {
+	return func(s *[]T, buffer *[]byte) C.ListRef {
+		if len(*s) == 0 {
+			return C.ListRef{
+				ptr: unsafe.Pointer(nil),
+				len: C.uintptr_t(0),
+			}
+		}
+		return C.ListRef{
+			ptr: unsafe.Pointer(&(*s)[0]),
+			len: C.uintptr_t(len(*s)),
+		}
+	}
+}
+func size_of[T any]() uint {
+	var t T
+	return uint(unsafe.Sizeof(t))
+}
+func cvt_ref[R, CR any](cnt_f func(s *R, cnt *uint) [0]CR, ref_f func(p *R, buffer *[]byte) CR) func(p *R) (CR, []byte) {
+	return func(p *R) (CR, []byte) {
+		var cnt uint
+		cnt_f(p, &cnt)
+		buffer := make([]byte, cnt)
+		return ref_f(p, &buffer), buffer
+	}
+}
+func cvt_ref_cap[R, CR any](cnt_f func(s *R, cnt *uint) [0]CR, ref_f func(p *R, buffer *[]byte) CR, add_cap uint) func(p *R) (CR, []byte) {
+	return func(p *R) (CR, []byte) {
+		var cnt uint
+		cnt_f(p, &cnt)
+		buffer := make([]byte, cnt, cnt+add_cap)
+		return ref_f(p, &buffer), buffer
+	}
+}
+
+func newC_uint8_t(n C.uint8_t) uint8    { return uint8(n) }
+func newC_uint16_t(n C.uint16_t) uint16 { return uint16(n) }
+func newC_uint32_t(n C.uint32_t) uint32 { return uint32(n) }
+func newC_uint64_t(n C.uint64_t) uint64 { return uint64(n) }
+func newC_int8_t(n C.int8_t) int8       { return int8(n) }
+func newC_int16_t(n C.int16_t) int16    { return int16(n) }
+func newC_int32_t(n C.int32_t) int32    { return int32(n) }
+func newC_int64_t(n C.int64_t) int64    { return int64(n) }
+func newC_bool(n C.bool) bool           { return bool(n) }
+func newC_uintptr_t(n C.uintptr_t) uint { return uint(n) }
+func newC_intptr_t(n C.intptr_t) int    { return int(n) }
+func newC_float(n C.float) float32      { return float32(n) }
+func newC_double(n C.double) float64    { return float64(n) }
+
+func cntC_uint8_t(_ *uint8, _ *uint) [0]C.uint8_t    { return [0]C.uint8_t{} }
+func cntC_uint16_t(_ *uint16, _ *uint) [0]C.uint16_t { return [0]C.uint16_t{} }
+func cntC_uint32_t(_ *uint32, _ *uint) [0]C.uint32_t { return [0]C.uint32_t{} }
+func cntC_uint64_t(_ *uint64, _ *uint) [0]C.uint64_t { return [0]C.uint64_t{} }
+func cntC_int8_t(_ *int8, _ *uint) [0]C.int8_t       { return [0]C.int8_t{} }
+func cntC_int16_t(_ *int16, _ *uint) [0]C.int16_t    { return [0]C.int16_t{} }
+func cntC_int32_t(_ *int32, _ *uint) [0]C.int32_t    { return [0]C.int32_t{} }
+func cntC_int64_t(_ *int64, _ *uint) [0]C.int64_t    { return [0]C.int64_t{} }
+func cntC_bool(_ *bool, _ *uint) [0]C.bool           { return [0]C.bool{} }
+func cntC_uintptr_t(_ *uint, _ *uint) [0]C.uintptr_t { return [0]C.uintptr_t{} }
+func cntC_intptr_t(_ *int, _ *uint) [0]C.intptr_t    { return [0]C.intptr_t{} }
+func cntC_float(_ *float32, _ *uint) [0]C.float      { return [0]C.float{} }
+func cntC_double(_ *float64, _ *uint) [0]C.double    { return [0]C.double{} }
+
+func refC_uint8_t(p *uint8, _ *[]byte) C.uint8_t    { return C.uint8_t(*p) }
+func refC_uint16_t(p *uint16, _ *[]byte) C.uint16_t { return C.uint16_t(*p) }
+func refC_uint32_t(p *uint32, _ *[]byte) C.uint32_t { return C.uint32_t(*p) }
+func refC_uint64_t(p *uint64, _ *[]byte) C.uint64_t { return C.uint64_t(*p) }
+func refC_int8_t(p *int8, _ *[]byte) C.int8_t       { return C.int8_t(*p) }
+func refC_int16_t(p *int16, _ *[]byte) C.int16_t    { return C.int16_t(*p) }
+func refC_int32_t(p *int32, _ *[]byte) C.int32_t    { return C.int32_t(*p) }
+func refC_int64_t(p *int64, _ *[]byte) C.int64_t    { return C.int64_t(*p) }
+func refC_bool(p *bool, _ *[]byte) C.bool           { return C.bool(*p) }
+func refC_uintptr_t(p *uint, _ *[]byte) C.uintptr_t { return C.uintptr_t(*p) }
+func refC_intptr_t(p *int, _ *[]byte) C.intptr_t    { return C.intptr_t(*p) }
+func refC_float(p *float32, _ *[]byte) C.float      { return C.float(*p) }
+func refC_double(p *float64, _ *[]byte) C.double    { return C.double(*p) }
+
+type EmptyReq struct {
+}
+
+func newEmptyReq(p C.EmptyReqRef) EmptyReq {
+	return EmptyReq{}
+}
+func cntEmptyReq(s *EmptyReq, cnt *uint) [0]C.EmptyReqRef {
+	return [0]C.EmptyReqRef{}
+}
+func refEmptyReq(p *EmptyReq, buffer *[]byte) C.EmptyReqRef {
+	return C.EmptyReqRef{}
+}

f3-sidecar/ffi_impl.go

@@ -0,0 +1,39 @@
+package main
+
+import (
+	"context"
+	"os"
+
+	logging "github.com/ipfs/go-log/v2"
+)
+
+func init() {
+	setGoDebugEnv()
+	logging.SetAllLoggers(logging.LevelWarn)
+	err := logging.SetLogLevel("f3/sidecar", "info")
+	checkError(err)
+	err = logging.SetLogLevel("f3", "info")
+	checkError(err)
+	GoF3NodeImpl = &f3Impl{ctx: context.Background()}
+}
+
+type f3Impl struct {
+	ctx context.Context
+}
+
+func (f3 *f3Impl) run(rpc_endpoint string, f3_rpc_endpoint string, finality int64, db string, manifest_server string) bool {
+	err := run(f3.ctx, rpc_endpoint, f3_rpc_endpoint, finality, db, manifest_server)
+	return err == nil
+}
+
+func checkError(err error) {
+	if err != nil {
+		panic(err)
+	}
+}
+
+// To avoid potential panics
+// See <https://github.com/ChainSafe/forest/pull/4636#issuecomment-2306500753>
+func setGoDebugEnv() {
+	os.Setenv("GODEBUG", "invalidptr=0,cgocheck=0")
+}

src/daemon/mod.rs

@@ -411,6 +411,20 @@ pub(super) async fn start(
             )
             .await
         });
+
+        let finality = chain_config.policy.chain_finality;
+        let chain = config.chain.to_string();
+        services.spawn_blocking(move || {
+            crate::f3::run_f3_sidecar_if_enabled(
+                format!("http://{rpc_address}/rpc/v1"),
+                crate::rpc::f3::get_f3_rpc_endpoint().to_string(),
+                finality,
+                std::env::var("FOREST_F3_DB_PATH")
+                    .unwrap_or_else(|_| format!("/var/tmp/f3-db-{chain}")),
+                std::env::var("FOREST_F3_MANIFEST_SERVER").unwrap_or_default(),
+            );
+            Ok(())
+        });
     } else {
         debug!("RPC disabled.");
     };