feat: build and run F3 sidecar in Forest process via FFI

Cargo.toml

@@ -153,6 +153,7 @@ reqwest = { version = "0.12", default-features = false, features = [
 rlimit = "0.10"
 rlp = "0.5"
 rs-car-ipfs = "0.3"
+rust2go = { version = "0.3" }
 schemars = { version = "0.8", features = ["chrono", "uuid1"] }
 scopeguard = "1"
 semver = "1"
@@ -228,6 +229,9 @@ regex-automata = "0.4"
 syn = { version = "2", default-features = false, features = ["full", "parsing", "visit", "printing", "extra-traits"] }
 tokio-test = "0.4"
 
+[build-dependencies]
+rust2go = { version = "0.3", features = ["build"] }
+
 # This needs to be set as default. Otherwise, a regular build or test will produce
 # gargantuan artifacts (around 70G for all tests). For a debugging session, you can
 # temporarily comment it out.

Makefile

@@ -82,6 +82,8 @@ lint-clippy:
 	cargo clippy --all-targets --no-default-features --features slim --quiet --no-deps -- --deny=warnings
 	cargo clippy --all-targets --no-default-features --quiet --no-deps -- --deny=warnings
 	cargo clippy --benches --features benchmark-private --quiet --no-deps -- --deny=warnings
+	# check docs.rs build
+	DOCS_RS=1 cargo clippy --all-targets --quiet --no-deps -- --deny=warnings
 
 DOCKERFILES=$(wildcard Dockerfile*)
 lint-docker: $(DOCKERFILES)

build.rs

@@ -0,0 +1,35 @@
+// Copyright 2019-2024 ChainSafe Systems
+// SPDX-License-Identifier: Apache-2.0, MIT
+
+fn main() {
+    // whitelist the cfg for cargo clippy
+    println!("cargo::rustc-check-cfg=cfg(f3sidecar)");
+
+    // Do not build f3-sidecar on docs.rs publishing
+    // No proper version of Go compiler is available.
+    if !is_docs_rs() && is_sidecar_ffi_enabled() {
+        println!("cargo:rustc-cfg=f3sidecar");
+        rust2go::Builder::default()
+            .with_go_src("./f3-sidecar")
+            .with_regen_arg(rust2go::RegenArgs {
+                src: "./src/f3/go_ffi.rs".into(),
+                dst: "./f3-sidecar/ffi_gen.go".into(),
+                without_main: true,
+                ..Default::default()
+            })
+            .build();
+    }
+}
+
+// See <https://docs.rs/about/builds#detecting-docsrs>
+fn is_docs_rs() -> bool {
+    std::env::var("DOCS_RS").is_ok()
+}
+
+fn is_sidecar_ffi_enabled() -> bool {
+    // Opt-out building the F3 sidecar staticlib
+    match std::env::var("FOREST_F3_SIDECAR_FFI_BUILD_OPT_OUT") {
+        Ok(value) => !matches!(value.to_lowercase().as_str(), "1" | "true"),
+        _ => true,
+    }
+}

f3-sidecar/.gitignore

@@ -44,3 +44,6 @@ Cargo.lock
 
 # The default f3 database folder
 /f3-db
+
+# generated by rust2go
+ffi_gen.go

f3-sidecar/README.md

@@ -47,3 +47,14 @@ flowchart TD
     A --> |storage backend| C[level db]
     A --> |dynamic manifest backend| D[manifest p2p server]
 ```
+
+### To build and run F3 sidecar within Forest via FFI
+
+By default, the Go F3-sidecar is built and linked into Forest binary unless
+environment variable `FOREST_F3_SIDECAR_FFI_BUILD_OPT_OUT=1` is set.
+
+F3 sidecar is not started by default, set `FOREST_F3_SIDECAR_FFI_ENABLED=1` to
+opt in.
+
+Set dynamic manifest server via `FOREST_F3_MANIFEST_SERVER`, e.g.
+`FOREST_F3_MANIFEST_SERVER=12D3KooWENMwUF9YxvQxar7uBWJtZkA6amvK4xWmKXfSiHUo2Qq7`

f3-sidecar/ffi_impl.go

@@ -0,0 +1,39 @@
+package main
+
+import (
+	"context"
+	"os"
+
+	logging "github.com/ipfs/go-log/v2"
+)
+
+func init() {
+	setGoDebugEnv()
+	logging.SetAllLoggers(logging.LevelWarn)
+	err := logging.SetLogLevel("f3/sidecar", "info")
+	checkError(err)
+	err = logging.SetLogLevel("f3", "info")
+	checkError(err)
+	GoF3NodeImpl = &f3Impl{ctx: context.Background()}
+}
+
+type f3Impl struct {
+	ctx context.Context
+}
+
+func (f3 *f3Impl) run(rpc_endpoint string, f3_rpc_endpoint string, finality int64, db string, manifest_server string) bool {
+	err := run(f3.ctx, rpc_endpoint, f3_rpc_endpoint, finality, db, manifest_server)
+	return err == nil
+}
+
+func checkError(err error) {
+	if err != nil {
+		panic(err)
+	}
+}
+
+// To avoid potential panics
+// See <https://github.com/ChainSafe/forest/pull/4636#issuecomment-2306500753>
+func setGoDebugEnv() {
+	os.Setenv("GODEBUG", "invalidptr=0,cgocheck=0")
+}

src/daemon/mod.rs

@@ -411,6 +411,20 @@ pub(super) async fn start(
             )
             .await
         });
+
+        let finality = chain_config.policy.chain_finality;
+        let chain = config.chain.to_string();
+        services.spawn_blocking(move || {
+            crate::f3::run_f3_sidecar_if_enabled(
+                format!("http://{rpc_address}/rpc/v1"),
+                crate::rpc::f3::get_f3_rpc_endpoint().to_string(),
+                finality,
+                std::env::var("FOREST_F3_DB_PATH")
+                    .unwrap_or_else(|_| format!("/var/tmp/f3-db-{chain}")),
+                std::env::var("FOREST_F3_MANIFEST_SERVER").unwrap_or_default(),
+            );
+            Ok(())
+        });
     } else {
         debug!("RPC disabled.");
     };

src/f3/go_ffi.rs

@@ -0,0 +1,21 @@
+// Copyright 2019-2024 ChainSafe Systems
+// SPDX-License-Identifier: Apache-2.0, MIT
+
+pub mod binding {
+    #![allow(warnings)]
+    rust2go::r2g_include_binding!();
+}
+
+#[derive(rust2go::R2G, Clone, Default)]
+pub struct EmptyReq {}
+
+#[rust2go::r2g]
+pub trait GoF3Node {
+    fn run(
+        rpc_endpoint: String,
+        f3_rpc_endpoint: String,
+        finality: i64,
+        db: String,
+        manifest_server: String,
+    ) -> bool;
+}

src/f3/mod.rs

@@ -0,0 +1,50 @@
+// Copyright 2019-2024 ChainSafe Systems
+// SPDX-License-Identifier: Apache-2.0, MIT
+
+#[cfg(f3sidecar)]
+mod go_ffi;
+#[cfg(f3sidecar)]
+use go_ffi::*;
+
+pub fn run_f3_sidecar_if_enabled(
+    _rpc_endpoint: String,
+    _f3_rpc_endpoint: String,
+    _finality: i64,
+    _db: String,
+    _manifest_server: String,
+) {
+    if is_sidecar_ffi_enabled() {
+        #[cfg(f3sidecar)]
+        {
+            GoF3NodeImpl::run(
+                _rpc_endpoint,
+                _f3_rpc_endpoint,
+                _finality,
+                _db,
+                _manifest_server,
+            );
+        }
+    }
+}
+
+// Use opt-in mode for now. Consider switching to opt-out mode once F3 is shipped.
+fn is_sidecar_ffi_enabled() -> bool {
+    // Opt-out building the F3 sidecar staticlib
+    match std::env::var("FOREST_F3_SIDECAR_FFI_ENABLED") {
+        Ok(value) => {
+            let enabled = matches!(value.to_lowercase().as_str(), "1" | "true");
+            cfg_if::cfg_if! {
+                if #[cfg(f3sidecar)] {
+                    enabled
+                }
+                else {
+                    if enabled {
+                        tracing::error!("Failed to enable F3 sidecar, the forerst binary is not compiled with f3-sidecar Go lib");
+                    }
+                    false
+                }
+            }
+        }
+        _ => false,
+    }
+}

src/lib.rs

@@ -49,6 +49,7 @@ mod daemon;
 mod db;
 mod documentation;
 mod eth;
+mod f3;
 mod fil_cns;
 mod genesis;
 mod health;

src/rpc/methods/f3.rs

@@ -583,7 +583,7 @@ impl RpcMethod<3> for F3Participate {
     }
 }
 
-fn get_f3_rpc_endpoint() -> Cow<'static, str> {
+pub fn get_f3_rpc_endpoint() -> Cow<'static, str> {
     if let Ok(host) = std::env::var("FOREST_F3_SIDECAR_RPC_ENDPOINT") {
         Cow::Owned(host)
     } else {