diff --git a/.github/workflows/forest.yml b/.github/workflows/forest.yml index 4897513636b..1569022fddb 100644 --- a/.github/workflows/forest.yml +++ b/.github/workflows/forest.yml @@ -44,6 +44,9 @@ jobs: uses: mozilla-actions/sccache-action@v0.0.5 timeout-minutes: '${{ fromJSON(env.CACHE_TIMEOUT_MINUTES) }}' continue-on-error: true + - uses: actions/setup-go@v5 + with: + go-version-file: "go.work" - name: Cargo Check run: cargo check build-ubuntu: @@ -68,6 +71,9 @@ jobs: uses: mozilla-actions/sccache-action@v0.0.5 timeout-minutes: '${{ fromJSON(env.CACHE_TIMEOUT_MINUTES) }}' continue-on-error: true + - uses: actions/setup-go@v5 + with: + go-version-file: "go.work" - name: Cargo Install env: # To minimize compile times: https://nnethercote.github.io/perf-book/build-configuration.html#minimizing-compile-times @@ -91,6 +97,9 @@ jobs: - name: Install Apt Dependencies run: | sudo make install-deps + - uses: actions/setup-go@v5 + with: + go-version-file: "go.work" - run: cargo publish --dry-run forest-cli-check: needs: diff --git a/Cargo.lock b/Cargo.lock index c37de9d2c27..4b375a62d93 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3281,6 +3281,7 @@ dependencies = [ "rlimit", "rlp", "rs-car-ipfs", + "rust2go", "schemars", "scopeguard", "semver", diff --git a/Cargo.toml b/Cargo.toml index 6a4b7849077..0a4be4b4e25 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,7 +6,7 @@ repository = "https://github.com/ChainSafe/forest" edition = "2021" license = "MIT OR Apache-2.0" description = "Rust Filecoin implementation." -exclude = [".config", ".github", ".maintain", "documentation", "scripts", "interop-tests"] +exclude = [".config", ".github", ".maintain", "documentation", "scripts", "interop-tests", "go.work*"] [dependencies] ahash = "0.8" @@ -153,6 +153,7 @@ reqwest = { version = "0.12", default-features = false, features = [ rlimit = "0.10" rlp = "0.5" rs-car-ipfs = "0.3" +rust2go = { version = "0.3" } schemars = { version = "0.8", features = ["chrono", "uuid1"] } scopeguard = "1" semver = "1" @@ -228,6 +229,9 @@ regex-automata = "0.4" syn = { version = "2", default-features = false, features = ["full", "parsing", "visit", "printing", "extra-traits"] } tokio-test = "0.4" +[build-dependencies] +rust2go = { version = "0.3", features = ["build"] } + # This needs to be set as default. Otherwise, a regular build or test will produce # gargantuan artifacts (around 70G for all tests). For a debugging session, you can # temporarily comment it out. @@ -264,6 +268,8 @@ tokio-console = ["dep:console-subscriber"] tracing-loki = ["dep:tracing-loki"] tracing-chrome = ["dep:tracing-chrome"] +no-f3-sidecar = [] + [[bench]] name = "example-benchmark" harness = false diff --git a/Dockerfile b/Dockerfile index cd80909cd52..5b05e7aa2da 100644 --- a/Dockerfile +++ b/Dockerfile @@ -46,6 +46,9 @@ RUN xx-apt-get update && \ WORKDIR /forest COPY . . +# TODO(forest): https://github.com/ChainSafe/forest/issues/4758 +ENV FOREST_F3_SIDECAR_FFI_BUILD_OPT_OUT=1 + # Install Forest. Move it out of the cache for the prod image. RUN --mount=type=cache,sharing=private,target=/root/.cargo/registry \ --mount=type=cache,sharing=private,target=/root/.rustup \ diff --git a/Makefile b/Makefile index 4ce6c80943e..7e296567fd5 100644 --- a/Makefile +++ b/Makefile @@ -82,6 +82,8 @@ lint-clippy: cargo clippy --all-targets --no-default-features --features slim --quiet --no-deps -- --deny=warnings cargo clippy --all-targets --no-default-features --quiet --no-deps -- --deny=warnings cargo clippy --benches --features benchmark-private --quiet --no-deps -- --deny=warnings + # check docs.rs build + DOCS_RS=1 cargo clippy --all-targets --quiet --no-deps -- --deny=warnings DOCKERFILES=$(wildcard Dockerfile*) lint-docker: $(DOCKERFILES) diff --git a/README.md b/README.md index 4e526343fe2..ed45d267666 100644 --- a/README.md +++ b/README.md @@ -58,9 +58,13 @@ tag. ## Dependencies - Rust (toolchain version is specified in `rust-toolchain.toml`) +- Go for building F3 sidecar module. (toolchain version is specified in + `go.work`) Install [rustup](https://rustup.rs/) +Install [Go](https://go.dev/doc/install) + - OS Base-Devel/Build-Essential - Clang compiler diff --git a/build.rs b/build.rs new file mode 100644 index 00000000000..be75387af6b --- /dev/null +++ b/build.rs @@ -0,0 +1,36 @@ +// Copyright 2019-2024 ChainSafe Systems +// SPDX-License-Identifier: Apache-2.0, MIT + +fn main() { + // whitelist the cfg for cargo clippy + println!("cargo::rustc-check-cfg=cfg(f3sidecar)"); + + // Do not build f3-sidecar on docs.rs publishing + // No proper version of Go compiler is available. + if !is_docs_rs() && is_sidecar_ffi_enabled() { + println!("cargo:rustc-cfg=f3sidecar"); + std::env::set_var("GOWORK", "off"); + rust2go::Builder::default() + .with_go_src("./f3-sidecar") + .with_regen_arg(rust2go::RegenArgs { + src: "./src/f3/go_ffi.rs".into(), + dst: "./f3-sidecar/ffi_gen.go".into(), + without_main: true, + ..Default::default() + }) + .build(); + } +} + +// See +fn is_docs_rs() -> bool { + std::env::var("DOCS_RS").is_ok() +} + +fn is_sidecar_ffi_enabled() -> bool { + // Opt-out building the F3 sidecar staticlib + match std::env::var("FOREST_F3_SIDECAR_FFI_BUILD_OPT_OUT") { + Ok(value) => !matches!(value.to_lowercase().as_str(), "1" | "true"), + _ => true, + } +} diff --git a/f3-sidecar/README.md b/f3-sidecar/README.md index 02e11e5c5ef..351ff880258 100644 --- a/f3-sidecar/README.md +++ b/f3-sidecar/README.md @@ -1,3 +1,11 @@ +### Dependencies + +In addition to the Rust toolchain, Go toolchain is required to build the +`f3-sidecar`. The Go version is specified in `go.mod`. + +Follow https://go.dev/doc/install or use one of the version managers of Go. +(e.g. https://github.com/voidint/g?tab=readme-ov-file#installation) + ### EC tests - run a forest node locally and expose RPC port at the default 2345 @@ -47,3 +55,14 @@ flowchart TD A --> |storage backend| C[level db] A --> |dynamic manifest backend| D[manifest p2p server] ``` + +### To build and run F3 sidecar within Forest via FFI + +By default, the Go F3-sidecar is built and linked into Forest binary unless +environment variable `FOREST_F3_SIDECAR_FFI_BUILD_OPT_OUT=1` is set. + +F3 sidecar is not started by default, set `FOREST_F3_SIDECAR_FFI_ENABLED=1` to +opt in. + +Set dynamic manifest server via `FOREST_F3_MANIFEST_SERVER`, e.g. +`FOREST_F3_MANIFEST_SERVER=12D3KooWENMwUF9YxvQxar7uBWJtZkA6amvK4xWmKXfSiHUo2Qq7` diff --git a/f3-sidecar/ffi_gen.go b/f3-sidecar/ffi_gen.go new file mode 100644 index 00000000000..211a86c3dc9 --- /dev/null +++ b/f3-sidecar/ffi_gen.go @@ -0,0 +1,193 @@ +package main + +/* +// Generated by rust2go. Please DO NOT edit this C part manually. + +#include +#include +#include +#include + +typedef struct ListRef { + const void *ptr; + uintptr_t len; +} ListRef; + +typedef struct StringRef { + const uint8_t *ptr; + uintptr_t len; +} StringRef; + +// hack from: https://stackoverflow.com/a/69904977 +__attribute__((weak)) +inline void GoF3Node_run_cb(const void *f_ptr, bool resp, const void *slot) { +((void (*)(bool, const void*))f_ptr)(resp, slot); +} +*/ +import "C" +import ( + "runtime" + "unsafe" +) + +var GoF3NodeImpl GoF3Node + +type GoF3Node interface { + run(rpc_endpoint string, f3_rpc_endpoint string, finality int64, db string, manifest_server string) bool +} + +//export CGoF3Node_run +func CGoF3Node_run(rpc_endpoint C.StringRef, f3_rpc_endpoint C.StringRef, finality C.int64_t, db C.StringRef, manifest_server C.StringRef, slot *C.void, cb *C.void) { + resp := GoF3NodeImpl.run(newString(rpc_endpoint), newString(f3_rpc_endpoint), newC_int64_t(finality), newString(db), newString(manifest_server)) + resp_ref, buffer := cvt_ref(cntC_bool, refC_bool)(&resp) + C.GoF3Node_run_cb(unsafe.Pointer(cb), resp_ref, unsafe.Pointer(slot)) + runtime.KeepAlive(resp) + runtime.KeepAlive(buffer) +} + +func newString(s_ref C.StringRef) string { + return unsafe.String((*byte)(unsafe.Pointer(s_ref.ptr)), s_ref.len) +} +func refString(s *string, _ *[]byte) C.StringRef { + return C.StringRef{ + ptr: (*C.uint8_t)(unsafe.StringData(*s)), + len: C.uintptr_t(len(*s)), + } +} + +func cntString(_ *string, _ *uint) [0]C.StringRef { return [0]C.StringRef{} } +func new_list_mapper[T1, T2 any](f func(T1) T2) func(C.ListRef) []T2 { + return func(x C.ListRef) []T2 { + input := unsafe.Slice((*T1)(unsafe.Pointer(x.ptr)), x.len) + output := make([]T2, len(input)) + for i, v := range input { + output[i] = f(v) + } + return output + } +} +func new_list_mapper_primitive[T1, T2 any](_ func(T1) T2) func(C.ListRef) []T2 { + return func(x C.ListRef) []T2 { + return unsafe.Slice((*T2)(unsafe.Pointer(x.ptr)), x.len) + } +} + +// only handle non-primitive type T +func cnt_list_mapper[T, R any](f func(s *T, cnt *uint) [0]R) func(s *[]T, cnt *uint) [0]C.ListRef { + return func(s *[]T, cnt *uint) [0]C.ListRef { + for _, v := range *s { + f(&v, cnt) + } + *cnt += uint(len(*s)) * size_of[R]() + return [0]C.ListRef{} + } +} + +// only handle primitive type T +func cnt_list_mapper_primitive[T, R any](_ func(s *T, cnt *uint) [0]R) func(s *[]T, cnt *uint) [0]C.ListRef { + return func(s *[]T, cnt *uint) [0]C.ListRef { return [0]C.ListRef{} } +} + +// only handle non-primitive type T +func ref_list_mapper[T, R any](f func(s *T, buffer *[]byte) R) func(s *[]T, buffer *[]byte) C.ListRef { + return func(s *[]T, buffer *[]byte) C.ListRef { + if len(*buffer) == 0 { + return C.ListRef{ + ptr: unsafe.Pointer(nil), + len: C.uintptr_t(len(*s)), + } + } + ret := C.ListRef{ + ptr: unsafe.Pointer(&(*buffer)[0]), + len: C.uintptr_t(len(*s)), + } + children_bytes := int(size_of[R]()) * len(*s) + children := (*buffer)[:children_bytes] + *buffer = (*buffer)[children_bytes:] + for _, v := range *s { + child := f(&v, buffer) + len := unsafe.Sizeof(child) + copy(children, unsafe.Slice((*byte)(unsafe.Pointer(&child)), len)) + children = children[len:] + } + return ret + } +} + +// only handle primitive type T +func ref_list_mapper_primitive[T, R any](_ func(s *T, buffer *[]byte) R) func(s *[]T, buffer *[]byte) C.ListRef { + return func(s *[]T, buffer *[]byte) C.ListRef { + if len(*s) == 0 { + return C.ListRef{ + ptr: unsafe.Pointer(nil), + len: C.uintptr_t(0), + } + } + return C.ListRef{ + ptr: unsafe.Pointer(&(*s)[0]), + len: C.uintptr_t(len(*s)), + } + } +} +func size_of[T any]() uint { + var t T + return uint(unsafe.Sizeof(t)) +} +func cvt_ref[R, CR any](cnt_f func(s *R, cnt *uint) [0]CR, ref_f func(p *R, buffer *[]byte) CR) func(p *R) (CR, []byte) { + return func(p *R) (CR, []byte) { + var cnt uint + cnt_f(p, &cnt) + buffer := make([]byte, cnt) + return ref_f(p, &buffer), buffer + } +} +func cvt_ref_cap[R, CR any](cnt_f func(s *R, cnt *uint) [0]CR, ref_f func(p *R, buffer *[]byte) CR, add_cap uint) func(p *R) (CR, []byte) { + return func(p *R) (CR, []byte) { + var cnt uint + cnt_f(p, &cnt) + buffer := make([]byte, cnt, cnt+add_cap) + return ref_f(p, &buffer), buffer + } +} + +func newC_uint8_t(n C.uint8_t) uint8 { return uint8(n) } +func newC_uint16_t(n C.uint16_t) uint16 { return uint16(n) } +func newC_uint32_t(n C.uint32_t) uint32 { return uint32(n) } +func newC_uint64_t(n C.uint64_t) uint64 { return uint64(n) } +func newC_int8_t(n C.int8_t) int8 { return int8(n) } +func newC_int16_t(n C.int16_t) int16 { return int16(n) } +func newC_int32_t(n C.int32_t) int32 { return int32(n) } +func newC_int64_t(n C.int64_t) int64 { return int64(n) } +func newC_bool(n C.bool) bool { return bool(n) } +func newC_uintptr_t(n C.uintptr_t) uint { return uint(n) } +func newC_intptr_t(n C.intptr_t) int { return int(n) } +func newC_float(n C.float) float32 { return float32(n) } +func newC_double(n C.double) float64 { return float64(n) } + +func cntC_uint8_t(_ *uint8, _ *uint) [0]C.uint8_t { return [0]C.uint8_t{} } +func cntC_uint16_t(_ *uint16, _ *uint) [0]C.uint16_t { return [0]C.uint16_t{} } +func cntC_uint32_t(_ *uint32, _ *uint) [0]C.uint32_t { return [0]C.uint32_t{} } +func cntC_uint64_t(_ *uint64, _ *uint) [0]C.uint64_t { return [0]C.uint64_t{} } +func cntC_int8_t(_ *int8, _ *uint) [0]C.int8_t { return [0]C.int8_t{} } +func cntC_int16_t(_ *int16, _ *uint) [0]C.int16_t { return [0]C.int16_t{} } +func cntC_int32_t(_ *int32, _ *uint) [0]C.int32_t { return [0]C.int32_t{} } +func cntC_int64_t(_ *int64, _ *uint) [0]C.int64_t { return [0]C.int64_t{} } +func cntC_bool(_ *bool, _ *uint) [0]C.bool { return [0]C.bool{} } +func cntC_uintptr_t(_ *uint, _ *uint) [0]C.uintptr_t { return [0]C.uintptr_t{} } +func cntC_intptr_t(_ *int, _ *uint) [0]C.intptr_t { return [0]C.intptr_t{} } +func cntC_float(_ *float32, _ *uint) [0]C.float { return [0]C.float{} } +func cntC_double(_ *float64, _ *uint) [0]C.double { return [0]C.double{} } + +func refC_uint8_t(p *uint8, _ *[]byte) C.uint8_t { return C.uint8_t(*p) } +func refC_uint16_t(p *uint16, _ *[]byte) C.uint16_t { return C.uint16_t(*p) } +func refC_uint32_t(p *uint32, _ *[]byte) C.uint32_t { return C.uint32_t(*p) } +func refC_uint64_t(p *uint64, _ *[]byte) C.uint64_t { return C.uint64_t(*p) } +func refC_int8_t(p *int8, _ *[]byte) C.int8_t { return C.int8_t(*p) } +func refC_int16_t(p *int16, _ *[]byte) C.int16_t { return C.int16_t(*p) } +func refC_int32_t(p *int32, _ *[]byte) C.int32_t { return C.int32_t(*p) } +func refC_int64_t(p *int64, _ *[]byte) C.int64_t { return C.int64_t(*p) } +func refC_bool(p *bool, _ *[]byte) C.bool { return C.bool(*p) } +func refC_uintptr_t(p *uint, _ *[]byte) C.uintptr_t { return C.uintptr_t(*p) } +func refC_intptr_t(p *int, _ *[]byte) C.intptr_t { return C.intptr_t(*p) } +func refC_float(p *float32, _ *[]byte) C.float { return C.float(*p) } +func refC_double(p *float64, _ *[]byte) C.double { return C.double(*p) } diff --git a/f3-sidecar/ffi_impl.go b/f3-sidecar/ffi_impl.go new file mode 100644 index 00000000000..6448a43fa6d --- /dev/null +++ b/f3-sidecar/ffi_impl.go @@ -0,0 +1,39 @@ +package main + +import ( + "context" + "os" + + logging "github.com/ipfs/go-log/v2" +) + +func init() { + setGoDebugEnv() + logging.SetAllLoggers(logging.LevelWarn) + err := logging.SetLogLevel("f3/sidecar", "info") + checkError(err) + err = logging.SetLogLevel("f3", "info") + checkError(err) + GoF3NodeImpl = &f3Impl{ctx: context.Background()} +} + +type f3Impl struct { + ctx context.Context +} + +func (f3 *f3Impl) run(rpc_endpoint string, f3_rpc_endpoint string, finality int64, db string, manifest_server string) bool { + err := run(f3.ctx, rpc_endpoint, f3_rpc_endpoint, finality, db, manifest_server) + return err == nil +} + +func checkError(err error) { + if err != nil { + panic(err) + } +} + +// To avoid potential panics +// See +func setGoDebugEnv() { + os.Setenv("GODEBUG", "invalidptr=0,cgocheck=0") +} diff --git a/interop-tests/Cargo.toml b/interop-tests/Cargo.toml index a9e8d400ddd..5aaa06da494 100644 --- a/interop-tests/Cargo.toml +++ b/interop-tests/Cargo.toml @@ -13,7 +13,7 @@ publish = false [dev-dependencies] anyhow = "1" flume = "0.11" -forest-filecoin = { path = "../", default-features = false, features = ["interop-tests-private"] } +forest-filecoin = { path = "../", default-features = false, features = ["interop-tests-private", "no-f3-sidecar"] } futures = "0.3" libipld = { version = "0.16", default-features = false } libp2p = { version = "0.54", default-features = false, features = [ diff --git a/src/daemon/mod.rs b/src/daemon/mod.rs index ead0db0fdff..283c391a842 100644 --- a/src/daemon/mod.rs +++ b/src/daemon/mod.rs @@ -411,6 +411,25 @@ pub(super) async fn start( ) .await }); + + services.spawn_blocking({ + let finality = chain_config.policy.chain_finality; + let default_f3_db_path = config + .client + .data_dir + .join(format!("f3-db/{}", config.chain)); + move || { + crate::f3::run_f3_sidecar_if_enabled( + format!("http://{rpc_address}/rpc/v1"), + crate::rpc::f3::get_f3_rpc_endpoint().to_string(), + finality, + std::env::var("FOREST_F3_DB_PATH") + .unwrap_or(default_f3_db_path.display().to_string()), + std::env::var("FOREST_F3_MANIFEST_SERVER").unwrap_or_default(), + ); + Ok(()) + } + }); } else { debug!("RPC disabled."); }; diff --git a/src/f3/go_ffi.rs b/src/f3/go_ffi.rs new file mode 100644 index 00000000000..fbecc861f84 --- /dev/null +++ b/src/f3/go_ffi.rs @@ -0,0 +1,19 @@ +// Copyright 2019-2024 ChainSafe Systems +// SPDX-License-Identifier: Apache-2.0, MIT + +pub mod binding { + #![allow(warnings)] + #![allow(clippy::indexing_slicing)] + rust2go::r2g_include_binding!(); +} + +#[rust2go::r2g] +pub trait GoF3Node { + fn run( + rpc_endpoint: String, + f3_rpc_endpoint: String, + finality: i64, + db: String, + manifest_server: String, + ) -> bool; +} diff --git a/src/f3/mod.rs b/src/f3/mod.rs new file mode 100644 index 00000000000..6e62703d3ee --- /dev/null +++ b/src/f3/mod.rs @@ -0,0 +1,47 @@ +// Copyright 2019-2024 ChainSafe Systems +// SPDX-License-Identifier: Apache-2.0, MIT + +#[cfg(all(f3sidecar, not(feature = "no-f3-sidecar")))] +mod go_ffi; +#[cfg(all(f3sidecar, not(feature = "no-f3-sidecar")))] +use go_ffi::*; + +use crate::utils::misc::env::is_env_truthy; + +pub fn run_f3_sidecar_if_enabled( + _rpc_endpoint: String, + _f3_rpc_endpoint: String, + _finality: i64, + _db: String, + _manifest_server: String, +) { + if is_sidecar_ffi_enabled() { + #[cfg(all(f3sidecar, not(feature = "no-f3-sidecar")))] + { + GoF3NodeImpl::run( + _rpc_endpoint, + _f3_rpc_endpoint, + _finality, + _db, + _manifest_server, + ); + } + } +} + +// Use opt-in mode for now. Consider switching to opt-out mode once F3 is shipped. +fn is_sidecar_ffi_enabled() -> bool { + // Opt-out building the F3 sidecar staticlib + let enabled = is_env_truthy("FOREST_F3_SIDECAR_FFI_ENABLED"); + cfg_if::cfg_if! { + if #[cfg(all(f3sidecar, not(feature = "no-f3-sidecar")))] { + enabled + } + else { + if enabled { + tracing::error!("Failed to enable F3 sidecar, the forerst binary is not compiled with f3-sidecar Go lib"); + } + false + } + } +} diff --git a/src/lib.rs b/src/lib.rs index be82e28f001..94093eba182 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -49,6 +49,7 @@ mod daemon; mod db; mod documentation; mod eth; +mod f3; mod fil_cns; mod genesis; mod health; diff --git a/src/rpc/methods/f3.rs b/src/rpc/methods/f3.rs index c83bda15c1a..8c8b0b8fba9 100644 --- a/src/rpc/methods/f3.rs +++ b/src/rpc/methods/f3.rs @@ -583,7 +583,7 @@ impl RpcMethod<3> for F3Participate { } } -fn get_f3_rpc_endpoint() -> Cow<'static, str> { +pub fn get_f3_rpc_endpoint() -> Cow<'static, str> { if let Ok(host) = std::env::var("FOREST_F3_SIDECAR_RPC_ENDPOINT") { Cow::Owned(host) } else {