Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: build and run F3 sidecar in Forest process via FFI #4727

Merged
merged 17 commits into from
Sep 11, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions .github/workflows/forest.yml
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good catch. Updated.

Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,9 @@ jobs:
uses: mozilla-actions/[email protected]
timeout-minutes: '${{ fromJSON(env.CACHE_TIMEOUT_MINUTES) }}'
continue-on-error: true
- uses: actions/setup-go@v5
with:
go-version-file: "go.work"
- name: Cargo Check
run: cargo check
build-ubuntu:
Expand All @@ -68,6 +71,9 @@ jobs:
uses: mozilla-actions/[email protected]
timeout-minutes: '${{ fromJSON(env.CACHE_TIMEOUT_MINUTES) }}'
continue-on-error: true
- uses: actions/setup-go@v5
with:
go-version-file: "go.work"
- name: Cargo Install
env:
# To minimize compile times: https://nnethercote.github.io/perf-book/build-configuration.html#minimizing-compile-times
Expand All @@ -91,6 +97,9 @@ jobs:
- name: Install Apt Dependencies
run: |
sudo make install-deps
- uses: actions/setup-go@v5
with:
go-version-file: "go.work"
- run: cargo publish --dry-run
forest-cli-check:
needs:
Expand Down
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

8 changes: 7 additions & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ repository = "https://github.com/ChainSafe/forest"
edition = "2021"
license = "MIT OR Apache-2.0"
description = "Rust Filecoin implementation."
exclude = [".config", ".github", ".maintain", "documentation", "scripts", "interop-tests"]
exclude = [".config", ".github", ".maintain", "documentation", "scripts", "interop-tests", "go.work*"]

[dependencies]
ahash = "0.8"
Expand Down Expand Up @@ -153,6 +153,7 @@ reqwest = { version = "0.12", default-features = false, features = [
rlimit = "0.10"
rlp = "0.5"
rs-car-ipfs = "0.3"
rust2go = { version = "0.3" }
schemars = { version = "0.8", features = ["chrono", "uuid1"] }
scopeguard = "1"
semver = "1"
Expand Down Expand Up @@ -228,6 +229,9 @@ regex-automata = "0.4"
syn = { version = "2", default-features = false, features = ["full", "parsing", "visit", "printing", "extra-traits"] }
tokio-test = "0.4"

[build-dependencies]
rust2go = { version = "0.3", features = ["build"] }

# This needs to be set as default. Otherwise, a regular build or test will produce
# gargantuan artifacts (around 70G for all tests). For a debugging session, you can
# temporarily comment it out.
Expand Down Expand Up @@ -264,6 +268,8 @@ tokio-console = ["dep:console-subscriber"]
tracing-loki = ["dep:tracing-loki"]
tracing-chrome = ["dep:tracing-chrome"]

no-f3-sidecar = []

[[bench]]
name = "example-benchmark"
harness = false
Expand Down
3 changes: 3 additions & 0 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,9 @@ RUN xx-apt-get update && \
WORKDIR /forest
COPY . .

# TODO(forest): https://github.com/ChainSafe/forest/issues/4758
ENV FOREST_F3_SIDECAR_FFI_BUILD_OPT_OUT=1

# Install Forest. Move it out of the cache for the prod image.
RUN --mount=type=cache,sharing=private,target=/root/.cargo/registry \
--mount=type=cache,sharing=private,target=/root/.rustup \
Expand Down
2 changes: 2 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,8 @@ lint-clippy:
cargo clippy --all-targets --no-default-features --features slim --quiet --no-deps -- --deny=warnings
cargo clippy --all-targets --no-default-features --quiet --no-deps -- --deny=warnings
cargo clippy --benches --features benchmark-private --quiet --no-deps -- --deny=warnings
# check docs.rs build
DOCS_RS=1 cargo clippy --all-targets --quiet --no-deps -- --deny=warnings

DOCKERFILES=$(wildcard Dockerfile*)
lint-docker: $(DOCKERFILES)
Expand Down
4 changes: 4 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -58,9 +58,13 @@ tag.
## Dependencies

- Rust (toolchain version is specified in `rust-toolchain.toml`)
- Go for building F3 sidecar module. (toolchain version is specified in
`go.work`)

Install [rustup](https://rustup.rs/)

Install [Go](https://go.dev/doc/install)

- OS Base-Devel/Build-Essential
- Clang compiler

Expand Down
36 changes: 36 additions & 0 deletions build.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
// Copyright 2019-2024 ChainSafe Systems
// SPDX-License-Identifier: Apache-2.0, MIT

fn main() {
// whitelist the cfg for cargo clippy
println!("cargo::rustc-check-cfg=cfg(f3sidecar)");

// Do not build f3-sidecar on docs.rs publishing
// No proper version of Go compiler is available.
if !is_docs_rs() && is_sidecar_ffi_enabled() {
println!("cargo:rustc-cfg=f3sidecar");
std::env::set_var("GOWORK", "off");
rust2go::Builder::default()
.with_go_src("./f3-sidecar")
.with_regen_arg(rust2go::RegenArgs {
src: "./src/f3/go_ffi.rs".into(),
dst: "./f3-sidecar/ffi_gen.go".into(),
without_main: true,
..Default::default()
})
.build();
}
}

// See <https://docs.rs/about/builds#detecting-docsrs>
fn is_docs_rs() -> bool {
std::env::var("DOCS_RS").is_ok()
}

fn is_sidecar_ffi_enabled() -> bool {
// Opt-out building the F3 sidecar staticlib
match std::env::var("FOREST_F3_SIDECAR_FFI_BUILD_OPT_OUT") {
Ok(value) => !matches!(value.to_lowercase().as_str(), "1" | "true"),
_ => true,
}
Comment on lines +32 to +35
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we use the function mentioned elsewhere in the PR for that in the build script?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The util is not available here in the build script context

}
19 changes: 19 additions & 0 deletions f3-sidecar/README.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,11 @@
### Dependencies

In addition to the Rust toolchain, Go toolchain is required to build the
`f3-sidecar`. The Go version is specified in `go.mod`.

Follow https://go.dev/doc/install or use one of the version managers of Go.
(e.g. https://github.com/voidint/g?tab=readme-ov-file#installation)

### EC tests

- run a forest node locally and expose RPC port at the default 2345
Expand Down Expand Up @@ -47,3 +55,14 @@ flowchart TD
A --> |storage backend| C[level db]
A --> |dynamic manifest backend| D[manifest p2p server]
```

### To build and run F3 sidecar within Forest via FFI

By default, the Go F3-sidecar is built and linked into Forest binary unless
environment variable `FOREST_F3_SIDECAR_FFI_BUILD_OPT_OUT=1` is set.

F3 sidecar is not started by default, set `FOREST_F3_SIDECAR_FFI_ENABLED=1` to
opt in.

Set dynamic manifest server via `FOREST_F3_MANIFEST_SERVER`, e.g.
`FOREST_F3_MANIFEST_SERVER=12D3KooWENMwUF9YxvQxar7uBWJtZkA6amvK4xWmKXfSiHUo2Qq7`
193 changes: 193 additions & 0 deletions f3-sidecar/ffi_gen.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,193 @@
package main
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Include this file so that the Go mod builds out of box without codegen from Rust build


/*
// Generated by rust2go. Please DO NOT edit this C part manually.

#include <stdarg.h>
#include <stdbool.h>
#include <stdint.h>
#include <stdlib.h>

typedef struct ListRef {
const void *ptr;
uintptr_t len;
} ListRef;

typedef struct StringRef {
const uint8_t *ptr;
uintptr_t len;
} StringRef;

// hack from: https://stackoverflow.com/a/69904977
__attribute__((weak))
inline void GoF3Node_run_cb(const void *f_ptr, bool resp, const void *slot) {
((void (*)(bool, const void*))f_ptr)(resp, slot);
}
*/
import "C"
import (
"runtime"
"unsafe"
)

var GoF3NodeImpl GoF3Node

type GoF3Node interface {
run(rpc_endpoint string, f3_rpc_endpoint string, finality int64, db string, manifest_server string) bool
}

//export CGoF3Node_run
func CGoF3Node_run(rpc_endpoint C.StringRef, f3_rpc_endpoint C.StringRef, finality C.int64_t, db C.StringRef, manifest_server C.StringRef, slot *C.void, cb *C.void) {
resp := GoF3NodeImpl.run(newString(rpc_endpoint), newString(f3_rpc_endpoint), newC_int64_t(finality), newString(db), newString(manifest_server))
resp_ref, buffer := cvt_ref(cntC_bool, refC_bool)(&resp)
C.GoF3Node_run_cb(unsafe.Pointer(cb), resp_ref, unsafe.Pointer(slot))
runtime.KeepAlive(resp)
runtime.KeepAlive(buffer)
}

func newString(s_ref C.StringRef) string {
return unsafe.String((*byte)(unsafe.Pointer(s_ref.ptr)), s_ref.len)
}
func refString(s *string, _ *[]byte) C.StringRef {
return C.StringRef{
ptr: (*C.uint8_t)(unsafe.StringData(*s)),
len: C.uintptr_t(len(*s)),
}
}

func cntString(_ *string, _ *uint) [0]C.StringRef { return [0]C.StringRef{} }
func new_list_mapper[T1, T2 any](f func(T1) T2) func(C.ListRef) []T2 {
return func(x C.ListRef) []T2 {
input := unsafe.Slice((*T1)(unsafe.Pointer(x.ptr)), x.len)
output := make([]T2, len(input))
for i, v := range input {
output[i] = f(v)
}
return output
}
}
func new_list_mapper_primitive[T1, T2 any](_ func(T1) T2) func(C.ListRef) []T2 {
return func(x C.ListRef) []T2 {
return unsafe.Slice((*T2)(unsafe.Pointer(x.ptr)), x.len)
}
}

// only handle non-primitive type T
func cnt_list_mapper[T, R any](f func(s *T, cnt *uint) [0]R) func(s *[]T, cnt *uint) [0]C.ListRef {
return func(s *[]T, cnt *uint) [0]C.ListRef {
for _, v := range *s {
f(&v, cnt)
}
*cnt += uint(len(*s)) * size_of[R]()
return [0]C.ListRef{}
}
}

// only handle primitive type T
func cnt_list_mapper_primitive[T, R any](_ func(s *T, cnt *uint) [0]R) func(s *[]T, cnt *uint) [0]C.ListRef {
return func(s *[]T, cnt *uint) [0]C.ListRef { return [0]C.ListRef{} }
}

// only handle non-primitive type T
func ref_list_mapper[T, R any](f func(s *T, buffer *[]byte) R) func(s *[]T, buffer *[]byte) C.ListRef {
return func(s *[]T, buffer *[]byte) C.ListRef {
if len(*buffer) == 0 {
return C.ListRef{
ptr: unsafe.Pointer(nil),
len: C.uintptr_t(len(*s)),
}
}
ret := C.ListRef{
ptr: unsafe.Pointer(&(*buffer)[0]),
len: C.uintptr_t(len(*s)),
}
children_bytes := int(size_of[R]()) * len(*s)
children := (*buffer)[:children_bytes]
*buffer = (*buffer)[children_bytes:]
for _, v := range *s {
child := f(&v, buffer)
len := unsafe.Sizeof(child)
copy(children, unsafe.Slice((*byte)(unsafe.Pointer(&child)), len))
children = children[len:]
}
return ret
}
}

// only handle primitive type T
func ref_list_mapper_primitive[T, R any](_ func(s *T, buffer *[]byte) R) func(s *[]T, buffer *[]byte) C.ListRef {
return func(s *[]T, buffer *[]byte) C.ListRef {
if len(*s) == 0 {
return C.ListRef{
ptr: unsafe.Pointer(nil),
len: C.uintptr_t(0),
}
}
return C.ListRef{
ptr: unsafe.Pointer(&(*s)[0]),
len: C.uintptr_t(len(*s)),
}
}
}
func size_of[T any]() uint {
var t T
return uint(unsafe.Sizeof(t))
}
func cvt_ref[R, CR any](cnt_f func(s *R, cnt *uint) [0]CR, ref_f func(p *R, buffer *[]byte) CR) func(p *R) (CR, []byte) {
return func(p *R) (CR, []byte) {
var cnt uint
cnt_f(p, &cnt)
buffer := make([]byte, cnt)
return ref_f(p, &buffer), buffer
}
}
func cvt_ref_cap[R, CR any](cnt_f func(s *R, cnt *uint) [0]CR, ref_f func(p *R, buffer *[]byte) CR, add_cap uint) func(p *R) (CR, []byte) {
return func(p *R) (CR, []byte) {
var cnt uint
cnt_f(p, &cnt)
buffer := make([]byte, cnt, cnt+add_cap)
return ref_f(p, &buffer), buffer
}
}

func newC_uint8_t(n C.uint8_t) uint8 { return uint8(n) }
func newC_uint16_t(n C.uint16_t) uint16 { return uint16(n) }
func newC_uint32_t(n C.uint32_t) uint32 { return uint32(n) }
func newC_uint64_t(n C.uint64_t) uint64 { return uint64(n) }
func newC_int8_t(n C.int8_t) int8 { return int8(n) }
func newC_int16_t(n C.int16_t) int16 { return int16(n) }
func newC_int32_t(n C.int32_t) int32 { return int32(n) }
func newC_int64_t(n C.int64_t) int64 { return int64(n) }
func newC_bool(n C.bool) bool { return bool(n) }
func newC_uintptr_t(n C.uintptr_t) uint { return uint(n) }
func newC_intptr_t(n C.intptr_t) int { return int(n) }
func newC_float(n C.float) float32 { return float32(n) }
func newC_double(n C.double) float64 { return float64(n) }

func cntC_uint8_t(_ *uint8, _ *uint) [0]C.uint8_t { return [0]C.uint8_t{} }
func cntC_uint16_t(_ *uint16, _ *uint) [0]C.uint16_t { return [0]C.uint16_t{} }
func cntC_uint32_t(_ *uint32, _ *uint) [0]C.uint32_t { return [0]C.uint32_t{} }
func cntC_uint64_t(_ *uint64, _ *uint) [0]C.uint64_t { return [0]C.uint64_t{} }
func cntC_int8_t(_ *int8, _ *uint) [0]C.int8_t { return [0]C.int8_t{} }
func cntC_int16_t(_ *int16, _ *uint) [0]C.int16_t { return [0]C.int16_t{} }
func cntC_int32_t(_ *int32, _ *uint) [0]C.int32_t { return [0]C.int32_t{} }
func cntC_int64_t(_ *int64, _ *uint) [0]C.int64_t { return [0]C.int64_t{} }
func cntC_bool(_ *bool, _ *uint) [0]C.bool { return [0]C.bool{} }
func cntC_uintptr_t(_ *uint, _ *uint) [0]C.uintptr_t { return [0]C.uintptr_t{} }
func cntC_intptr_t(_ *int, _ *uint) [0]C.intptr_t { return [0]C.intptr_t{} }
func cntC_float(_ *float32, _ *uint) [0]C.float { return [0]C.float{} }
func cntC_double(_ *float64, _ *uint) [0]C.double { return [0]C.double{} }

func refC_uint8_t(p *uint8, _ *[]byte) C.uint8_t { return C.uint8_t(*p) }
func refC_uint16_t(p *uint16, _ *[]byte) C.uint16_t { return C.uint16_t(*p) }
func refC_uint32_t(p *uint32, _ *[]byte) C.uint32_t { return C.uint32_t(*p) }
func refC_uint64_t(p *uint64, _ *[]byte) C.uint64_t { return C.uint64_t(*p) }
func refC_int8_t(p *int8, _ *[]byte) C.int8_t { return C.int8_t(*p) }
func refC_int16_t(p *int16, _ *[]byte) C.int16_t { return C.int16_t(*p) }
func refC_int32_t(p *int32, _ *[]byte) C.int32_t { return C.int32_t(*p) }
func refC_int64_t(p *int64, _ *[]byte) C.int64_t { return C.int64_t(*p) }
func refC_bool(p *bool, _ *[]byte) C.bool { return C.bool(*p) }
func refC_uintptr_t(p *uint, _ *[]byte) C.uintptr_t { return C.uintptr_t(*p) }
func refC_intptr_t(p *int, _ *[]byte) C.intptr_t { return C.intptr_t(*p) }
func refC_float(p *float32, _ *[]byte) C.float { return C.float(*p) }
func refC_double(p *float64, _ *[]byte) C.double { return C.double(*p) }
Loading
Loading