-
Notifications
You must be signed in to change notification settings - Fork 152
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
feat: build and run F3 sidecar in Forest process via FFI #4727
Changes from all commits
7b4642b
356ceb3
2640a6f
b7c004b
8d798ca
8ab1f9c
e5e73bc
a6498c5
9fc2ba8
1018b7f
2113f95
936c448
3080061
293d5ff
a99c8c1
eff7286
bb05630
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -44,6 +44,9 @@ jobs: | |
uses: mozilla-actions/[email protected] | ||
timeout-minutes: '${{ fromJSON(env.CACHE_TIMEOUT_MINUTES) }}' | ||
continue-on-error: true | ||
- uses: actions/setup-go@v5 | ||
with: | ||
go-version-file: "go.work" | ||
- name: Cargo Check | ||
run: cargo check | ||
build-ubuntu: | ||
|
@@ -68,6 +71,9 @@ jobs: | |
uses: mozilla-actions/[email protected] | ||
timeout-minutes: '${{ fromJSON(env.CACHE_TIMEOUT_MINUTES) }}' | ||
continue-on-error: true | ||
- uses: actions/setup-go@v5 | ||
with: | ||
go-version-file: "go.work" | ||
- name: Cargo Install | ||
env: | ||
# To minimize compile times: https://nnethercote.github.io/perf-book/build-configuration.html#minimizing-compile-times | ||
|
@@ -91,6 +97,9 @@ jobs: | |
- name: Install Apt Dependencies | ||
run: | | ||
sudo make install-deps | ||
- uses: actions/setup-go@v5 | ||
with: | ||
go-version-file: "go.work" | ||
- run: cargo publish --dry-run | ||
forest-cli-check: | ||
needs: | ||
|
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,36 @@ | ||
// Copyright 2019-2024 ChainSafe Systems | ||
// SPDX-License-Identifier: Apache-2.0, MIT | ||
|
||
fn main() { | ||
// whitelist the cfg for cargo clippy | ||
println!("cargo::rustc-check-cfg=cfg(f3sidecar)"); | ||
|
||
// Do not build f3-sidecar on docs.rs publishing | ||
// No proper version of Go compiler is available. | ||
if !is_docs_rs() && is_sidecar_ffi_enabled() { | ||
println!("cargo:rustc-cfg=f3sidecar"); | ||
std::env::set_var("GOWORK", "off"); | ||
rust2go::Builder::default() | ||
.with_go_src("./f3-sidecar") | ||
.with_regen_arg(rust2go::RegenArgs { | ||
src: "./src/f3/go_ffi.rs".into(), | ||
dst: "./f3-sidecar/ffi_gen.go".into(), | ||
without_main: true, | ||
..Default::default() | ||
}) | ||
.build(); | ||
} | ||
} | ||
|
||
// See <https://docs.rs/about/builds#detecting-docsrs> | ||
fn is_docs_rs() -> bool { | ||
std::env::var("DOCS_RS").is_ok() | ||
} | ||
|
||
fn is_sidecar_ffi_enabled() -> bool { | ||
// Opt-out building the F3 sidecar staticlib | ||
match std::env::var("FOREST_F3_SIDECAR_FFI_BUILD_OPT_OUT") { | ||
Ok(value) => !matches!(value.to_lowercase().as_str(), "1" | "true"), | ||
_ => true, | ||
} | ||
Comment on lines
+32
to
+35
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Can we use the function mentioned elsewhere in the PR for that in the build script? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The util is not available here in the build script context |
||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,193 @@ | ||
package main | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Include this file so that the Go mod builds out of box without codegen from Rust build |
||
|
||
/* | ||
// Generated by rust2go. Please DO NOT edit this C part manually. | ||
|
||
#include <stdarg.h> | ||
#include <stdbool.h> | ||
#include <stdint.h> | ||
#include <stdlib.h> | ||
|
||
typedef struct ListRef { | ||
const void *ptr; | ||
uintptr_t len; | ||
} ListRef; | ||
|
||
typedef struct StringRef { | ||
const uint8_t *ptr; | ||
uintptr_t len; | ||
} StringRef; | ||
|
||
// hack from: https://stackoverflow.com/a/69904977 | ||
__attribute__((weak)) | ||
inline void GoF3Node_run_cb(const void *f_ptr, bool resp, const void *slot) { | ||
((void (*)(bool, const void*))f_ptr)(resp, slot); | ||
} | ||
*/ | ||
import "C" | ||
import ( | ||
"runtime" | ||
"unsafe" | ||
) | ||
|
||
var GoF3NodeImpl GoF3Node | ||
|
||
type GoF3Node interface { | ||
run(rpc_endpoint string, f3_rpc_endpoint string, finality int64, db string, manifest_server string) bool | ||
} | ||
|
||
//export CGoF3Node_run | ||
func CGoF3Node_run(rpc_endpoint C.StringRef, f3_rpc_endpoint C.StringRef, finality C.int64_t, db C.StringRef, manifest_server C.StringRef, slot *C.void, cb *C.void) { | ||
resp := GoF3NodeImpl.run(newString(rpc_endpoint), newString(f3_rpc_endpoint), newC_int64_t(finality), newString(db), newString(manifest_server)) | ||
resp_ref, buffer := cvt_ref(cntC_bool, refC_bool)(&resp) | ||
C.GoF3Node_run_cb(unsafe.Pointer(cb), resp_ref, unsafe.Pointer(slot)) | ||
runtime.KeepAlive(resp) | ||
runtime.KeepAlive(buffer) | ||
} | ||
|
||
func newString(s_ref C.StringRef) string { | ||
return unsafe.String((*byte)(unsafe.Pointer(s_ref.ptr)), s_ref.len) | ||
} | ||
func refString(s *string, _ *[]byte) C.StringRef { | ||
return C.StringRef{ | ||
ptr: (*C.uint8_t)(unsafe.StringData(*s)), | ||
len: C.uintptr_t(len(*s)), | ||
} | ||
} | ||
|
||
func cntString(_ *string, _ *uint) [0]C.StringRef { return [0]C.StringRef{} } | ||
func new_list_mapper[T1, T2 any](f func(T1) T2) func(C.ListRef) []T2 { | ||
return func(x C.ListRef) []T2 { | ||
input := unsafe.Slice((*T1)(unsafe.Pointer(x.ptr)), x.len) | ||
output := make([]T2, len(input)) | ||
for i, v := range input { | ||
output[i] = f(v) | ||
} | ||
return output | ||
} | ||
} | ||
func new_list_mapper_primitive[T1, T2 any](_ func(T1) T2) func(C.ListRef) []T2 { | ||
return func(x C.ListRef) []T2 { | ||
return unsafe.Slice((*T2)(unsafe.Pointer(x.ptr)), x.len) | ||
} | ||
} | ||
|
||
// only handle non-primitive type T | ||
func cnt_list_mapper[T, R any](f func(s *T, cnt *uint) [0]R) func(s *[]T, cnt *uint) [0]C.ListRef { | ||
return func(s *[]T, cnt *uint) [0]C.ListRef { | ||
for _, v := range *s { | ||
f(&v, cnt) | ||
} | ||
*cnt += uint(len(*s)) * size_of[R]() | ||
return [0]C.ListRef{} | ||
} | ||
} | ||
|
||
// only handle primitive type T | ||
func cnt_list_mapper_primitive[T, R any](_ func(s *T, cnt *uint) [0]R) func(s *[]T, cnt *uint) [0]C.ListRef { | ||
return func(s *[]T, cnt *uint) [0]C.ListRef { return [0]C.ListRef{} } | ||
} | ||
|
||
// only handle non-primitive type T | ||
func ref_list_mapper[T, R any](f func(s *T, buffer *[]byte) R) func(s *[]T, buffer *[]byte) C.ListRef { | ||
return func(s *[]T, buffer *[]byte) C.ListRef { | ||
if len(*buffer) == 0 { | ||
return C.ListRef{ | ||
ptr: unsafe.Pointer(nil), | ||
len: C.uintptr_t(len(*s)), | ||
} | ||
} | ||
ret := C.ListRef{ | ||
ptr: unsafe.Pointer(&(*buffer)[0]), | ||
len: C.uintptr_t(len(*s)), | ||
} | ||
children_bytes := int(size_of[R]()) * len(*s) | ||
children := (*buffer)[:children_bytes] | ||
*buffer = (*buffer)[children_bytes:] | ||
for _, v := range *s { | ||
child := f(&v, buffer) | ||
len := unsafe.Sizeof(child) | ||
copy(children, unsafe.Slice((*byte)(unsafe.Pointer(&child)), len)) | ||
children = children[len:] | ||
} | ||
return ret | ||
} | ||
} | ||
|
||
// only handle primitive type T | ||
func ref_list_mapper_primitive[T, R any](_ func(s *T, buffer *[]byte) R) func(s *[]T, buffer *[]byte) C.ListRef { | ||
return func(s *[]T, buffer *[]byte) C.ListRef { | ||
if len(*s) == 0 { | ||
return C.ListRef{ | ||
ptr: unsafe.Pointer(nil), | ||
len: C.uintptr_t(0), | ||
} | ||
} | ||
return C.ListRef{ | ||
ptr: unsafe.Pointer(&(*s)[0]), | ||
len: C.uintptr_t(len(*s)), | ||
} | ||
} | ||
} | ||
func size_of[T any]() uint { | ||
var t T | ||
return uint(unsafe.Sizeof(t)) | ||
} | ||
func cvt_ref[R, CR any](cnt_f func(s *R, cnt *uint) [0]CR, ref_f func(p *R, buffer *[]byte) CR) func(p *R) (CR, []byte) { | ||
return func(p *R) (CR, []byte) { | ||
var cnt uint | ||
cnt_f(p, &cnt) | ||
buffer := make([]byte, cnt) | ||
return ref_f(p, &buffer), buffer | ||
} | ||
} | ||
func cvt_ref_cap[R, CR any](cnt_f func(s *R, cnt *uint) [0]CR, ref_f func(p *R, buffer *[]byte) CR, add_cap uint) func(p *R) (CR, []byte) { | ||
return func(p *R) (CR, []byte) { | ||
var cnt uint | ||
cnt_f(p, &cnt) | ||
buffer := make([]byte, cnt, cnt+add_cap) | ||
return ref_f(p, &buffer), buffer | ||
} | ||
} | ||
|
||
func newC_uint8_t(n C.uint8_t) uint8 { return uint8(n) } | ||
func newC_uint16_t(n C.uint16_t) uint16 { return uint16(n) } | ||
func newC_uint32_t(n C.uint32_t) uint32 { return uint32(n) } | ||
func newC_uint64_t(n C.uint64_t) uint64 { return uint64(n) } | ||
func newC_int8_t(n C.int8_t) int8 { return int8(n) } | ||
func newC_int16_t(n C.int16_t) int16 { return int16(n) } | ||
func newC_int32_t(n C.int32_t) int32 { return int32(n) } | ||
func newC_int64_t(n C.int64_t) int64 { return int64(n) } | ||
func newC_bool(n C.bool) bool { return bool(n) } | ||
func newC_uintptr_t(n C.uintptr_t) uint { return uint(n) } | ||
func newC_intptr_t(n C.intptr_t) int { return int(n) } | ||
func newC_float(n C.float) float32 { return float32(n) } | ||
func newC_double(n C.double) float64 { return float64(n) } | ||
|
||
func cntC_uint8_t(_ *uint8, _ *uint) [0]C.uint8_t { return [0]C.uint8_t{} } | ||
func cntC_uint16_t(_ *uint16, _ *uint) [0]C.uint16_t { return [0]C.uint16_t{} } | ||
func cntC_uint32_t(_ *uint32, _ *uint) [0]C.uint32_t { return [0]C.uint32_t{} } | ||
func cntC_uint64_t(_ *uint64, _ *uint) [0]C.uint64_t { return [0]C.uint64_t{} } | ||
func cntC_int8_t(_ *int8, _ *uint) [0]C.int8_t { return [0]C.int8_t{} } | ||
func cntC_int16_t(_ *int16, _ *uint) [0]C.int16_t { return [0]C.int16_t{} } | ||
func cntC_int32_t(_ *int32, _ *uint) [0]C.int32_t { return [0]C.int32_t{} } | ||
func cntC_int64_t(_ *int64, _ *uint) [0]C.int64_t { return [0]C.int64_t{} } | ||
func cntC_bool(_ *bool, _ *uint) [0]C.bool { return [0]C.bool{} } | ||
func cntC_uintptr_t(_ *uint, _ *uint) [0]C.uintptr_t { return [0]C.uintptr_t{} } | ||
func cntC_intptr_t(_ *int, _ *uint) [0]C.intptr_t { return [0]C.intptr_t{} } | ||
func cntC_float(_ *float32, _ *uint) [0]C.float { return [0]C.float{} } | ||
func cntC_double(_ *float64, _ *uint) [0]C.double { return [0]C.double{} } | ||
|
||
func refC_uint8_t(p *uint8, _ *[]byte) C.uint8_t { return C.uint8_t(*p) } | ||
func refC_uint16_t(p *uint16, _ *[]byte) C.uint16_t { return C.uint16_t(*p) } | ||
func refC_uint32_t(p *uint32, _ *[]byte) C.uint32_t { return C.uint32_t(*p) } | ||
func refC_uint64_t(p *uint64, _ *[]byte) C.uint64_t { return C.uint64_t(*p) } | ||
func refC_int8_t(p *int8, _ *[]byte) C.int8_t { return C.int8_t(*p) } | ||
func refC_int16_t(p *int16, _ *[]byte) C.int16_t { return C.int16_t(*p) } | ||
func refC_int32_t(p *int32, _ *[]byte) C.int32_t { return C.int32_t(*p) } | ||
func refC_int64_t(p *int64, _ *[]byte) C.int64_t { return C.int64_t(*p) } | ||
func refC_bool(p *bool, _ *[]byte) C.bool { return C.bool(*p) } | ||
func refC_uintptr_t(p *uint, _ *[]byte) C.uintptr_t { return C.uintptr_t(*p) } | ||
func refC_intptr_t(p *int, _ *[]byte) C.intptr_t { return C.intptr_t(*p) } | ||
func refC_float(p *float32, _ *[]byte) C.float { return C.float(*p) } | ||
func refC_double(p *float64, _ *[]byte) C.double { return C.double(*p) } |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Should we mention that
Go
is required in https://github.com/ChainSafe/forest?tab=readme-ov-file#dependencies?There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Good catch. Updated.