From 6ebcf740c5705b925ac2edc162b7ca3af249c422 Mon Sep 17 00:00:00 2001 From: Emiliano Bonassi Date: Mon, 20 May 2024 22:26:41 -0700 Subject: [PATCH 1/4] feat(das): add support to configure http body limit --- cmd/daserver/daserver.go | 13 ++++++++----- cmd/genericconf/server.go | 2 ++ das/dasRpcServer.go | 9 ++++++--- das/rpc_test.go | 2 +- system_tests/common_test.go | 2 +- system_tests/das_test.go | 4 ++-- 6 files changed, 20 insertions(+), 12 deletions(-) diff --git a/cmd/daserver/daserver.go b/cmd/daserver/daserver.go index 8036487d26..48cc5546de 100644 --- a/cmd/daserver/daserver.go +++ b/cmd/daserver/daserver.go @@ -31,10 +31,11 @@ import ( ) type DAServerConfig struct { - EnableRPC bool `koanf:"enable-rpc"` - RPCAddr string `koanf:"rpc-addr"` - RPCPort uint64 `koanf:"rpc-port"` - RPCServerTimeouts genericconf.HTTPServerTimeoutConfig `koanf:"rpc-server-timeouts"` + EnableRPC bool `koanf:"enable-rpc"` + RPCAddr string `koanf:"rpc-addr"` + RPCPort uint64 `koanf:"rpc-port"` + RPCServerTimeouts genericconf.HTTPServerTimeoutConfig `koanf:"rpc-server-timeouts"` + RPCServerBodyLimit int `koanf:"rpc-server-body-limit"` EnableREST bool `koanf:"enable-rest"` RESTAddr string `koanf:"rest-addr"` @@ -58,6 +59,7 @@ var DefaultDAServerConfig = DAServerConfig{ RPCAddr: "localhost", RPCPort: 9876, RPCServerTimeouts: genericconf.HTTPServerTimeoutConfigDefault, + RPCServerBodyLimit: genericconf.HTTPServerBodyLimitDefault, EnableREST: false, RESTAddr: "localhost", RESTPort: 9877, @@ -88,6 +90,7 @@ func parseDAServer(args []string) (*DAServerConfig, error) { f.Bool("enable-rpc", DefaultDAServerConfig.EnableRPC, "enable the HTTP-RPC server listening on rpc-addr and rpc-port") f.String("rpc-addr", DefaultDAServerConfig.RPCAddr, "HTTP-RPC server listening interface") f.Uint64("rpc-port", DefaultDAServerConfig.RPCPort, "HTTP-RPC server listening port") + f.Int("rpc-server-body-limit", DefaultDAServerConfig.RPCServerBodyLimit, "HTTP-RPC server maximum request body size in bytes") genericconf.HTTPServerTimeoutConfigAddOptions("rpc-server-timeouts", f) f.Bool("enable-rest", DefaultDAServerConfig.EnableREST, "enable the REST server listening on rest-addr and rest-port") @@ -250,7 +253,7 @@ func startup() error { if serverConfig.EnableRPC { log.Info("Starting HTTP-RPC server", "addr", serverConfig.RPCAddr, "port", serverConfig.RPCPort, "revision", vcsRevision, "vcs.time", vcsTime) - rpcServer, err = das.StartDASRPCServer(ctx, serverConfig.RPCAddr, serverConfig.RPCPort, serverConfig.RPCServerTimeouts, daReader, daWriter, daHealthChecker) + rpcServer, err = das.StartDASRPCServer(ctx, serverConfig.RPCAddr, serverConfig.RPCPort, serverConfig.RPCServerTimeouts, serverConfig.RPCServerBodyLimit, daReader, daWriter, daHealthChecker) if err != nil { return err } diff --git a/cmd/genericconf/server.go b/cmd/genericconf/server.go index 7550791d6d..18f13dd204 100644 --- a/cmd/genericconf/server.go +++ b/cmd/genericconf/server.go @@ -48,6 +48,8 @@ var HTTPServerTimeoutConfigDefault = HTTPServerTimeoutConfig{ IdleTimeout: 120 * time.Second, } +var HTTPServerBodyLimitDefault = 0 // Use default from go-ethereum + func (c HTTPConfig) Apply(stackConf *node.Config) { stackConf.HTTPHost = c.Addr stackConf.HTTPPort = c.Port diff --git a/das/dasRpcServer.go b/das/dasRpcServer.go index 2f1fc1fd42..8bab8f0b6d 100644 --- a/das/dasRpcServer.go +++ b/das/dasRpcServer.go @@ -36,19 +36,22 @@ type DASRPCServer struct { daHealthChecker DataAvailabilityServiceHealthChecker } -func StartDASRPCServer(ctx context.Context, addr string, portNum uint64, rpcServerTimeouts genericconf.HTTPServerTimeoutConfig, daReader DataAvailabilityServiceReader, daWriter DataAvailabilityServiceWriter, daHealthChecker DataAvailabilityServiceHealthChecker) (*http.Server, error) { +func StartDASRPCServer(ctx context.Context, addr string, portNum uint64, rpcServerTimeouts genericconf.HTTPServerTimeoutConfig, RPCServerBodyLimit int, daReader DataAvailabilityServiceReader, daWriter DataAvailabilityServiceWriter, daHealthChecker DataAvailabilityServiceHealthChecker) (*http.Server, error) { listener, err := net.Listen("tcp", fmt.Sprintf("%s:%d", addr, portNum)) if err != nil { return nil, err } - return StartDASRPCServerOnListener(ctx, listener, rpcServerTimeouts, daReader, daWriter, daHealthChecker) + return StartDASRPCServerOnListener(ctx, listener, rpcServerTimeouts, RPCServerBodyLimit, daReader, daWriter, daHealthChecker) } -func StartDASRPCServerOnListener(ctx context.Context, listener net.Listener, rpcServerTimeouts genericconf.HTTPServerTimeoutConfig, daReader DataAvailabilityServiceReader, daWriter DataAvailabilityServiceWriter, daHealthChecker DataAvailabilityServiceHealthChecker) (*http.Server, error) { +func StartDASRPCServerOnListener(ctx context.Context, listener net.Listener, rpcServerTimeouts genericconf.HTTPServerTimeoutConfig, RPCServerBodyLimit int, daReader DataAvailabilityServiceReader, daWriter DataAvailabilityServiceWriter, daHealthChecker DataAvailabilityServiceHealthChecker) (*http.Server, error) { if daWriter == nil { return nil, errors.New("No writer backend was configured for DAS RPC server. Has the BLS signing key been set up (--data-availability.key.key-dir or --data-availability.key.priv-key options)?") } rpcServer := rpc.NewServer() + if RPCServerBodyLimit > 0 { + rpcServer.SetHTTPBodyLimit(RPCServerBodyLimit) + } err := rpcServer.RegisterName("das", &DASRPCServer{ daReader: daReader, daWriter: daWriter, diff --git a/das/rpc_test.go b/das/rpc_test.go index 044ba597be..658592cc0b 100644 --- a/das/rpc_test.go +++ b/das/rpc_test.go @@ -55,7 +55,7 @@ func TestRPC(t *testing.T) { testhelpers.RequireImpl(t, err) localDas, err := NewSignAfterStoreDASWriterWithSeqInboxCaller(privKey, nil, storageService, "") testhelpers.RequireImpl(t, err) - dasServer, err := StartDASRPCServerOnListener(ctx, lis, genericconf.HTTPServerTimeoutConfigDefault, storageService, localDas, storageService) + dasServer, err := StartDASRPCServerOnListener(ctx, lis, genericconf.HTTPServerTimeoutConfigDefault, genericconf.HTTPServerBodyLimitDefault, storageService, localDas, storageService) defer func() { if err := dasServer.Shutdown(ctx); err != nil { panic(err) diff --git a/system_tests/common_test.go b/system_tests/common_test.go index f6bfde2108..04b91d6a18 100644 --- a/system_tests/common_test.go +++ b/system_tests/common_test.go @@ -1127,7 +1127,7 @@ func setupConfigWithDAS( Require(t, err) restLis, err := net.Listen("tcp", "localhost:0") Require(t, err) - _, err = das.StartDASRPCServerOnListener(ctx, rpcLis, genericconf.HTTPServerTimeoutConfigDefault, daReader, daWriter, daHealthChecker) + _, err = das.StartDASRPCServerOnListener(ctx, rpcLis, genericconf.HTTPServerTimeoutConfigDefault, genericconf.HTTPServerBodyLimitDefault, daReader, daWriter, daHealthChecker) Require(t, err) _, err = das.NewRestfulDasServerOnListener(restLis, genericconf.HTTPServerTimeoutConfigDefault, daReader, daHealthChecker) Require(t, err) diff --git a/system_tests/das_test.go b/system_tests/das_test.go index bb09cc9880..a5ce02d87b 100644 --- a/system_tests/das_test.go +++ b/system_tests/das_test.go @@ -74,7 +74,7 @@ func startLocalDASServer( Require(t, err) rpcLis, err := net.Listen("tcp", "localhost:0") Require(t, err) - rpcServer, err := das.StartDASRPCServerOnListener(ctx, rpcLis, genericconf.HTTPServerTimeoutConfigDefault, storageService, daWriter, storageService) + rpcServer, err := das.StartDASRPCServerOnListener(ctx, rpcLis, genericconf.HTTPServerTimeoutConfigDefault, genericconf.HTTPServerBodyLimitDefault, storageService, daWriter, storageService) Require(t, err) restLis, err := net.Listen("tcp", "localhost:0") Require(t, err) @@ -283,7 +283,7 @@ func TestDASComplexConfigAndRestMirror(t *testing.T) { defer lifecycleManager.StopAndWaitUntil(time.Second) rpcLis, err := net.Listen("tcp", "localhost:0") Require(t, err) - _, err = das.StartDASRPCServerOnListener(ctx, rpcLis, genericconf.HTTPServerTimeoutConfigDefault, daReader, daWriter, daHealthChecker) + _, err = das.StartDASRPCServerOnListener(ctx, rpcLis, genericconf.HTTPServerTimeoutConfigDefault, genericconf.HTTPServerBodyLimitDefault, daReader, daWriter, daHealthChecker) Require(t, err) restLis, err := net.Listen("tcp", "localhost:0") Require(t, err) From 751ff4a3d8bf4d79d4523d5dca4c382d0a60a668 Mon Sep 17 00:00:00 2001 From: Emiliano Bonassi Date: Sat, 25 May 2024 09:52:56 -0700 Subject: [PATCH 2/4] Update daserver.go Co-authored-by: Tristan-Wilson <87238672+Tristan-Wilson@users.noreply.github.com> --- cmd/daserver/daserver.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/daserver/daserver.go b/cmd/daserver/daserver.go index 48cc5546de..1a3fd435b8 100644 --- a/cmd/daserver/daserver.go +++ b/cmd/daserver/daserver.go @@ -90,7 +90,7 @@ func parseDAServer(args []string) (*DAServerConfig, error) { f.Bool("enable-rpc", DefaultDAServerConfig.EnableRPC, "enable the HTTP-RPC server listening on rpc-addr and rpc-port") f.String("rpc-addr", DefaultDAServerConfig.RPCAddr, "HTTP-RPC server listening interface") f.Uint64("rpc-port", DefaultDAServerConfig.RPCPort, "HTTP-RPC server listening port") - f.Int("rpc-server-body-limit", DefaultDAServerConfig.RPCServerBodyLimit, "HTTP-RPC server maximum request body size in bytes") + f.Int("rpc-server-body-limit", DefaultDAServerConfig.RPCServerBodyLimit, "HTTP-RPC server maximum request body size in bytes; the default (0) uses geth's 5MB limit") genericconf.HTTPServerTimeoutConfigAddOptions("rpc-server-timeouts", f) f.Bool("enable-rest", DefaultDAServerConfig.EnableREST, "enable the REST server listening on rest-addr and rest-port") From 1d088e5f882eb2fe270c7ef61e84e41deb44554a Mon Sep 17 00:00:00 2001 From: Emiliano Bonassi Date: Sat, 25 May 2024 09:53:12 -0700 Subject: [PATCH 3/4] Update server.go Co-authored-by: Tristan-Wilson <87238672+Tristan-Wilson@users.noreply.github.com> --- cmd/genericconf/server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/genericconf/server.go b/cmd/genericconf/server.go index 18f13dd204..9b8acd5f71 100644 --- a/cmd/genericconf/server.go +++ b/cmd/genericconf/server.go @@ -48,7 +48,7 @@ var HTTPServerTimeoutConfigDefault = HTTPServerTimeoutConfig{ IdleTimeout: 120 * time.Second, } -var HTTPServerBodyLimitDefault = 0 // Use default from go-ethereum +const HTTPServerBodyLimitDefault = 0 // Use default from go-ethereum func (c HTTPConfig) Apply(stackConf *node.Config) { stackConf.HTTPHost = c.Addr From 5931051f80e6fb1acedd9bf341a0cba6335a084d Mon Sep 17 00:00:00 2001 From: Emiliano Bonassi Date: Sat, 25 May 2024 09:56:31 -0700 Subject: [PATCH 4/4] nits dasRpcServer.go --- das/dasRpcServer.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/das/dasRpcServer.go b/das/dasRpcServer.go index 8bab8f0b6d..03f755b90e 100644 --- a/das/dasRpcServer.go +++ b/das/dasRpcServer.go @@ -36,21 +36,21 @@ type DASRPCServer struct { daHealthChecker DataAvailabilityServiceHealthChecker } -func StartDASRPCServer(ctx context.Context, addr string, portNum uint64, rpcServerTimeouts genericconf.HTTPServerTimeoutConfig, RPCServerBodyLimit int, daReader DataAvailabilityServiceReader, daWriter DataAvailabilityServiceWriter, daHealthChecker DataAvailabilityServiceHealthChecker) (*http.Server, error) { +func StartDASRPCServer(ctx context.Context, addr string, portNum uint64, rpcServerTimeouts genericconf.HTTPServerTimeoutConfig, rpcServerBodyLimit int, daReader DataAvailabilityServiceReader, daWriter DataAvailabilityServiceWriter, daHealthChecker DataAvailabilityServiceHealthChecker) (*http.Server, error) { listener, err := net.Listen("tcp", fmt.Sprintf("%s:%d", addr, portNum)) if err != nil { return nil, err } - return StartDASRPCServerOnListener(ctx, listener, rpcServerTimeouts, RPCServerBodyLimit, daReader, daWriter, daHealthChecker) + return StartDASRPCServerOnListener(ctx, listener, rpcServerTimeouts, rpcServerBodyLimit, daReader, daWriter, daHealthChecker) } -func StartDASRPCServerOnListener(ctx context.Context, listener net.Listener, rpcServerTimeouts genericconf.HTTPServerTimeoutConfig, RPCServerBodyLimit int, daReader DataAvailabilityServiceReader, daWriter DataAvailabilityServiceWriter, daHealthChecker DataAvailabilityServiceHealthChecker) (*http.Server, error) { +func StartDASRPCServerOnListener(ctx context.Context, listener net.Listener, rpcServerTimeouts genericconf.HTTPServerTimeoutConfig, rpcServerBodyLimit int, daReader DataAvailabilityServiceReader, daWriter DataAvailabilityServiceWriter, daHealthChecker DataAvailabilityServiceHealthChecker) (*http.Server, error) { if daWriter == nil { return nil, errors.New("No writer backend was configured for DAS RPC server. Has the BLS signing key been set up (--data-availability.key.key-dir or --data-availability.key.priv-key options)?") } rpcServer := rpc.NewServer() - if RPCServerBodyLimit > 0 { - rpcServer.SetHTTPBodyLimit(RPCServerBodyLimit) + if rpcServerBodyLimit > 0 { + rpcServer.SetHTTPBodyLimit(rpcServerBodyLimit) } err := rpcServer.RegisterName("das", &DASRPCServer{ daReader: daReader,