add pruning vs archive node sharding

.env

@@ -27,18 +27,20 @@ KAVA_PRUNING_HOST_EVM_RPC_PORT=8555
 KAVA_PRUNING_HOST_COSMOS_RPC_PORT=26667
 
 PROXY_CONTAINER_PORT=7777
-PROXY_CONTAINER_EVM_RPC_DATA_PORT=7778
+PROXY_CONTAINER_EVM_RPC_PRUNING_PORT=7778
 PROXY_HOST_PORT=7777
 PROXY_CONTAINER_DEBUG_PORT=2345
 PROXY_HOST_DEBUG_PORT=2345
 
 ##### E2E Testing Config
 TEST_PROXY_SERVICE_EVM_RPC_URL=http://localhost:7777
 TEST_PROXY_SERVICE_EVM_RPC_HOSTNAME=localhost:7777
-TEST_PROXY_SERVICE_EVM_RPC_DATA_URL=http://localhost:7778
+TEST_PROXY_SERVICE_EVM_RPC_PRUNING_URL=http://localhost:7778
 TEST_PROXY_BACKEND_EVM_RPC_HOST_URL=http://localhost:8545
 TEST_DATABASE_ENDPOINT_URL=localhost:5432
 TEST_PROXY_BACKEND_HOST_URL_MAP=localhost:7777>http://kava-validator:8545,localhost:7778>http://kava-pruning:8545
+TEST_PROXY_HEIGHT_BASED_ROUTING_ENABLED=true
+TEST_PROXY_PRUNING_BACKEND_HOST_URL_MAP=localhost:7777>http://kava-pruning:8545,localhost:7778>http://kava-pruning:8545
 # What level of logging to use for service objects constructed during
 # unit tests
 TEST_SERVICE_LOG_LEVEL=ERROR
@@ -57,6 +59,11 @@ HTTP_READ_TIMEOUT_SECONDS=30
 HTTP_WRITE_TIMEOUT_SECONDS=60
 # Address of the origin server to proxy all requests to
 PROXY_BACKEND_HOST_URL_MAP=localhost:7777>http://kava-validator:8545,localhost:7778>http://kava-pruning:8545
+# height-based routing will look at the height of an incoming EVM request
+# iff. the height is "latest", it routes to the corresponding PROXY_PRUNING_BACKEND_HOST_URL_MAP value
+# otherwise, it falls back to the value in PROXY_BACKEND_HOST_URL_MAP
+PROXY_HEIGHT_BASED_ROUTING_ENABLED=true
+PROXY_PRUNING_BACKEND_HOST_URL_MAP=localhost:7777>http://kava-pruning:8545,localhost:7778>http://kava-pruning:8545
 # Configuration for the servcie to connect to it's database
 DATABASE_NAME=postgres
 DATABASE_ENDPOINT_URL=postgres:5432

DEVELOPMENT.md

@@ -98,6 +98,17 @@ make ready e2e-test
 
 For details on the local E2E setup, see [the `docker` directory](./docker/README.md).
 
+#### Against testnet
+
+The Continuous Integration (CI) for this repo is setup to run a local proxy service with database & redis, but configures the service to use public testnet urls for the kava requests. This allows for testing the proxy service against a production-like environment (requests are routed to public testnet).
+
+You can emulate the CI configuration in your local environment:
+```bash
+make ci-setup
+```
+
+At that point, running `make e2e-test` will run the end-to-end tests with requests routing to public testnet.
+
 ## Test Coverage Report
 
 The test commands `make test`, `make unit-test`, and `make e2e-test` generate a `cover.out` raw test coverage report. The coverage can be converted into a user-friendly webpage:

Makefile

@@ -44,6 +44,14 @@ unit-test:
 e2e-test:
 	go test -count=1 -v -cover -coverprofile cover.out --race ./... -run "^TestE2ETest*"
 
+.PHONY: ci-setup
+# set up your local environment such that running `make e2e-test` runs against testnet (like in CI)
+ci-setup:
+	docker compose -f ci.docker-compose.yml pull
+	docker compose -f ci.docker-compose.yml up -d --build --force-recreate
+	PROXY_CONTAINER_PORT=7777 bash scripts/wait-for-proxy-service-running.sh
+	PROXY_CONTAINER_PORT=7777 MINIMUM_REQUIRED_PARTITIONS=30 bash scripts/wait-for-proxy-service-running.sh
+
 .PHONY: it
 # run any test matching the provided pattern, can pass a regex or a string
 # of the exact test to run

architecture/MIDDLEWARE.MD

@@ -61,27 +61,7 @@ server := &http.Server{
 
 1. Times the roundtrip latency for the response from the backend origin server and stores the latency in the context key `X-KAVA-PROXY-ORIGIN-ROUNDTRIP-LATENCY-MILLISECONDS` for use by other middleware.
 
-#### Configuration
-
-The proxy is configured to route requests based on their incoming Host. These are controlled via the
-`PROXY_BACKEND_HOST_URL_MAP` environment variable.
-
-As an example, consider a value of `localhost:7777>http://kava:8545,localhost:7778>http://kava:8545`.
-This value is parsed into a map that looks like the following:
-```
-{
-    "localhost:7777" => "http://kava:8545",
-    "localhost:7778" => "http://kava:8545",
-}
-```
-Any request to the service will be routed according to this map.
-ie. all requests to local ports 7777 & 7778 get forwarded to `http://kava:8545`
-
-Implementations of the [`Proxies` interface](../service/proxy.go#L13) contain logic for deciding
-the backend host url to which a request is routed. This is used in the ProxyRequestMiddleware to
-route requests.
-
-Any request made to a host not in the map responds 502 Bad Gateway.
+See [Proxy Routing](./PROXY_ROUTING.md) for details on configuration and how requests are routed.
 
 ### After Proxy Middleware
 

architecture/PROXY_ROUTING.md

@@ -0,0 +1,152 @@
+# Proxy Routing
+
+The proxy chooses where to route a request primarily by the incoming Host URL to which the client
+originally made their request. The routing is configured by maps of the Host to a backend url in
+the environment variables.
+
+All possible configurations use the `PROXY_BACKEND_HOST_URL_MAP` environment variable. This encodes
+the default backend to route all requests from a given host. Additional functionality is available
+via the `PROXY_HEIGHT_BASED_ROUTING_ENABLED` env variable (see [Rudimentary Sharding](#rudimentary-sharding)).
+
+Consider the simplest case: a host-based-only routing proxy configured for one host
+```
+PROXY_HEIGHT_BASED_ROUTING_ENABLED=false
+PROXY_BACKEND_HOST_URL_MAP=localhost:7777>http://kava:8545
+```
+This value is parsed into a map that looks like the following:
+```
+{
+    "localhost:7777" => "http://kava:8545",
+}
+```
+Any request to the service will be routed according to this map.
+ie. all requests to local port 7777 get forwarded to `http://kava:8545`
+
+Implementations of the [`Proxies` interface](../service/proxy.go#L13) contain logic for deciding
+the backend host url to which a request is routed. This is used in the ProxyRequestMiddleware to
+route requests.
+
+Any request made to a host not in the map responds 502 Bad Gateway.
+
+## More Examples of Host-only routing
+
+Here is a diagram of the above network topology:
+
+![Proxy Service configured for one host](images/proxy_service_simple_one_host.jpg)
+
+In this simple configuration of only having default hosts, you can do many things:
+
+**Many hosts -> One backend**
+
+```
+PROXY_HEIGHT_BASED_ROUTING_ENABLED=false
+PROXY_BACKEND_HOST_URL_MAP=localhost:7777>http://kava:8545,localhost:7778>http://kava:8545
+```
+This value is parsed into a map that looks like the following:
+```
+{
+    "localhost:7777" => "http://kava:8545",
+    "localhost:7778" => "http://kava:8545",
+}
+```
+All requests to local ports 7777 & 7778 route to the same cluster at kava:8545
+
+![Proxy Service configured for many hosts for one backend](images/proxy_service_many_hosts_one_backend.jpg)
+
+**Many hosts -> Many backends**
+
+```
+PROXY_HEIGHT_BASED_ROUTING_ENABLED=false
+PROXY_BACKEND_HOST_URL_MAP=evm.kava.io>http://kava-pruning:8545,evm.data.kava.io>http://kava-archive:8545
+```
+This value is parsed into a map that looks like the following:
+```
+{
+    "evm.kava.io"      => "http://kava-pruning:8545",
+    "evm.data.kava.io" => "http://kava-archive:8545",
+}
+```
+Requests made to evm.kava.io route to a pruning cluster.
+Those made to evm.data.kava.io route to an archive cluster.
+
+![Proxy Service configured for many hosts with many backends](images/proxy_service_many_hosts_many_backends.jpg)
+
+## Rudimentary Sharding
+
+Now suppose you want multiple backends for the same host.
+
+The proxy service supports height-based routing to direct requests that only require the most recent
+block to a different cluster.
+This support is handled via the [`HeightShardingProxies` implementation](../service/shard.go#L16).
+
+This is configured via the `PROXY_HEIGHT_BASED_ROUTING_ENABLED` and `PROXY_PRUNING_BACKEND_HOST_URL_MAP`
+environment variables.
+* `PROXY_HEIGHT_BASED_ROUTING_ENABLED` - flag to toggle this functionality
+* `PROXY_PRUNING_BACKEND_HOST_URL_MAP` - like `PROXY_BACKEND_HOST_URL_MAP`, but only used for JSON-RPC
+  requests that target the latest block (or are stateless, like `eth_chainId`, `eth_coinbase`, etc).
+
+For example, to lighten the load for your resource-intensive (& expensive) archive cluster, you can
+route all requests for the "latest" block to a less resource-intensive (& cheaper) pruning cluster:
+```
+PROXY_HEIGHT_BASED_ROUTING_ENABLED=true
+PROXY_BACKEND_HOST_URL_MAP=evm.data.kava.io>http://kava-archive:8545
+PROXY_PRUNING_BACKEND_HOST_URL_MAP=evm.data.kava.io>http://kava-pruning:8545
+```
+This value is parsed into a map that looks like the following:
+```
+{
+  "default": {
+    "evm.data.kava.io" => "http://kava-archive:8545",
+  },
+  "pruning": {
+    "evm.data.kava.io" => "http://kava-pruning:8545",
+  }
+}
+```
+All traffic to evm.data.kava.io that targets the latest block (or requires no history) routes to the pruning cluster.
+Otherwise, all traffic is sent to the archive cluster.
+
+![Proxy Service configured with rudimentary sharding](images/proxy_service_rudimentary_sharding.jpg)
+
+### Default vs Pruning Backend Routing
+
+When `PROXY_HEIGHT_BASED_ROUTING_ENABLED` is `true`, the following cases will cause requests to route
+to the the backend url defined in `PROXY_PRUNING_BACKEND_HOST_URL_MAP` (if present):
+* requests that include any of the following block tags:
+  * `"latest"`
+  * `"finalized"`
+  * `"pending"`
+  * `"safe"`
+  * empty/missing block tag (interpreted as `"latest"`)
+* requests for methods that require no historic state, including transaction broadcasting
+  * for a full list of methods, see [`NoHistoryMethods`](../decode/evm_rpc.go#L89)
+
+All other requests fallback to the default backend url defined in `PROXY_BACKEND_HOST_URL_MAP`.
+This includes
+* requests for hosts not included in `PROXY_PRUNING_BACKEND_HOST_URL_MAP`
+* requests targeting any specific height by number
+  * NOTE: the service does not track the current height of the chain. if the tip of the chain is at
+    block 1000, a query for block 1000 will still route to the default (not pruning) backend
+* requests for methods that use block hash, like `eth_getBlockByHash`
+* requests with unparsable (invalid) block numbers
+* requests for block tag `"earliest"`
+
+The service will panic on startup if a host in `PROXY_PRUNING_BACKEND_HOST_URL_MAP` is not present
+in `PROXY_BACKEND_HOST_URL_MAP`.
+
+Any request made to a host not in the `PROXY_BACKEND_HOST_URL_MAP` map responds 502 Bad Gateway.
+
+## Metrics
+
+When metrics are enabled, the `proxied_request_metrics` table tracks the backend to which requests
+are routed in the `response_backend` column.
+
+When height-based sharding is disabled (`PROXY_HEIGHT_BASED_ROUTING_ENABLED=false`), the value is
+always `DEFAULT`.
+
+When enabled, the column will have one of the following values:
+* `DEFAULT` - the request was routed to the backend defined in `PROXY_BACKEND_HOST_URL_MAP`
+* `PRUNING` - the request was routed to the backend defined in `PROXY_PRUNING_BACKEND_HOST_URL_MAP`
+
+Additionally, the actual URL to which the request is routed to is tracked in the
+`response_backend_route` column.

ci.docker-compose.yml

@@ -22,12 +22,14 @@ services:
       dockerfile: ci.Dockerfile
     env_file: .env
     environment:
+      PROXY_HEIGHT_BASED_ROUTING_ENABLED: true
       # use public testnet as backend origin server to avoid having
       # to self-host a beefy Github Action runner
       # to build and run a kava node each execution
-      PROXY_BACKEND_HOST_URL_MAP: localhost:7777>https://evmrpc.internal.testnet.proxy.kava.io,localhost:7778>https://evmrpcdata.internal.testnet.proxy.kava.io
+      PROXY_BACKEND_HOST_URL_MAP: localhost:7777>https://evmrpcdata.internal.testnet.proxy.kava.io,localhost:7778>https://evmrpc.internal.testnet.proxy.kava.io
+      PROXY_PRUNING_BACKEND_HOST_URL_MAP: localhost:7777>https://evmrpc.internal.testnet.proxy.kava.io
       EVM_QUERY_SERVICE_URL: https://evmrpc.internal.testnet.proxy.kava.io
     ports:
       - "${PROXY_HOST_PORT}:${PROXY_CONTAINER_PORT}"
-      - "${PROXY_CONTAINER_EVM_RPC_DATA_PORT}:${PROXY_CONTAINER_PORT}"
+      - "${PROXY_CONTAINER_EVM_RPC_PRUNING_PORT}:${PROXY_CONTAINER_PORT}"
       - "${PROXY_HOST_DEBUG_PORT}:${PROXY_CONTAINER_DEBUG_PORT}"

clients/database/migrations/20231013122742_add_response_backend.up.sql

@@ -0,0 +1,9 @@
+-- add response backend column, backfilling with "DEFAULT" (the only value to exist up until now)
+-- new metrics that omit its value are assumed to have been routed to "DEFAULT" backend
+-- also add response backend route. this is the backend url the request was routed to.
+ALTER TABLE
+  IF EXISTS proxied_request_metrics
+ADD
+  response_backend character varying DEFAULT 'DEFAULT',
+ADD
+  response_backend_route character varying;

clients/database/request_metric.go

@@ -27,6 +27,8 @@ type ProxiedRequestMetric struct {
 	UserAgent                   *string
 	Referer                     *string
 	Origin                      *string
+	ResponseBackend             string
+	ResponseBackendRoute        string
 }
 
 // Save saves the current ProxiedRequestMetric to

config/config.go

@@ -17,6 +17,9 @@ type Config struct {
 	LogLevel                               string
 	ProxyBackendHostURLMapRaw              string
 	ProxyBackendHostURLMapParsed           map[string]url.URL
+	EnableHeightBasedRouting               bool
+	ProxyPruningBackendHostURLMapRaw       string
+	ProxyPruningBackendHostURLMap          map[string]url.URL
 	EvmQueryServiceURL                     string
 	DatabaseName                           string
 	DatabaseEndpointURL                    string
@@ -43,6 +46,8 @@ const (
 	LOG_LEVEL_ENVIRONMENT_KEY                          = "LOG_LEVEL"
 	DEFAULT_LOG_LEVEL                                  = "INFO"
 	PROXY_BACKEND_HOST_URL_MAP_ENVIRONMENT_KEY         = "PROXY_BACKEND_HOST_URL_MAP"
+	PROXY_HEIGHT_BASED_ROUTING_ENABLED_KEY             = "PROXY_HEIGHT_BASED_ROUTING_ENABLED"
+	PROXY_PRUNING_BACKEND_HOST_URL_MAP_ENVIRONMENT_KEY = "PROXY_PRUNING_BACKEND_HOST_URL_MAP"
 	PROXY_SERVICE_PORT_ENVIRONMENT_KEY                 = "PROXY_SERVICE_PORT"
 	DATABASE_NAME_ENVIRONMENT_KEY                      = "DATABASE_NAME"
 	DATABASE_ENDPOINT_URL_ENVIRONMENT_KEY              = "DATABASE_ENDPOINT_URL"
@@ -81,6 +86,8 @@ const (
 	DEFAULT_DATABASE_WRITE_TIMEOUT_SECONDS                      = 10
 )
 
+var ErrEmptyHostMap = errors.New("backend host url map is empty")
+
 // EnvOrDefault fetches an environment variable value, or if not set returns the fallback value
 func EnvOrDefault(key string, fallback string) string {
 	if val, ok := os.LookupEnv(key); ok {
@@ -141,8 +148,9 @@ func ParseRawProxyBackendHostURLMap(raw string) (map[string]url.URL, error) {
 
 	entries := strings.Split(raw, PROXY_BACKEND_HOST_URL_MAP_ENTRY_DELIMITER)
 
-	if len(entries) < 1 {
-		return hostURLMap, fmt.Errorf("found zero mappings delimited by %s in %s", PROXY_BACKEND_HOST_URL_MAP_ENTRY_DELIMITER, raw)
+	if raw == "" || len(entries) < 1 {
+		extraErr := fmt.Errorf("found zero mappings delimited by %s in %s", PROXY_BACKEND_HOST_URL_MAP_ENTRY_DELIMITER, raw)
+		return hostURLMap, errors.Join(ErrEmptyHostMap, extraErr)
 	}
 
 	for _, entry := range entries {
@@ -175,15 +183,20 @@ func ParseRawProxyBackendHostURLMap(raw string) (map[string]url.URL, error) {
 // function of the Config package before use
 func ReadConfig() Config {
 	rawProxyBackendHostURLMap := os.Getenv(PROXY_BACKEND_HOST_URL_MAP_ENVIRONMENT_KEY)
-	// best effort to pares, callers are responsible for validating
+	rawProxyPruningBackendHostURLMap := os.Getenv(PROXY_PRUNING_BACKEND_HOST_URL_MAP_ENVIRONMENT_KEY)
+	// best effort to parse, callers are responsible for validating
 	// before using any values read
 	parsedProxyBackendHostURLMap, _ := ParseRawProxyBackendHostURLMap(rawProxyBackendHostURLMap)
+	parsedProxyPruningBackendHostURLMap, _ := ParseRawProxyBackendHostURLMap(rawProxyPruningBackendHostURLMap)
 
 	return Config{
 		ProxyServicePort:                       os.Getenv(PROXY_SERVICE_PORT_ENVIRONMENT_KEY),
 		LogLevel:                               EnvOrDefault(LOG_LEVEL_ENVIRONMENT_KEY, DEFAULT_LOG_LEVEL),
 		ProxyBackendHostURLMapRaw:              rawProxyBackendHostURLMap,
 		ProxyBackendHostURLMapParsed:           parsedProxyBackendHostURLMap,
+		EnableHeightBasedRouting:               EnvOrDefaultBool(PROXY_HEIGHT_BASED_ROUTING_ENABLED_KEY, false),
+		ProxyPruningBackendHostURLMapRaw:       rawProxyPruningBackendHostURLMap,
+		ProxyPruningBackendHostURLMap:          parsedProxyPruningBackendHostURLMap,
 		DatabaseName:                           os.Getenv(DATABASE_NAME_ENVIRONMENT_KEY),
 		DatabaseEndpointURL:                    os.Getenv(DATABASE_ENDPOINT_URL_ENVIRONMENT_KEY),
 		DatabaseUserName:                       os.Getenv(DATABASE_USERNAME_ENVIRONMENT_KEY),

config/config_test.go

@@ -9,9 +9,11 @@ import (
 )
 
 var (
-	proxyServicePort              = "7777"
-	randomEnvironmentVariableKey  = "TEST_KAVA_RANDOM_VALUE"
-	proxyServiceBackendHostURLMap = os.Getenv("TEST_PROXY_BACKEND_HOST_URL_MAP")
+	proxyServicePort                     = "7777"
+	randomEnvironmentVariableKey         = "TEST_KAVA_RANDOM_VALUE"
+	proxyServiceBackendHostURLMap        = os.Getenv("TEST_PROXY_BACKEND_HOST_URL_MAP")
+	proxyServiceHeightBasedRouting       = os.Getenv("TEST_PROXY_HEIGHT_BASED_ROUTING_ENABLED")
+	proxyServicePruningBackendHostURLMap = os.Getenv("TEST_PROXY_PRUNING_BACKEND_HOST_URL_MAP")
 )
 
 func TestUnitTestEnvODefaultReturnsDefaultIfEnvironmentVariableNotSet(t *testing.T) {
@@ -46,8 +48,15 @@ func TestUnitTestReadConfigReturnsConfigWithValuesFromEnv(t *testing.T) {
 	assert.Equal(t, proxyServicePort, readConfig.ProxyServicePort)
 }
 
+func TestUnitTestParseHostMapReturnsErrEmptyHostMapWhenEmpty(t *testing.T) {
+	_, err := config.ParseRawProxyBackendHostURLMap("")
+	assert.ErrorIs(t, err, config.ErrEmptyHostMap)
+}
+
 func setDefaultEnv() {
 	os.Setenv(config.PROXY_BACKEND_HOST_URL_MAP_ENVIRONMENT_KEY, proxyServiceBackendHostURLMap)
+	os.Setenv(config.PROXY_HEIGHT_BASED_ROUTING_ENABLED_KEY, proxyServiceHeightBasedRouting)
+	os.Setenv(config.PROXY_PRUNING_BACKEND_HOST_URL_MAP_ENVIRONMENT_KEY, proxyServicePruningBackendHostURLMap)
 	os.Setenv(config.PROXY_SERVICE_PORT_ENVIRONMENT_KEY, proxyServicePort)
 	os.Setenv(config.LOG_LEVEL_ENVIRONMENT_KEY, config.DEFAULT_LOG_LEVEL)
 }