Merge pull request #101 from EspressoSystems/jh/sync-20240313

Sync upstream 20240313

.github/workflows/ci.yml

@@ -132,8 +132,7 @@ jobs:
           skip-pkg-cache: true
       - name: Custom Lint
         run: |
-          go run ./linter/koanf ./...
-          go run ./linter/pointercheck ./...
+          go run ./linters ./...
 
       - name: Set environment variables
         run: |

.github/workflows/release-ci.yml

@@ -0,0 +1,30 @@
+name: Release CI
+run-name: Release CI triggered from @${{ github.actor }} of ${{ github.head_ref }}
+
+on:
+  workflow_dispatch:
+
+jobs:
+  build_and_run:
+    runs-on: ubuntu-8
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+      with:
+        submodules: recursive
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+      with:
+        driver-opts: network=host
+
+    - name: Cache Docker layers
+      uses: actions/cache@v3
+      with:
+        path: /tmp/.buildx-cache
+        key: ${{ runner.os }}-buildx-${{ hashFiles('Dockerfile') }}
+        restore-keys: ${{ runner.os }}-buildx-
+
+    - name:  Startup Nitro testnode
+      run: ./scripts/startup-testnode.bash

Dockerfile

@@ -171,6 +171,7 @@ RUN ./download-machine.sh consensus-v10.2 0x0754e09320c381566cc0449904c377a52bd3
 RUN ./download-machine.sh consensus-v10.3 0xf559b6d4fa869472dabce70fe1c15221bdda837533dfd891916836975b434dec
 RUN ./download-machine.sh consensus-v11 0xf4389b835497a910d7ba3ebfb77aa93da985634f3c052de1290360635be40c4a
 RUN ./download-machine.sh consensus-v11.1 0x68e4fe5023f792d4ef584796c84d710303a5e12ea02d6e37e2b5e9c4332507c4
+RUN ./download-machine.sh consensus-v20 0x8b104a2e80ac6165dc58b9048de12f301d70b02a0ab51396c22b4b4b802a16a4
 
 FROM golang:1.20-bookworm as node-builder
 WORKDIR /workspace
@@ -207,7 +208,7 @@ FROM debian:bookworm-slim as nitro-fuzzer
 COPY --from=fuzz-builder /workspace/fuzzers/*.fuzz /usr/local/bin/
 COPY ./scripts/fuzz.bash /usr/local/bin
 RUN mkdir /fuzzcache
-ENTRYPOINT [ "/usr/local/bin/fuzz.bash", "--binary-path", "/usr/local/bin/", "--fuzzcache-path", "/fuzzcache" ]
+ENTRYPOINT [ "/usr/local/bin/fuzz.bash", "FuzzStateTransition", "--binary-path", "/usr/local/bin/", "--fuzzcache-path", "/fuzzcache" ]
 
 FROM debian:bookworm-slim as nitro-node-slim
 WORKDIR /home/user

Makefile

@@ -319,8 +319,7 @@ contracts/test/prover/proofs/%.json: $(arbitrator_cases)/%.wasm $(arbitrator_pro
 # strategic rules to minimize dependency building
 
 .make/lint: $(DEP_PREDICATE) build-node-deps $(ORDER_ONLY_PREDICATE) .make
-	go run ./linter/koanf ./...
-	go run ./linter/pointercheck ./...
+	go run ./linters ./...
 	golangci-lint run --fix
 	yarn --cwd contracts solhint
 	@touch $@

arbnode/batch_poster.go

@@ -41,6 +41,7 @@ import (
 	"github.com/offchainlabs/nitro/cmd/chaininfo"
 	"github.com/offchainlabs/nitro/cmd/genericconf"
 	"github.com/offchainlabs/nitro/das"
+	"github.com/offchainlabs/nitro/execution"
 	"github.com/offchainlabs/nitro/solgen/go/bridgegen"
 	"github.com/offchainlabs/nitro/util"
 	"github.com/offchainlabs/nitro/util/arbmath"
@@ -73,22 +74,24 @@ type batchPosterPosition struct {
 
 type BatchPoster struct {
 	stopwaiter.StopWaiter
-	l1Reader         *headerreader.HeaderReader
-	inbox            *InboxTracker
-	streamer         *TransactionStreamer
-	config           BatchPosterConfigFetcher
-	seqInbox         *bridgegen.SequencerInbox
-	bridge           *bridgegen.Bridge
-	syncMonitor      *SyncMonitor
-	seqInboxABI      *abi.ABI
-	seqInboxAddr     common.Address
-	bridgeAddr       common.Address
-	gasRefunderAddr  common.Address
-	building         *buildingBatch
-	daWriter         das.DataAvailabilityServiceWriter
-	dataPoster       *dataposter.DataPoster
-	redisLock        *redislock.Simple
-	messagesPerBatch *arbmath.MovingAverage[uint64]
+	l1Reader           *headerreader.HeaderReader
+	inbox              *InboxTracker
+	streamer           *TransactionStreamer
+	arbOSVersionGetter execution.FullExecutionClient
+	config             BatchPosterConfigFetcher
+	seqInbox           *bridgegen.SequencerInbox
+	bridge             *bridgegen.Bridge
+	syncMonitor        *SyncMonitor
+	seqInboxABI        *abi.ABI
+	seqInboxAddr       common.Address
+	bridgeAddr         common.Address
+	gasRefunderAddr    common.Address
+	building           *buildingBatch
+	daWriter           das.DataAvailabilityServiceWriter
+	dataPoster         *dataposter.DataPoster
+	redisLock          *redislock.Simple
+	messagesPerBatch   *arbmath.MovingAverage[uint64]
+	non4844BatchCount  int // Count of consecutive non-4844 batches posted
 	// This is an atomic variable that should only be accessed atomically.
 	// An estimate of the number of batches we want to post but haven't yet.
 	// This doesn't include batches which we don't want to post yet due to the L1 bounds.
@@ -136,7 +139,7 @@ type BatchPosterConfig struct {
 	RedisLock          redislock.SimpleCfg         `koanf:"redis-lock" reload:"hot"`
 	ExtraBatchGas      uint64                      `koanf:"extra-batch-gas" reload:"hot"`
 	Post4844Blobs      bool                        `koanf:"post-4844-blobs" reload:"hot"`
-	ForcePost4844Blobs bool                        `koanf:"force-post-4844-blobs" reload:"hot"`
+	IgnoreBlobPrice    bool                        `koanf:"ignore-blob-price" reload:"hot"`
 	ParentChainWallet  genericconf.WalletConfig    `koanf:"parent-chain-wallet"`
 	L1BlockBound       string                      `koanf:"l1-block-bound" reload:"hot"`
 	L1BlockBoundBypass time.Duration               `koanf:"l1-block-bound-bypass" reload:"hot"`
@@ -186,7 +189,7 @@ func BatchPosterConfigAddOptions(prefix string, f *pflag.FlagSet) {
 	f.String(prefix+".gas-refunder-address", DefaultBatchPosterConfig.GasRefunderAddress, "The gas refunder contract address (optional)")
 	f.Uint64(prefix+".extra-batch-gas", DefaultBatchPosterConfig.ExtraBatchGas, "use this much more gas than estimation says is necessary to post batches")
 	f.Bool(prefix+".post-4844-blobs", DefaultBatchPosterConfig.Post4844Blobs, "if the parent chain supports 4844 blobs and they're well priced, post EIP-4844 blobs")
-	f.Bool(prefix+".force-post-4844-blobs", DefaultBatchPosterConfig.ForcePost4844Blobs, "if the parent chain supports 4844 blobs and post-4844-blobs is true, post 4844 blobs even if it's not price efficient")
+	f.Bool(prefix+".ignore-blob-price", DefaultBatchPosterConfig.IgnoreBlobPrice, "if the parent chain supports 4844 blobs and ignore-blob-price is true, post 4844 blobs even if it's not price efficient")
 	f.String(prefix+".redis-url", DefaultBatchPosterConfig.RedisUrl, "if non-empty, the Redis URL to store queued transactions in")
 	f.String(prefix+".l1-block-bound", DefaultBatchPosterConfig.L1BlockBound, "only post messages to batches when they're within the max future block/timestamp as of this L1 block tag (\"safe\", \"finalized\", \"latest\", or \"ignore\" to ignore this check)")
 	f.Duration(prefix+".l1-block-bound-bypass", DefaultBatchPosterConfig.L1BlockBoundBypass, "post batches even if not within the layer 1 future bounds if we're within this margin of the max delay")
@@ -202,7 +205,7 @@ var DefaultBatchPosterConfig = BatchPosterConfig{
 	// This default is overridden for L3 chains in applyChainParameters in cmd/nitro/nitro.go
 	MaxSize: 100000,
 	// TODO: is 1000 bytes an appropriate margin for error vs blob space efficiency?
-	Max4844BatchSize:   (254 * params.BlobTxFieldElementsPerBlob / 8 * (params.MaxBlobGasPerBlock / params.BlobTxBlobGasPerBlob)) - 1000,
+	Max4844BatchSize:   blobs.BlobEncodableData*(params.MaxBlobGasPerBlock/params.BlobTxBlobGasPerBlob) - 1000,
 	PollInterval:       time.Second * 10,
 	ErrorDelay:         time.Second * 10,
 	MaxDelay:           time.Hour,
@@ -212,7 +215,7 @@ var DefaultBatchPosterConfig = BatchPosterConfig{
 	GasRefunderAddress: "",
 	ExtraBatchGas:      50_000,
 	Post4844Blobs:      false,
-	ForcePost4844Blobs: false,
+	IgnoreBlobPrice:    false,
 	DataPoster:         dataposter.DefaultDataPosterConfig,
 	ParentChainWallet:  DefaultBatchPosterL1WalletConfig,
 	L1BlockBound:       "",
@@ -242,7 +245,7 @@ var TestBatchPosterConfig = BatchPosterConfig{
 	GasRefunderAddress: "",
 	ExtraBatchGas:      10_000,
 	Post4844Blobs:      true,
-	ForcePost4844Blobs: false,
+	IgnoreBlobPrice:    false,
 	DataPoster:         dataposter.TestDataPosterConfig,
 	ParentChainWallet:  DefaultBatchPosterL1WalletConfig,
 	L1BlockBound:       "",
@@ -255,6 +258,7 @@ type BatchPosterOpts struct {
 	L1Reader      *headerreader.HeaderReader
 	Inbox         *InboxTracker
 	Streamer      *TransactionStreamer
+	VersionGetter execution.FullExecutionClient
 	SyncMonitor   *SyncMonitor
 	Config        BatchPosterConfigFetcher
 	DeployInfo    *chaininfo.RollupAddresses
@@ -293,19 +297,20 @@ func NewBatchPoster(ctx context.Context, opts *BatchPosterOpts) (*BatchPoster, e
 		return nil, err
 	}
 	b := &BatchPoster{
-		l1Reader:        opts.L1Reader,
-		inbox:           opts.Inbox,
-		streamer:        opts.Streamer,
-		syncMonitor:     opts.SyncMonitor,
-		config:          opts.Config,
-		bridge:          bridge,
-		seqInbox:        seqInbox,
-		seqInboxABI:     seqInboxABI,
-		seqInboxAddr:    opts.DeployInfo.SequencerInbox,
-		gasRefunderAddr: opts.Config().gasRefunder,
-		bridgeAddr:      opts.DeployInfo.Bridge,
-		daWriter:        opts.DAWriter,
-		redisLock:       redisLock,
+		l1Reader:           opts.L1Reader,
+		inbox:              opts.Inbox,
+		streamer:           opts.Streamer,
+		arbOSVersionGetter: opts.VersionGetter,
+		syncMonitor:        opts.SyncMonitor,
+		config:             opts.Config,
+		bridge:             bridge,
+		seqInbox:           seqInbox,
+		seqInboxABI:        seqInboxABI,
+		seqInboxAddr:       opts.DeployInfo.SequencerInbox,
+		gasRefunderAddr:    opts.Config().gasRefunder,
+		bridgeAddr:         opts.DeployInfo.Bridge,
+		daWriter:           opts.DAWriter,
+		redisLock:          redisLock,
 	}
 	b.messagesPerBatch, err = arbmath.NewMovingAverage[uint64](20)
 	if err != nil {
@@ -947,7 +952,6 @@ func (b *BatchPoster) maybePostSequencerBatch(ctx context.Context) (bool, error)
 	if dbBatchCount > batchPosition.NextSeqNum {
 		return false, fmt.Errorf("attempting to post batch %v, but the local inbox tracker database already has %v batches", batchPosition.NextSeqNum, dbBatchCount)
 	}
-
 	if b.building == nil || b.building.startMsgCount != batchPosition.MessageCount {
 		latestHeader, err := b.l1Reader.LastHeader(ctx)
 		if err != nil {
@@ -956,17 +960,34 @@ func (b *BatchPoster) maybePostSequencerBatch(ctx context.Context) (bool, error)
 		var use4844 bool
 		config := b.config()
 		if config.Post4844Blobs && latestHeader.ExcessBlobGas != nil && latestHeader.BlobGasUsed != nil {
-			if config.ForcePost4844Blobs {
-				use4844 = true
-			} else {
-				blobFeePerByte := eip4844.CalcBlobFee(eip4844.CalcExcessBlobGas(*latestHeader.ExcessBlobGas, *latestHeader.BlobGasUsed))
-				blobFeePerByte.Mul(blobFeePerByte, blobTxBlobGasPerBlob)
-				blobFeePerByte.Div(blobFeePerByte, usableBytesInBlob)
-
-				calldataFeePerByte := arbmath.BigMulByUint(latestHeader.BaseFee, 16)
-				use4844 = arbmath.BigLessThan(blobFeePerByte, calldataFeePerByte)
+			arbOSVersion, err := b.arbOSVersionGetter.ArbOSVersionForMessageNumber(arbutil.MessageIndex(arbmath.SaturatingUSub(uint64(batchPosition.MessageCount), 1)))
+			if err != nil {
+				return false, err
+			}
+			if arbOSVersion >= 20 {
+				if config.IgnoreBlobPrice {
+					use4844 = true
+				} else {
+					backlog := atomic.LoadUint64(&b.backlog)
+					// Logic to prevent switching from non-4844 batches to 4844 batches too often,
+					// so that blocks can be filled efficiently. The geth txpool rejects txs for
+					// accounts that already have the other type of txs in the pool with
+					// "address already reserved". This logic makes sure that, if there is a backlog,
+					// that enough non-4844 batches have been posted to fill a block before switching.
+					if backlog == 0 ||
+						b.non4844BatchCount == 0 ||
+						b.non4844BatchCount > 16 {
+						blobFeePerByte := eip4844.CalcBlobFee(eip4844.CalcExcessBlobGas(*latestHeader.ExcessBlobGas, *latestHeader.BlobGasUsed))
+						blobFeePerByte.Mul(blobFeePerByte, blobTxBlobGasPerBlob)
+						blobFeePerByte.Div(blobFeePerByte, usableBytesInBlob)
+
+						calldataFeePerByte := arbmath.BigMulByUint(latestHeader.BaseFee, 16)
+						use4844 = arbmath.BigLessThan(blobFeePerByte, calldataFeePerByte)
+					}
+				}
 			}
 		}
+
 		b.building = &buildingBatch{
 			segments:      newBatchSegments(batchPosition.DelayedMessageCount, b.config(), b.GetBacklogEstimate(), use4844),
 			msgCount:      batchPosition.MessageCount,
@@ -1198,9 +1219,15 @@ func (b *BatchPoster) maybePostSequencerBatch(ctx context.Context) (bool, error)
 		"totalSegments", len(b.building.segments.rawSegments),
 		"numBlobs", len(kzgBlobs),
 	)
+
 	recentlyHitL1Bounds := time.Since(b.lastHitL1Bounds) < config.PollInterval*3
 	postedMessages := b.building.msgCount - batchPosition.MessageCount
 	b.messagesPerBatch.Update(uint64(postedMessages))
+	if b.building.use4844 {
+		b.non4844BatchCount = 0
+	} else {
+		b.non4844BatchCount++
+	}
 	unpostedMessages := msgCount - b.building.msgCount
 	messagesPerBatch := b.messagesPerBatch.Average()
 	if messagesPerBatch == 0 {
@@ -1340,3 +1367,56 @@ func (b *BatchPoster) StopAndWait() {
 	b.dataPoster.StopAndWait()
 	b.redisLock.StopAndWait()
 }
+
+type BoolRing struct {
+	buffer         []bool
+	bufferPosition int
+}
+
+func NewBoolRing(size int) *BoolRing {
+	return &BoolRing{
+		buffer: make([]bool, 0, size),
+	}
+}
+
+func (b *BoolRing) Update(value bool) {
+	period := cap(b.buffer)
+	if period == 0 {
+		return
+	}
+	if len(b.buffer) < period {
+		b.buffer = append(b.buffer, value)
+	} else {
+		b.buffer[b.bufferPosition] = value
+	}
+	b.bufferPosition = (b.bufferPosition + 1) % period
+}
+
+func (b *BoolRing) Empty() bool {
+	return len(b.buffer) == 0
+}
+
+// Peek returns the most recently inserted value.
+// Assumes not empty, check Empty() first
+func (b *BoolRing) Peek() bool {
+	lastPosition := b.bufferPosition - 1
+	if lastPosition < 0 {
+		// This is the case where we have wrapped around, since Peek() shouldn't
+		// be called without checking Empty(), so we can just use capactity.
+		lastPosition = cap(b.buffer) - 1
+	}
+	return b.buffer[lastPosition]
+}
+
+// All returns true if the BoolRing is full and all values equal value.
+func (b *BoolRing) All(value bool) bool {
+	if len(b.buffer) < cap(b.buffer) {
+		return false
+	}
+	for _, v := range b.buffer {
+		if v != value {
+			return false
+		}
+	}
+	return true
+}