Local Head Tracking ; Bug Fixes

op-e2e/interop/contracts/build/emit.sol/EmitEvent.abi

@@ -0,0 +1,28 @@
+[
+  {
+    "type": "function",
+    "name": "emitData",
+    "inputs": [
+      {
+        "name": "_data",
+        "type": "bytes",
+        "internalType": "bytes"
+      }
+    ],
+    "outputs": [],
+    "stateMutability": "nonpayable"
+  },
+  {
+    "type": "event",
+    "name": "DataEmitted",
+    "inputs": [
+      {
+        "name": "_data",
+        "type": "bytes",
+        "indexed": true,
+        "internalType": "bytes"
+      }
+    ],
+    "anonymous": false
+  }
+]

op-e2e/interop/contracts/build/emit.sol/EmitEvent.bin

@@ -0,0 +1 @@
+0x6080604052348015600e575f80fd5b5060ff8061001b5f395ff3fe6080604052348015600e575f80fd5b50600436106026575f3560e01c8063d836083e14602a575b5f80fd5b60396035366004607c565b603b565b005b8181604051604992919060e3565b604051908190038120907fe00bbfe6f6f8f1bbed2da38e3f5a139c6f9da594ab248a3cf8b44fc73627772c905f90a25050565b5f8060208385031215608c575f80fd5b823567ffffffffffffffff8082111560a2575f80fd5b818501915085601f83011260b4575f80fd5b81358181111560c1575f80fd5b86602082850101111560d1575f80fd5b60209290920196919550909350505050565b818382375f910190815291905056fea164736f6c6343000819000a

op-e2e/interop/contracts/build/emit.sol/EmitEvent.json

@@ -0,0 +1 @@
+{"abi":[{"type":"function","name":"emitData","inputs":[{"name":"_data","type":"bytes","internalType":"bytes"}],"outputs":[],"stateMutability":"nonpayable"},{"type":"event","name":"DataEmitted","inputs":[{"name":"_data","type":"bytes","indexed":true,"internalType":"bytes"}],"anonymous":false}],"bytecode":{"object":"0x6080604052348015600e575f80fd5b5060ff8061001b5f395ff3fe6080604052348015600e575f80fd5b50600436106026575f3560e01c8063d836083e14602a575b5f80fd5b60396035366004607c565b603b565b005b8181604051604992919060e3565b604051908190038120907fe00bbfe6f6f8f1bbed2da38e3f5a139c6f9da594ab248a3cf8b44fc73627772c905f90a25050565b5f8060208385031215608c575f80fd5b823567ffffffffffffffff8082111560a2575f80fd5b818501915085601f83011260b4575f80fd5b81358181111560c1575f80fd5b86602082850101111560d1575f80fd5b60209290920196919550909350505050565b818382375f910190815291905056fea164736f6c6343000819000a","sourceMap":"58:278:0:-:0;;;;;;;;;;;;;;;;;;;","linkReferences":{}},"deployedBytecode":{"object":"0x6080604052348015600e575f80fd5b50600436106026575f3560e01c8063d836083e14602a575b5f80fd5b60396035366004607c565b603b565b005b8181604051604992919060e3565b604051908190038120907fe00bbfe6f6f8f1bbed2da38e3f5a139c6f9da594ab248a3cf8b44fc73627772c905f90a25050565b5f8060208385031215608c575f80fd5b823567ffffffffffffffff8082111560a2575f80fd5b818501915085601f83011260b4575f80fd5b81358181111560c1575f80fd5b86602082850101111560d1575f80fd5b60209290920196919550909350505050565b818382375f910190815291905056fea164736f6c6343000819000a","sourceMap":"58:278:0:-:0;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;245:89;;;;;;:::i;:::-;;:::i;:::-;;;321:5;;309:18;;;;;;;:::i;:::-;;;;;;;;;;;;;;;245:89;;:::o;14:591:1:-;84:6;92;145:2;133:9;124:7;120:23;116:32;113:52;;;161:1;158;151:12;113:52;201:9;188:23;230:18;271:2;263:6;260:14;257:34;;;287:1;284;277:12;257:34;325:6;314:9;310:22;300:32;;370:7;363:4;359:2;355:13;351:27;341:55;;392:1;389;382:12;341:55;432:2;419:16;458:2;450:6;447:14;444:34;;;474:1;471;464:12;444:34;519:7;514:2;505:6;501:2;497:15;493:24;490:37;487:57;;;540:1;537;530:12;487:57;571:2;563:11;;;;;593:6;;-1:-1:-1;14:591:1;;-1:-1:-1;;;;14:591:1:o;610:271::-;793:6;785;780:3;767:33;749:3;819:16;;844:13;;;819:16;610:271;-1:-1:-1;610:271:1:o","linkReferences":{}},"methodIdentifiers":{"emitData(bytes)":"d836083e"},"rawMetadata":"{\"compiler\":{\"version\":\"0.8.25+commit.b61c2a91\"},\"language\":\"Solidity\",\"output\":{\"abi\":[{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"bytes\",\"name\":\"_data\",\"type\":\"bytes\"}],\"name\":\"DataEmitted\",\"type\":\"event\"},{\"inputs\":[{\"internalType\":\"bytes\",\"name\":\"_data\",\"type\":\"bytes\"}],\"name\":\"emitData\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"}],\"devdoc\":{\"kind\":\"dev\",\"methods\":{},\"version\":1},\"userdoc\":{\"kind\":\"user\",\"methods\":{},\"version\":1}},\"settings\":{\"compilationTarget\":{\"src/emit.sol\":\"EmitEvent\"},\"evmVersion\":\"cancun\",\"libraries\":{},\"metadata\":{\"bytecodeHash\":\"none\"},\"optimizer\":{\"enabled\":true,\"runs\":999999},\"remappings\":[]},\"sources\":{\"src/emit.sol\":{\"keccak256\":\"0xe078378fd445ed0cbbf1087c5013110412ab6a44850af24a15bcde945467accc\",\"license\":\"MIT\",\"urls\":[\"bzz-raw://66c218de0059688c75903c2ba6d4066661dc6f5fa17a329dd7c385d151f3993a\",\"dweb:/ipfs/Qmby4S4N44naZ2miw3Tgdpq5Qbj4DwzJbcGapgqtd7qd26\"]}},\"version\":1}","metadata":{"compiler":{"version":"0.8.25+commit.b61c2a91"},"language":"Solidity","output":{"abi":[{"inputs":[{"internalType":"bytes","name":"_data","type":"bytes","indexed":true}],"type":"event","name":"DataEmitted","anonymous":false},{"inputs":[{"internalType":"bytes","name":"_data","type":"bytes"}],"stateMutability":"nonpayable","type":"function","name":"emitData"}],"devdoc":{"kind":"dev","methods":{},"version":1},"userdoc":{"kind":"user","methods":{},"version":1}},"settings":{"remappings":[],"optimizer":{"enabled":true,"runs":999999},"metadata":{"bytecodeHash":"none"},"compilationTarget":{"src/emit.sol":"EmitEvent"},"evmVersion":"cancun","libraries":{}},"sources":{"src/emit.sol":{"keccak256":"0xe078378fd445ed0cbbf1087c5013110412ab6a44850af24a15bcde945467accc","urls":["bzz-raw://66c218de0059688c75903c2ba6d4066661dc6f5fa17a329dd7c385d151f3993a","dweb:/ipfs/Qmby4S4N44naZ2miw3Tgdpq5Qbj4DwzJbcGapgqtd7qd26"],"license":"MIT"}},"version":1},"storageLayout":{"storage":[],"types":{}},"userdoc":{"version":1,"kind":"user"},"devdoc":{"version":1,"kind":"dev"},"ast":{"absolutePath":"src/emit.sol","id":17,"exportedSymbols":{"EmitEvent":[16]},"nodeType":"SourceUnit","src":"32:305:0","nodes":[{"id":1,"nodeType":"PragmaDirective","src":"32:24:0","nodes":[],"literals":["solidity","^","0.8",".15"]},{"id":16,"nodeType":"ContractDefinition","src":"58:278:0","nodes":[{"id":5,"nodeType":"EventDefinition","src":"133:39:0","nodes":[],"anonymous":false,"eventSelector":"e00bbfe6f6f8f1bbed2da38e3f5a139c6f9da594ab248a3cf8b44fc73627772c","name":"DataEmitted","nameLocation":"139:11:0","parameters":{"id":4,"nodeType":"ParameterList","parameters":[{"constant":false,"id":3,"indexed":true,"mutability":"mutable","name":"_data","nameLocation":"165:5:0","nodeType":"VariableDeclaration","scope":5,"src":"151:19:0","stateVariable":false,"storageLocation":"default","typeDescriptions":{"typeIdentifier":"t_bytes_memory_ptr","typeString":"bytes"},"typeName":{"id":2,"name":"bytes","nodeType":"ElementaryTypeName","src":"151:5:0","typeDescriptions":{"typeIdentifier":"t_bytes_storage_ptr","typeString":"bytes"}},"visibility":"internal"}],"src":"150:21:0"}},{"id":15,"nodeType":"FunctionDefinition","src":"245:89:0","nodes":[],"body":{"id":14,"nodeType":"Block","src":"294:40:0","nodes":[],"statements":[{"eventCall":{"arguments":[{"id":11,"name":"_data","nodeType":"Identifier","overloadedDeclarations":[],"referencedDeclaration":7,"src":"321:5:0","typeDescriptions":{"typeIdentifier":"t_bytes_calldata_ptr","typeString":"bytes calldata"}}],"expression":{"argumentTypes":[{"typeIdentifier":"t_bytes_calldata_ptr","typeString":"bytes calldata"}],"id":10,"name":"DataEmitted","nodeType":"Identifier","overloadedDeclarations":[],"referencedDeclaration":5,"src":"309:11:0","typeDescriptions":{"typeIdentifier":"t_function_event_nonpayable$_t_bytes_memory_ptr_$returns$__$","typeString":"function (bytes memory)"}},"id":12,"isConstant":false,"isLValue":false,"isPure":false,"kind":"functionCall","lValueRequested":false,"nameLocations":[],"names":[],"nodeType":"FunctionCall","src":"309:18:0","tryCall":false,"typeDescriptions":{"typeIdentifier":"t_tuple$__$","typeString":"tuple()"}},"id":13,"nodeType":"EmitStatement","src":"304:23:0"}]},"functionSelector":"d836083e","implemented":true,"kind":"function","modifiers":[],"name":"emitData","nameLocation":"254:8:0","parameters":{"id":8,"nodeType":"ParameterList","parameters":[{"constant":false,"id":7,"mutability":"mutable","name":"_data","nameLocation":"278:5:0","nodeType":"VariableDeclaration","scope":15,"src":"263:20:0","stateVariable":false,"storageLocation":"calldata","typeDescriptions":{"typeIdentifier":"t_bytes_calldata_ptr","typeString":"bytes"},"typeName":{"id":6,"name":"bytes","nodeType":"ElementaryTypeName","src":"263:5:0","typeDescriptions":{"typeIdentifier":"t_bytes_storage_ptr","typeString":"bytes"}},"visibility":"internal"}],"src":"262:22:0"},"returnParameters":{"id":9,"nodeType":"ParameterList","parameters":[],"src":"294:0:0"},"scope":16,"stateMutability":"nonpayable","virtual":false,"visibility":"external"}],"abstract":false,"baseContracts":[],"canonicalName":"EmitEvent","contractDependencies":[],"contractKind":"contract","fullyImplemented":true,"linearizedBaseContracts":[16],"name":"EmitEvent","nameLocation":"67:9:0","scope":17,"usedErrors":[],"usedEvents":[5]}],"license":"MIT"},"id":0}

op-e2e/interop/interop_test.go

@@ -2,6 +2,7 @@ package interop
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"math/big"
 	"testing"
@@ -86,14 +87,35 @@ func TestInteropTrivial(t *testing.T) {
 	require.Equal(t, expectedBalance, bobBalance)
 
 	s2.DeployEmitterContract(chainA, "Alice")
-	rec := s2.EmitData(chainA, "Alice", "0x1234567890abcdef")
-
-	fmt.Println("Result of emitting event:", rec)
-
 	s2.DeployEmitterContract(chainB, "Alice")
-	rec = s2.EmitData(chainB, "Alice", "0x1234567890abcdef")
+	for i := 0; i < 5; i++ {
+		s2.EmitData(chainA, "Alice", fmt.Sprintf("0x1234567890abcdef%d", i))
+		s2.EmitData(chainA, "Alice", fmt.Sprintf("0x1234567890abcdef%d", i))
+		s2.EmitData(chainA, "Alice", fmt.Sprintf("0x1234567890abcdef%d", i))
+		s2.EmitData(chainA, "Alice", fmt.Sprintf("0x1234567890abcdef%d", i))
+		s2.EmitData(chainA, "Alice", fmt.Sprintf("0x1234567890abcdef%d", i))
+
+		s2.EmitData(chainB, "Alice", fmt.Sprintf("0x1234567890abcdef%d", i))
+		s2.EmitData(chainB, "Alice", fmt.Sprintf("0x1234567890abcdef%d", i))
+		s2.EmitData(chainB, "Alice", fmt.Sprintf("0x1234567890abcdef%d", i))
+		s2.EmitData(chainB, "Alice", fmt.Sprintf("0x1234567890abcdef%d", i))
+		s2.EmitData(chainB, "Alice", fmt.Sprintf("0x1234567890abcdef%d", i))
+	}
 
-	fmt.Println("Result of emitting event:", rec)
+	{
+		rec := s2.EmitData(chainA, "Alice", "0x1234567890abcdef")
+		out, err := json.MarshalIndent(rec, "  ", "  ")
+		require.NoError(t, err)
+		t.Log("Result on chain A:")
+		t.Log(string(out))
+	}
+	{
+		rec := s2.EmitData(chainB, "Alice", "0x1234567890abcdef")
+		out, err := json.MarshalIndent(rec, "  ", "  ")
+		require.NoError(t, err)
+		t.Log("Result on chain B:")
+		t.Log(string(out))
+	}
 
 	time.Sleep(10 * time.Second)
 

op-supervisor/supervisor/backend/db/db.go

@@ -30,6 +30,7 @@ type LogStorage interface {
 	Contains(blockNum uint64, logIdx uint32, loghash backendTypes.TruncatedHash) (bool, entrydb.EntryIdx, error)
 	LastCheckpointBehind(entrydb.EntryIdx) (logs.Iterator, error)
 	NextExecutingMessage(logs.Iterator) (backendTypes.ExecutingMessage, error)
+	LastEntryIdx() entrydb.EntryIdx
 }
 
 type HeadsStorage interface {
@@ -159,18 +160,22 @@ func (db *ChainsDB) UpdateCrossHeadsForChain(chainID types.ChainID, checker Safe
 	// - if an error occurs
 	for {
 		exec, err := db.logDBs[chainID].NextExecutingMessage(i)
+		// if we have reached the end of the database,
+		// the x-head is at most the end of the database
 		if err == io.EOF {
-			break
+			xHead = db.LastEntryIdx(chainID)
 		} else if err != nil {
 			return fmt.Errorf("failed to read next executing message for chain %v: %w", chainID, err)
 		}
-		// if we are now beyond the local head, stop
-		if i.Index() > localHead {
+		// if we are now beyond the local head,
+		// trim the x-head to the local head and stop advancing
+		if i.Index() >= localHead {
+			xHead = localHead
 			break
 		}
 		// use the checker to determine if this message is safe
 		safe := checker.Check(
-			types.ChainIDFromUInt64(uint64(exec.Chain)),
+			chainID,
 			exec.BlockNum,
 			exec.LogIdx,
 			exec.Hash)
@@ -197,6 +202,12 @@ func (db *ChainsDB) UpdateCrossHeadsForChain(chainID types.ChainID, checker Safe
 	return nil
 }
 
+func (db *ChainsDB) Apply(op heads.OperationFn) error {
+	err := db.heads.Apply(op)
+	fmt.Println("XLXL Latest Chain Heads: ", db.heads.Current())
+	return err
+}
+
 // UpdateCrossHeads updates the cross-heads of all chains
 // based on the provided SafetyChecker. The SafetyChecker is used to determine
 // the safety of each log entry in the database, and the cross-head associated with it.
@@ -210,6 +221,10 @@ func (db *ChainsDB) UpdateCrossHeads(checker SafetyChecker) error {
 	return nil
 }
 
+func (db *ChainsDB) LastEntryIdx(chain types.ChainID) entrydb.EntryIdx {
+	return db.logDBs[chain].LastEntryIdx()
+}
+
 // LastLogInBlock scans through the logs of the given chain starting from the given block number,
 // and returns the index of the last log entry in that block.
 func (db *ChainsDB) LastLogInBlock(chain types.ChainID, blockNum uint64) (entrydb.EntryIdx, error) {
@@ -224,7 +239,7 @@ func (db *ChainsDB) LastLogInBlock(chain types.ChainID, blockNum uint64) (entryd
 	ret := entrydb.EntryIdx(0)
 	// scan through using the iterator until the block number exceeds the target
 	for {
-		bn, index, _, err := iter.NextLog()
+		bn, _, _, err := iter.NextLog()
 		// if we have reached the end of the database, stop
 		if err == io.EOF {
 			break
@@ -235,12 +250,21 @@ func (db *ChainsDB) LastLogInBlock(chain types.ChainID, blockNum uint64) (entryd
 		}
 		// if we are now beyond the target block, stop withour updating the return value
 		if bn > blockNum {
+			// if we jumped over the target block without seeing it, use the first available index
+			// Note: this makes the assumption that local safety is implied to be the *next* block
+			// if no logs are present in the target block. This is like tracking "log finality"
+			// and it might indicate final blocks which are beyond the standard definition
+			if ret == 0 {
+				// using the iter.Index() instead of the potential return index from NextLog
+				// because the NextLog index tends to be 0 unexpectedly. Will need to investigate that
+				ret = iter.Index()
+			}
 			break
 		}
 		// only update the return value if the block number is the same
 		// it is possible the iterator started before the target block, or that the target block is not in the db
 		if bn == blockNum {
-			ret = entrydb.EntryIdx(index)
+			ret = iter.Index()
 		}
 	}
 	// if we never found the block, return an error

op-supervisor/supervisor/backend/db/db_test.go

@@ -425,6 +425,10 @@ func (s *stubLogDB) LastCheckpointBehind(entrydb.EntryIdx) (logs.Iterator, error
 	return s.lastCheckpointBehind, nil
 }
 
+func (s *stubLogDB) LastEntryIdx() entrydb.EntryIdx {
+	return 0
+}
+
 func (s *stubLogDB) ClosestBlockIterator(blockNum uint64) (logs.Iterator, error) {
 	return &stubIterator{
 		index:    entrydb.EntryIdx(99),

op-supervisor/supervisor/backend/db/init_test.go

@@ -62,6 +62,10 @@ func (s *stubLogStore) LastCheckpointBehind(entrydb.EntryIdx) (logs.Iterator, er
 	panic("not supported")
 }
 
+func (s *stubLogStore) LastEntryIdx() entrydb.EntryIdx {
+	panic("not supported")
+}
+
 func (s *stubLogStore) ClosestBlockInfo(blockNum uint64) (uint64, types.TruncatedHash, error) {
 	if s.closestBlockErr != nil {
 		return 0, types.TruncatedHash{}, s.closestBlockErr

op-supervisor/supervisor/backend/db/logs/db.go

@@ -14,7 +14,7 @@ import (
 )
 
 const (
-	searchCheckpointFrequency = 256
+	searchCheckpointFrequency = 4
 
 	eventFlagIncrementLogIdx     = byte(1)
 	eventFlagHasExecutingMessage = byte(1) << 1
@@ -111,7 +111,7 @@ func NewFromEntryStore(logger log.Logger, m Metrics, store EntryStore) (*DB, err
 	return db, nil
 }
 
-func (db *DB) lastEntryIdx() entrydb.EntryIdx {
+func (db *DB) LastEntryIdx() entrydb.EntryIdx {
 	return db.store.LastEntryIdx()
 }
 
@@ -120,12 +120,12 @@ func (db *DB) init() error {
 	if err := db.trimInvalidTrailingEntries(); err != nil {
 		return fmt.Errorf("failed to trim invalid trailing entries: %w", err)
 	}
-	if db.lastEntryIdx() < 0 {
+	if db.LastEntryIdx() < 0 {
 		// Database is empty so no context to load
 		return nil
 	}
 
-	lastCheckpoint := (db.lastEntryIdx() / searchCheckpointFrequency) * searchCheckpointFrequency
+	lastCheckpoint := (db.LastEntryIdx() / searchCheckpointFrequency) * searchCheckpointFrequency
 	i, err := db.newIterator(lastCheckpoint)
 	if err != nil {
 		return fmt.Errorf("failed to create iterator at last search checkpoint: %w", err)
@@ -144,7 +144,7 @@ func (db *DB) init() error {
 }
 
 func (db *DB) trimInvalidTrailingEntries() error {
-	i := db.lastEntryIdx()
+	i := db.LastEntryIdx()
 	for ; i >= 0; i-- {
 		entry, err := db.store.Read(i)
 		if err != nil {
@@ -166,8 +166,8 @@ func (db *DB) trimInvalidTrailingEntries() error {
 			}
 		}
 	}
-	if i < db.lastEntryIdx() {
-		db.log.Warn("Truncating unexpected trailing entries", "prev", db.lastEntryIdx(), "new", i)
+	if i < db.LastEntryIdx() {
+		db.log.Warn("Truncating unexpected trailing entries", "prev", db.LastEntryIdx(), "new", i)
 		return db.store.Truncate(i)
 	}
 	return nil
@@ -208,7 +208,7 @@ func (db *DB) ClosestBlockIterator(blockNum uint64) (Iterator, error) {
 	db.rwLock.RLock()
 	defer db.rwLock.RUnlock()
 	checkpointIdx, err := db.searchCheckpoint(blockNum, math.MaxUint32)
-	if err != nil {
+	if err != nil && err != io.EOF {
 		return nil, fmt.Errorf("no checkpoint at or before block %v found: %w", blockNum, err)
 	}
 	return db.newIterator(checkpointIdx)
@@ -353,7 +353,7 @@ func (db *DB) newIterator(startCheckpointEntry entrydb.EntryIdx) (*iterator, err
 // the requested log.
 // Returns the index of the searchCheckpoint to begin reading from or an error
 func (db *DB) searchCheckpoint(blockNum uint64, logIdx uint32) (entrydb.EntryIdx, error) {
-	n := (db.lastEntryIdx() / searchCheckpointFrequency) + 1
+	n := (db.LastEntryIdx() / searchCheckpointFrequency) + 1
 	// Define x[-1] < target and x[n] >= target.
 	// Invariant: x[i-1] < target, x[j] >= target.
 	i, j := entrydb.EntryIdx(0), n
@@ -409,7 +409,7 @@ func (db *DB) AddLog(logHash types.TruncatedHash, block eth.BlockID, timestamp u
 	}
 	var entriesToAdd []entrydb.Entry
 	newContext := db.lastEntryContext
-	lastEntryIdx := db.lastEntryIdx()
+	lastEntryIdx := db.LastEntryIdx()
 
 	addEntry := func(entry entrydb.Entry) {
 		entriesToAdd = append(entriesToAdd, entry)

op-supervisor/supervisor/backend/db/logs/iterator.go

@@ -27,8 +27,9 @@ type iterator struct {
 
 // NextLog returns the next log in the iterator.
 // It scans forward until it finds an initiating event, returning the block number, log index, and event hash.
+// TODO(#12004): the returned index seems to be 0, which is unexpected
 func (i *iterator) NextLog() (blockNum uint64, logIdx uint32, evtHash types.TruncatedHash, outErr error) {
-	for i.nextEntryIdx <= i.db.lastEntryIdx() {
+	for i.nextEntryIdx <= i.db.LastEntryIdx() {
 		entryIdx := i.nextEntryIdx
 		entry, err := i.db.store.Read(entryIdx)
 		if err != nil {