From 0ec372c2af4a84a7e2e3a9b4018ec955d82f4dee Mon Sep 17 00:00:00 2001 From: Niven Date: Tue, 30 Jul 2024 15:36:12 +0800 Subject: [PATCH] Add nubit integration fixes --- cmd/run.go | 14 +- config/config.go | 3 + config/default.go | 7 + dataavailability/config.go | 3 +- dataavailability/dataavailability.go | 6 +- .../datacommittee/datacommittee.go | 10 +- dataavailability/interfaces.go | 2 +- dataavailability/nubit/abi.go | 28 ++ dataavailability/nubit/backend.go | 244 +++++++----------- dataavailability/nubit/backend_test.go | 140 ++++++++++ dataavailability/nubit/blob.go | 103 ++++++++ dataavailability/nubit/blob_test.go | 35 +++ dataavailability/nubit/chain.go | 42 --- dataavailability/nubit/config.go | 42 +-- dataavailability/nubit/encoding.go | 75 ++++++ dataavailability/nubit/encoding_test.go | 85 ++++++ dataavailability/nubit/nubit_test.go | 36 --- dataavailability/nubit/types.go | 22 -- sequencesender/interfaces.go | 2 +- sequencesender/sequencesender.go | 4 +- 20 files changed, 591 insertions(+), 312 deletions(-) create mode 100644 dataavailability/nubit/abi.go create mode 100644 dataavailability/nubit/backend_test.go create mode 100644 dataavailability/nubit/blob.go create mode 100644 dataavailability/nubit/blob_test.go delete mode 100644 dataavailability/nubit/chain.go create mode 100644 dataavailability/nubit/encoding.go create mode 100644 dataavailability/nubit/encoding_test.go delete mode 100644 dataavailability/nubit/nubit_test.go delete mode 100644 dataavailability/nubit/types.go diff --git a/cmd/run.go b/cmd/run.go index 9209b662e4..8f15f40acb 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -328,7 +328,6 @@ func newDataAvailability(c config.Config, st *state.State, etherman *etherman.Cl return nil, fmt.Errorf("error getting data availability protocol name: %v", err) } var daBackend dataavailability.DABackender - daProtocolName = string(dataavailability.Nubit) switch daProtocolName { case string(dataavailability.DataAvailabilityCommittee): var ( @@ -355,7 +354,7 @@ func newDataAvailability(c config.Config, st *state.State, etherman *etherman.Cl if err != nil { return nil, err } - case string(dataavailability.Nubit): + case string(dataavailability.DataAvailabilityNubitDA): var ( pk *ecdsa.PrivateKey err error @@ -367,16 +366,7 @@ func newDataAvailability(c config.Config, st *state.State, etherman *etherman.Cl } } - dacAddr, err := etherman.GetDAProtocolAddr() - if err != nil { - return nil, fmt.Errorf("error getting trusted sequencer URI. Error: %v", err) - } - - daBackend, err = nubit.NewNubitDABackend( - c.Etherman.URL, - dacAddr, - pk, - ) + daBackend, err = nubit.NewNubitDABackend(&c.DataAvailability, pk) if err != nil { return nil, err } diff --git a/config/config.go b/config/config.go index cb329b6a0c..964f77d98c 100644 --- a/config/config.go +++ b/config/config.go @@ -9,6 +9,7 @@ import ( "github.com/0xPolygonHermez/zkevm-node/aggregator" "github.com/0xPolygonHermez/zkevm-node/config/types" + "github.com/0xPolygonHermez/zkevm-node/dataavailability/nubit" "github.com/0xPolygonHermez/zkevm-node/db" "github.com/0xPolygonHermez/zkevm-node/etherman" "github.com/0xPolygonHermez/zkevm-node/ethtxmanager" @@ -105,6 +106,8 @@ type Config struct { SequenceSender sequencesender.Config // Configuration of the aggregator service Aggregator aggregator.Config + // Configuration of the NubitDA data availability service + DataAvailability nubit.Config // Configuration of the genesis of the network. This is used to known the initial state of the network NetworkConfig NetworkConfig // Configuration of the gas price suggester service diff --git a/config/default.go b/config/default.go index 9b7ec56291..5d1623046c 100644 --- a/config/default.go +++ b/config/default.go @@ -181,6 +181,13 @@ AggLayerTxTimeout = "5m" AggLayerURL = "http://zkevm-agglayer" SequencerPrivateKey = {Path = "/pk/sequencer.keystore", Password = "testonly"} +[DataAvailability] +NubitRpcURL = "http://127.0.0.1:26658" +NubitAuthKey = "" +NubitNamespace = "xlayer" +NubitGetProofMaxRetry = "10" +NubitGetProofWaitPeriod = "5s" + [L2GasPriceSuggester] Type = "follower" UpdatePeriod = "10s" diff --git a/dataavailability/config.go b/dataavailability/config.go index 0d2cd5e8fb..b0f6a0c85f 100644 --- a/dataavailability/config.go +++ b/dataavailability/config.go @@ -6,5 +6,6 @@ type DABackendType string const ( // DataAvailabilityCommittee is the DAC protocol backend DataAvailabilityCommittee DABackendType = "DataAvailabilityCommittee" - Nubit DABackendType = "Nubit" + // DataAvailabilityNubitDA is the NubitDA protocol backend + DataAvailabilityNubitDA DABackendType = "NubitDA" ) diff --git a/dataavailability/dataavailability.go b/dataavailability/dataavailability.go index 1ddd564a47..489c4c4a3d 100644 --- a/dataavailability/dataavailability.go +++ b/dataavailability/dataavailability.go @@ -44,7 +44,7 @@ func New( // PostSequence sends the sequence data to the data availability backend, and returns the dataAvailabilityMessage // as expected by the contract -func (d *DataAvailability) PostSequence(ctx context.Context, sequences []types.Sequence) ([]byte, []byte, error) { +func (d *DataAvailability) PostSequence(ctx context.Context, sequences []types.Sequence) ([]byte, error) { batchesData := [][]byte{} for _, batch := range sequences { // Do not send to the DA backend data that will be stored to L1 @@ -113,7 +113,7 @@ func (d *DataAvailability) trustedSequencerData(batchNums []uint64, expectedHash return nil, fmt.Errorf("invalid arguments, len of batch numbers does not equal length of expected hashes: %d != %d", len(batchNums), len(expectedHashes)) } - nums := make([]*big.Int, 0, len(batchNums)) + var nums []*big.Int for _, n := range batchNums { nums = append(nums, new(big.Int).SetUint64(n)) } @@ -124,7 +124,7 @@ func (d *DataAvailability) trustedSequencerData(batchNums []uint64, expectedHash if len(batchData) != len(batchNums) { return nil, fmt.Errorf("missing batch data, expected %d, got %d", len(batchNums), len(batchData)) } - result := make([][]byte, 0, len(batchNums)) + var result [][]byte for i := 0; i < len(batchNums); i++ { number := batchNums[i] batch := batchData[i] diff --git a/dataavailability/datacommittee/datacommittee.go b/dataavailability/datacommittee/datacommittee.go index 966ddca473..c79dfb645c 100644 --- a/dataavailability/datacommittee/datacommittee.go +++ b/dataavailability/datacommittee/datacommittee.go @@ -152,11 +152,11 @@ type signatureMsg struct { // PostSequence sends the sequence data to the data availability backend, and returns the dataAvailabilityMessage // as expected by the contract -func (s *DataCommitteeBackend) PostSequence(ctx context.Context, batchesData [][]byte) ([]byte, []byte, error) { +func (s *DataCommitteeBackend) PostSequence(ctx context.Context, batchesData [][]byte) ([]byte, error) { // Get current committee committee, err := s.getCurrentDataCommittee() if err != nil { - return nil, nil, err + return nil, err } // Authenticate as trusted sequencer by signing the sequences @@ -166,7 +166,7 @@ func (s *DataCommitteeBackend) PostSequence(ctx context.Context, batchesData [][ } signedSequence, err := sequence.Sign(s.privKey) if err != nil { - return nil, nil, err + return nil, err } // Request signatures to all members in parallel @@ -189,7 +189,7 @@ func (s *DataCommitteeBackend) PostSequence(ctx context.Context, batchesData [][ failedToCollect++ if len(committee.Members)-int(failedToCollect) < int(committee.RequiredSignatures) { cancelSignatureCollection() - return nil, nil, errors.New("too many members failed to send their signature") + return nil, errors.New("too many members failed to send their signature") } } else { log.Infof("received signature from %s", msg.addr) @@ -201,7 +201,7 @@ func (s *DataCommitteeBackend) PostSequence(ctx context.Context, batchesData [][ // Stop requesting as soon as we have N valid signatures cancelSignatureCollection() - return buildSignaturesAndAddrs(signatureMsgs(msgs), committee.Members), nil, nil + return buildSignaturesAndAddrs(signatureMsgs(msgs), committee.Members), nil } func requestSignatureFromMember(ctx context.Context, signedSequence daTypes.SignedSequence, member DataCommitteeMember, ch chan signatureMsg) { diff --git a/dataavailability/interfaces.go b/dataavailability/interfaces.go index b08e42c494..d45b41b358 100644 --- a/dataavailability/interfaces.go +++ b/dataavailability/interfaces.go @@ -22,7 +22,7 @@ type DABackender interface { type SequenceSender interface { // PostSequence sends the sequence data to the data availability backend, and returns the dataAvailabilityMessage // as expected by the contract - PostSequence(ctx context.Context, batchesData [][]byte) ([]byte, []byte, error) + PostSequence(ctx context.Context, batchesData [][]byte) ([]byte, error) } // SequenceRetriever is used to retrieve batch data diff --git a/dataavailability/nubit/abi.go b/dataavailability/nubit/abi.go new file mode 100644 index 0000000000..e0e6f0db61 --- /dev/null +++ b/dataavailability/nubit/abi.go @@ -0,0 +1,28 @@ +package nubit + +const blobDataABI = `[ + { + "type": "function", + "name": "BlobData", + "inputs": [ + { + "name": "blobData", + "type": "tuple", + "internalType": "struct NubitDAVerifier.BlobData", + "components": [ + { + "name": "blobID", + "type": "bytes", + "internalType": "bytes" + }, + { + "name": "signature", + "type": "bytes", + "internalType": "bytes" + } + ] + } + ], + "stateMutability": "pure" + } +]` diff --git a/dataavailability/nubit/backend.go b/dataavailability/nubit/backend.go index e846f29f26..8321af9aa1 100644 --- a/dataavailability/nubit/backend.go +++ b/dataavailability/nubit/backend.go @@ -4,209 +4,139 @@ import ( "context" "crypto/ecdsa" "encoding/hex" - "fmt" + "strings" + "time" + daTypes "github.com/0xPolygon/cdk-data-availability/types" - "github.com/0xPolygonHermez/zkevm-node/etherman/smartcontracts/polygondatacommittee" "github.com/0xPolygonHermez/zkevm-node/log" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/ethclient" "github.com/rollkit/go-da" "github.com/rollkit/go-da/proxy" - "time" ) -// // DABackender is an interface for components that store and retrieve batch data -// type DABackender interface { -// SequenceRetriever -// SequenceSender -// // Init initializes the DABackend -// Init() error -// } - -// // SequenceSender is used to send provided sequence of batches -// type SequenceSender interface { -// // PostSequence sends the sequence data to the data availability backend, and returns the dataAvailabilityMessage -// // as expected by the contract -// PostSequence(ctx context.Context, batchesData [][]byte) ([]byte, error) -// } - -// // SequenceRetriever is used to retrieve batch data -// type SequenceRetriever interface { -// // GetSequence retrieves the sequence data from the data availability backend -// GetSequence(ctx context.Context, batchHashes []common.Hash, dataAvailabilityMessage []byte) ([][]byte, error) -// } - +// NubitDABackend implements the DA integration with Nubit DA layer type NubitDABackend struct { - commitTime time.Time - config *Config - attestationContract *polygondatacommittee.Polygondatacommittee - ns da.Namespace - privKey *ecdsa.PrivateKey - client da.DA - dataCommitteeContract *polygondatacommittee.Polygondatacommittee + client da.DA + config *Config + namespace da.Namespace + privKey *ecdsa.PrivateKey + commitTime time.Time } -func NewNubitDABackend(l1RPCURL string, dataCommitteeAddr common.Address, privKey *ecdsa.PrivateKey) (*NubitDABackend, error) { - var config Config - err := config.GetConfig("/app/nubit-config.json") - if err != nil { - log.Fatalf("cannot get config:%w", err) - } - - ethClient, err := ethclient.Dial(l1RPCURL) +// NewNubitDABackend is the factory method to create a new instance of NubitDABackend +func NewNubitDABackend( + cfg *Config, + privKey *ecdsa.PrivateKey, +) (*NubitDABackend, error) { + log.Infof("NubitDABackend config: %#v ", cfg) + cn, err := proxy.NewClient(cfg.NubitRpcURL, cfg.NubitAuthKey) if err != nil { - log.Errorf("error connecting to %s: %+v", l1RPCURL, err) return nil, err } - log.Infof("⚙️ Nubit config : %#v ", config) - - attestationContract, err := polygondatacommittee.NewPolygondatacommittee(dataCommitteeAddr, ethClient) + hexStr := hex.EncodeToString([]byte(cfg.NubitNamespace)) + name, err := hex.DecodeString(strings.Repeat("0", NubitNamespaceBytesLength-len(hexStr)) + hexStr) if err != nil { + log.Errorf("error decoding NubitDA namespace config: %+v", err) return nil, err } - cn, err := proxy.NewClient(config.RpcURL, config.AuthKey) if err != nil { return nil, err } - name, err := hex.DecodeString("00000000000000000000000000000000000000000000706f6c79676f6e") - dataCommittee, err := polygondatacommittee.NewPolygondatacommittee(dataCommitteeAddr, ethClient) + log.Infof("NubitDABackend namespace: %s ", string(name)) - log.Infof("⚙️ Nubit Namespace : %s ", string(name)) return &NubitDABackend{ - dataCommitteeContract: dataCommittee, - config: &config, - attestationContract: attestationContract, - privKey: privKey, - ns: name, - client: cn, - commitTime: time.Now(), - }, nil -} - -func NewNubitDABackendTest(url string, authKey string, pk *ecdsa.PrivateKey) (*NubitDABackend, error) { - cn, err := proxy.NewClient(url, authKey) - if err != nil || cn == nil { - return nil, err - } - - name, err := hex.DecodeString("00000000000000000000000000000000000000000000706f6c79676f6e") - if err != nil { - return nil, err - } - - log.Infof("⚙️ Nubit Namespace : %s ", string(name)) - return &NubitDABackend{ - ns: name, + config: cfg, + privKey: privKey, + namespace: name, client: cn, - privKey: pk, commitTime: time.Now(), }, nil } -func (a *NubitDABackend) Init() error { +// Init initializes the NubitDA backend +func (backend *NubitDABackend) Init() error { return nil } // PostSequence sends the sequence data to the data availability backend, and returns the dataAvailabilityMessage // as expected by the contract - -var BatchsDataCache = [][]byte{} -var BatchsSize = 0 - -func (a *NubitDABackend) PostSequence(ctx context.Context, batchesData [][]byte) ([]byte, []byte, error) { - encodedData, err := MarshalBatchData(batchesData) - if err != nil { - log.Errorf("🏆 NubitDABackend.MarshalBatchData:%s", err) - return encodedData, nil, err - } - - BatchsDataCache = append(BatchsDataCache, encodedData) - BatchsSize += len(encodedData) - if BatchsSize < 100*1024 { - log.Infof("🏆 Nubit BatchsDataCache:%+v", len(encodedData)) - return nil, nil, nil - } - if time.Since(a.commitTime) < 12*time.Second { - time.Sleep(time.Since(a.commitTime)) - } - - BatchsData, err := MarshalBatchData(BatchsDataCache) - if err != nil { - log.Errorf("🏆 NubitDABackend.MarshalBatchData:%s", err) - return nil, nil, err - } - id, err := a.client.Submit(ctx, [][]byte{BatchsData}, -1, a.ns) - if err != nil { - log.Errorf("🏆 NubitDABackend.Submit:%s", err) - return nil, nil, err +func (backend *NubitDABackend) PostSequence(ctx context.Context, batchesData [][]byte) ([]byte, error) { + // Check commit time interval validation + lastCommitTime := time.Since(backend.commitTime) + if lastCommitTime < NubitMinCommitTime { + time.Sleep(NubitMinCommitTime - lastCommitTime) + } + + // Encode NubitDA blob data + data := EncodeSequence(batchesData) + ids, err := backend.client.Submit(ctx, [][]byte{data}, -1, backend.namespace) + // Ensure only a single blob ID returned + if err != nil || len(ids) != 1 { + log.Errorf("Submit batch data with NubitDA client failed: %s", err) + return nil, err } - - log.Infof("🏆 Nubit Data submitted by sequencer: %d bytes against namespace %v sent with id %#x", len(BatchsDataCache), a.ns, id) - a.commitTime = time.Now() - BatchsDataCache = [][]byte{} - BatchsSize = 0 - // todo: May be need to sleep - //dataProof, err := a.client.Blob.GetProof(ctx, uint64(blockNumber), a.ns.Bytes(), body.Commitment) - //if err != nil { - // log.Errorf("🏆 NubitDABackend.GetProof:%s", err) - // return nil, err - //} - // - //log.Infof("🏆 Nubit received data proof:%+v", dataProof) - - var batchDAData BatchDAData - batchDAData.ID = id - log.Infof("🏆 Nubit prepared DA data:%+v", batchDAData) - - // todo: use bridge API data - returnData, err := batchDAData.Encode() - if err != nil { - return nil, nil, fmt.Errorf("🏆 Nubit cannot encode batch data:%w", err) + blobID := ids[0] + backend.commitTime = time.Now() + log.Infof("Data submitted to Nubit DA: %d bytes against namespace %v sent with id %#x", len(data), backend.namespace, blobID) + + // Get proof of batches data on NubitDA layer + tries := uint64(0) + posted := false + for tries < backend.config.NubitGetProofMaxRetry { + dataProof, err := backend.client.GetProofs(ctx, [][]byte{blobID}, backend.namespace) + if err != nil { + log.Infof("Proof not available: %s", err) + } + if len(dataProof) == 1 { + // TODO: add data proof to DA message + log.Infof("Data proof from Nubit DA received: %+v", dataProof) + posted = true + break + } + + // Retries + tries += 1 + time.Sleep(backend.config.NubitGetProofWaitPeriod) + } + if !posted { + log.Errorf("Get blob proof on Nubit DA failed: %s", err) + return nil, err } + // Get abi-encoded data availability message sequence := daTypes.Sequence{} for _, seq := range batchesData { sequence = append(sequence, seq) } - signedSequence, err := sequence.Sign(a.privKey) //todo - sequence.HashToSign() - - Signature := append(sequence.HashToSign(), signedSequence.Signature...) + signedSequence, err := sequence.Sign(backend.privKey) + if err != nil { + log.Errorf("Failed to sign sequence with pk: %v", err) + return nil, err + } + signature := append(sequence.HashToSign(), signedSequence.Signature...) + blobData := BlobData{ + BlobID: blobID, + Signature: signature, + } - return returnData, Signature, nil + return TryEncodeToDataAvailabilityMessage(blobData) } -func (a *NubitDABackend) GetSequence(ctx context.Context, batchHashes []common.Hash, dataAvailabilityMessage []byte) ([][]byte, error) { - batchDAData := BatchDAData{} - err := batchDAData.Decode(dataAvailabilityMessage) +// GetSequence gets the sequence data from NubitDA layer +func (backend *NubitDABackend) GetSequence(ctx context.Context, batchHashes []common.Hash, dataAvailabilityMessage []byte) ([][]byte, error) { + blobData, err := TryDecodeFromDataAvailabilityMessage(dataAvailabilityMessage) if err != nil { - log.Errorf("🏆 NubitDABackend.GetSequence.Decode:%s", err) + log.Error("Error decoding from da message: ", err) return nil, err } - log.Infof("🏆 Nubit GetSequence batchDAData:%+v", batchDAData) - blob, err := a.client.Get(context.TODO(), batchDAData.ID, a.ns) - if err != nil { - log.Errorf("🏆 NubitDABackend.GetSequence.Blob.Get:%s", err) + + reply, err := backend.client.Get(ctx, [][]byte{blobData.BlobID}, backend.namespace) + if err != nil || len(reply) != 1 { + log.Error("Error retrieving blob from NubitDA client: ", err) return nil, err } - log.Infof("🏆 Nubit GetSequence blob.data:%+v", len(blob)) - byteBlob := make([][]byte, len(blob)) - for _, b := range blob { - byteBlob = append(byteBlob, b) - } - return byteBlob, nil -} - -// DataCommitteeMember represents a member of the Data Committee -type DataCommitteeMember struct { - Addr common.Address - URL string -} -// DataCommittee represents a specific committee -type DataCommittee struct { - AddressesHash common.Hash - Members []DataCommitteeMember - RequiredSignatures uint64 + batchesData, _ := DecodeSequence(reply[0]) + return batchesData, nil } diff --git a/dataavailability/nubit/backend_test.go b/dataavailability/nubit/backend_test.go new file mode 100644 index 0000000000..121743c840 --- /dev/null +++ b/dataavailability/nubit/backend_test.go @@ -0,0 +1,140 @@ +package nubit + +import ( + "context" + "crypto/ecdsa" + "crypto/elliptic" + crand "crypto/rand" + "encoding/hex" + "fmt" + "math/rand" + "testing" + "time" + + "github.com/0xPolygonHermez/zkevm-node/log" + "github.com/ethereum/go-ethereum/common" + "github.com/rollkit/go-da/proxy" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestOffchainPipeline(t *testing.T) { + cfg := Config{ + NubitRpcURL: "http://127.0.0.1:26658", + NubitAuthKey: "", + NubitNamespace: "xlayer", + NubitGetProofMaxRetry: 10, + NubitGetProofWaitPeriod: 5 * time.Second, + } + pk, err := ecdsa.GenerateKey(elliptic.P256(), crand.Reader) + require.NoError(t, err) + + backend, err := NewNubitDABackend(&cfg, pk) + require.NoError(t, err) + + // Generate mock string batch data + stringData := "hihihihihihihihihihihihihihihihihihi" + data := []byte(stringData) + + // Generate mock string sequence + mockBatches := [][]byte{} + for i := 0; i < 1; i++ { + mockBatches = append(mockBatches, data) + } + + msg, err := backend.PostSequence(context.Background(), mockBatches) + fmt.Println("DA msg: ", msg) + require.NoError(t, err) + time.Sleep(600 * time.Millisecond) + + blobData, err := TryDecodeFromDataAvailabilityMessage(msg) + require.NoError(t, err) + require.NotNil(t, blobData.BlobID) + require.NotNil(t, blobData.Signature) + require.NotZero(t, len(blobData.BlobID)) + require.NotZero(t, len(blobData.Signature)) + fmt.Println("Decoding DA msg successful") + + // Retrieve sequence with provider + returnData, err := backend.GetSequence(context.Background(), []common.Hash{}, msg) + + // Validate retrieved data + require.NoError(t, err) + require.Equal(t, 10, len(returnData)) + for _, batchData := range returnData { + assert.Equal(t, stringData, string(batchData)) + } +} + +func TestOffchainPipelineWithRandomData(t *testing.T) { + cfg := Config{ + NubitRpcURL: "http://127.0.0.1:26658", + NubitAuthKey: "", + NubitNamespace: "xlayer", + NubitGetProofMaxRetry: 10, + NubitGetProofWaitPeriod: 5 * time.Second, + } + pk, err := ecdsa.GenerateKey(elliptic.P256(), crand.Reader) + require.NoError(t, err) + + backend, err := NewNubitDABackend(&cfg, pk) + require.NoError(t, err) + + // Define Different DataSizes + dataSize := []int{100000, 200000, 1000, 80, 30000} + + // Disperse Blob with different DataSizes + rand.Seed(time.Now().UnixNano()) //nolint:gosec,staticcheck + data := make([]byte, dataSize[rand.Intn(len(dataSize))]) //nolint:gosec,staticcheck + _, err = rand.Read(data) //nolint:gosec,staticcheck + assert.NoError(t, err) + + // Generate mock string sequence + mockBatches := [][]byte{} + for i := 0; i < 10; i++ { + mockBatches = append(mockBatches, data) + } + + msg, err := backend.PostSequence(context.Background(), mockBatches) + fmt.Println("DA msg: ", msg) + require.NoError(t, err) + time.Sleep(600 * time.Millisecond) + + blobData, err := TryDecodeFromDataAvailabilityMessage(msg) + require.NoError(t, err) + require.NotNil(t, blobData.BlobID) + require.NotNil(t, blobData.Signature) + require.NotZero(t, len(blobData.BlobID)) + require.NotZero(t, len(blobData.Signature)) + fmt.Println("Decoding DA msg successful") + + // Retrieve sequence with provider + returnData, err := backend.GetSequence(context.Background(), []common.Hash{}, msg) + + // Validate retrieved data + require.NoError(t, err) + require.Equal(t, 10, len(returnData)) + for idx, batchData := range returnData { + assert.Equal(t, mockBatches[idx], batchData) + } +} + +func NewMockNubitDABackend(url string, authKey string, pk *ecdsa.PrivateKey) (*NubitDABackend, error) { + cn, err := proxy.NewClient(url, authKey) + if err != nil || cn == nil { + return nil, err + } + + name, err := hex.DecodeString("xlayer") + if err != nil { + return nil, err + } + + log.Infof("Nubit Namespace: %s ", string(name)) + return &NubitDABackend{ + namespace: name, + client: cn, + privKey: pk, + commitTime: time.Now(), + }, nil +} diff --git a/dataavailability/nubit/blob.go b/dataavailability/nubit/blob.go new file mode 100644 index 0000000000..19a74ac19d --- /dev/null +++ b/dataavailability/nubit/blob.go @@ -0,0 +1,103 @@ +package nubit + +import ( + "bytes" + "errors" + "fmt" + "reflect" + + "github.com/ethereum/go-ethereum/accounts/abi" +) + +// ErrConvertFromABIInterface is used when there is a decoding error +var ErrConvertFromABIInterface = errors.New("conversion from abi interface error") + +// BlobData is the NubitDA blob data +type BlobData struct { + BlobID []byte `abi:"blobID"` + Signature []byte `abi:"signature"` +} + +// TryEncodeToDataAvailabilityMessage is a fallible encoding method to encode +// Nubit blob data into data availability message represented as byte array. +func TryEncodeToDataAvailabilityMessage(blobData BlobData) ([]byte, error) { + parsedABI, err := abi.JSON(bytes.NewReader([]byte(blobDataABI))) + if err != nil { + return nil, err + } + + // Encode the data + method, exist := parsedABI.Methods["BlobData"] + if !exist { + return nil, fmt.Errorf("abi error, BlobData method not found") + } + + encoded, err := method.Inputs.Pack(blobData) + if err != nil { + return nil, err + } + + return encoded, nil +} + +// TryDecodeFromDataAvailabilityMessage is a fallible decoding method to +// decode data availability message into Nubit blob data. +func TryDecodeFromDataAvailabilityMessage(msg []byte) (BlobData, error) { + // Parse the ABI + parsedABI, err := abi.JSON(bytes.NewReader([]byte(blobDataABI))) + if err != nil { + return BlobData{}, err + } + + // Decode the data + method, exist := parsedABI.Methods["BlobData"] + if !exist { + return BlobData{}, fmt.Errorf("abi error, BlobData method not found") + } + + unpackedMap := make(map[string]interface{}) + err = method.Inputs.UnpackIntoMap(unpackedMap, msg) + if err != nil { + return BlobData{}, err + } + unpacked, ok := unpackedMap["blobData"] + if !ok { + return BlobData{}, fmt.Errorf("abi error, failed to unpack to BlobData") + } + + val := reflect.ValueOf(unpacked) + typ := reflect.TypeOf(unpacked) + + blobData := BlobData{} + + for i := 0; i < typ.NumField(); i++ { + field := typ.Field(i) + value := val.Field(i) + + switch field.Name { + case "BlobID": + blobData.BlobID, err = convertBlobID(value) + if err != nil { + return BlobData{}, ErrConvertFromABIInterface + } + case "Signature": + blobData.Signature, err = convertSignature(value) + if err != nil { + return BlobData{}, ErrConvertFromABIInterface + } + default: + return BlobData{}, ErrConvertFromABIInterface + } + } + + return blobData, nil +} + +// -------- Helper fallible conversion methods -------- +func convertBlobID(val reflect.Value) ([]byte, error) { + return val.Interface().([]byte), nil +} + +func convertSignature(val reflect.Value) ([]byte, error) { + return val.Interface().([]byte), nil +} diff --git a/dataavailability/nubit/blob_test.go b/dataavailability/nubit/blob_test.go new file mode 100644 index 0000000000..24f6898d70 --- /dev/null +++ b/dataavailability/nubit/blob_test.go @@ -0,0 +1,35 @@ +package nubit + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestEncodeBlobData(t *testing.T) { + data := BlobData{ + BlobID: []byte{10}, + Signature: []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06}, + } + msg, err := TryEncodeToDataAvailabilityMessage(data) + assert.NoError(t, err) + assert.NotNil(t, msg) + assert.NotEmpty(t, msg) +} + +func TestEncodeDecodeBlobData(t *testing.T) { + data := BlobData{ + BlobID: []byte{10}, + Signature: []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06}, + } + msg, err := TryEncodeToDataAvailabilityMessage(data) + assert.NoError(t, err) + assert.NotNil(t, msg) + assert.NotEmpty(t, msg) + + // Check blob ID + decoded_data, err := TryDecodeFromDataAvailabilityMessage(msg) + assert.NoError(t, err) + assert.Equal(t, data.BlobID, decoded_data.BlobID) + assert.Equal(t, data.Signature, decoded_data.Signature) +} diff --git a/dataavailability/nubit/chain.go b/dataavailability/nubit/chain.go deleted file mode 100644 index c24de40243..0000000000 --- a/dataavailability/nubit/chain.go +++ /dev/null @@ -1,42 +0,0 @@ -package nubit - -import ( - "fmt" - - "github.com/ethereum/go-ethereum/accounts/abi" -) - -// We could pack the batchesData to many formats. -// For Polygon CDK, it shall be EVM-compatible. - -func MarshalBatchData(batchData [][]byte) ([]byte, error) { - byteArrayType, _ := abi.NewType("bytes[]", "", nil) - args := abi.Arguments{ - {Type: byteArrayType, Name: "batchData"}, - } - res, err := args.Pack(&batchData) - if err != nil { - return make([]byte, 0), fmt.Errorf("cannot pack batchData:%w", err) - } - return res, nil -} - -func UnmarshalBatchData(encodedData []byte) ([][]byte, error) { - byteArrayType, _ := abi.NewType("bytes[]", "", nil) - args := abi.Arguments{ - {Type: byteArrayType, Name: "batchData"}, - } - res, err := args.Unpack(encodedData) - if err != nil { - return nil, fmt.Errorf("cannot unpack batchData: %w", err) - } - batchData := make([][]byte, len(res)) - for i, v := range res { - byteSlice, ok := v.([]byte) - if !ok { - return nil, fmt.Errorf("element at index %d is not a []byte", i) - } - batchData[i] = byteSlice - } - return batchData, nil -} diff --git a/dataavailability/nubit/config.go b/dataavailability/nubit/config.go index 787c668287..8230c23b89 100644 --- a/dataavailability/nubit/config.go +++ b/dataavailability/nubit/config.go @@ -1,36 +1,18 @@ package nubit -import ( - "encoding/json" - "io" - "os" +import "time" - "github.com/0xPolygonHermez/zkevm-node/log" -) +// NubitNamespaceBytesLength is the fixed-size bytes array. +const NubitNamespaceBytesLength = 58 -type Config struct { - RpcURL string `json:"rpcURL"` - Namespace string `json:"modularAppName"` - AuthKey string `json:"authKey"` -} - -func (c *Config) GetConfig(configFileName string) error { - jsonFile, err := os.Open(configFileName) - if err != nil { - return err - } - defer jsonFile.Close() +// NubitMinCommitTime is the minimum commit time interval between blob submissions to NubitDA. +const NubitMinCommitTime time.Duration = 12 * time.Second - byteValue, err := io.ReadAll(jsonFile) - if err != nil { - return err - } - - log.Infof("⚙️ Nubit byteValue : %s ", string(byteValue)) - err = json.Unmarshal(byteValue, &c) - if err != nil { - return err - } - log.Infof("⚙️ Nubit GetConfig : %#v ", c) - return nil +// Config is the NubitDA backend configurations +type Config struct { + NubitRpcURL string `mapstructure:"NubitRpcURL"` + NubitAuthKey string `mapstructure:"NubitAuthKey"` + NubitNamespace string `mapstructure:"NubitNamespace"` + NubitGetProofMaxRetry uint64 `mapstructure:"NubitGetProofMaxRetry"` + NubitGetProofWaitPeriod time.Duration `mapstructure:"NubitGetProofWaitPeriod"` } diff --git a/dataavailability/nubit/encoding.go b/dataavailability/nubit/encoding.go new file mode 100644 index 0000000000..9d56c2d9c2 --- /dev/null +++ b/dataavailability/nubit/encoding.go @@ -0,0 +1,75 @@ +package nubit + +import ( + "encoding/binary" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" +) + +// EncodeSequence is the helper function to encode sequence data and their metadata into 1D byte array. +// The encoding scheme is ensured to be lossless. +// +// When encoding the blob data, the first 8-bytes stores the size of the batches (n) in the sequence. The +// next n slots of sized 40 bytes stores the metadata of the batches data. +// The first 8-bytes of the batches metadata stores the batches data length, and the next 32-bytes stores +// the batches hash. +// +// The remaining n slots contains the batches data, each slot length is specified in the retrieved batch +// metadata. +func EncodeSequence(batchesData [][]byte) []byte { + sequence := []byte{} + metadata := []byte{} + n := uint64(len(batchesData)) + bn := make([]byte, 8) //nolint:gomnd + binary.BigEndian.PutUint64(bn, n) + metadata = append(metadata, bn...) + + for _, seq := range batchesData { + // Add batch data to byte array + sequence = append(sequence, seq...) + + // Add batch metadata to byte array + // Batch metadata contains the byte array length and the Keccak256 hash of the + // batch data + n := uint64(len(seq)) + bn := make([]byte, 8) //nolint:gomnd + binary.BigEndian.PutUint64(bn, n) + hash := crypto.Keccak256Hash(seq) + metadata = append(metadata, bn...) + metadata = append(metadata, hash.Bytes()...) + } + sequence = append(metadata, sequence...) + + return sequence +} + +// DecodeSequence is the helper function to decode the 1D byte array into sequence data and the batches +// metadata. The decoding sceheme is ensured to be lossless and follows the encoding scheme specified in +// the EncodeSequence function. +func DecodeSequence(blobData []byte) ([][]byte, []common.Hash) { + bn := blobData[:8] + n := binary.BigEndian.Uint64(bn) + // Each batch metadata contains the batch data byte array length (8 byte) and the + // batch data hash (32 byte) + metadata := blobData[8 : 40*n+8] + sequence := blobData[40*n+8:] + + batchesData := [][]byte{} + batchesHash := []common.Hash{} + idx := uint64(0) + for i := uint64(0); i < n; i++ { + // Get batch metadata + bn := metadata[40*i : 40*i+8] + n := binary.BigEndian.Uint64(bn) + + hash := common.BytesToHash(metadata[40*i+8 : 40*(i+1)]) + batchesHash = append(batchesHash, hash) + + // Get batch data + batchesData = append(batchesData, sequence[idx:idx+n]) + idx += n + } + + return batchesData, batchesHash +} diff --git a/dataavailability/nubit/encoding_test.go b/dataavailability/nubit/encoding_test.go new file mode 100644 index 0000000000..7d71f0c8ac --- /dev/null +++ b/dataavailability/nubit/encoding_test.go @@ -0,0 +1,85 @@ +package nubit + +import ( + "math/rand" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" + "github.com/stretchr/testify/assert" +) + +func TestEncodeDecodeSequenceToAndFromStringBlob(t *testing.T) { + mock_string_data := "hihihihihihihihihihihihihihihihihihi" + data := []byte(mock_string_data) + hash := crypto.Keccak256Hash(data) + + // Generate mock sequence data + mockSeqData := [][]byte{} + for i := 0; i < 10; i++ { + mockSeqData = append(mockSeqData, data) + } + blob := EncodeSequence(mockSeqData) + + // Decode blob + decodedBatchesData, decodedBatchesHash := DecodeSequence(blob) + + // Assert decoded sequence length is correct + n_data := len(decodedBatchesData) + n_hash := len(decodedBatchesHash) + assert.Equal(t, 10, n_data) + assert.Equal(t, 10, n_hash) + + // Assert decoded sequence data is correct + for _, batchData := range decodedBatchesData { + data_decoded := string(batchData) + assert.Equal(t, mock_string_data, data_decoded) + } + + // Assert decoded batches' hash is correct + for _, batchHash := range decodedBatchesHash { + assert.Equal(t, hash, batchHash) + } +} + +func TestEncodeDecodeSequenceToAndFromRandomBlob(t *testing.T) { + // Define Different DataSizes + dataSize := []int{100000, 200000, 1000, 80, 30000} + + // Generate mock sequence data + mockSeqData := [][]byte{} + mockSeqHash := []common.Hash{} + for i := 0; i < 10; i++ { + // Disperse Blob with different DataSizes + rand.Seed(time.Now().UnixNano()) //nolint:gosec,staticcheck + data := make([]byte, dataSize[rand.Intn(len(dataSize))]) //nolint:gosec,staticcheck + _, err := rand.Read(data) //nolint:gosec,staticcheck + assert.NoError(t, err) + mockSeqData = append(mockSeqData, data) + + // Get batch hash + hash := crypto.Keccak256Hash(data) + mockSeqHash = append(mockSeqHash, hash) + } + blob := EncodeSequence(mockSeqData) + + // Decode blob + decodedBatchesData, decodedBatchesHash := DecodeSequence(blob) + + // Assert decoded sequence length is correct + n_data := len(decodedBatchesData) + n_hash := len(decodedBatchesHash) + assert.Equal(t, 10, n_data) + assert.Equal(t, 10, n_hash) + + // Assert decoded sequence data is correct + for i := 0; i < n_data; i++ { + assert.Equal(t, mockSeqData[i], decodedBatchesData[i]) + } + + // Assert decoded batches' hash is correct + for i := 0; i < n_hash; i++ { + assert.Equal(t, mockSeqHash[i], decodedBatchesHash[i]) + } +} diff --git a/dataavailability/nubit/nubit_test.go b/dataavailability/nubit/nubit_test.go deleted file mode 100644 index 0ec4a6e824..0000000000 --- a/dataavailability/nubit/nubit_test.go +++ /dev/null @@ -1,36 +0,0 @@ -package nubit - -import ( - "context" - "crypto/ecdsa" - "crypto/elliptic" - "crypto/rand" - "github.com/ethereum/go-ethereum/common" - "github.com/stretchr/testify/require" - "testing" - "time" -) - -func TestStoreDetailsOnChain(t *testing.T) { - pk, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) - if err != nil { - t.Fatal(err) - } - nubit, err := NewNubitDABackendTest("http://127.0.0.1:26658", "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJBbGxvdyI6WyJwdWJsaWMiLCJyZWFkIiwid3JpdGUiLCJhZG1pbiJdfQ.DAMv0s7915Ahx-kDFSzDT1ATz4Q9WwktWcHmjp7_99Q", pk) - if err != nil { - t.Fatal(err) - } - var returnData []byte - for i := 0; i < 10; i++ { - txs := []byte("test txs") - r, _, err := nubit.PostSequence(context.TODO(), [][]byte{txs}) - require.NoError(t, err) - if r != nil { - returnData = r - } - time.Sleep(600 * time.Millisecond) - } - - _, err = nubit.GetSequence(context.TODO(), []common.Hash{}, returnData) - require.NoError(t, err) -} diff --git a/dataavailability/nubit/types.go b/dataavailability/nubit/types.go deleted file mode 100644 index 168b6499eb..0000000000 --- a/dataavailability/nubit/types.go +++ /dev/null @@ -1,22 +0,0 @@ -package nubit - -import ( - "encoding/json" - "github.com/rollkit/go-da" - "reflect" -) - -type BatchDAData struct { - ID []da.ID `json:"id,omitempty"` -} - -// write a function that encode batchDAData struct into ABI-encoded bytes -func (b *BatchDAData) Encode() ([]byte, error) { - return json.Marshal(b) -} -func (b *BatchDAData) Decode(data []byte) error { - return json.Unmarshal(data, &b) -} -func (b *BatchDAData) IsEmpty() bool { - return reflect.DeepEqual(b, BatchDAData{}) -} diff --git a/sequencesender/interfaces.go b/sequencesender/interfaces.go index 8830e409c1..a49070ce39 100644 --- a/sequencesender/interfaces.go +++ b/sequencesender/interfaces.go @@ -43,5 +43,5 @@ type ethTxManager interface { } type dataAbilitier interface { - PostSequence(ctx context.Context, sequences []ethmanTypes.Sequence) ([]byte, []byte, error) + PostSequence(ctx context.Context, sequences []ethmanTypes.Sequence) ([]byte, error) } diff --git a/sequencesender/sequencesender.go b/sequencesender/sequencesender.go index 15321ca017..754c0cad6e 100644 --- a/sequencesender/sequencesender.go +++ b/sequencesender/sequencesender.go @@ -185,14 +185,14 @@ func (s *SequenceSender) tryToSendSequence(ctx context.Context) { } // add sequence to be monitored - _, Signature, err := s.da.PostSequence(ctx, sequences) + dataAvailabilityMessage, err := s.da.PostSequence(ctx, sequences) if err != nil { log.Error("error posting sequences to the data availability protocol: ", err) return } firstSequence := sequences[0] - to, data, err := s.etherman.BuildSequenceBatchesTxData(s.cfg.SenderAddress, sequences, uint64(lastSequence.LastL2BLockTimestamp), firstSequence.BatchNumber-1, s.cfg.L2Coinbase, Signature) + to, data, err := s.etherman.BuildSequenceBatchesTxData(s.cfg.SenderAddress, sequences, uint64(lastSequence.LastL2BLockTimestamp), firstSequence.BatchNumber-1, s.cfg.L2Coinbase, dataAvailabilityMessage) if err != nil { log.Error("error estimating new sequenceBatches to add to eth tx manager: ", err) return