diff --git a/e2e-tests/storage/backup-restore/run-test.sh b/e2e-tests/storage/backup-restore/run-test.sh index b8ce612391..e4b1df1730 100755 --- a/e2e-tests/storage/backup-restore/run-test.sh +++ b/e2e-tests/storage/backup-restore/run-test.sh @@ -65,15 +65,6 @@ runOnAlpine "$(pwd):/host/" rm -rf /host/node-data runOnAlpine "$(pwd):/host/" mv -f /host/node-backup /host/node-data BACKUP_INTERVAL=0 docker compose up --wait -echo "Rebuilding data" -# Sanity check for test; assert that the data is not there before rebuild -assertDiagnostic "http://localhost:11323" "revocations_count: 0" -docker compose exec nodeA nuts network reprocess "application/vc+json" -docker compose exec nodeA nuts network reprocess "application/ld+json;type=revocation" - -# Wait for reprocess to finish -waitForDiagnostic "nodeA" revocations_count 1 - assertDiagnostic "http://localhost:11323" "transaction_count: 5" assertDiagnostic "http://localhost:11323" "credential_count: 2" assertDiagnostic "http://localhost:11323" "issued_credentials_count: 2" diff --git a/vcr/issuer/leia_store.go b/vcr/issuer/leia_store.go index 802a68fd3b..d663ac1b0e 100644 --- a/vcr/issuer/leia_store.go +++ b/vcr/issuer/leia_store.go @@ -41,9 +41,7 @@ const revocationBackupShelf = "revocations" // leiaIssuerStore implements the issuer Store interface. It is a simple and fast JSON store. // Note: It can not be used in a clustered setup. type leiaIssuerStore struct { - issuedCredentials leia.Collection - revokedCredentials leia.Collection - store storage.KVBackedLeiaStore + store storage.KVBackedLeiaStore } // NewLeiaIssuerStore creates a new instance of leiaIssuerStore which implements the Store interface. @@ -72,21 +70,15 @@ func NewLeiaIssuerStore(dbPath string, backupStore stoabs.KVStore) (Store, error SearchQuery: leia.NewJSONPath(credential.RevocationSubjectPath), }) - // collections - issuedCollection := kvBackedStore.Collection(leia.JSONCollection, "issuedCredentials") - revokedCollection := kvBackedStore.Collection(leia.JSONCollection, "revokedCredentials") - newLeiaStore := &leiaIssuerStore{ - issuedCredentials: issuedCollection, - revokedCredentials: revokedCollection, - store: kvBackedStore, + store: kvBackedStore, } // initialize indices, this is required for handleRestore. Without the index metadata it can't iterate and detect data entries. - if err = newLeiaStore.createIssuedIndices(issuedCollection); err != nil { + if err = newLeiaStore.createIssuedIndices(); err != nil { return nil, err } - if err = newLeiaStore.createRevokedIndices(revokedCollection); err != nil { + if err = newLeiaStore.createRevokedIndices(); err != nil { return nil, err } @@ -98,7 +90,7 @@ func NewLeiaIssuerStore(dbPath string, backupStore stoabs.KVStore) (Store, error func (s leiaIssuerStore) StoreCredential(vc vc.VerifiableCredential) error { vcAsBytes, _ := json.Marshal(vc) - return s.issuedCredentials.Add([]leia.Document{vcAsBytes}) + return s.issuedCollection().Add([]leia.Document{vcAsBytes}) } func (s leiaIssuerStore) SearchCredential(credentialType ssi.URI, issuer did.DID, subject *ssi.URI) ([]vc.VerifiableCredential, error) { @@ -111,7 +103,7 @@ func (s leiaIssuerStore) SearchCredential(credentialType ssi.URI, issuer did.DID } } - docs, err := s.issuedCredentials.Find(context.Background(), query) + docs, err := s.issuedCollection().Find(context.Background(), query) if err != nil { return nil, err } @@ -128,7 +120,7 @@ func (s leiaIssuerStore) SearchCredential(credentialType ssi.URI, issuer did.DID func (s leiaIssuerStore) GetCredential(id ssi.URI) (*vc.VerifiableCredential, error) { query := leia.New(leia.Eq(leia.NewJSONPath("id"), leia.MustParseScalar(id.String()))) - results, err := s.issuedCredentials.Find(context.Background(), query) + results, err := s.issuedCollection().Find(context.Background(), query) if err != nil { return nil, fmt.Errorf("could not get credential by id: %w", err) } @@ -148,13 +140,13 @@ func (s leiaIssuerStore) GetCredential(id ssi.URI) (*vc.VerifiableCredential, er func (s leiaIssuerStore) StoreRevocation(revocation credential.Revocation) error { revocationAsBytes, _ := json.Marshal(revocation) - return s.revokedCredentials.Add([]leia.Document{revocationAsBytes}) + return s.revokedCollection().Add([]leia.Document{revocationAsBytes}) } func (s leiaIssuerStore) GetRevocation(subject ssi.URI) (*credential.Revocation, error) { query := leia.New(leia.Eq(leia.NewJSONPath(credential.RevocationSubjectPath), leia.MustParseScalar(subject.String()))) - results, err := s.revokedCredentials.Find(context.Background(), query) + results, err := s.revokedCollection().Find(context.Background(), query) if err != nil { return nil, fmt.Errorf("error while getting revocation by id: %w", err) } @@ -177,34 +169,42 @@ func (s leiaIssuerStore) Close() error { return s.store.Close() } +func (s leiaIssuerStore) issuedCollection() leia.Collection { + return s.store.Collection(leia.JSONCollection, "issuedCredentials") +} + +func (s leiaIssuerStore) revokedCollection() leia.Collection { + return s.store.Collection(leia.JSONCollection, "revokedCredentials") +} + // createIssuedIndices creates the needed indices for the issued VC store // It allows faster searching on context, type issuer and subject values. -func (s leiaIssuerStore) createIssuedIndices(collection leia.Collection) error { - searchIndex := collection.NewIndex("issuedVCs", +func (s leiaIssuerStore) createIssuedIndices() error { + searchIndex := s.issuedCollection().NewIndex("issuedVCs", leia.NewFieldIndexer(leia.NewJSONPath("issuer")), leia.NewFieldIndexer(leia.NewJSONPath("type")), leia.NewFieldIndexer(leia.NewJSONPath("credentialSubject.id")), ) // Index used for getting issued VCs by id - idIndex := collection.NewIndex("issuedVCByID", + idIndex := s.issuedCollection().NewIndex("issuedVCByID", leia.NewFieldIndexer(leia.NewJSONPath("id"))) - return s.issuedCredentials.AddIndex(searchIndex, idIndex) + return s.issuedCollection().AddIndex(searchIndex, idIndex) } // createRevokedIndices creates the needed indices for the issued VC store // It allows faster searching on context, type issuer and subject values. -func (s leiaIssuerStore) createRevokedIndices(collection leia.Collection) error { +func (s leiaIssuerStore) createRevokedIndices() error { // Index used for getting issued VCs by id - revocationBySubjectIDIndex := collection.NewIndex("revocationBySubjectIDIndex", + revocationBySubjectIDIndex := s.revokedCollection().NewIndex("revocationBySubjectIDIndex", leia.NewFieldIndexer(leia.NewJSONPath(credential.RevocationSubjectPath))) - return s.revokedCredentials.AddIndex(revocationBySubjectIDIndex) + return s.revokedCollection().AddIndex(revocationBySubjectIDIndex) } func (s leiaIssuerStore) Diagnostics() []core.DiagnosticResult { var err error var issuedCredentialCount int - issuedCredentialCount, err = s.issuedCredentials.DocumentCount() + issuedCredentialCount, err = s.issuedCollection().DocumentCount() if err != nil { issuedCredentialCount = -1 log.Logger(). @@ -212,7 +212,7 @@ func (s leiaIssuerStore) Diagnostics() []core.DiagnosticResult { Warn("unable to retrieve issuedCredentials document count") } var revokedCredentialsCount int - revokedCredentialsCount, err = s.revokedCredentials.DocumentCount() + revokedCredentialsCount, err = s.revokedCollection().DocumentCount() if err != nil { revokedCredentialsCount = -1 log.Logger(). diff --git a/vcr/issuer/leia_store_test.go b/vcr/issuer/leia_store_test.go index 89a91d250d..af7d8162f2 100644 --- a/vcr/issuer/leia_store_test.go +++ b/vcr/issuer/leia_store_test.go @@ -188,7 +188,7 @@ func Test_leiaStore_GetCredential(t *testing.T) { ID *ssi.URI `json:"id,omitempty"` }{ID: vcToGet.ID} asBytes, _ := json.Marshal(rawStructWithSameID) - lstore.issuedCredentials.Add([]leia.Document{asBytes}) + lstore.issuedCollection().Add([]leia.Document{asBytes}) t.Run("it fails", func(t *testing.T) { foundCredential, err := store.GetCredential(*vcToGet.ID) @@ -257,10 +257,12 @@ func TestNewLeiaIssuerStore(t *testing.T) { backupMockStore := stoabs.NewMockKVStore(ctrl) backupStorePath := path.Join(t.TempDir(), "backup-issued-credentials.db") emptyBackupStore, err := bbolt.CreateBBoltStore(backupStorePath) + require.NoError(t, err) dbPath := path.Join(t.TempDir(), "issuer.db") // first create a store with 1 credential store, err := NewLeiaIssuerStore(dbPath, emptyBackupStore) + defer store.Close() require.NoError(t, err) vc := vc.VerifiableCredential{} _ = json.Unmarshal([]byte(jsonld.TestCredential), &vc) @@ -334,5 +336,6 @@ func newStoreInDir(t *testing.T, testDir string) Store { require.NoError(t, err) store, err := NewLeiaIssuerStore(issuerStorePath, backupStore) require.NoError(t, err) + t.Cleanup(func() { _ = store.Close() }) return store } diff --git a/vcr/verifier/leia_store.go b/vcr/verifier/leia_store.go index 057a9a709f..29a1dc3809 100644 --- a/vcr/verifier/leia_store.go +++ b/vcr/verifier/leia_store.go @@ -37,9 +37,7 @@ const revocationBackupShelf = "revocations" // leiaVerifierStore implements the verifier Store interface. It is a simple and fast JSON store. // Note: It can not be used in a clustered setup. type leiaVerifierStore struct { - // revocations is a leia collection containing all the revocations - revocations leia.Collection - store storage.KVBackedLeiaStore + store storage.KVBackedLeiaStore } // NewLeiaVerifierStore creates a new instance of leiaVerifierStore which implements the Store interface. @@ -63,12 +61,10 @@ func NewLeiaVerifierStore(dbPath string, backupStore stoabs.KVStore) (Store, err SearchQuery: leia.NewJSONPath(credential.RevocationSubjectPath), }) - revocations := store.Collection(leia.JSONCollection, "revocations") newLeiaStore := &leiaVerifierStore{ - revocations: revocations, - store: kvBackedStore, + store: kvBackedStore, } - if err = newLeiaStore.createIndices(revocations); err != nil { + if err = newLeiaStore.createIndices(); err != nil { return nil, err } @@ -81,13 +77,13 @@ func NewLeiaVerifierStore(dbPath string, backupStore stoabs.KVStore) (Store, err func (s leiaVerifierStore) StoreRevocation(revocation credential.Revocation) error { revocationAsBytes, _ := json.Marshal(revocation) - return s.revocations.Add([]leia.Document{revocationAsBytes}) + return s.revocationCollection().Add([]leia.Document{revocationAsBytes}) } func (s leiaVerifierStore) GetRevocations(id ssi.URI) ([]*credential.Revocation, error) { query := leia.New(leia.Eq(leia.NewJSONPath(credential.RevocationSubjectPath), leia.MustParseScalar(id.String()))) - results, err := s.revocations.Find(context.Background(), query) + results, err := s.revocationCollection().Find(context.Background(), query) if err != nil { return nil, fmt.Errorf("error while getting revocation by id: %w", err) } @@ -111,19 +107,23 @@ func (s leiaVerifierStore) Close() error { return s.store.Close() } +func (s leiaVerifierStore) revocationCollection() leia.Collection { + return s.store.Collection(leia.JSONCollection, "revocations") +} + // createIndices creates the needed indices for the issued VC store // It allows faster searching on context, type issuer and subject values. -func (s leiaVerifierStore) createIndices(collection leia.Collection) error { +func (s leiaVerifierStore) createIndices() error { // Index used for getting issued VCs by id - revocationBySubjectIDIndex := collection.NewIndex("revocationBySubjectIDIndex", + revocationBySubjectIDIndex := s.revocationCollection().NewIndex("revocationBySubjectIDIndex", leia.NewFieldIndexer(leia.NewJSONPath(credential.RevocationSubjectPath))) - return s.revocations.AddIndex(revocationBySubjectIDIndex) + return s.revocationCollection().AddIndex(revocationBySubjectIDIndex) } func (s leiaVerifierStore) Diagnostics() []core.DiagnosticResult { var count int var err error - count, err = s.revocations.DocumentCount() + count, err = s.revocationCollection().DocumentCount() if err != nil { count = -1 log.Logger(). diff --git a/vcr/verifier/leia_store_test.go b/vcr/verifier/leia_store_test.go index 12594edb2c..070d74b30c 100644 --- a/vcr/verifier/leia_store_test.go +++ b/vcr/verifier/leia_store_test.go @@ -67,18 +67,33 @@ func TestLeiaStore_Close(t *testing.T) { func Test_leiaVerifierStore_StoreRevocation(t *testing.T) { testDir := io.TestDirectory(t) verifierStorePath := path.Join(testDir, "vcr", "verifier-store.db") - backupStore := newMockKVStore(t) + backupStorePath := path.Join(testDir, "vcr", "backup-revoked-credentials.db") + backupStore, err := bbolt.CreateBBoltStore(backupStorePath) + require.NoError(t, err) sut, _ := NewLeiaVerifierStore(verifierStorePath, backupStore) + defer sut.Close() - t.Run("it stores a revocation and can find it back", func(t *testing.T) { - subjectID := ssi.MustParseURI("did:nuts:123#ab-c") - revocation := &credential.Revocation{Subject: subjectID} - err := sut.StoreRevocation(*revocation) - assert.NoError(t, err) - result, err := sut.GetRevocations(subjectID) - assert.NoError(t, err) - assert.Equal(t, []*credential.Revocation{revocation}, result) + // store revocation + subjectID := ssi.MustParseURI("did:nuts:123#ab-c") + revocation := &credential.Revocation{Subject: subjectID} + err = sut.StoreRevocation(*revocation) + assert.NoError(t, err) + + // exists in verifier-store.db + result, err := sut.GetRevocations(subjectID) + assert.NoError(t, err) + assert.Equal(t, []*credential.Revocation{revocation}, result) + + // exists in backup-revoked-credentials.db + revocationBytes, _ := json.Marshal(revocation) + ref := sha1.Sum(revocationBytes) + var backupBytes []byte + err = backupStore.ReadShelf(context.Background(), revocationBackupShelf, func(reader stoabs.Reader) error { + backupBytes, err = reader.Get(stoabs.BytesKey(ref[:])) + return err }) + assert.NoError(t, err) + assert.Equal(t, revocationBytes, backupBytes) } func Test_leiaVerifierStore_GetRevocation(t *testing.T) { @@ -147,6 +162,7 @@ func Test_restoreFromShelf(t *testing.T) { store, err := NewLeiaVerifierStore(issuerStorePath, backupStore) require.NoError(t, err) + defer store.Close() result, err := store.GetRevocations(subjectID) require.NoError(t, err) assert.Equal(t, []*credential.Revocation{revocation}, result) @@ -154,13 +170,8 @@ func Test_restoreFromShelf(t *testing.T) { // newMockKVStore create a mockStore with contents func newMockKVStore(t *testing.T) stoabs.KVStore { - ctrl := gomock.NewController(t) - backupStore := stoabs.NewMockKVStore(ctrl) - mockReader := stoabs.NewMockReader(ctrl) - mockReader.EXPECT().Empty().Return(false, nil).AnyTimes() - mockReader.EXPECT().Iterate(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() - backupStore.EXPECT().ReadShelf(context.Background(), revocationBackupShelf, gomock.Any()).Do(func(arg0 context.Context, arg1 string, arg2 func(reader stoabs.Reader) error) error { - return arg2(mockReader) - }).AnyTimes() + backupStore := stoabs.NewMockKVStore(gomock.NewController(t)) + backupStore.EXPECT().ReadShelf(context.Background(), revocationBackupShelf, gomock.Any()).AnyTimes() + backupStore.EXPECT().WriteShelf(context.Background(), revocationBackupShelf, gomock.Any()).AnyTimes() return backupStore }