diff --git a/delta/plugins/elasticsearch/src/main/resources/metrics/metrics-mapping.json b/delta/plugins/elasticsearch/src/main/resources/metrics/metrics-mapping.json index d1c42b4f22..b8be7b0d1a 100644 --- a/delta/plugins/elasticsearch/src/main/resources/metrics/metrics-mapping.json +++ b/delta/plugins/elasticsearch/src/main/resources/metrics/metrics-mapping.json @@ -8,5 +8,10 @@ } } } - ] + ], + "properties": { + "rev": { + "type": "integer" + } + } } \ No newline at end of file diff --git a/delta/plugins/elasticsearch/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/elasticsearch/ElasticSearchPluginModule.scala b/delta/plugins/elasticsearch/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/elasticsearch/ElasticSearchPluginModule.scala index 03283385f3..69c8c432dc 100644 --- a/delta/plugins/elasticsearch/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/elasticsearch/ElasticSearchPluginModule.scala +++ b/delta/plugins/elasticsearch/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/elasticsearch/ElasticSearchPluginModule.scala @@ -7,9 +7,10 @@ import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.client.ElasticSearchC import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.config.ElasticSearchViewsConfig import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.deletion.{ElasticSearchDeletionTask, EventMetricsDeletionTask} import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.indexing.{ElasticSearchCoordinator, ElasticSearchDefaultViewsResetter} +import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.metrics.{EventMetricsProjection, EventMetricsQuery} import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.model.{contexts, schema => viewsSchemaId, ElasticSearchFiles, ElasticSearchView, ElasticSearchViewEvent} import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.query.DefaultViewsQuery -import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.routes._ +import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.routes.{ElasticSearchHistoryRoutes, _} import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.api.JsonLdApi import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.ContextValue.ContextObject @@ -305,6 +306,21 @@ class ElasticSearchPluginModule(priority: Int) extends ModuleDef { ) } + make[EventMetricsQuery].from { (client: ElasticSearchClient, config: ElasticSearchViewsConfig) => + EventMetricsQuery(client, config.prefix) + } + + make[ElasticSearchHistoryRoutes].from { + ( + identities: Identities, + aclCheck: AclCheck, + metricsQuery: EventMetricsQuery, + rcr: RemoteContextResolution @Id("aggregate"), + ordering: JsonKeyOrdering + ) => + new ElasticSearchHistoryRoutes(identities, aclCheck, metricsQuery)(rcr, ordering) + } + make[ElasticSearchScopeInitialization] .from { (views: ElasticSearchViews, serviceAccount: ServiceAccount, config: ElasticSearchViewsConfig) => new ElasticSearchScopeInitialization(views, serviceAccount, config.defaults) @@ -372,6 +388,7 @@ class ElasticSearchPluginModule(priority: Int) extends ModuleDef { query: ElasticSearchQueryRoutes, indexing: ElasticSearchIndexingRoutes, idResolutionRoute: IdResolutionRoutes, + historyRoutes: ElasticSearchHistoryRoutes, schemeDirectives: DeltaSchemeDirectives, baseUri: BaseUri ) => @@ -382,7 +399,8 @@ class ElasticSearchPluginModule(priority: Int) extends ModuleDef { es.routes, query.routes, indexing.routes, - idResolutionRoute.routes + idResolutionRoute.routes, + historyRoutes.routes )(baseUri), requiresStrictEntity = true ) diff --git a/delta/plugins/elasticsearch/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/elasticsearch/client/QueryBuilder.scala b/delta/plugins/elasticsearch/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/elasticsearch/client/QueryBuilder.scala index e8b2bb743c..f779ba1367 100644 --- a/delta/plugins/elasticsearch/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/elasticsearch/client/QueryBuilder.scala +++ b/delta/plugins/elasticsearch/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/elasticsearch/client/QueryBuilder.scala @@ -199,6 +199,8 @@ object QueryBuilder { */ val empty: QueryBuilder = QueryBuilder(JsonObject.empty) + def unsafe(jsonObject: JsonObject): QueryBuilder = QueryBuilder(jsonObject) + /** * A [[QueryBuilder]] using the filter ''params''. */ diff --git a/delta/plugins/elasticsearch/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/elasticsearch/deletion/EventMetricsDeletionTask.scala b/delta/plugins/elasticsearch/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/elasticsearch/deletion/EventMetricsDeletionTask.scala index 358249f6f0..43c76e47d2 100644 --- a/delta/plugins/elasticsearch/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/elasticsearch/deletion/EventMetricsDeletionTask.scala +++ b/delta/plugins/elasticsearch/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/elasticsearch/deletion/EventMetricsDeletionTask.scala @@ -1,8 +1,8 @@ package ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.deletion import cats.effect.IO -import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.EventMetricsProjection import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.client.ElasticSearchClient +import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.metrics.eventMetricsIndex import ch.epfl.bluebrain.nexus.delta.sdk.deletion.ProjectDeletionTask import ch.epfl.bluebrain.nexus.delta.sdk.deletion.model.ProjectDeletionReport import ch.epfl.bluebrain.nexus.delta.sourcing.model.{Identity, ProjectRef} @@ -17,7 +17,7 @@ import io.circe.parser.parse */ final class EventMetricsDeletionTask(client: ElasticSearchClient, prefix: String) extends ProjectDeletionTask { - private val index = EventMetricsProjection.eventMetricsIndex(prefix) + private val index = eventMetricsIndex(prefix) override def apply(project: ProjectRef)(implicit subject: Identity.Subject): IO[ProjectDeletionReport.Stage] = searchByProject(project).flatMap { search => diff --git a/delta/plugins/elasticsearch/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/elasticsearch/EventMetricsProjection.scala b/delta/plugins/elasticsearch/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/elasticsearch/metrics/EventMetricsProjection.scala similarity index 86% rename from delta/plugins/elasticsearch/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/elasticsearch/EventMetricsProjection.scala rename to delta/plugins/elasticsearch/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/elasticsearch/metrics/EventMetricsProjection.scala index a4d2aa4c5a..65a86924eb 100644 --- a/delta/plugins/elasticsearch/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/elasticsearch/EventMetricsProjection.scala +++ b/delta/plugins/elasticsearch/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/elasticsearch/metrics/EventMetricsProjection.scala @@ -1,11 +1,11 @@ -package ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch +package ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.metrics import cats.data.NonEmptyChain import cats.effect.IO import cats.effect.std.Env import ch.epfl.bluebrain.nexus.delta.kernel.Logger +import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.client.ElasticSearchClient import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.client.ElasticSearchClient.Refresh -import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.client.{ElasticSearchClient, IndexLabel} import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.indexing.ElasticSearchSink import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.model.{MetricsMapping, MetricsSettings} import ch.epfl.bluebrain.nexus.delta.sdk.model.metrics.EventMetric._ @@ -25,8 +25,7 @@ trait EventMetricsProjection object EventMetricsProjection { private val logger = Logger[EventMetricsProjection] - val projectionMetadata: ProjectionMetadata = ProjectionMetadata("system", "event-metrics", None, None) - val eventMetricsIndex: String => IndexLabel = prefix => IndexLabel.unsafe(s"${prefix}_project_metrics") + val projectionMetadata: ProjectionMetadata = ProjectionMetadata("system", "event-metrics", None, None) // We need a value to return to Distage private val dummy = new EventMetricsProjection {} @@ -81,14 +80,13 @@ object EventMetricsProjection { for { shouldRestart <- Env[IO].get("RESET_EVENT_METRICS").map(_.getOrElse("false").toBoolean) - _ <- IO.whenA(shouldRestart) { - client.deleteIndex(index) >> - logger.warn("Resetting event metrics as the env RESET_EVENT_METRICS is set") >> projections.reset( - projectionMetadata.name - ) - } + _ <- IO.whenA(shouldRestart)( + logger.warn("Resetting event metrics as the env RESET_EVENT_METRICS is set...") >> + client.deleteIndex(index) >> + projections.reset(projectionMetadata.name) + ) metricsProjection <- apply(sink, supervisor, metrics, createIndex) - } yield (metricsProjection) + } yield metricsProjection } else IO.pure(dummy) diff --git a/delta/plugins/elasticsearch/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/elasticsearch/metrics/EventMetricsQuery.scala b/delta/plugins/elasticsearch/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/elasticsearch/metrics/EventMetricsQuery.scala new file mode 100644 index 0000000000..f1920b950a --- /dev/null +++ b/delta/plugins/elasticsearch/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/elasticsearch/metrics/EventMetricsQuery.scala @@ -0,0 +1,60 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.metrics + +import akka.http.scaladsl.model.Uri +import cats.effect.IO +import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.client.{ElasticSearchClient, QueryBuilder} +import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri +import ch.epfl.bluebrain.nexus.delta.sdk.model.search.SearchResults +import ch.epfl.bluebrain.nexus.delta.sourcing.model.ProjectRef +import io.circe.JsonObject +import io.circe.literal.JsonStringContext +import io.circe.syntax.EncoderOps + +trait EventMetricsQuery { + + def history(project: ProjectRef, id: Iri): IO[SearchResults[JsonObject]] + +} + +object EventMetricsQuery { + + def apply(client: ElasticSearchClient, prefix: String): EventMetricsQuery = new EventMetricsQuery { + + val index = eventMetricsIndex(prefix) + + private def searchQuery(project: ProjectRef, id: Iri) = + json"""{ + "query": { + "bool": { + "must": [ + { + "term": { + "project": ${project.asJson} + } + }, + { + "term": { + "@id": ${id.asJson} + } + } + ] + } + }, + "size": 2000, + "from": 0, + "sort": [ + { "rev": { "order" : "asc" } } + ] + } + """.asObject.toRight(new IllegalStateException("Should not happen, an es query is an object")) + + override def history(project: ProjectRef, id: Iri): IO[SearchResults[JsonObject]] = { + for { + jsonQuery <- IO.fromEither(searchQuery(project, id)) + queryBuilder = QueryBuilder.unsafe(jsonQuery) + results <- client.search(queryBuilder, Set(index.value), Uri.Query.Empty) + } yield results + } + } + +} diff --git a/delta/plugins/elasticsearch/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/elasticsearch/metrics/package.scala b/delta/plugins/elasticsearch/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/elasticsearch/metrics/package.scala new file mode 100644 index 0000000000..b4ecc0ecac --- /dev/null +++ b/delta/plugins/elasticsearch/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/elasticsearch/metrics/package.scala @@ -0,0 +1,8 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch + +import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.client.IndexLabel + +package object metrics { + + val eventMetricsIndex: String => IndexLabel = prefix => IndexLabel.unsafe(s"${prefix}_project_metrics") +} diff --git a/delta/plugins/elasticsearch/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/elasticsearch/routes/ElasticSearchHistoryRoutes.scala b/delta/plugins/elasticsearch/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/elasticsearch/routes/ElasticSearchHistoryRoutes.scala new file mode 100644 index 0000000000..026a63a0d1 --- /dev/null +++ b/delta/plugins/elasticsearch/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/elasticsearch/routes/ElasticSearchHistoryRoutes.scala @@ -0,0 +1,42 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.routes + +import akka.http.scaladsl.server.Route +import cats.syntax.all._ +import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.metrics.EventMetricsQuery +import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.query.ElasticSearchQueryError +import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.RemoteContextResolution +import ch.epfl.bluebrain.nexus.delta.rdf.utils.JsonKeyOrdering +import ch.epfl.bluebrain.nexus.delta.sdk.acls.AclCheck +import ch.epfl.bluebrain.nexus.delta.sdk.directives.AuthDirectives +import ch.epfl.bluebrain.nexus.delta.sdk.directives.DeltaDirectives.{emit, projectRef} +import ch.epfl.bluebrain.nexus.delta.sdk.directives.UriDirectives.iriSegment +import ch.epfl.bluebrain.nexus.delta.sdk.identities.Identities +import ch.epfl.bluebrain.nexus.delta.sdk.marshalling.RdfMarshalling +import ch.epfl.bluebrain.nexus.delta.sdk.model.search.SearchResults +import ch.epfl.bluebrain.nexus.delta.sdk.model.search.SearchResults.searchResultsEncoder +import ch.epfl.bluebrain.nexus.delta.sdk.permissions.Permissions.resources.{read => Read} +import io.circe.syntax.EncoderOps +import io.circe.{Encoder, JsonObject} + +class ElasticSearchHistoryRoutes(identities: Identities, aclCheck: AclCheck, metricsQuery: EventMetricsQuery)(implicit + cr: RemoteContextResolution, + ordering: JsonKeyOrdering +) extends AuthDirectives(identities, aclCheck) + with RdfMarshalling { + implicit private val searchEncoder: Encoder.AsObject[SearchResults[JsonObject]] = searchResultsEncoder(_ => None) + + def routes: Route = + pathPrefix("history") { + pathPrefix("resources") { + extractCaller { implicit caller => + projectRef.apply { project => + authorizeFor(project, Read).apply { + (get & iriSegment & pathEndOrSingleSlash) { id => + emit(metricsQuery.history(project, id).map(_.asJson).attemptNarrow[ElasticSearchQueryError]) + } + } + } + } + } + } +} diff --git a/delta/plugins/elasticsearch/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/elasticsearch/deletion/EventMetricsDeletionTaskSuite.scala b/delta/plugins/elasticsearch/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/elasticsearch/deletion/EventMetricsDeletionTaskSuite.scala index 957c2af2c3..ce3e248ea3 100644 --- a/delta/plugins/elasticsearch/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/elasticsearch/deletion/EventMetricsDeletionTaskSuite.scala +++ b/delta/plugins/elasticsearch/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/elasticsearch/deletion/EventMetricsDeletionTaskSuite.scala @@ -2,7 +2,8 @@ package ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.deletion import akka.http.scaladsl.model.Uri.Query import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.client.{ElasticSearchAction, QueryBuilder} -import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.{ElasticSearchClientSetup, EventMetricsProjection, Fixtures} +import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.metrics.eventMetricsIndex +import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.{ElasticSearchClientSetup, Fixtures} import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.{Anonymous, Subject} import ch.epfl.bluebrain.nexus.delta.sourcing.model.ProjectRef import ch.epfl.bluebrain.nexus.testkit.CirceLiteral @@ -23,7 +24,7 @@ class EventMetricsDeletionTaskSuite test("Delete all entries for a given project") { val prefix = "test" - val index = EventMetricsProjection.eventMetricsIndex(prefix) + val index = eventMetricsIndex(prefix) val projectToDelete = ProjectRef.unsafe("org", "marked-for-deletion") val anotherProject = ProjectRef.unsafe("org", "another") diff --git a/delta/plugins/elasticsearch/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/elasticsearch/metrics/EventMetricsProjectionSuite.scala b/delta/plugins/elasticsearch/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/elasticsearch/metrics/EventMetricsProjectionSuite.scala index 0728821343..afdb7e871c 100644 --- a/delta/plugins/elasticsearch/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/elasticsearch/metrics/EventMetricsProjectionSuite.scala +++ b/delta/plugins/elasticsearch/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/elasticsearch/metrics/EventMetricsProjectionSuite.scala @@ -1,49 +1,90 @@ package ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.metrics -import cats.effect.IO -import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.metrics.MetricsStream._ -import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.{EventMetricsProjection, Fixtures} +import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.client.ElasticSearchClient.Refresh +import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.indexing.ElasticSearchSink +import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.metrics.EventMetricsProjectionSuite.{metric1, metric2} +import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.{ElasticSearchClientSetup, Fixtures} +import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary.nxv +import ch.epfl.bluebrain.nexus.delta.rdf.syntax.iriStringContextSyntax +import ch.epfl.bluebrain.nexus.delta.sdk.metrics.ProjectScopedMetricStream +import ch.epfl.bluebrain.nexus.delta.sdk.model.metrics.EventMetric.{Created, ProjectScopedMetric, Updated} +import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.Anonymous +import ch.epfl.bluebrain.nexus.delta.sourcing.model.{EntityType, Label, ProjectRef} import ch.epfl.bluebrain.nexus.delta.sourcing.offset.Offset -import ch.epfl.bluebrain.nexus.delta.sourcing.stream.{CacheSink, ProjectionProgress, SupervisorSetup} +import ch.epfl.bluebrain.nexus.delta.sourcing.stream.{ProjectionProgress, SupervisorSetup} import ch.epfl.bluebrain.nexus.testkit.mu.NexusSuite import ch.epfl.bluebrain.nexus.testkit.mu.ce.PatienceConfig -import io.circe.Json -import io.circe.syntax.EncoderOps +import io.circe.syntax.{EncoderOps, KeyOps} +import io.circe.{Json, JsonObject} import munit.AnyFixture import java.time.Instant import scala.concurrent.duration.DurationInt -class EventMetricsProjectionSuite extends NexusSuite with SupervisorSetup.Fixture with Fixtures { +class EventMetricsProjectionSuite + extends NexusSuite + with SupervisorSetup.Fixture + with ElasticSearchClientSetup.Fixture + with Fixtures { - override def munitFixtures: Seq[AnyFixture[_]] = List(supervisor) + override def munitFixtures: Seq[AnyFixture[_]] = List(supervisor, esClient) implicit private val patienceConfig: PatienceConfig = PatienceConfig(2.seconds, 10.millis) - private lazy val sv = supervisor().supervisor - private val sink = CacheSink.events[Json] + private val index = eventMetricsIndex("nexus") - test("Start the metrics projection") { + private lazy val sv = supervisor().supervisor + private lazy val client = esClient() + private lazy val sink = ElasticSearchSink.events(client, 2, 50.millis, index, Refresh.True) + + test("Start the metrics projection and index metrics") { + def createIndex = client + .createIndex(index, Some(metricsMapping.value), Some(metricsSettings.value)) + .assertEquals(true) for { - _ <- EventMetricsProjection( - sink, - sv, - _ => metricsStream.take(2), - IO.unit - ) + _ <- EventMetricsProjection(sink, sv, _ => EventMetricsProjectionSuite.stream, createIndex) _ <- sv.describe(EventMetricsProjection.projectionMetadata.name) .map(_.map(_.progress)) .assertEquals(Some(ProjectionProgress(Offset.at(2L), Instant.EPOCH, 2, 0, 0))) .eventually + _ <- client.count(index.value).assertEquals(2L) + // Asserting the sources + _ <- client.getSource[Json](index, metric1.eventId).assertEquals(metric1.asJson) + _ <- client.getSource[Json](index, metric2.eventId).assertEquals(metric2.asJson) } yield () } +} - test("Sink has the correct metrics") { - assertEquals(sink.successes.size, 2) - assert(sink.dropped.isEmpty) - assert(sink.failed.isEmpty) - assert(sink.successes.values.toSet.contains(metric1.asJson)) - assert(sink.successes.values.toSet.contains(metric2.asJson)) - } +object EventMetricsProjectionSuite { + private val org = Label.unsafe("org") + private val proj1 = Label.unsafe("proj1") + private val projectRef1: ProjectRef = ProjectRef(org, proj1) + + private val metric1: ProjectScopedMetric = ProjectScopedMetric( + Instant.EPOCH, + Anonymous, + 1, + Set(Created), + projectRef1, + org, + iri"http://bbp.epfl.ch/file1", + Set(nxv + "Resource1", nxv + "Resource2"), + JsonObject("extraField" := "extraValue") + ) + private val metric2: ProjectScopedMetric = ProjectScopedMetric( + Instant.EPOCH, + Anonymous, + 2, + Set(Updated), + projectRef1, + org, + iri"http://bbp.epfl.ch/file1", + Set(nxv + "Resource1", nxv + "Resource3"), + JsonObject( + "extraField" := "extraValue", + "extraField2" := 42 + ) + ) + private val stream = ProjectScopedMetricStream(EntityType("entity"), metric1, metric2) } diff --git a/delta/plugins/elasticsearch/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/elasticsearch/metrics/EventMetricsQuerySuite.scala b/delta/plugins/elasticsearch/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/elasticsearch/metrics/EventMetricsQuerySuite.scala new file mode 100644 index 0000000000..dca08a85ed --- /dev/null +++ b/delta/plugins/elasticsearch/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/elasticsearch/metrics/EventMetricsQuerySuite.scala @@ -0,0 +1,68 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.metrics + +import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.client.ElasticSearchAction +import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.client.ElasticSearchClient.BulkResponse +import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.{ElasticSearchClientSetup, Fixtures} +import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary.nxv +import ch.epfl.bluebrain.nexus.delta.sourcing.model.ProjectRef +import ch.epfl.bluebrain.nexus.testkit.mu.NexusSuite +import io.circe.JsonObject +import io.circe.syntax.{EncoderOps, KeyOps} +import munit.AnyFixture + +class EventMetricsQuerySuite extends NexusSuite with ElasticSearchClientSetup.Fixture with Fixtures { + + override def munitFixtures: Seq[AnyFixture[_]] = List(esClient) + + private lazy val client = esClient() + + private val prefix = "nexus" + + private val index = eventMetricsIndex(prefix) + + private lazy val eventMetricsQuery = EventMetricsQuery(client, prefix) + + private val project1 = ProjectRef.unsafe("org", "project1") + private val project2 = ProjectRef.unsafe("org", "project2") + + private val id1 = nxv + "id1" + private val id2 = nxv + "id1" + + private val event11 = JsonObject( + "project" := project1, + "@type" := Set(nxv + "Type11"), + "@id" := id1, + "rev" := 1 + ) + + private val event12 = JsonObject( + "project" := project1, + "@type" := Set(nxv + "Type11", nxv + "Type12"), + "@id" := id1, + "rev" := 2 + ) + + private val event21 = JsonObject( + "project" := project2, + "@type" := nxv + "Type2", + "@id" := id2, + "rev" := 1 + ) + + test("Query for a resource history") { + for { + _ <- client.createIndex(index, Some(metricsMapping.value), Some(metricsSettings.value)).assertEquals(true) + bulk = List(event11, event12, event21).zipWithIndex.map { case (event, i) => + ElasticSearchAction.Index(index, i.toString, event.asJson) + } + _ <- client.bulk(bulk).assertEquals(BulkResponse.Success) + _ <- client.refresh(index) + _ <- client.count(index.value).assertEquals(3L) + result <- eventMetricsQuery.history(project1, id1) + } yield { + assertEquals(result.total, 2L) + assertEquals(result.sources, List(event11, event12)) + } + } + +} diff --git a/delta/plugins/elasticsearch/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/elasticsearch/metrics/MetricsStream.scala b/delta/plugins/elasticsearch/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/elasticsearch/metrics/MetricsStream.scala deleted file mode 100644 index 48853a9131..0000000000 --- a/delta/plugins/elasticsearch/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/elasticsearch/metrics/MetricsStream.scala +++ /dev/null @@ -1,130 +0,0 @@ -package ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.metrics - -import cats.effect.IO -import ch.epfl.bluebrain.nexus.delta.rdf.Vocabulary.nxv -import ch.epfl.bluebrain.nexus.delta.rdf.syntax.iriStringContextSyntax -import ch.epfl.bluebrain.nexus.delta.sdk.model.metrics.EventMetric.{ProjectScopedMetric, _} -import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.Anonymous -import ch.epfl.bluebrain.nexus.delta.sourcing.model.{EntityType, Label, ProjectRef} -import ch.epfl.bluebrain.nexus.delta.sourcing.offset.Offset -import ch.epfl.bluebrain.nexus.delta.sourcing.stream.Elem -import fs2.Stream -import io.circe.{Json, JsonObject} - -import java.time.Instant - -object MetricsStream { - - private val org = Label.unsafe("org") - private val proj1 = Label.unsafe("proj1") - private val proj2 = Label.unsafe("proj2") - val projectRef1: ProjectRef = ProjectRef(org, proj1) - val projectRef2: ProjectRef = ProjectRef(org, proj2) - - val metric1: ProjectScopedMetric = ProjectScopedMetric( - Instant.EPOCH, - Anonymous, - 1, - Set(Created), - projectRef1, - org, - iri"http://bbp.epfl.ch/file1", - Set(iri"File"), - JsonObject.apply( - "storage" -> Json.fromString("storageId"), - "newFileWritten" -> Json.fromInt(1), - "bytes" -> Json.fromLong(10L) - ) - ) - val metric2: ProjectScopedMetric = ProjectScopedMetric( - Instant.EPOCH, - Anonymous, - 2, - Set(Updated), - projectRef1, - org, - iri"http://bbp.epfl.ch/file1", - Set(iri"File"), - JsonObject.apply( - "storage" -> Json.fromString("storageId"), - "newFileWritten" -> Json.fromInt(1), - "bytes" -> Json.fromLong(20L) - ) - ) - val metric3: ProjectScopedMetric = ProjectScopedMetric( - Instant.EPOCH, - Anonymous, - 3, - Set(Tagged), - projectRef1, - org, - iri"http://bbp.epfl.ch/file1", - Set(iri"File"), - JsonObject.apply( - "storage" -> Json.fromString("storageId") - ) - ) - val metric4: ProjectScopedMetric = ProjectScopedMetric( - Instant.EPOCH, - Anonymous, - 4, - Set(TagDeleted), - projectRef1, - org, - iri"http://bbp.epfl.ch/file1", - Set(iri"File"), - JsonObject.apply( - "storage" -> Json.fromString("storageId") - ) - ) - val metric5: ProjectScopedMetric = ProjectScopedMetric( - Instant.EPOCH, - Anonymous, - 1, - Set(Created), - projectRef2, - org, - iri"http://bbp.epfl.ch/file2", - Set(iri"File"), - JsonObject.apply( - "storage" -> Json.fromString("storageId"), - "newFileWritten" -> Json.fromInt(1), - "bytes" -> Json.fromLong(20L) - ) - ) - val metric6: ProjectScopedMetric = ProjectScopedMetric( - Instant.EPOCH, - Anonymous, - 2, - Set(Deprecated), - projectRef2, - org, - iri"http://bbp.epfl.ch/file2", - Set(iri"File"), - JsonObject.apply( - "storage" -> Json.fromString("storageId") - ) - ) - - private val elems = List( - // Create file in proj1 - elem("1", 1L, metric1), - // Update file in proj1 - elem("2", 2L, metric2), - // Tag file in proj1 - elem("3", 3L, metric3), - // Delete file tag in proj 1 - elem("4", 4L, metric4), - // Create file in proj 2 - elem("5", 5L, metric5), - // Deprecate file in proj 2 - elem("6", 6L, metric6) - ) - - def elem(idSuffix: String, offset: Long, metric: ProjectScopedMetric) = - Elem.SuccessElem(EntityType("entity"), nxv + idSuffix, None, Instant.EPOCH, Offset.At(offset), metric, 1) - - val metricsStream: Stream[IO, Elem.SuccessElem[ProjectScopedMetric]] = - Stream.emits(elems) - -} diff --git a/delta/plugins/elasticsearch/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/elasticsearch/routes/ElasticSearchHistoryRoutesSpec.scala b/delta/plugins/elasticsearch/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/elasticsearch/routes/ElasticSearchHistoryRoutesSpec.scala new file mode 100644 index 0000000000..e27deab196 --- /dev/null +++ b/delta/plugins/elasticsearch/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/elasticsearch/routes/ElasticSearchHistoryRoutesSpec.scala @@ -0,0 +1,54 @@ +package ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.routes + +import akka.http.scaladsl.model.StatusCodes +import akka.http.scaladsl.server.Route +import cats.effect.IO +import ch.epfl.bluebrain.nexus.delta.kernel.utils.UrlUtils +import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.metrics.EventMetricsQuery +import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode +import ch.epfl.bluebrain.nexus.delta.rdf.syntax.iriStringContextSyntax +import ch.epfl.bluebrain.nexus.delta.sdk.acls.model.AclAddress +import ch.epfl.bluebrain.nexus.delta.sdk.model.search.SearchResults +import ch.epfl.bluebrain.nexus.delta.sdk.permissions.Permissions.resources +import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.Anonymous +import ch.epfl.bluebrain.nexus.delta.sourcing.model.ProjectRef +import io.circe.JsonObject +import io.circe.syntax.KeyOps + +class ElasticSearchHistoryRoutesSpec extends ElasticSearchViewsRoutesFixtures { + + private val myId = iri"""https://bbp.epfl.ch/data/myid""" + private val myIdEncoded = UrlUtils.encode(myId.toString) + + private val eventMetricsQuery = new EventMetricsQuery { + override def history(project: ProjectRef, id: IriOrBNode.Iri): IO[SearchResults[JsonObject]] = { + IO.pure(SearchResults(1L, List(JsonObject("project" := project, "@id" := id)))) + } + } + + private lazy val routes = + Route.seal( + new ElasticSearchHistoryRoutes( + identities, + aclCheck, + eventMetricsQuery + ).routes + ) + + "Fail to access the history of a resource if the user has no access" in { + Get(s"/history/resources/org/proj/$myIdEncoded") ~> routes ~> check { + response.status shouldEqual StatusCodes.Forbidden + } + } + + "Return the history if no access if the user has access" in { + aclCheck.append(AclAddress.Root, Anonymous -> Set(resources.read)).accepted + Get(s"/history/resources/org/proj/$myIdEncoded") ~> routes ~> check { + response.status shouldEqual StatusCodes.OK + val expected = + json"""{ "_total" : 1, "_results" : [{ "@id" : "https://bbp.epfl.ch/data/myid", "project" : "org/proj" } ]}""" + response.asJson shouldEqual expected + } + } + +} diff --git a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/StoragesStatistics.scala b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/StoragesStatistics.scala index e047e2c92e..83a1949f2d 100644 --- a/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/StoragesStatistics.scala +++ b/delta/plugins/storage/src/main/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/storages/StoragesStatistics.scala @@ -2,8 +2,9 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.storages import akka.http.scaladsl.model.Uri.Query import cats.effect.IO -import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.EventMetricsProjection.eventMetricsIndex +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.nxvFile import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.client.ElasticSearchClient +import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.metrics.eventMetricsIndex import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageStatEntry import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri import ch.epfl.bluebrain.nexus.delta.sdk.model.IdSegment @@ -64,7 +65,7 @@ object StoragesStatistics { "query": { "bool": { "filter": [ - { "term": { "@type.short": "File" } }, + { "term": { "@type": $nxvFile } }, { "term": { "project": $projectRef } }, { "term": { "storage": $storageId } } ] diff --git a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/statistics/StoragesStatisticsSuite.scala b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/statistics/StoragesStatisticsSuite.scala index 2e8e1241e1..48945974df 100644 --- a/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/statistics/StoragesStatisticsSuite.scala +++ b/delta/plugins/storage/src/test/scala/ch/epfl/bluebrain/nexus/delta/plugins/storage/statistics/StoragesStatisticsSuite.scala @@ -1,21 +1,30 @@ package ch.epfl.bluebrain.nexus.delta.plugins.storage.statistics import cats.effect.IO -import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.EventMetricsProjection.eventMetricsIndex import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.client.ElasticSearchClient import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.client.ElasticSearchClient.Refresh import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.indexing.ElasticSearchSink -import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.metrics.MetricsStream.{metricsStream, projectRef1, projectRef2} -import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.{ElasticSearchClientSetup, EventMetricsProjection, Fixtures} +import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.metrics.{eventMetricsIndex, EventMetricsProjection} +import ch.epfl.bluebrain.nexus.delta.plugins.elasticsearch.{ElasticSearchClientSetup, Fixtures} +import ch.epfl.bluebrain.nexus.delta.plugins.storage.files.nxvFile +import ch.epfl.bluebrain.nexus.delta.plugins.storage.statistics.StoragesStatisticsSuite.{metricsStream, projectRef1, projectRef2} import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.StoragesStatistics import ch.epfl.bluebrain.nexus.delta.plugins.storage.storages.model.StorageStatEntry import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri +import ch.epfl.bluebrain.nexus.delta.rdf.syntax.iriStringContextSyntax +import ch.epfl.bluebrain.nexus.delta.sdk.metrics.ProjectScopedMetricStream +import ch.epfl.bluebrain.nexus.delta.sdk.model.metrics.EventMetric.{Created, Deprecated, ProjectScopedMetric, TagDeleted, Tagged, Updated} +import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.Anonymous +import ch.epfl.bluebrain.nexus.delta.sourcing.model.{EntityType, Label, ProjectRef} import ch.epfl.bluebrain.nexus.delta.sourcing.stream.SupervisorSetup import ch.epfl.bluebrain.nexus.delta.sourcing.stream.SupervisorSetup.unapply import ch.epfl.bluebrain.nexus.testkit.mu.NexusSuite import ch.epfl.bluebrain.nexus.testkit.mu.ce.PatienceConfig +import io.circe.JsonObject +import io.circe.syntax.KeyOps import munit.AnyFixture +import java.time.Instant import scala.concurrent.duration.DurationInt class StoragesStatisticsSuite @@ -57,3 +66,99 @@ class StoragesStatisticsSuite } } + +object StoragesStatisticsSuite { + private val org = Label.unsafe("org") + private val proj1 = Label.unsafe("proj1") + private val proj2 = Label.unsafe("proj2") + val projectRef1: ProjectRef = ProjectRef(org, proj1) + val projectRef2: ProjectRef = ProjectRef(org, proj2) + + private val metric1 = ProjectScopedMetric( + Instant.EPOCH, + Anonymous, + 1, + Set(Created), + projectRef1, + org, + iri"http://bbp.epfl.ch/file1", + Set(nxvFile), + JsonObject( + "storage" := "storageId", + "newFileWritten" := 1, + "bytes" := 10L + ) + ) + + private val metric2 = ProjectScopedMetric( + Instant.EPOCH, + Anonymous, + 2, + Set(Updated), + projectRef1, + org, + iri"http://bbp.epfl.ch/file1", + Set(nxvFile), + JsonObject( + "storage" := "storageId", + "newFileWritten" := 1, + "bytes" := 20L + ) + ) + + private val metric3 = ProjectScopedMetric( + Instant.EPOCH, + Anonymous, + 3, + Set(Tagged), + projectRef1, + org, + iri"http://bbp.epfl.ch/file1", + Set(nxvFile), + JsonObject("storage" := "storageId") + ) + + private val metric4 = ProjectScopedMetric( + Instant.EPOCH, + Anonymous, + 4, + Set(TagDeleted), + projectRef1, + org, + iri"http://bbp.epfl.ch/file1", + Set(nxvFile), + JsonObject("storage" := "storageId") + ) + + private val metric5 = ProjectScopedMetric( + Instant.EPOCH, + Anonymous, + 1, + Set(Created), + projectRef2, + org, + iri"http://bbp.epfl.ch/file2", + Set(nxvFile), + JsonObject( + "storage" := "storageId", + "newFileWritten" := 1, + "bytes" := 20L + ) + ) + + private val metric6 = ProjectScopedMetric( + Instant.EPOCH, + Anonymous, + 2, + Set(Deprecated), + projectRef2, + org, + iri"http://bbp.epfl.ch/file2", + Set(nxvFile), + JsonObject("storage" := "storageId") + ) + + private val metricsStream = + ProjectScopedMetricStream(EntityType("entity"), metric1, metric2, metric3, metric4, metric5, metric6) + +} diff --git a/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/model/metrics/EventMetric.scala b/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/model/metrics/EventMetric.scala index 2f3d2a393b..d7f536653f 100644 --- a/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/model/metrics/EventMetric.scala +++ b/delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/model/metrics/EventMetric.scala @@ -2,14 +2,13 @@ package ch.epfl.bluebrain.nexus.delta.sdk.model.metrics import ch.epfl.bluebrain.nexus.delta.rdf.IriOrBNode.Iri import ch.epfl.bluebrain.nexus.delta.rdf.jsonld.context.JsonLdContext.keywords -import ch.epfl.bluebrain.nexus.delta.sdk.implicits._ import ch.epfl.bluebrain.nexus.delta.sourcing.event.Event.ScopedEvent import ch.epfl.bluebrain.nexus.delta.sourcing.model.Identity.Subject import ch.epfl.bluebrain.nexus.delta.sourcing.model.{Label, ProjectRef} import io.circe.generic.extras.Configuration import io.circe.generic.extras.semiauto.deriveConfiguredEncoder -import io.circe.syntax.EncoderOps -import io.circe.{Encoder, Json, JsonObject} +import io.circe.syntax.{EncoderOps, KeyOps} +import io.circe.{Encoder, JsonObject} import java.time.Instant @@ -51,7 +50,7 @@ sealed trait EventMetric extends Product with Serializable { * @return * the id of the underlying resource */ - def resourceId: Iri + def id: Iri /** * @return @@ -86,11 +85,11 @@ object EventMetric { subject: Subject, rev: Int, action: Set[Label], - resourceId: Iri, + id: Iri, types: Set[Iri], additionalFields: JsonObject ) extends EventMetric { - def eventId: String = s"$resourceId-$rev" + def eventId: String = s"$id-$rev" } /** @@ -102,11 +101,11 @@ object EventMetric { rev: Int, action: Set[Label], organization: Label, - resourceId: Iri, + id: Iri, types: Set[Iri], additionalFields: JsonObject ) extends EventMetric { - def eventId: String = s"$organization-$resourceId-$rev" + def eventId: String = s"$organization-$id-$rev" } /** @@ -119,11 +118,11 @@ object EventMetric { action: Set[Label], project: ProjectRef, organization: Label, - resourceId: Iri, + id: Iri, types: Set[Iri], additionalFields: JsonObject ) extends EventMetric { - def eventId: String = s"$project-$resourceId-$rev" + def eventId: String = s"$project/$id:$rev" } object ProjectScopedMetric { @@ -172,16 +171,12 @@ object EventMetric { implicit val subjectCodec: Encoder[Subject] = deriveConfiguredEncoder[Subject] Encoder.AsObject.instance { e => val common = JsonObject( - "instant" -> e.instant.asJson, - "subject" -> e.subject.asJson, - "action" -> e.action.asJson, - "@id" -> e.resourceId.asJson, - "@type" -> e.types.map { tpe => - Json.obj( - "raw" -> tpe.asJson, - "short" -> tpe.toUri.toOption.flatMap { uri => uri.fragment.orElse(uri.path.lastSegment) }.asJson - ) - }.asJson + "instant" := e.instant, + "subject" := e.subject, + "action" := e.action, + "@id" := e.id, + "rev" := e.rev, + "@type" := e.types ) val scoped = e match { diff --git a/delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/metrics/ProjectScopedMetricStream.scala b/delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/metrics/ProjectScopedMetricStream.scala new file mode 100644 index 0000000000..2bf95b69ea --- /dev/null +++ b/delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/metrics/ProjectScopedMetricStream.scala @@ -0,0 +1,22 @@ +package ch.epfl.bluebrain.nexus.delta.sdk.metrics + +import cats.effect.IO +import ch.epfl.bluebrain.nexus.delta.sdk.model.metrics.EventMetric.ProjectScopedMetric +import ch.epfl.bluebrain.nexus.delta.sourcing.model.EntityType +import ch.epfl.bluebrain.nexus.delta.sourcing.offset.Offset +import ch.epfl.bluebrain.nexus.delta.sourcing.stream.Elem +import fs2.Stream + +object ProjectScopedMetricStream { + + def apply(entityType: EntityType, metrics: ProjectScopedMetric*): Stream[IO, Elem.SuccessElem[ProjectScopedMetric]] = + Stream.emits( + metrics.zipWithIndex.map { case (metric, index) => + elem(entityType, metric, index + 1L) + } + ) + + private def elem(entityType: EntityType, metric: ProjectScopedMetric, offset: Long) = + Elem.SuccessElem(entityType, metric.id, Some(metric.project), metric.instant, Offset.At(offset), metric, metric.rev) + +} diff --git a/delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/model/metrics/EventMetricSpec.scala b/delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/model/metrics/EventMetricSpec.scala index cd7670d604..0d3ad82b75 100644 --- a/delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/model/metrics/EventMetricSpec.scala +++ b/delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/model/metrics/EventMetricSpec.scala @@ -56,15 +56,10 @@ class EventMetricSpec extends BaseSpec with CirceLiteral { }, "action" : ["Created"], "@id" : "https://bluebrain.github.io/nexus/vocabulary/id", + "rev": 2, "@type" : [ - { - "raw": "https://bluebrain.github.io/nexus/vocabulary/Type1", - "short": "Type1" - }, - { - "raw": "https://bluebrain.github.io/nexus/vocabulary/Type1#Type2", - "short": "Type2" - } + "https://bluebrain.github.io/nexus/vocabulary/Type1", + "https://bluebrain.github.io/nexus/vocabulary/Type1#Type2" ], "project" : "org/proj", "organization" : "org", diff --git a/docs/src/main/paradox/docs/delta/api/assets/history/request.sh b/docs/src/main/paradox/docs/delta/api/assets/history/request.sh new file mode 100644 index 0000000000..f4cd539483 --- /dev/null +++ b/docs/src/main/paradox/docs/delta/api/assets/history/request.sh @@ -0,0 +1 @@ +curl "http://localhost:8080/v1/org/project/https%3A%2F%2Fexample.com%2FAlice" \ No newline at end of file diff --git a/docs/src/main/paradox/docs/delta/api/assets/history/response.json b/docs/src/main/paradox/docs/delta/api/assets/history/response.json new file mode 100644 index 0000000000..c724dc9ea6 --- /dev/null +++ b/docs/src/main/paradox/docs/delta/api/assets/history/response.json @@ -0,0 +1,40 @@ +{ + "_total": 2, + "_results": [ + { + "@id": "https://example.com/Alice", + "@type": [ + "http://schema.org/Person" + ], + "action": [ + "Created" + ], + "organization": "org", + "project": "org/project", + "rev": 1, + "subject": { + "@type": "User", + "realm": "realm", + "subject": "user" + } + }, + { + "@id": "https://example.com/Alice", + "@type": [ + "http://schema.org/Person" + ], + "action": [ + "Tagged" + ], + "instant": "1970-01-01T00:00:00Z", + "organization": "org", + "project": "org/project", + "rev": 2, + "subject": { + "@type": "User", + "realm": "realm", + "subject": "user" + } + } + ] +} \ No newline at end of file diff --git a/docs/src/main/paradox/docs/delta/api/history.md b/docs/src/main/paradox/docs/delta/api/history.md new file mode 100644 index 0000000000..1e2aebccf4 --- /dev/null +++ b/docs/src/main/paradox/docs/delta/api/history.md @@ -0,0 +1,36 @@ +# History of resources + +The history endpoint allows to get an overview of the differents events and actions which occured +during the lifetime of a resource (creation/update/tag/deprecation) for the different types of resources +(generic resources, schemas, files). + +@@@ note { .tip title="Authorization notes" } + +When performing a request, the caller must have `resources/read` permission on the project the resource belongs to. + +Please visit @ref:[Authentication & authorization](authentication.md) section to learn more about it. + +@@@ + +@@@ note { .warning } + +The described endpoints are experimental and the responses structure might change in the future. + +@@@ + +``` +GET /v1/history/{org_label}/{project_label}/{id} +``` + +where... + +- `{id}`: the identifier of the resource to resolve (URL encoded value). + +**Example** + +Request +: @@snip [request.sh](assets/history/request.sh) + +Response +: @@snip [response.json](assets/history/response.json) + diff --git a/docs/src/main/paradox/docs/delta/api/index.md b/docs/src/main/paradox/docs/delta/api/index.md index 443f04e6d3..5592d412e8 100644 --- a/docs/src/main/paradox/docs/delta/api/index.md +++ b/docs/src/main/paradox/docs/delta/api/index.md @@ -17,6 +17,7 @@ * @ref:[Practice](trial.md) * @ref:[Multi-fetch](multi-fetch.md) * @ref:[Id Resolution](id-resolution.md) +* @ref:[History](history.md) * @ref:[Resolvers](resolvers-api.md) * @ref:[Views](views/index.md) * @ref:[Storages](storages-api.md) diff --git a/tests/src/test/resources/resources/history-file.json b/tests/src/test/resources/resources/history-file.json new file mode 100644 index 0000000000..cdd7b6a532 --- /dev/null +++ b/tests/src/test/resources/resources/history-file.json @@ -0,0 +1,48 @@ +{ + "_total": 2, + "_results": [ + { + "@id": "https://bbp.epfl.ch/data/my-file", + "@type": [ + "https://bluebrain.github.io/nexus/vocabulary/File" + ], + "action": [ + "Created" + ], + "bytes": 28, + "extension": "json", + "mediaType": "application/json", + "newFileWritten": 1, + "organization": "{{org}}", + "origin": "Client", + "project": "{{org}}/{{project}}", + "rev": 1, + "storage": "https://bluebrain.github.io/nexus/vocabulary/diskStorageDefault", + "storageType": "DiskStorage", + "subject": { + "@type": "User", + "realm": "{{realm}}", + "subject": "{{user}}" + } + }, + { + "@id": "https://bbp.epfl.ch/data/my-file", + "@type": [ + "https://bluebrain.github.io/nexus/vocabulary/File" + ], + "action": [ + "Tagged" + ], + "organization": "{{org}}", + "project": "{{org}}/{{project}}", + "rev": 2, + "storage": "https://bluebrain.github.io/nexus/vocabulary/diskStorageDefault", + "storageType": "DiskStorage", + "subject": { + "@type": "User", + "realm": "{{realm}}", + "subject": "{{user}}" + } + } + ] +} \ No newline at end of file diff --git a/tests/src/test/resources/resources/history-resource.json b/tests/src/test/resources/resources/history-resource.json new file mode 100644 index 0000000000..a66cc9f1ad --- /dev/null +++ b/tests/src/test/resources/resources/history-resource.json @@ -0,0 +1,73 @@ +{ + "_total": 4, + "_results": [ + { + "@id": "https://bbp.epfl.ch/data/my-resource", + "@type": [ + "http://schema.org/TestResource" + ], + "action": [ + "Created" + ], + "organization": "{{org}}", + "project": "{{org}}/{{project}}", + "rev": 1, + "subject": { + "@type": "User", + "realm": "{{realm}}", + "subject": "{{user}}" + } + }, + { + "@id": "https://bbp.epfl.ch/data/my-resource", + "@type": [ + "http://schema.org/TestResource" + ], + "action": [ + "Updated" + ], + "organization": "{{org}}", + "project": "{{org}}/{{project}}", + "rev": 2, + "subject": { + "@type": "User", + "realm": "{{realm}}", + "subject": "{{user}}" + } + }, + { + "@id": "https://bbp.epfl.ch/data/my-resource", + "@type": [ + "http://schema.org/TestResource" + ], + "action": [ + "Tagged" + ], + "organization": "{{org}}", + "project": "{{org}}/{{project}}", + "rev": 3, + "subject": { + "@type": "User", + "realm": "{{realm}}", + "subject": "{{user}}" + } + }, + { + "@id": "https://bbp.epfl.ch/data/my-resource", + "@type": [ + "http://schema.org/TestResource" + ], + "action": [ + "Deprecated" + ], + "organization": "{{org}}", + "project": "{{org}}/{{project}}", + "rev": 4, + "subject": { + "@type": "User", + "realm": "{{realm}}", + "subject": "{{user}}" + } + } + ] +} \ No newline at end of file diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/HttpClient.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/HttpClient.scala index aef30c474f..f9aa082de9 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/HttpClient.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/HttpClient.scala @@ -14,6 +14,7 @@ import akka.stream.alpakka.sse.scaladsl.EventSource import akka.stream.scaladsl.Sink import cats.effect.IO import cats.effect.unsafe.implicits._ +import ch.epfl.bluebrain.nexus.delta.kernel.utils.UrlUtils import ch.epfl.bluebrain.nexus.tests.HttpClient.{jsonHeaders, rdfApplicationSqlQuery, tokensMap} import ch.epfl.bluebrain.nexus.tests.Identity.Anonymous import ch.epfl.bluebrain.nexus.tests.kg.files.model.FileInput @@ -130,7 +131,7 @@ class HttpClient private (baseUrl: Uri, httpExt: HttpExt)(implicit val storageParam = storage.map { s => s"storage=nxv:$s" } val revParam = rev.map { r => s"&rev=$r" } val params = (storageParam ++ revParam).mkString("?", "&", "") - val requestPath = s"/files/$project/${file.fileId}$params" + val requestPath = s"/files/$project/${UrlUtils.encode(file.fileId)}$params" def buildClue(a: Json, response: HttpResponse) = s""" |Endpoint: PUT $requestPath diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/files/S3DelegationFileSpec.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/files/S3DelegationFileSpec.scala index b246748ae7..296a9e54ef 100644 --- a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/files/S3DelegationFileSpec.scala +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/kg/files/S3DelegationFileSpec.scala @@ -120,7 +120,7 @@ class S3DelegationFileSpec extends BaseIntegrationSpec with S3ClientFixtures { val fileId = s"https://bluebrain.github.io/nexus/vocabulary/${genId()}" val filename = genString() val updatedFilename = genString() - val originalFile = FileInput(UrlUtils.encode(fileId), filename, ContentTypes.`text/plain(UTF-8)`, "test") + val originalFile = FileInput(fileId, filename, ContentTypes.`text/plain(UTF-8)`, "test") val delegationPayload = Json.obj("filename" := updatedFilename, "mediaType" := "image/png") for { _ <- deltaClient.uploadFile(projectRef, "defaultS3Storage", originalFile, None) { expectCreated }(Coyote) diff --git a/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/resources/ResourceHistorySpec.scala b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/resources/ResourceHistorySpec.scala new file mode 100644 index 0000000000..8bf8e3c62f --- /dev/null +++ b/tests/src/test/scala/ch/epfl/bluebrain/nexus/tests/resources/ResourceHistorySpec.scala @@ -0,0 +1,85 @@ +package ch.epfl.bluebrain.nexus.tests.resources + +import akka.http.scaladsl.model.{ContentTypes, StatusCodes} +import ch.epfl.bluebrain.nexus.delta.kernel.utils.UrlUtils +import ch.epfl.bluebrain.nexus.tests.Identity.Anonymous +import ch.epfl.bluebrain.nexus.tests.Identity.resources.Rick +import ch.epfl.bluebrain.nexus.tests.kg.files.model.FileInput +import ch.epfl.bluebrain.nexus.tests.kg.files.model.FileInput.CustomMetadata +import ch.epfl.bluebrain.nexus.tests.{BaseIntegrationSpec, Identity, Optics} +import io.circe.Json + +class ResourceHistorySpec extends BaseIntegrationSpec { + + implicit val currentUser: Identity.UserCredentials = Rick + + private val orgId = genId() + private val projId = genId() + private val project = s"$orgId/$projId" + + private val resourceId = "https://bbp.epfl.ch/data/my-resource" + private val encodedResourceId = UrlUtils.encode(resourceId) + + private val fileInput = FileInput( + "https://bbp.epfl.ch/data/my-file", + "attachment.json", + ContentTypes.NoContentType, + """{ "content: "Some content" }""", + CustomMetadata("Crb 2", "A cerebellum file", Map("brainRegion" -> "cerebellum")) + ) + private val encodedFileId = UrlUtils.encode(fileInput.fileId) + + private val mapping = replacements( + Rick, + "org" -> orgId, + "project" -> projId + ) + + override def beforeAll(): Unit = { + super.beforeAll() + val setup = for { + _ <- createOrg(Rick, orgId) + _ <- createProjects(Rick, orgId, projId) + resourcePayload <- SimpleResource.sourcePayload(resourceId, 42) + // Creating, updating, tagging and deprecating a resource + _ <- deltaClient.post[Json](s"/resources/$project/_/", resourcePayload, Rick) { expectCreated } + updatedPayload <- SimpleResource.sourcePayload(resourceId, 9000) + _ <- deltaClient.put[Json](s"/resources/$project/_/$encodedResourceId?rev=1", updatedPayload, Rick) { + expectOk + } + _ <- deltaClient.post[Json](s"/resources/$project/_/$encodedResourceId/tags?rev=2", tag("v1.0.0", 1), Rick) { + expectCreated + } + _ <- deltaClient.delete(s"/resources/$project/_/$encodedResourceId?rev=3", Rick) { expectOk } + _ <- deltaClient.uploadFile(project, None, fileInput, None) { expectCreated } + _ <- deltaClient.post[Json](s"/files/$project/$encodedFileId/tags?rev=1", tag("v1.0.0", 1), Rick) { + expectCreated + } + } yield () + setup.accepted + } + + private def filterInstantField = Optics.filterNestedKeys("instant") + + "Getting the history of a resource" should { + "fail for a user without access" in { + deltaClient.get[Json](s"/history/resources/$project/$encodedResourceId", Anonymous) { expectForbidden } + } + + "succeed on a resource for a user with access" in eventually { + deltaClient.get[Json](s"/history/resources/$project/$encodedResourceId", Rick) { (json, response) => + response.status shouldEqual StatusCodes.OK + val expected = jsonContentOf("resources/history-resource.json", mapping: _*) + filterInstantField(json) shouldEqual expected + } + } + + "succeed on a file for a user with access" in eventually { + deltaClient.get[Json](s"/history/resources/$project/$encodedFileId", Rick) { (json, response) => + val expected = jsonContentOf("resources/history-file.json", mapping: _*) + response.status shouldEqual StatusCodes.OK + filterInstantField(json) shouldEqual expected + } + } + } +}