forbid nexus metadata fields in user payloads (#4013)

* forbid nexus metadata fields in user payloads

* add config to turn off metadata field rejection

* add second underscore check

* add integration test

* change approach

* add back testing for the decoding lenience switch

* add docs

* add release notes

* fix problems with config loading

* fix test with underscore in payload

* docs update

delta/app/src/main/resources/app.conf

@@ -256,6 +256,8 @@ app {
   resources {
     # the resources event-log configuration
     event-log = ${app.defaults.event-log}
+    # Reject payloads which contain nexus metadata fields (any field beginning with _)
+    decoding-option = "strict"
   }
 
   # Schemas configuration

delta/app/src/main/scala/ch/epfl/bluebrain/nexus/delta/routes/ResourcesRoutes.scala

@@ -22,7 +22,8 @@ import ch.epfl.bluebrain.nexus.delta.sdk.marshalling.RdfMarshalling
 import ch.epfl.bluebrain.nexus.delta.sdk.model.routes.Tag
 import ch.epfl.bluebrain.nexus.delta.sdk.model.{BaseUri, ResourceF}
 import ch.epfl.bluebrain.nexus.delta.sdk.permissions.Permissions.resources.{read => Read, write => Write}
-import ch.epfl.bluebrain.nexus.delta.sdk.resources.Resources
+import ch.epfl.bluebrain.nexus.delta.sdk.resources.NexusSource.DecodingOption
+import ch.epfl.bluebrain.nexus.delta.sdk.resources.{NexusSource, Resources}
 import ch.epfl.bluebrain.nexus.delta.sdk.resources.model.ResourceRejection.{InvalidJsonLdFormat, InvalidSchemaRejection, ResourceNotFound}
 import ch.epfl.bluebrain.nexus.delta.sdk.resources.model.{Resource, ResourceRejection}
 import io.circe.{Json, Printer}
@@ -55,7 +56,8 @@ final class ResourcesRoutes(
     s: Scheduler,
     cr: RemoteContextResolution,
     ordering: JsonKeyOrdering,
-    fusionConfig: FusionConfig
+    fusionConfig: FusionConfig,
+    decodingOption: DecodingOption
 ) extends AuthDirectives(identities, aclCheck)
     with CirceUnmarshalling
     with RdfMarshalling {
@@ -75,15 +77,16 @@ final class ResourcesRoutes(
           resolveProjectRef.apply { ref =>
             concat(
               // Create a resource without schema nor id segment
-              (post & pathEndOrSingleSlash & noParameter("rev") & entity(as[Json]) & indexingMode) { (source, mode) =>
-                operationName(s"$prefixSegment/resources/{org}/{project}") {
-                  authorizeFor(ref, Write).apply {
-                    emit(
-                      Created,
-                      resources.create(ref, resourceSchema, source).tapEval(index(ref, _, mode)).map(_.void)
-                    )
+              (post & pathEndOrSingleSlash & noParameter("rev") & entity(as[NexusSource]) & indexingMode) {
+                (source, mode) =>
+                  operationName(s"$prefixSegment/resources/{org}/{project}") {
+                    authorizeFor(ref, Write).apply {
+                      emit(
+                        Created,
+                        resources.create(ref, resourceSchema, source.value).tapEval(index(ref, _, mode)).map(_.void)
+                      )
+                    }
                   }
-                }
               },
               (idSegment & indexingMode) { (schema, mode) =>
                 val schemaOpt = underscoreToOption(schema)
@@ -92,11 +95,11 @@ final class ResourcesRoutes(
                   (post & pathEndOrSingleSlash & noParameter("rev")) {
                     operationName(s"$prefixSegment/resources/{org}/{project}/{schema}") {
                       authorizeFor(ref, Write).apply {
-                        entity(as[Json]) { source =>
+                        entity(as[NexusSource]) { source =>
                           emit(
                             Created,
                             resources
-                              .create(ref, schema, source)
+                              .create(ref, schema, source.value)
                               .tapEval(index(ref, _, mode))
                               .map(_.void)
                               .rejectWhen(wrongJsonOrNotFound)
@@ -113,13 +116,13 @@ final class ResourcesRoutes(
                             // Create or update a resource
                             put {
                               authorizeFor(ref, Write).apply {
-                                (parameter("rev".as[Int].?) & pathEndOrSingleSlash & entity(as[Json])) {
+                                (parameter("rev".as[Int].?) & pathEndOrSingleSlash & entity(as[NexusSource])) {
                                   case (None, source)      =>
                                     // Create a resource with schema and id segments
                                     emit(
                                       Created,
                                       resources
-                                        .create(id, ref, schema, source)
+                                        .create(id, ref, schema, source.value)
                                         .tapEval(index(ref, _, mode))
                                         .map(_.void)
                                         .rejectWhen(wrongJsonOrNotFound)
@@ -128,7 +131,7 @@ final class ResourcesRoutes(
                                     // Update a resource
                                     emit(
                                       resources
-                                        .update(id, ref, schemaOpt, rev, source)
+                                        .update(id, ref, schemaOpt, rev, source.value)
                                         .tapEval(index(ref, _, mode))
                                         .map(_.void)
                                         .rejectWhen(wrongJsonOrNotFound)
@@ -284,7 +287,8 @@ object ResourcesRoutes {
       s: Scheduler,
       cr: RemoteContextResolution,
       ordering: JsonKeyOrdering,
-      fusionConfig: FusionConfig
+      fusionConfig: FusionConfig,
+      decodingOption: DecodingOption
   ): Route = new ResourcesRoutes(identities, aclCheck, resources, projectsDirectives, index).routes
 
   implicit private val api: JsonLdApi = JsonLdJavaApi.lenient

delta/app/src/main/scala/ch/epfl/bluebrain/nexus/delta/wiring/ResourcesModule.scala

@@ -81,14 +81,16 @@ object ResourcesModule extends ModuleDef {
         s: Scheduler,
         cr: RemoteContextResolution @Id("aggregate"),
         ordering: JsonKeyOrdering,
-        fusionConfig: FusionConfig
+        fusionConfig: FusionConfig,
+        config: AppConfig
     ) =>
       new ResourcesRoutes(identities, aclCheck, resources, schemeDirectives, indexingAction(_, _, _)(shift, cr))(
         baseUri,
         s,
         cr,
         ordering,
-        fusionConfig
+        fusionConfig,
+        config.resources.decodingOption
       )
   }
 

delta/app/src/test/resources/resources/errors/underscore-fields.json

@@ -0,0 +1,6 @@
+{
+  "@context" : "https://bluebrain.github.io/nexus/contexts/error.json",
+  "@type" : "MalformedRequestContentRejection",
+  "reason" : "The request content was malformed.",
+  "details" : "DecodingFailure at : Field(s) starting with _ found in payload: _createdAt"
+}

delta/app/src/test/scala/ch/epfl/bluebrain/nexus/delta/routes/ResourcesRoutesSpec.scala

@@ -22,6 +22,7 @@ import ch.epfl.bluebrain.nexus.delta.sdk.projects.model.ApiMappings
 import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResolverContextResolution
 import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResolverResolution.FetchResource
 import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.model.ResourceResolutionReport
+import ch.epfl.bluebrain.nexus.delta.sdk.resources.NexusSource.DecodingOption
 import ch.epfl.bluebrain.nexus.delta.sdk.resources.model.ResourceRejection.ProjectContextRejection
 import ch.epfl.bluebrain.nexus.delta.sdk.resources.{Resources, ResourcesConfig, ResourcesImpl, ValidateResource, ValidateResourceImpl}
 import ch.epfl.bluebrain.nexus.delta.sdk.schemas.model.Schema
@@ -59,15 +60,17 @@ class ResourcesRoutesSpec extends BaseRouteSpec {
   private val schema1      = SchemaGen.schema(nxv + "myschema", project.value.ref, schemaSource.removeKeys(keywords.id))
   private val schema2      = SchemaGen.schema(schema.Person, project.value.ref, schemaSource.removeKeys(keywords.id))
 
-  private val myId                = nxv + "myid"  // Resource created against no schema with id present on the payload
-  private val myId2               = nxv + "myid2" // Resource created against schema1 with id present on the payload
-  private val myId3               = nxv + "myid3" // Resource created against no schema with id passed and present on the payload
-  private val myId4               = nxv + "myid4" // Resource created against schema1 with id passed and present on the payload
-  private val myIdEncoded         = UrlUtils.encode(myId.toString)
-  private val myId2Encoded        = UrlUtils.encode(myId2.toString)
-  private val payload             = jsonContentOf("resources/resource.json", "id" -> myId)
-  private val payloadWithBlankId  = jsonContentOf("resources/resource.json", "id" -> "")
-  private val payloadWithMetadata = jsonContentOf("resources/resource-with-metadata.json", "id" -> myId)
+  private val myId                        = nxv + "myid"  // Resource created against no schema with id present on the payload
+  private val myId2                       = nxv + "myid2" // Resource created against schema1 with id present on the payload
+  private val myId3                       = nxv + "myid3" // Resource created against no schema with id passed and present on the payload
+  private val myId4                       = nxv + "myid4" // Resource created against schema1 with id passed and present on the payload
+  private val myIdEncoded                 = UrlUtils.encode(myId.toString)
+  private val myId2Encoded                = UrlUtils.encode(myId2.toString)
+  private val payload                     = jsonContentOf("resources/resource.json", "id" -> myId)
+  private val payloadWithBlankId          = jsonContentOf("resources/resource.json", "id" -> "")
+  private val payloadWithUnderscoreFields =
+    jsonContentOf("resources/resource-with-underscore-fields.json", "id" -> myId)
+  private val payloadWithMetadata         = jsonContentOf("resources/resource-with-metadata.json", "id" -> myId)
 
   private val aclCheck = AclSimpleCheck().accepted
 
@@ -83,18 +86,26 @@ class ResourcesRoutesSpec extends BaseRouteSpec {
     rcr,
     (_, _, _) => IO.raiseError(ResourceResolutionReport())
   )
-  private val config                                                          = ResourcesConfig(eventLogConfig)
 
-  private lazy val routes =
+  private def routesWithDecodingOption(implicit decodingOption: DecodingOption) = {
     Route.seal(
       ResourcesRoutes(
         IdentitiesDummy(caller),
         aclCheck,
-        ResourcesImpl(validator, fetchContext, resolverContextResolution, config, xas),
+        ResourcesImpl(
+          validator,
+          fetchContext,
+          resolverContextResolution,
+          ResourcesConfig(eventLogConfig, decodingOption),
+          xas
+        ),
         DeltaSchemeDirectives(fetchContext, ioFromMap(uuid -> projectRef.organization), ioFromMap(uuid -> projectRef)),
         IndexingAction.noop
       )
     )
+  }
+
+  private lazy val routes = routesWithDecodingOption(DecodingOption.Strict)
 
   private val payloadUpdated = payload deepMerge json"""{"name": "Alice", "address": null}"""
 
@@ -171,6 +182,26 @@ class ResourcesRoutesSpec extends BaseRouteSpec {
       }
     }
 
+    "fail if underscore fields are present" in {
+      Post("/v1/resources/myorg/myproject/_/", payloadWithUnderscoreFields.toEntity) ~> routes ~> check {
+        response.status shouldEqual StatusCodes.BadRequest
+        response.asJson shouldEqual jsonContentOf(
+          "/resources/errors/underscore-fields.json"
+        )
+      }
+    }
+
+    "succeed if underscore fields are present but the decoding is set to lenient" in {
+      val lenientDecodingRoutes = routesWithDecodingOption(DecodingOption.Lenient)
+
+      Post("/v1/resources/myorg/myproject/_/", payloadWithUnderscoreFields.toEntity) ~> lenientDecodingRoutes ~> check {
+        response.status shouldEqual StatusCodes.BadRequest
+        response.asJson shouldEqual jsonContentOf(
+          "/resources/errors/underscore-fields.json"
+        )
+      }
+    }
+
     "fail to update a resource without resources/write permission" in {
       aclCheck.subtract(AclAddress.Root, Anonymous -> Set(resources.write)).accepted
       Put("/v1/resources/myorg/myproject/_/myid?rev=1", payload.toEntity) ~> routes ~> check {

delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/jsonld/JsonLdRejection.scala

@@ -22,7 +22,7 @@ object JsonLdRejection {
       extends InvalidJsonLdRejection(s"Id '$id' does not match the id on payload '$payloadId'.")
 
   /**
-    * Rejection returned when the passed id is blaNK
+    * Rejection returned when the passed id is blank
     */
   case object BlankId extends InvalidJsonLdRejection(s"Id was blank.")
 

delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/jsonld/JsonLdSourceProcessor.scala

@@ -68,7 +68,10 @@ object JsonLdSourceProcessor {
   /**
     * Allows to parse the given json source to JsonLD compacted and expanded using static contexts
     */
-  final class JsonLdSourceParser[R](contextIri: Seq[Iri], override val uuidF: UUIDF)(implicit
+  final class JsonLdSourceParser[R](
+      contextIri: Seq[Iri],
+      override val uuidF: UUIDF
+  )(implicit
       api: JsonLdApi,
       rejectionMapper: Mapper[InvalidJsonLdRejection, R]
   ) extends JsonLdSourceProcessor {

delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/resources/NexusSource.scala

@@ -0,0 +1,70 @@
+package ch.epfl.bluebrain.nexus.delta.sdk.resources
+
+import io.circe.Decoder.Result
+import io.circe.{Decoder, DecodingFailure, HCursor, Json}
+import pureconfig.error.{CannotConvert, ConfigReaderFailures, ConvertFailure}
+import pureconfig.{ConfigCursor, ConfigReader}
+
+final case class NexusSource(value: Json) extends AnyVal
+
+object NexusSource {
+
+  sealed trait DecodingOption
+
+  object DecodingOption {
+    final case object Strict extends DecodingOption
+
+    final case object Lenient extends DecodingOption
+
+    implicit val decodingOptionConfigReader: ConfigReader[DecodingOption] = {
+      new ConfigReader[DecodingOption] {
+        private val stringReader = implicitly[ConfigReader[String]]
+        override def from(cur: ConfigCursor): ConfigReader.Result[DecodingOption] = {
+          stringReader.from(cur).flatMap {
+            case "strict"  => Right(Strict)
+            case "lenient" => Right(Lenient)
+            case other     =>
+              Left(
+                ConfigReaderFailures(
+                  ConvertFailure(
+                    CannotConvert(
+                      other,
+                      "DecodingOption",
+                      s"values can only be 'strict' or 'lenient'"
+                    ),
+                    cur
+                  )
+                )
+              )
+          }
+        }
+      }
+    }
+  }
+
+  implicit def nexusSourceDecoder(implicit decodingOption: DecodingOption): Decoder[NexusSource] = {
+
+    new Decoder[NexusSource] {
+      private val decoder = implicitly[Decoder[Json]]
+
+      println(decodingOption)
+
+      override def apply(c: HCursor): Result[NexusSource] = {
+        decoder(c).flatMap { json =>
+          val underscoreFields = json.asObject.toList.flatMap(_.keys).filter(_.startsWith("_"))
+          if (underscoreFields.nonEmpty) {
+            Left(
+              DecodingFailure(
+                s"Field(s) starting with _ found in payload: ${underscoreFields.mkString(", ")}",
+                c.history
+              )
+            )
+          } else {
+            Right(NexusSource(json))
+          }
+        }
+      }
+    }
+  }
+
+}

delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/resources/ResourcesConfig.scala

@@ -1,5 +1,6 @@
 package ch.epfl.bluebrain.nexus.delta.sdk.resources
 
+import ch.epfl.bluebrain.nexus.delta.sdk.resources.NexusSource.DecodingOption
 import ch.epfl.bluebrain.nexus.delta.sourcing.config.EventLogConfig
 import pureconfig.ConfigReader
 import pureconfig.generic.semiauto.deriveReader
@@ -10,7 +11,7 @@ import pureconfig.generic.semiauto.deriveReader
   * @param eventLog
   *   configuration of the event log
   */
-final case class ResourcesConfig(eventLog: EventLogConfig)
+final case class ResourcesConfig(eventLog: EventLogConfig, decodingOption: DecodingOption)
 
 object ResourcesConfig {
   implicit final val resourcesConfigReader: ConfigReader[ResourcesConfig] =

delta/sdk/src/main/scala/ch/epfl/bluebrain/nexus/delta/sdk/schemas/model/SchemaRejection.scala

@@ -228,7 +228,7 @@ object SchemaRejection {
   implicit val schemaJsonLdRejectionMapper: Mapper[InvalidJsonLdRejection, SchemaRejection] = {
     case UnexpectedId(id, payloadIri)                      => UnexpectedSchemaId(id, payloadIri)
     case JsonLdRejection.InvalidJsonLdFormat(id, rdfError) => InvalidJsonLdFormat(id, rdfError)
-    case BlankId                                           => InvalidSchemaId("")
+    case BlankId                                           => BlankSchemaId
   }
 
   implicit val responseFieldsSchemas: HttpResponseFields[SchemaRejection] =

delta/sdk/src/test/resources/resources/resource-with-underscore-fields.json

@@ -0,0 +1,11 @@
+{
+  "@context": {
+    "@vocab": "https://bluebrain.github.io/nexus/vocabulary/"
+  },
+  "@id": "{{id}}",
+  "@type": "Custom",
+  "name": "Alex",
+  "number": 24,
+  "bool": false,
+  "_createdAt": "1970-01-01T00:00:00Z"
+}

delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/resources/ResourcesImplSpec.scala

@@ -15,6 +15,7 @@ import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResolverContextResolution
 import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.ResolverResolution.{FetchResource, ResourceResolution}
 import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.model.ResourceResolutionReport.ResolverReport
 import ch.epfl.bluebrain.nexus.delta.sdk.resolvers.model.{ResolverResolutionRejection, ResourceResolutionReport}
+import ch.epfl.bluebrain.nexus.delta.sdk.resources.NexusSource.DecodingOption
 import ch.epfl.bluebrain.nexus.delta.sdk.resources.model.ResourceRejection.{BlankResourceId, IncorrectRev, InvalidJsonLdFormat, InvalidResource, InvalidSchemaRejection, ProjectContextRejection, ResourceAlreadyExists, ResourceIsDeprecated, ResourceNotFound, RevisionNotFound, SchemaIsDeprecated, TagNotFound, UnexpectedResourceId, UnexpectedResourceSchema}
 import ch.epfl.bluebrain.nexus.delta.sdk.schemas.model.Schema
 import ch.epfl.bluebrain.nexus.delta.sdk.syntax._
@@ -84,7 +85,7 @@ class ResourcesImplSpec
     Set(projectDeprecated.ref),
     ProjectContextRejection
   )
-  private val config       = ResourcesConfig(eventLogConfig)
+  private val config       = ResourcesConfig(eventLogConfig, DecodingOption.Strict)
 
   private val resolverContextResolution: ResolverContextResolution = new ResolverContextResolution(
     res,
@@ -672,8 +673,6 @@ class ResourcesImplSpec
       "reject if the tag doesn't exist" in {
         resources.deleteTag(myId, projectRef, Some(schemas.resources), tag, 3).rejectedWith[TagNotFound]
       }
-
     }
   }
-
 }

delta/sdk/src/test/scala/ch/epfl/bluebrain/nexus/delta/sdk/schemas/SchemasImplSpec.scala

@@ -85,7 +85,8 @@ class SchemasImplSpec
     FetchContextDummy(Map(project.ref -> project.context), Set(projectDeprecated.ref), ProjectContextRejection)
   private val config       = SchemasConfig(eventLogConfig)
 
-  private lazy val schemas: Schemas = SchemasImpl(fetchContext, schemaImports, resolverContextResolution, config, xas)
+  private lazy val schemas: Schemas =
+    SchemasImpl(fetchContext, schemaImports, resolverContextResolution, config, xas)
 
   private def schemaSourceWithId(id: Iri) = {
     source deepMerge json"""{"@id": "$id"}"""