Merge branch 'conduktor:master' into master

.github/workflows/scala.yml

@@ -54,7 +54,6 @@ jobs:
         PRE_RELEASE: "false"
         CHANGELOG_FILE: "CHANGELOG.md"
         ALLOW_EMPTY_CHANGELOG: "true"
-        ALLOW_TAG_PREFIX: "true"
       with:
         args: |
             target/universal/kafka-security-manager-*.zip

.gitignore

@@ -1,7 +1,8 @@
+.bsp/
 target/
 .idea/
 local.*
 src/main/resources/specs/KsmService.yml
 .metals/
 .bloop/
-data/
+data/

CHANGELOG.md

@@ -1,11 +1,15 @@
 # Changelog
 All notable changes to this project will be documented in this file.
 
-The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
+The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.1/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
-## [1.0.0] - SNAPSHOT
+## [1.1.0-SNAPSHOT]
+- TODO
 
+## [1.0.1] - 06/02/2022
+- Fixed log4j dependency
+- Added Google IAM
 
 ## [0.11.0] - 31/06/2021
 - Renamed packages to `io.conduktor`

README.md

@@ -1,10 +1,13 @@
 [![Actions Status](https://github.com/conduktor/kafka-security-manager/workflows/ScalaCI/badge.svg)](https://github.com/conduktor/kafka-security-manager/actions)
 
-# An open-source project by   [![Conduktor.io](https://www.conduktor.io/uploads/conduktor.svg)](https://conduktor.io/)
+# An open-source project by   [![Conduktor.io](https://www.conduktor.io/images/logo.svg)](https://conduktor.io/)
 
 This project is sponsored by [Conduktor.io](https://www.conduktor.io/), a graphical desktop user interface for Apache Kafka.
 With Conduktor you can visualize your ACLs in your Apache Kafka cluster!
 
+![Kafka Security Manager Diagram](https://i.imgur.com/DdVhUYQ.png)
+
+
 # Kafka Security Manager
 
 Kafka Security Manager (KSM) allows you to manage your Kafka ACLs at scale by leveraging an external source as the source of truth. 
@@ -189,6 +192,14 @@ The [default configurations](src/main/resources/application.conf) can be overwri
       - `SOURCE_S3_OBJECTKEY` The Object containing the ACL CSV in S3
     - `io.conduktor.ksm.source.BitbucketServerSourceAcl`: get the ACL from Bitbucket Server using the v1 REST API. Great if you have private repos in Bitbucket.
     - `io.conduktor.ksm.source.BitbucketCloudSourceAcl`: get the ACL from Bitbucket Cloud using the Bitbucket Cloud REST API v2.
+    - `io.conduktor.ksm.source.HttpSourceAcl`: get the ACL from an HTTP endpoint. You can enable [Google OAuth OIDC Token Authentication](https://cloud.google.com/docs/authentication/production).
+        - `SOURCE_HTTP_URL` HTTP endpoint to retrieve ACL data.
+        - `SOURCE_HTTP_METHOD` HTTP Method. Default is `GET`.
+        - `SOURCE_HTTP_AUTH_TYPE` To enable Http Authentication. `googleiam` for Google IAM. Default is `NONE`.
+        - `SOURCE_HTTP_AUTH_GOOGLEIAM_SERVICE_ACCOUNT` Google Service Account name.
+        - `SOURCE_HTTP_AUTH_GOOGLEIAM_SERVICE_ACCOUNT_KEY` Google Service Account Key in JSON string encoded. If not the key isn't configured, it'll try to get the token from environment.
+        - `SOURCE_HTTP_AUTH_GOOGLEIAM_TARGET_AUDIENCE` Google Target Audience for token authentication.
+
 - `NOTIFICATION_CLASS`: Class for notification in case of ACL changes in Kafka.
     - `io.conduktor.ksm.notification.ConsoleNotification` (default): Print changes to the console. Useful for logging
     - `io.conduktor.ksm.notification.SlackNotification`: Send notifications to a Slack channel (useful for devops / admin team)
@@ -252,7 +263,8 @@ You can then use place this CSV anywhere and use it as your source of truth.
 
 KSM Version | Kafka Version | Notes
 --- | --- | ---
-1.0.0-SNAPSHOT | 2.5.x | 
+1.1.0-SNAPSHOT | 2.8.x | updated log4j dependency
+1.0.1 | 2.8.x | updated log4j dependency
 0.11.0 | 2.5.x | renamed packages to `io.conduktor`. Breaking change on extract config name
 0.10.0 | 2.5.x | YAML support<br>Add configurable num failed refreshes before notification 
 0.9 | 2.5.x | Upgrade to Kafka 2.5.x
@@ -283,4 +295,4 @@ Please open an issue before opening a PR.
 - push the tag (eg: `v0.10.0`)
 - update version in [build.sbt] to the next snapshot version
 
-That's it !
+That's it !

build.sbt

@@ -1,30 +1,54 @@
-inThisBuild(List(
-  organization := "io.conduktor",
-  homepage := Some(url("https://github.com/conduktor/kafka-security-manager")),
-  licenses := List("Apache-2.0" -> url("http://www.apache.org/licenses/LICENSE-2.0")),
-  developers := List(
-    Developer(
-      "conduktor",
-      "Stephane Maarek",
-      "[email protected]",
-      url("https://github.com/conduktor")
-    )
+ThisBuild / organization := "io.conduktor"
+ThisBuild / homepage := Some(url("https://github.com/conduktor/kafka-security-manager"))
+ThisBuild / licenses := List("Apache-2.0" -> url("http://www.apache.org/licenses/LICENSE-2.0"))
+ThisBuild /developers := List(
+  Developer(
+    "conduktor",
+    "Stephane Maarek",
+    "[email protected]",
+    url("https://github.com/conduktor")
   )
-))
-
+)
+import scala.sys.process.Process
 
 name := "kafka-security-manager"
 
-
-version := "1.0.0-SNAPSHOT"
-
-scalaVersion := "2.12.8"
+version := "1.1.0-SNAPSHOT"
+
+scalaVersion := "2.12.15"
+
+lazy val ensureDockerBuildx = taskKey[Unit]("Ensure that docker buildx configuration exists")
+lazy val dockerBuildWithBuildx = taskKey[Unit]("Build docker images using buildx")
+lazy val dockerBuildxSettings = Seq(
+  ensureDockerBuildx := {
+    if (Process("docker buildx inspect multi-arch-builder").! == 1) {
+      Process("docker buildx create --use --name multi-arch-builder", baseDirectory.value).!
+    }
+  },
+  dockerBuildWithBuildx := {
+    streams.value.log("Building and pushing image with Buildx")
+    dockerAliases.value.foreach(alias =>
+      Process(
+        "docker buildx build --platform=linux/arm64,linux/amd64 --push -t " + alias + " .",
+        baseDirectory.value / "target" / "docker" / "stage"
+      ).!
+    )
+  },
+  Docker / publish := Def
+    .sequential(
+      Docker / publishLocal,
+      ensureDockerBuildx,
+      dockerBuildWithBuildx
+    )
+    .value
+)
 
 lazy val root = (project in file("."))
   .enablePlugins(JavaAppPackaging)
   .enablePlugins(DockerPlugin)
   .enablePlugins(ClasspathJarPlugin)
-
+  .enablePlugins(AshScriptPlugin)
+  .settings(dockerBuildxSettings)
 
 resolvers ++= Seq(
   "Artima Maven Repository" at "https://repo.artima.com/releases",
@@ -33,52 +57,59 @@ resolvers ++= Seq(
 
 libraryDependencies ++= Seq(
   // kafka
-  "org.apache.kafka" %% "kafka" % "2.5.0",
-  "io.github.embeddedkafka" %% "embedded-kafka" % "2.5.0" % "test",
+  "org.apache.kafka" %% "kafka" % "2.8.1",
+  "io.github.embeddedkafka" %% "embedded-kafka" % "2.8.1" % "test",
   "io.findify" %% "s3mock" % "0.2.6" % "test",
 
+  //netty
+  "io.netty" % "netty-handler" % "4.1.72.Final",
+
   "org.apache.kafka" % "kafka-clients" % "2.5.0", // needed explicitly for proper classPath
   "org.apache.kafka" % "kafka-clients" % "2.5.0" % Test classifier "test",
 
   // test
   "org.scalatest" %% "scalatest" % "3.0.5" % Test,
   "org.scalamock" %% "scalamock" % "5.1.0" % Test,
+  "com.github.tomakehurst" % "wiremock" % "2.27.2" % Test,
 
   // logging
-  "org.slf4j" % "slf4j-api" % "1.7.25",
-  "org.slf4j" % "slf4j-log4j12" % "1.7.25",
+  "org.slf4j" % "slf4j-api" % "1.7.32",
+  "org.slf4j" % "slf4j-log4j12" % "1.7.32",
+  "org.apache.logging.log4j" % "log4j" % "2.17.1",
 
   // config
   "com.typesafe" % "config" % "1.3.3",
 
   // parsers
   "com.github.tototoshi" %% "scala-csv" % "1.3.5",
-  "io.circe" %% "circe-yaml" % "0.12.0",
-  "io.circe" %% "circe-generic" % "0.12.0",
+  "io.circe" %% "circe-yaml" % "0.14.1",
+  "io.circe" %% "circe-generic" % "0.14.1",
 
   // APIs
   "org.skinny-framework" %% "skinny-http-client" % "2.3.7",
-  "com.fasterxml.jackson.core" % "jackson-core" % "2.9.4",
+  "com.fasterxml.jackson.core" % "jackson-core" % "2.13.1",
 
   // AWS SDK to access S3
-  "com.amazonaws" % "aws-java-sdk-s3" % "1.11.385"
+  "com.amazonaws" % "aws-java-sdk-s3" % "1.11.385",
+
+  // Google Auth
+  "com.google.auth" % "google-auth-library-oauth2-http" % "0.25.5"
 
 )
 
-mainClass in Compile := Some("io.conduktor.ksm.KafkaSecurityManager")
+Compile / mainClass := Some("io.conduktor.ksm.KafkaSecurityManager")
 
-parallelExecution in Test := false
+Test / parallelExecution := false
 
 // Docker stuff
 dockerRepository := Some("conduktor")
 dockerUpdateLatest := true
-dockerBaseImage := "openjdk:8-jre-slim"
-
+dockerBaseImage := "openjdk:11-jre-slim"
 
 
-assemblyMergeStrategy in assembly := {
-  case PathList("META-INF", _ @ _*) => MergeStrategy.discard
+assembly / assemblyMergeStrategy := {
+  case PathList("META-INF", _@_*) => MergeStrategy.discard
   case _ => MergeStrategy.first
 }
 
-assemblyJarName in assembly := s"${name.value}-${version.value}.jar"
+assembly / assemblyJarName := s"${name.value}-${version.value}.jar"

project/build.properties

@@ -1 +1 @@
-sbt.version = 1.5.2
+sbt.version = 1.6.2

project/plugins.sbt

@@ -2,16 +2,18 @@ resolvers += Resolver.typesafeRepo("releases")
 
 addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.3.1")
 
-addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.7.6")
+addSbtPlugin("com.github.sbt" % "sbt-native-packager" % "1.9.9")
 
 resolvers += "Artima Maven Repository" at "https://repo.artima.com/releases"
 
-addSbtPlugin("com.artima.supersafe" % "sbtplugin" % "1.1.7")
+addSbtPlugin("com.artima.supersafe" % "sbtplugin" % "1.1.12")
 
 addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "3.9.7")
 
 addSbtPlugin("com.github.sbt" % "sbt-pgp" % "2.1.2")
 
-addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.10")
+addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "1.2.0")
 
 addSbtPlugin("com.geirsson" % "sbt-ci-release" % "1.5.7")
+
+addSbtPlugin("com.timushev.sbt"          % "sbt-updates"         % "0.6.1")

src/main/resources/application.conf

@@ -134,6 +134,31 @@ source {
       password = ${?SOURCE_BITBUCKET_CLOUD_AUTH_PASSWORD}
     }
   }
+  http {
+    url = ${?SOURCE_HTTP_URL}
+
+    parser = "csv"
+    parser = ${?SOURCE_HTTP_PARSER}
+
+
+    method = "GET"
+    method = ${?SOURCE_HTTP_METHOD}
+
+    auth {
+        type = "NONE"
+        type = ${?SOURCE_HTTP_AUTH_TYPE}
+        googleiam {
+            service-account = ""
+            service-account = ${?SOURCE_HTTP_AUTH_GOOGLEIAM_SERVICE_ACCOUNT}
+
+            service-account_key = ""
+            service-account-key = ${?SOURCE_HTTP_AUTH_GOOGLEIAM_SERVICE_ACCOUNT_KEY}
+
+            target-audience = ""
+            target-audience = ${?SOURCE_HTTP_AUTH_GOOGLEIAM_TARGET_AUDIENCE}
+        }
+    }
+  }
 }
 
 notification {

src/main/scala/io/conduktor/ksm/notification/SlackNotification.scala

@@ -45,6 +45,7 @@ class SlackNotification extends Notification {
       val messages = acls.map {
         case (resource, acl) =>
           val message = Notification.printAcl(acl, resource)
+          log.info(s"$action $message")
           s"$action $message"
       }.toList