From 883a17281a7635bc5f7c50538fcd5c05e076dff0 Mon Sep 17 00:00:00 2001 From: Sameera Priyatham Tadikonda Date: Wed, 2 Oct 2024 13:26:32 -0700 Subject: [PATCH 1/6] MLE-17173: Run sonar scans on flux project --- Jenkinsfile | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index e9c29513..19107fee 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -42,6 +42,15 @@ def postCleanup(){ echo "y" | docker volume prune --filter all=1 || true; ''' } +def runSonarScan(String javaVersion){ + sh label:'test', script: '''#!/bin/bash + export JAVA_HOME=$'''+javaVersion+''' + export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR + export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH + cd flux + ./gradlew sonar -Dsonar.projectKey='ML-DevExp-marklogic-flux' -Dsonar.projectName='ML-DevExp-marklogic-flux' || true + ''' +} pipeline{ agent none options { @@ -50,15 +59,22 @@ pipeline{ } environment{ JAVA_HOME_DIR="/home/builder/java/jdk-11.0.2" + JAVA17_HOME_DIR="/home/builder/java/jdk-17.0.2" GRADLE_DIR =".gradle" DMC_USER = credentials('MLBUILD_USER') DMC_PASSWORD = credentials('MLBUILD_PASSWORD') } stages{ stage('tests'){ + environment{ + scannerHome = tool 'SONAR_Progress' + } agent{ label 'devExpLinuxPool'} steps{ runtests() + withSonarQubeEnv('SONAR_Progress') { + runSonarScan('JAVA17_HOME_DIR') + } } post{ always{ From c7ee80e6d89220769a2bb522b89848842726ba5c Mon Sep 17 00:00:00 2001 From: Sameera Priyatham Tadikonda Date: Wed, 2 Oct 2024 14:05:27 -0700 Subject: [PATCH 2/6] Testing non-root user --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index a1c7a94e..2fdf689c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,7 +15,7 @@ services: - 8007:8007 marklogic: - image: "progressofficial/marklogic-db:latest" + image: "progressofficial/marklogic-db:11.3.0-ubi" platform: linux/amd64 environment: - MARKLOGIC_INIT=true From eaa47591914f610dcc5b6fdd93fb31d587c242df Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Thu, 3 Oct 2024 09:02:59 -0400 Subject: [PATCH 3/6] Bumped to 1.1-SNAPSHOT Adjust --- CONTRIBUTING.md | 6 +++--- build.gradle | 8 ++++---- gradle.properties | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7f3f75e1..93b5c49c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -256,7 +256,7 @@ are all synonyms): ./gradlew shadowJar -This will produce an assembly jar at `./flux-cli/build/libs/marklogic-flux-1.1.0-all.jar`. +This will produce an assembly jar at `./flux-cli/build/libs/marklogic-flux-1.1-SNAPSHOT-all.jar`. You can now run any CLI command via spark-submit. This is an example of previewing an import of files - change the value of `--path`, as an absolute path is needed, and of course change the value of `--master` to match that of your Spark @@ -264,7 +264,7 @@ cluster: ``` $SPARK_HOME/bin/spark-submit --class com.marklogic.flux.spark.Submit \ ---master spark://NYWHYC3G0W:7077 flux-cli/build/libs/marklogic-flux-1.1.0-all.jar \ +--master spark://NYWHYC3G0W:7077 flux-cli/build/libs/marklogic-flux-1.1-SNAPSHOT-all.jar \ import-files --path /Users/rudin/workspace/flux/flux-cli/src/test/resources/mixed-files \ --connection-string "admin:admin@localhost:8000" \ --preview 5 --preview-drop content @@ -281,7 +281,7 @@ to something you can access): $SPARK_HOME/bin/spark-submit --class com.marklogic.flux.spark.Submit \ --packages org.apache.hadoop:hadoop-aws:3.3.4,org.apache.hadoop:hadoop-client:3.3.4 \ --master spark://NYWHYC3G0W:7077 \ -flux-cli/build/libs/marklogic-flux-1.1.0-all.jar \ +flux-cli/build/libs/marklogic-flux-1.1-SNAPSHOT-all.jar \ import-files --path "s3a://changeme/" \ --connection-string "admin:admin@localhost:8000" \ --s3-add-credentials \ diff --git a/build.gradle b/build.gradle index ec4fa3c6..e2ce7c34 100644 --- a/build.gradle +++ b/build.gradle @@ -10,10 +10,10 @@ subprojects { repositories { mavenCentral() -// mavenLocal() -// maven { -// url "https://bed-artifactory.bedford.progress.com:443/artifactory/ml-maven-snapshots/" -// } + mavenLocal() + maven { + url "https://bed-artifactory.bedford.progress.com:443/artifactory/ml-maven-snapshots/" + } } test { diff --git a/gradle.properties b/gradle.properties index 5dea9f33..ec1090d2 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ -version=1.1.0 +version=1.1-SNAPSHOT # Define these on the command line to publish to OSSRH # See https://central.sonatype.org/publish/publish-gradle/#credentials for more information From 4243c843cba9e74cca4546a2865c6d290811235e Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Thu, 3 Oct 2024 11:08:34 -0400 Subject: [PATCH 4/6] Adjusting sonarqube logs --- docker-compose.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 2fdf689c..93da8bd1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -63,13 +63,12 @@ services: volumes: - sonarqube_data:/opt/sonarqube/data - sonarqube_extensions:/opt/sonarqube/extensions - - sonarqube_logs:/opt/sonarqube/logs + - ./docker/sonarqube/logs:/opt/sonarqube/logs ports: - "9000:9000" volumes: sonarqube_data: sonarqube_extensions: - sonarqube_logs: postgresql: postgresql_data: From abad9b661a90c8412b14b3e4fa8dd49a73c186c9 Mon Sep 17 00:00:00 2001 From: Sameera Priyatham Tadikonda Date: Thu, 3 Oct 2024 15:08:40 -0700 Subject: [PATCH 5/6] MLE-17198: updating the permissions of sonar logs --- Jenkinsfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index 19107fee..382c07f6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -3,6 +3,7 @@ def runtests(){ cd $WORKSPACE/flux; sudo /usr/local/sbin/mladmin stop; sudo /usr/local/sbin/mladmin remove; + mkdir -p $WORKSPACE/flux/docker/sonarqube; docker-compose up -d --build; sleep 30s; ''' @@ -38,6 +39,7 @@ def postCleanup(){ cd $WORKSPACE/flux; sudo /usr/local/sbin/mladmin delete $WORKSPACE/flux/docker/marklogic/logs/; docker exec -i --privileged --user root flux-caddy-load-balancer-1 /bin/sh -c "chmod -R 777 /data" || true; + docker exec -i --privileged --user root flux-sonarqube-1 /bin/sh -c "chmod -R 777 /opt/sonarqube" || true; docker-compose rm -fsv || true; echo "y" | docker volume prune --filter all=1 || true; ''' From 4e7410058e7ee0575951fe6f1b1c8a0f5c078d4a Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Mon, 7 Oct 2024 19:26:16 -0400 Subject: [PATCH 6/6] Bumped to 1.1.1 Depending Spark connector 2.4.1, which includes the fix for logging by excluding slf4j-api 1.x. --- flux-cli/build.gradle | 2 +- gradle.properties | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/flux-cli/build.gradle b/flux-cli/build.gradle index 800bdd5c..0551f091 100644 --- a/flux-cli/build.gradle +++ b/flux-cli/build.gradle @@ -17,7 +17,7 @@ dependencies { // The rocksdbjni dependency weighs in at 50mb and so far does not appear necessary for our use of Spark. exclude module: "rocksdbjni" } - implementation "com.marklogic:marklogic-spark-connector:2.4.0" + implementation "com.marklogic:marklogic-spark-connector:2.4.1" implementation "info.picocli:picocli:4.7.6" // Spark 3.4.3 depends on Hadoop 3.3.4, which depends on AWS SDK 1.12.262. As of August 2024, all public releases of diff --git a/gradle.properties b/gradle.properties index ec1090d2..7a84c88c 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ -version=1.1-SNAPSHOT +version=1.1.1 # Define these on the command line to publish to OSSRH # See https://central.sonatype.org/publish/publish-gradle/#credentials for more information