build.gradle

/*
 * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License").
 * You may not use this file except in compliance with the License.
 * A copy of the License is located at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * or in the "license" file accompanying this file. This file is distributed
 * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
 * express or implied. See the License for the specific language governing
 * permissions and limitations under the License.
 */

buildscript {
    dependencies {
        classpath group: 'com.google.guava', name: 'guava', version: '28.2-jre'
        classpath 'org.ajoberstar:gradle-git:0.2.3'
    }
}

plugins {
    id 'java'
    id 'application'
    id 'maven-publish'
    id 'com.google.protobuf' version '0.8.8'
    id 'jacoco'
    id 'idea'
    id 'com.github.spotbugs' version '4.6.0'
    id "de.undercouch.download" version "4.0.4"
    id 'com.adarshr.test-logger' version '2.1.0'
    id 'org.gradle.test-retry' version '1.2.0'
    id 'com.diffplug.spotless' version '5.11.0'
}

application {
    mainClassName = 'com.amazon.opendistro.elasticsearch.performanceanalyzer.PerformanceAnalyzerApp'
    applicationDefaultJvmArgs = ['-Xms64M',
                                 '-Xmx64M',
                                 '-XX:+UseSerialGC',
                                 '-XX:CICompilerCount=1',
                                 '-XX:-TieredCompilation',
                                 '-XX:InitialCodeCacheSize=4096',
                                 '-XX:InitialBootClassLoaderMetaspaceSize=30720',
                                 '-XX:MaxRAM=400m']
}

// Include pa_config folder in the distribution.
applicationDistribution.from(".") {
    include 'pa_config/*'
    include 'pa_bin/*'
}

distributions {
    main {
        contents {
            eachFile {
                it.path = it.path.replace("-$version/", '/')
            }
        }
    }
}

distZip {
    archiveName 'performance-analyzer-rca.zip'
}

publishing {
    publications {
        maven(MavenPublication) {
            groupId = 'com.amazon.opendistro.elasticsearch'
            artifactId = 'performanceanalyzer-rca'
            version = '1.13'
            from components.java
        }
    }
}

ext {
    opendistroVersion = '1.13'
    isSnapshot = "true" == System.getProperty("build.snapshot", "true")
    gitDefaultBranch = 'main'
}

spotless {
    java {
        // only apply formatting rules to updated files
        ratchetFrom "origin/${gitDefaultBranch}"

        licenseHeaderFile(file('license-header'))
        googleJavaFormat().aosp()
        importOrder()
        removeUnusedImports()
        trimTrailingWhitespace()
        endWithNewline()

        // add support for spotless:off and spotless:on tags to exclude sections of code
        toggleOffOn()
    }
}

tasks.withType(Checkstyle) {
    reports {
        xml.enabled false
        html.enabled true
        html.stylesheet resources.text.fromFile('configs/xsl/severity-sorted.xsl')
    }
}

testlogger {
    theme 'standard'
    showExceptions true
    showStackTraces false
    showFullStackTraces false
    showCauses true
    slowThreshold 4000
    showSummary true
    showSimpleNames false
    showPassed true
    showSkipped true
    showFailed true
    showStandardStreams false
    showPassedStandardStreams true
    showSkippedStandardStreams true
    showFailedStandardStreams true
    logLevel 'lifecycle'
}

spotbugsMain {
    excludeFilter = file("checkstyle/findbugs-exclude.xml")
    effort = 'max'
    ignoreFailures = false

    reports {
        xml.enabled = false
        html.enabled = true
    }
}

spotbugsTest {
    ignoreFailures = true
}

check {
    dependsOn spotbugsMain
    //dependsOn spotbugsTest
}

jacoco {
    toolVersion = "0.8.5"
}

jacocoTestReport {
    reports {
        xml.enabled true
        html.enabled true
        csv.enabled false
    }

    afterEvaluate {
        classDirectories.from = files(classDirectories.files.collect {
            fileTree(dir: it,
                    include: [
                            '**/com/amazon/opendistro/elasticsearch/performanceanalyzer/**'
                    ],
                    exclude: [
                            '**/com/amazon/opendistro/elasticsearch/performanceanalyzer/grpc/**',
                            '**/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/framework/api/metrics/**',
                            '**/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/net/requests**'
                    ])
        })
    }
}

jacocoTestCoverageVerification {
    afterEvaluate {
        classDirectories.from = files(classDirectories.files.collect {
            fileTree(dir: it,
                    include: [
                            '**/com/amazon/opendistro/elasticsearch/performanceanalyzer/rca/**',
                            '**/com/amazon/opendistro/elasticsearch/performanceanalyzer/reader/**'
                    ],
                    exclude: [
                            '**/com/amazon/opendistro/elasticsearch/performanceanalyzer/grpc/**'])
        })
    }
    violationRules {
        rule {
            limit {
                minimum = 0.8
            }
        }
    }
}

// to run coverage verification during the build (and fail when appropriate)
check.dependsOn jacocoTestCoverageVerification


version = "${opendistroVersion}.0"
if (isSnapshot) {
    version += "-SNAPSHOT"
}

test {
    enabled = true
    retry {
        maxRetries = 2
    }
}

task rcaTest(type: Test) {
    retry {
        maxRetries = 2
    }
    useJUnit {
        includeCategories 'com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.GradleTaskForRca'
    }
    testLogging.showStandardStreams = true
}

task rcaIt(type: Test) {
    dependsOn spotbugsMain
    retry {
        maxRetries = 2
    }
    useJUnit {
        includeCategories 'com.amazon.opendistro.elasticsearch.performanceanalyzer.rca.integTests.framework.RcaItMarker'
    }
    //testLogging.showStandardStreams = true
}

sourceCompatibility = 1.8
targetCompatibility = 1.8

compileJava {
    dependsOn spotlessApply
    JavaVersion targetVersion = JavaVersion.toVersion(targetCompatibility);
    if (targetVersion.isJava9Compatible()) {
        options.compilerArgs += ["--add-exports", "jdk.attach/sun.tools.attach=ALL-UNNAMED"]
    }
}

javadoc {
    options.addStringOption("-add-exports", "jdk.attach/sun.tools.attach=ALL-UNNAMED")
}

project.afterEvaluate {
    JavaVersion targetVersion = JavaVersion.toVersion(targetCompatibility)

    // cannot contain the first version
    // should be '8' or '11' etc.
    String version = targetVersion.toString()
    if (version.length() > 2) {
        version = targetVersion.toString().substring(2)
    }

    compileJava.options.compilerArgs.removeAll(['--release', version])
}

repositories {
    mavenCentral()
}


configurations {
    includeJars
}

tasks.withType(JavaCompile) {
    options.warnings = false
}

dependencies {
    if (JavaVersion.current() <= JavaVersion.VERSION_1_8) {
        compile files("${System.properties['java.home']}/../lib/tools.jar")
    }

    compile 'org.jooq:jooq:3.10.8'
    compile 'org.bouncycastle:bcprov-jdk15on:1.68'
    compile 'org.bouncycastle:bcpkix-jdk15on:1.68'
    compile 'org.xerial:sqlite-jdbc:3.32.3.2'
    compile 'com.google.guava:guava:28.2-jre'
    compile 'com.fasterxml.jackson.core:jackson-annotations:2.12.6'
    compile 'com.fasterxml.jackson.core:jackson-databind:2.12.6.1'
    compile group: 'org.apache.logging.log4j', name: 'log4j-api', version: '2.16.0'
    compile group: 'org.apache.logging.log4j', name: 'log4j-core', version: '2.16.0'
    compile group: 'org.apache.commons', name: 'commons-lang3', version: '3.9'
    compile group: 'commons-io', name: 'commons-io', version: '2.3'
    implementation 'io.grpc:grpc-netty-shaded:1.28.0'
    implementation 'io.grpc:grpc-protobuf:1.28.0'
    implementation 'io.grpc:grpc-stub:1.28.0'
    implementation 'javax.annotation:javax.annotation-api:1.3.2'

    // JDK9+ has to run powermock 2+. https://github.com/powermock/powermock/issues/888
    testCompile group: 'org.powermock', name: 'powermock-api-mockito2', version: '2.0.0'
    testCompile group: 'org.powermock', name: 'powermock-module-junit4', version: '2.0.0'
    testCompile group: 'org.mockito', name: 'mockito-core', version: '2.23.0'
    testCompile group: 'org.powermock', name: 'powermock-core', version: '2.0.0'
    testCompile group: 'org.powermock', name: 'powermock-api-support', version: '2.0.0'
    testCompile group: 'org.powermock', name: 'powermock-module-junit4-common', version: '2.0.0'
    testCompile group: 'org.javassist', name: 'javassist', version: '3.24.0-GA'
    testCompile group: 'org.powermock', name: 'powermock-reflect', version: '2.0.0'
    testCompile group: 'net.bytebuddy', name: 'byte-buddy', version: '1.9.3'
    testCompile group: 'org.objenesis', name: 'objenesis', version: '3.0.1'
    testCompile group: 'org.hamcrest', name: 'hamcrest-library', version: '2.1'
    testCompile group: 'org.hamcrest', name: 'hamcrest', version: '2.1'
    testCompile group: 'junit', name: 'junit', version: '4.12'
}

protobuf {
    protoc {
        artifact = "com.google.protobuf:protoc:3.16.1"
    }

    plugins {
        grpc {
            artifact = 'io.grpc:protoc-gen-grpc-java:1.28.0'
        }

    }
    generateProtoTasks {
        all()*.plugins {
            grpc {}
        }
    }
}

idea {
    module {
        sourceDirs += file("${projectDir}/build/generated/source/proto/main/java");
        sourceDirs += file("${projectDir}/build/generated/source/proto/main/grpc");
    }
}

import groovy.json.JsonSlurper
import java.nio.file.Paths
import org.ajoberstar.gradle.git.tasks.GitClone

String paDir
String dockerBuildDir
String dockerArtifactsDir

task cloneGitRepo(type: GitClone) {
    dockerBuildDir = Paths.get(getProject().getBuildDir().toString(), "docker-build").toString()
    mkdir dockerBuildDir
    if (localPaDir != null && !localPaDir.empty) {
        // If you already have the performance Analyzer sources checked out then update the
        // localPaDir in gradle.properties with the path.
        paDir = localPaDir
    } else {
        paDir = Paths.get(dockerBuildDir, "pa").toString()
    }

    def destination = file(paDir)
    uri = "https://github.com/opendistro-for-elasticsearch/performance-analyzer.git"
    branch = gitDefaultBranch
    destinationPath = destination
    bare = false
    enabled = !destination.exists() //to clone only once
}

task removeLicenses(type: Delete) {
    dependsOn(cloneGitRepo)
    def toDelete = Paths.get(paDir, 'licenses', 'performanceanalyzer-rca-1.13.jar.sha1')
    delete(toDelete)
}

task regenerateLicenses(type: Exec) {
    dependsOn(removeLicenses)
    workingDir(paDir)
    commandLine './gradlew', 'updateShas'
}

task buildPa(type: Exec) {
    dependsOn(assemble, publishToMavenLocal, regenerateLicenses)
    workingDir paDir
    println String.format('pa in dir: (%s) will be built.', paDir)
    commandLine './gradlew', 'assemble'
    println "PA repo located at '" + paDir + "' will be used."
}

task copyAllArtifacts(type: Copy) {
    dependsOn(buildPa)
    def projectPathStr = getProjectDir().toPath().toString()
    def dockerArtifacts = Paths.get(projectPathStr, 'docker').toString()
    def rcaArtifacts = Paths.get(projectPathStr, 'build', 'distributions', 'performance-analyzer-rca.zip')
    def paArtifacts = Paths.get(paDir, 'build', 'distributions', 'opendistro-performance-analyzer-1.13.3.0-SNAPSHOT.zip')

    dockerArtifactsDir = Paths.get(dockerBuildDir, 'rca-docker')
    from(dockerArtifacts)
    from(rcaArtifacts)
    from(paArtifacts)
    into(dockerArtifactsDir)
}

task copyReaderMetricsFiles(type: Copy) {
    dependsOn(buildPa)
    copy{
        from ('src/test/resources/reader'){
        }
        into 'build/private/test_resources'
    }
}

task buildDocker(type: Exec) {
    dependsOn(copyAllArtifacts)

    workingDir(dockerArtifactsDir)
    commandLine 'docker', 'build', '-t', 'odfe-es/pa-rca:1.0', '.'
}

task runDocker(type: Exec) {
    dependsOn(buildDocker)

    workingDir(dockerArtifactsDir)

    // This block is included to make the runDocker task work with Github Actions
    // It sets the path to the docker-compose program from an environment variable
    // The DOCKER_COMPOSE_LOCATION environment variable is set in the gradle.yml file inside the
    // performance-analyzer repository.
    def docker_compose_location = "docker-compose"
    if (System.getenv("DOCKER_COMPOSE_LOCATION") != null) {
        docker_compose_location = System.getenv("DOCKER_COMPOSE_LOCATION")
    }

    environment 'DATA_VOLUME1', 'esdata1'
    environment 'DATA_VOLUME2', 'esdata2'

    commandLine(docker_compose_location,
            '-f', 'docker-compose.yml',
            '-f', 'docker-compose.hostports.yml',
            '-f', 'docker-compose.cluster.yml',
            'up', 'elasticsearch1', 'elasticsearch2')

}

def printCommandOutput(proc) {
    Thread.start {
        BufferedReader stdInput = new BufferedReader(new
                InputStreamReader(proc.getInputStream()));

        BufferedReader stdError = new BufferedReader(new
                InputStreamReader(proc.getErrorStream()));

        System.out.println("Here is the standard output of the command:\n");
        String s = null;
        while ((s = stdInput.readLine()) != null) {
            System.out.println(s);
        }

        System.out.println("Here is the standard error of the command (if any):\n");
        while ((s = stdError.readLine()) != null) {
            System.out.println(s);
        }
    }
    int exitVal = proc.waitFor()
    System.out.println("Process exitValue: " + exitVal);
}

def runCommand(commandArr, envArr, workingDir, printOutput) {
    try {
        Process proc = Runtime.getRuntime().exec(commandArr, envArr, file(workingDir))
        if (printOutput) {
            printCommandOutput(proc)
        }
    } catch (Throwable th) {
        th.printStackTrace()
    }
}

def runInProcess(commandArr) {
    Process proc = new ProcessBuilder(commandArr).start();
    printCommandOutput(proc)
}

def runDockerCompose = { executionPath ->
    // This block is included to make the runDocker task work with Github Actions
    // It sets the path to the docker-compose program from an environment variable
    // The DOCKER_COMPOSE_LOCATION environment variable is set in the gradle.yml file inside the
    // performance-analyzer repository.
    String dockerCompose = System.getenv("DOCKER_COMPOSE_LOCATION") == null ?
            "docker-compose" : System.getenv("DOCKER_COMPOSE_LOCATION")
    String[] commandArray = [dockerCompose,
                             '-f', 'docker-compose.yml',
                             '-f', 'docker-compose.hostports.yml',
                             '-f', 'docker-compose.cluster.yml',
                             'up', 'elasticsearch1', 'elasticsearch2']
    String[] env = ['DATA_VOLUME1=esdata1',
                    'DATA_VOLUME2=esdata2']
    runCommand(commandArray, env, file(executionPath), true)
}

def getHttpResponse(server, timeoutSeconds) {
    URL url = new URL(server)
    long timeout = timeoutSeconds * 1000
    long increments = 1000
    long passed = 0
    println 'waiting for ' + server
    while (passed < timeout) {
        try {
            HttpURLConnection con = (HttpURLConnection) url.openConnection()
            con.setRequestMethod("GET")
            InputStreamReader isr = new InputStreamReader(con.getInputStream())
            BufferedReader bufR = new BufferedReader(isr)
            String inputLine;
            StringBuffer content = new StringBuffer()
            while ((inputLine = bufR.readLine()) != null) {
                content.append(inputLine);
            }
            println(content.toString())
            bufR.close()
            break
        } catch (IOException) {
            Thread.sleep(increments);
            passed += increments
        }
    }
    if (passed == timeout) {
        throw new GradleException(String.format("ERROR:  %s not ready.", server));
    }
}

/**
 * enableComponent is used to enable PA or RCA based on the given parameters
 * @param endpoint the endpoint to make the request against e.g. localhost:9200/pa/rca/example
 * @param desiredState
 *          The list of enabled components is represented as an n-bit binary number
 *          The LSB represents the state of PA and the LSB+1th bit is RCA
 *          0x03 (00000011 in binary) is therefore a state where both PA and RCA are enabled
 * @param timeoutSeconds we will attempt to enable the component for this many seconds before
 *          giving up and throwing a GradleException
 */
def enableComponent(endpoint, desiredState, timeoutSeconds) {
    long timeout = timeoutSeconds * 1000
    long increments = 1000
    long passed = 0
    println 'waiting for ' + endpoint
    while (passed < timeout) {
        try {
            def p = ['curl', endpoint, '-H', 'Content-Type: application/json', '-d', '{"enabled": true}'].execute()
            def json = new JsonSlurper().parseText(p.text)
            if (json.get("currentPerformanceAnalyzerClusterState").equals(desiredState) ||
                    // desiredState+2 means that the "next" significant component is enabled
                    json.get("currentPerformanceAnalyzerClusterState").equals(desiredState + 2)) {
                break
            }
        } catch (Exception) {
        }
        Thread.sleep(increments);
        passed += increments
    }
    if (passed == timeout) {
        throw new GradleException(String.format("ERROR:  %s not ready.", endpoint));
    }
}

Boolean esIsUp

def esUpChecker = {
    String server = "http://localhost:9200"
    int timeoutSeconds = 2 * 60
    getHttpResponse(server, timeoutSeconds)
    esIsUp = true
}

// Attempts to enable PA for up to 2 minutes. Returns when PA is successfully enabled or
// throws an Exception if the timeout is exceeded
def paUpChecker = {
    String server = "localhost:9200/_opendistro/_performanceanalyzer/cluster/config"
    int timeoutSeconds = 2 * 60
    enableComponent(server, 1, timeoutSeconds)
}

// Attempts to enable RCA for up to 2 minutes. Returns when RCA is successfully enabled or
// throws an Exception if the timeout is exceeded
def rcaUpChecker = {
    String server = "localhost:9200/_opendistro/_performanceanalyzer/rca/cluster/config"
    int timeoutSeconds = 2 * 60
    enableComponent(server, 3, timeoutSeconds)
}

Thread esRunnerThread

def runInThread(cl, input) {
    return Thread.start {
        cl(input)
    }
}

task waitForES {
    dependsOn(buildDocker)
    doLast {
        esRunnerThread = runInThread(runDockerCompose, dockerArtifactsDir)
        esUpChecker()
    }
}

String perfTopDir
String perfTopZip
String perfTopBin

task downloadPerfTop(type: Download) {
    String zip = 'perf-top-1.9.0.0-MACOS.zip'
    perfTopDir = Paths.get(dockerBuildDir, 'perftop').toString()
    mkdir perfTopDir

    src 'https://d3g5vo6xdbdb9a.cloudfront.net/downloads/perftop/' + zip
    dest perfTopDir
    overwrite false

    perfTopZip = Paths.get(perfTopDir, zip).toString()
    File f = file(perfTopZip)
    enabled = !f.exists() //to clone only once
}

task unzipPerfTop(type: Copy) {
    dependsOn(downloadPerfTop)

    perfTopBin = Paths.get(perfTopDir, 'perf-top-macos').toString()

    def zipFile = file(perfTopZip)
    def outputDir = file(perfTopDir)

    from zipTree(zipFile)
    into outputDir

    enabled = !file(perfTopBin).exists()
}

// Attempts to enable PA for up to 2 minutes. Returns when PA is successfully enabled or
// throws an Exception if the timeout is exceeded
task enablePa() {
    dependsOn(waitForES)
    doLast {
        paUpChecker()
    }
}

// Attempts to enable RCA for up to 2 minutes. Returns when RCA is successfully enabled or
// throws an Exception if the timeout is exceeded
task enableRca() {
    dependsOn(enablePa)
    doLast {
        rcaUpChecker()
    }
}

task runPerftop {
    dependsOn(enableRca, unzipPerfTop)
    doLast {
        println '============================================================================'
        println 'invoke perftop as: '
        println String.format("%s --dashboard %s", Paths.get(perfTopBin).toString(), 'ClusterOverview')
        println '============================================================================'
    }
}

task generateLoad {
    dependsOn(enableRca, runPerftop)

    doLast {
        String[] deleteIndex = ['curl', '-X', 'DELETE', 'localhost:9200/accounts?pretty']
        String[] bulk = ['curl', '-s', '-H', "Content-Type: application/x-ndjson",
                         '-XPOST', 'localhost:9200/_bulk', '--data-binary', '@accounts.json']
        String[] search = ['curl', '-X', 'GET', 'localhost:9200/_search?pretty', '-H',
                           'Content-Type:application/json', '-d',
                           '{"query": {"range": {"balance": {"gte": 100, "lte": 100000}}}}']

        String wdir = Paths.get(getProjectDir().getAbsolutePath(), 'src', 'test', 'resources')
                .toString()
        while (true) {
            // runCommand(deleteIndex, null, wdir, false)
            //println("starting indexing: " + System.currentTimeMillis())
            runCommand(bulk, null, wdir, false)
            //println("completed indexing: " + System.currentTimeMillis())
            Thread.sleep(3000)
            //println("starting search" + System.currentTimeMillis())
            runCommand(search, null, wdir, false)
            Thread.sleep(1000)
            //println("completed search" + System.currentTimeMillis())
            runCommand(deleteIndex, null, wdir, false)
            Thread.sleep(1000)
        }
    }
}

task runRally {
    dependsOn(enableRca, runPerftop)

    doLast {
        String rallyTrack='pmc'

        String[] esrally = ['docker', 'run', '--network=host', 'elastic/rally',
                                '--track=' + rallyTrack, '--test-mode', '--pipeline=benchmark-only',
                '--target-hosts=localhost:9200']

        String wdir = Paths.get(getProjectDir().getAbsolutePath(), 'src', 'test', 'resources')
                .toString()
        while (true) {
            runCommand(esrally, null, wdir, true)
            println('Get all RCAs using:')
            println('curl localhost:9600/_opendistro/_performanceanalyzer/rca?all | python -m json.tool')
            Thread.sleep(1000)
        }
    }
}