build.gradle

import java.nio.file.Files
import org.opensearch.gradle.testclusters.OpenSearchCluster
import org.opensearch.gradle.test.RestIntegTestTask
import java.util.concurrent.Callable
import java.nio.file.Paths

buildscript {
    ext {
        opensearch_version = System.getProperty("opensearch.version", "3.0.0-SNAPSHOT")
        buildVersionQualifier = System.getProperty("build.version_qualifier", "")
        isSnapshot = "true" == System.getProperty("build.snapshot", "true")
        version_tokens = opensearch_version.tokenize('-')
        opensearch_build = version_tokens[0] + '.0'
        plugin_no_snapshot = opensearch_build
        if (buildVersionQualifier) {
            opensearch_build += "-${buildVersionQualifier}"
            plugin_no_snapshot += "-${buildVersionQualifier}"
        }
        if (isSnapshot) {
            opensearch_build += "-SNAPSHOT"
        }
        opensearch_group = "org.opensearch"
        opensearch_no_snapshot = opensearch_build.replace("-SNAPSHOT","")
        System.setProperty('tests.security.manager', 'false')
        common_utils_version = System.getProperty("common_utils.version", opensearch_build)
    }

    repositories {
        mavenLocal()
        maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" }
        mavenCentral()
        maven { url "https://plugins.gradle.org/m2/" }
        maven { url 'https://jitpack.io' }
    }

    dependencies {
        classpath "org.opensearch.gradle:build-tools:${opensearch_version}"
        classpath "com.diffplug.spotless:spotless-plugin-gradle:6.25.0"
        classpath "com.github.form-com.diff-coverage-gradle:diff-coverage:0.9.5"
    }
}

plugins {
    id "de.undercouch.download" version "5.5.0"
    id "org.gradle.test-retry" version "1.5.8" apply false
}

apply plugin: 'java'
apply plugin: 'idea'
apply plugin: 'opensearch.opensearchplugin'
apply plugin: 'opensearch.yaml-rest-test'
apply plugin: 'opensearch.pluginzip'
// for formatting and license headers
apply plugin: 'com.diffplug.spotless'
apply from: 'formatter/formatting.gradle'
// for javadocs and checks spotless doesn't do
apply plugin: 'checkstyle'
// for coverage and diff
apply plugin: 'jacoco'
apply plugin: 'com.form.diff-coverage'

def pluginName = 'opensearch-flow-framework'
def pluginDescription = 'OpenSearch plugin that enables builders to innovate AI apps on OpenSearch'
def projectPath = 'org.opensearch'
def pathToPlugin = 'flowframework'
def pluginClassName = 'FlowFrameworkPlugin'
group = "org.opensearch.flowframework"

opensearchplugin {
    name pluginName
    description pluginDescription
    classname "${projectPath}.${pathToPlugin}.${pluginClassName}"
    licenseFile rootProject.file('LICENSE')
    noticeFile rootProject.file('NOTICE')
}

dependencyLicenses.enabled = false
// This requires an additional Jar not published as part of build-tools
loggerUsageCheck.enabled = false
thirdPartyAudit.enabled = false

// No need to validate pom, as we do not upload to maven/sonatype
validateNebulaPom.enabled = false

allprojects {
    // Default to the apache license
    project.ext.licenseName = 'The Apache Software License, Version 2.0'
    project.ext.licenseUrl = 'http://www.apache.org/licenses/LICENSE-2.0.txt'
}

publishing {
    publications {
        pluginZip(MavenPublication) { publication ->

            pom {
                name = pluginName
                description = pluginDescription
                groupId = "org.opensearch.plugin"

                withXml { XmlProvider xml ->
                    Node node = xml.asNode()
                    node.appendNode('inceptionYear', '2021')

                    Node license = node.appendNode('licenses').appendNode('license')
                    license.appendNode('name', project.licenseName)
                    license.appendNode('url', project.licenseUrl)

                    Node developer = node.appendNode('developers').appendNode('developer')
                    developer.appendNode('name', 'OpenSearch')
                    developer.appendNode('url', 'https://github.com/opensearch-project/flow-framework')
                }
            }
        }
    }

    repositories {
        maven {
            name = 'staging'
            url = "${rootProject.buildDir}/local-staging-repo"
        }
        maven {
            name = "Snapshots"
            url = "https://aws.oss.sonatype.org/content/repositories/snapshots"
            credentials {
                username "$System.env.SONATYPE_USERNAME"
                password "$System.env.SONATYPE_PASSWORD"
            }
        }
    }
}

allprojects {
    group = opensearch_group
    version = "${opensearch_build}"
}

java {
    targetCompatibility = JavaVersion.VERSION_11
    sourceCompatibility = JavaVersion.VERSION_11
}

repositories {
    mavenLocal()
    maven { url "https://aws.oss.sonatype.org/content/repositories/snapshots" }
    mavenCentral()
    maven { url "https://plugins.gradle.org/m2/" }
}

configurations {
    zipArchive
    secureIntegTestPluginArchive
}

dependencies {
    implementation "org.opensearch:opensearch:${opensearch_version}"
    implementation 'org.junit.jupiter:junit-jupiter:5.10.2'
    api group: 'org.opensearch', name:'opensearch-ml-client', version: "${opensearch_build}"
    api group: 'org.opensearch.client', name: 'opensearch-rest-client', version: "${opensearch_version}"
    implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.14.0'
    implementation "org.opensearch:common-utils:${common_utils_version}"
    implementation 'com.amazonaws:aws-encryption-sdk-java:2.4.1'
    implementation 'org.bouncycastle:bcprov-jdk18on:1.77'
    api "org.apache.httpcomponents.core5:httpcore5:5.2.2"

    // ZipArchive dependencies used for integration tests
    zipArchive group: 'org.opensearch.plugin', name:'opensearch-ml-plugin', version: "${opensearch_build}"
    secureIntegTestPluginArchive group: 'org.opensearch.plugin', name:'opensearch-security', version: "${opensearch_build}"

    configurations.all {
        resolutionStrategy {
            force("com.google.guava:guava:33.0.0-jre") // CVE for 31.1, keep to force transitive dependencies
            force("org.eclipse.platform:org.eclipse.core.runtime:3.30.0") // CVE for < 3.29.0
            force("com.fasterxml.jackson.core:jackson-core:2.16.1") // Dependency Jar Hell
        }
    }
}


def opensearch_tmp_dir = rootProject.file('build/private/opensearch_tmp').absoluteFile
opensearch_tmp_dir.mkdirs()
def _numNodes = findProperty('numNodes') as Integer ?: 1

test {
    include '**/*Tests.class'
}

jacocoTestReport {
    dependsOn test
    reports {
        xml.required = true
    }
}
tasks.named("check").configure { dependsOn(jacocoTestReport) }


// Set up integration tests
task integTest(type: RestIntegTestTask) {
    description = "Run tests against a cluster"
    testClassesDirs = sourceSets.test.output.classesDirs
    classpath = sourceSets.test.runtimeClasspath
}
tasks.named("check").configure { dependsOn(integTest) }

integTest {

    dependsOn "bundlePlugin"
    systemProperty 'tests.security.manager', 'false'
    systemProperty 'java.io.tmpdir', opensearch_tmp_dir.absolutePath
    systemProperty('project.root', project.rootDir.absolutePath)
    systemProperty 'security.enabled', System.getProperty('security.enabled')
    var is_https = System.getProperty('https')
    var user = System.getProperty('user')
    var password = System.getProperty('password')
    // Using default admin credentials since the install_plugin_configuration script is not used to configure the security plugin
    if (System.getProperty('security.enabled') != null) {
        is_https = is_https == null ? 'true' : is_https
        user = user == null ? 'admin' : user
        password = password == null ? 'admin' : password
        System.setProperty("https", is_https)
        System.setProperty("user", user)
        System.setProperty("password", password)
    }
    systemProperty('https', is_https)
    systemProperty('user', user)
    systemProperty('password', password)

    // Only rest case can run with remote cluster
    if (System.getProperty("tests.rest.cluster") != null) {
        filter {
            includeTestsMatching "org.opensearch.flowframework.rest.*IT"
        }
    }

    // Exclude integration tests that require security plugin
    if (System.getProperty("https") == null || System.getProperty("https") == "false") {
        filter {
            excludeTestsMatching "org.opensearch.flowframework.rest.FlowFrameworkSecureRestApiIT"
        }
    }

    // Include only secure integration tests in security enabled clusters
    if (System.getProperty("https") != null && System.getProperty("https") == "true") {
        filter {
            includeTestsMatching "org.opensearch.flowframework.rest.FlowFrameworkSecureRestApiIT"
            excludeTestsMatching "org.opensearch.flowframework.rest.FlowFrameworkRestApiIT"
        }
    }

    // doFirst delays this block until execution time
    doFirst {
        // Tell the test JVM if the cluster JVM is running under a debugger so that tests can
        // use longer timeouts for requests.
        def isDebuggingCluster = getDebug() || System.getProperty("test.debug") != null
        systemProperty 'cluster.debug', isDebuggingCluster
        // Set number of nodes system property to be used in tests
        systemProperty 'cluster.number_of_nodes', "${_numNodes}"
        // There seems to be an issue when running multi node run or integ tasks with unicast_hosts
        // not being written, the waitForAllConditions ensures it's written
        getClusters().forEach { cluster ->
            cluster.waitForAllConditions()
        }
    }

    // The --debug-jvm command-line option makes the cluster debuggable; this makes the tests debuggable
    if (System.getProperty("test.debug") != null) {
        jvmArgs '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005'
    }
}

// Set up integration test clusters, installs all zipArchive dependencies and Flow Framework
testClusters.integTest {
    testDistribution = "ARCHIVE"

    // Optionally install security
    if (System.getProperty("https") != null && System.getProperty("https") == "true") {
        // Retrieve Security Plugin Zip from zipArchive
        configurations.secureIntegTestPluginArchive.asFileTree.each {
            if(it.name.contains("opensearch-security")) {
                plugin(provider(new Callable<RegularFile>(){
                        @Override
                        RegularFile call() throws Exception {
                            return new RegularFile() {
                                @Override
                                File getAsFile() {
                                    return it
                                }
                            }
                        }
                    })
                )
            }
        }

        // Using default admin credentials since the install_plugin_configuration script is not used to configure the security plugin
        getNodes().forEach { node ->
            var creds = node.getCredentials()
            if (creds.isEmpty()) {
                creds.add(Map.of('username', 'admin', 'password', 'admin'))
            } else {
                creds.get(0).putAll(Map.of('username', 'admin', 'password', 'admin'))
            }
        }

        // Config below including files are copied from security demo configuration
        ['esnode.pem', 'esnode-key.pem', 'root-ca.pem'].forEach { file ->
            File local = Paths.get(opensearch_tmp_dir.absolutePath, file).toFile()
            download.run {
                src "https://raw.githubusercontent.com/opensearch-project/security/main/bwc-test/src/test/resources/security/" + file
                dest local
                overwrite false
            }
            extraConfigFile(file, local)
        }

        // This configuration is copied from the security plugins demo install:
        // https://github.com/opensearch-project/security/blob/2.11.1.0/tools/install_demo_configuration.sh#L365-L388
        setting("plugins.security.ssl.transport.pemcert_filepath", "esnode.pem")
        setting("plugins.security.ssl.transport.pemkey_filepath", "esnode-key.pem")
        setting("plugins.security.ssl.transport.pemtrustedcas_filepath", "root-ca.pem")
        setting("plugins.security.ssl.transport.enforce_hostname_verification", "false")
        setting("plugins.security.ssl.http.enabled", "true")
        setting("plugins.security.ssl.http.pemcert_filepath", "esnode.pem")
        setting("plugins.security.ssl.http.pemkey_filepath", "esnode-key.pem")
        setting("plugins.security.ssl.http.pemtrustedcas_filepath", "root-ca.pem")
        setting("plugins.security.allow_unsafe_democertificates", "true")
        setting("plugins.security.allow_default_init_securityindex", "true")
        setting("plugins.security.unsupported.inject_user.enabled", "true")

        setting("plugins.security.authcz.admin_dn", "\n- CN=kirk,OU=client,O=client,L=test, C=de")
        setting('plugins.security.restapi.roles_enabled', '["all_access", "security_rest_api_access"]')
        setting('plugins.security.system_indices.enabled', "true")
        setting('plugins.security.system_indices.indices', '[' +
                '".plugins-ml-config", ' +
                '".plugins-ml-connector", ' +
                '".plugins-ml-model-group", ' +
                '".plugins-ml-model", ' +
                '".plugins-ml-task", ' +
                '".plugins-ml-conversation-meta", ' +
                '".plugins-ml-conversation-interactions", ' +
                '".plugins-flow-framework-config", ' +
                '".plugins-flow-framework-templates", ' +
                '".plugins-flow-framework-state"' +
                ']'
        )
        setSecure(true)
    }

    // Installs all registered zipArchive dependencies on integTest cluster nodes except security
    configurations.zipArchive.asFileTree.each {
            plugin(provider(new Callable<RegularFile>(){
                    @Override
                    RegularFile call() throws Exception {
                        return new RegularFile() {
                            @Override
                            File getAsFile() {
                                return it
                            }
                        }
                    }
                })
            )

    }

    // Install Flow Framwork Plugin on integTest cluster nodes
    plugin(project.tasks.bundlePlugin.archiveFile)

    // Cluster shrink exception thrown if we try to set numberOfNodes to 1, so only apply if > 1
    if (_numNodes > 1) numberOfNodes = _numNodes

    // When running integration tests it doesn't forward the --debug-jvm to the cluster anymore
    // i.e. we have to use a custom property to flag when we want to debug OpenSearch JVM
    // since we also support multi node integration tests we increase debugPort per node
    if (System.getProperty("opensearch.debug") != null) {
        def debugPort = 5005
        nodes.forEach { node ->
            node.jvmArgs("-agentlib:jdwp=transport=dt_socket,server=n,suspend=y,address=*:${debugPort}")
            debugPort += 1
        }
    }
}

// Remote Integration Tests
task integTestRemote(type: RestIntegTestTask) {
    testClassesDirs = sourceSets.test.output.classesDirs
    classpath = sourceSets.test.runtimeClasspath
    var is_https = System.getProperty('https')
    var user = System.getProperty('user')
    var password = System.getProperty('password')
    // Using default admin credentials since the install_plugin_configuration script is not used to configure the security plugin
    if (System.getProperty('security.enabled') != null) {
        is_https = is_https == null ? 'true' : is_https
        user = user == null ? 'admin' : user
        password = password == null ? 'admin' : password
        System.setProperty("https", is_https)
        System.setProperty("user", user)
        System.setProperty("password", password)
    }
    systemProperty('https', is_https)
    systemProperty('user', user)
    systemProperty('password', password)
    systemProperty 'cluster.number_of_nodes', "${_numNodes}"
    systemProperty 'tests.security.manager', 'false'

    // Run tests with remote cluster only if rest case is defined
    if (System.getProperty("tests.rest.cluster") != null) {
        filter {
            includeTestsMatching "org.opensearch.flowframework.rest.*IT"
        }
    }

    // Exclude integration tests that require security plugin
    if (System.getProperty("https") == null || System.getProperty("https") == "false") {
        filter {
            excludeTestsMatching "org.opensearch.flowframework.rest.FlowFrameworkSecureRestApiIT"
        }
    }

    // Include only secure integration tests in security enabled clusters
    if (System.getProperty("https") != null && System.getProperty("https") == "true") {
        filter {
            includeTestsMatching "org.opensearch.flowframework.rest.FlowFrameworkSecureRestApiIT"
            excludeTestsMatching "org.opensearch.flowframework.rest.FlowFrameworkRestApiIT"
        }
    }
}


// test retry configuration
allprojects {
    apply plugin: "org.gradle.test-retry"
    tasks.withType(RestIntegTestTask).configureEach {
        retry {
            if (System.getenv().containsKey("CI")) {
                maxRetries = 1
                maxFailures = 4
                failOnPassedAfterRetry = false
            }
        }
    }
}

// Automatically sets up the integration test cluster locally
run {
    doFirst {
        // There seems to be an issue when running multi node run or integ tasks with unicast_hosts
        // not being written, the waitForAllConditions ensures it's written
        getClusters().forEach { cluster ->
            cluster.waitForAllConditions()
        }
    }

    useCluster testClusters.integTest
}

// updateVersion: Task to auto update version to the next development iteration
task updateVersion {
    onlyIf { System.getProperty('newVersion') }
    doLast {
        ext.newVersion = System.getProperty('newVersion')
        println "Setting version to ${newVersion}."
        // String tokenization to support -SNAPSHOT
        ant.replaceregexp(file:'build.gradle', match: '"opensearch.version", "\\d.*"', replace: '"opensearch.version", "' + newVersion.tokenize('-')[0] + '-SNAPSHOT"', flags:'g', byline:true)
    }
}

diffCoverageReport {
    afterEvaluate {
        // Get uncommitted files via git diff
        // https://github.com/form-com/diff-coverage-gradle/issues/73
        def file = Files.createTempFile(URLEncoder.encode(project.name, 'UTF-8'), '.diff').toFile()
        def diffBase = 'refs/remotes/origin/main'
        // Only run locally
        if (!System.getenv('CI')) {
            file.withOutputStream { out ->
                exec {
                    ignoreExitValue true
                    commandLine 'git', 'diff', '--no-color', '--minimal', diffBase
                    standardOutput = out
                }
            }
        }
        diffSource.file = file
    }

    // View report at build/reports/jacoco/diffCoverage/html/index.html
    reports {
        html = true
    }

    violationRules {
        minBranches = 0.60
        minLines = 0.75
        failOnViolation = true
    }
}


tasks.withType(AbstractPublishToMaven) {
    def predicate = provider {
        publication.name == "pluginZip"
    }
    onlyIf("Publishing only ZIP distributions") {
        predicate.get()
    }
}