Skip to content

Commit

Permalink
Add created, updated, and provisioned timestamps to saved template (o…
Browse files Browse the repository at this point in the history
…pensearch-project#551)

* Add created, last updated, and last provisioned fields to Template

Signed-off-by: Daniel Widdis <[email protected]>

* Change last updated timestamp when updating workflow

Signed-off-by: Daniel Widdis <[email protected]>

* Change last provisioned timestamp when provisioning workflow

Signed-off-by: Daniel Widdis <[email protected]>

* Allow overriding template not started check

Signed-off-by: Daniel Widdis <[email protected]>

* Use java.time and not joda time

Signed-off-by: Daniel Widdis <[email protected]>

* Preserve timestamps when encrypting and redacting template

Signed-off-by: Daniel Widdis <[email protected]>

* Add bwc tests, more timestamp testing

Signed-off-by: Daniel Widdis <[email protected]>

* Build a Template from an existing one

Signed-off-by: Daniel Widdis <[email protected]>

* Rename param, add comments

Signed-off-by: Daniel Widdis <[email protected]>

---------

Signed-off-by: Daniel Widdis <[email protected]>
  • Loading branch information
dbwiddis authored Mar 9, 2024
1 parent e9686f7 commit 2707210
Show file tree
Hide file tree
Showing 21 changed files with 875 additions and 126 deletions.
45 changes: 45 additions & 0 deletions .github/workflows/test_bwc.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
name: BWC
on:
push:
branches:
- "**"
pull_request:
branches:
- "**"

jobs:
Build-ff-linux:
strategy:
matrix:
java: [11,17,21]
fail-fast: false

name: Test Flow Framework BWC
runs-on: ubuntu-latest

steps:
- name: Setup Java ${{ matrix.java }}
uses: actions/setup-java@v4
with:
distribution: 'temurin'
java-version: ${{ matrix.java }}

- name: Checkout Flow Framework
uses: actions/checkout@v4

- name: Assemble Flow Framework
run: |
plugin_version=`./gradlew properties -q | grep "opensearch_build:" | awk '{print $2}'`
echo plugin_version $plugin_version
./gradlew assemble
echo "Creating ./src/test/resources/org/opensearch/flowframework/bwc/flow-framework/$plugin_version ..."
mkdir -p ./src/test/resources/org/opensearch/flowframework/bwc/flow-framework/$plugin_version
echo "Copying ./build/distributions/*.zip to ./src/test/resources/org/opensearch/flowframework/bwc/flow-framework/$plugin_version ..."
ls ./build/distributions/
cp ./build/distributions/*.zip ./src/test/resources/org/opensearch/flowframework/bwc/flow-framework/$plugin_version
echo "Copied ./build/distributions/*.zip to ./src/test/resources/org/opensearch/flowframework/bwc/flow-framework/$plugin_version ..."
ls ./src/test/resources/org/opensearch/flowframework/bwc/flow-framework/$plugin_version
- name: Run Flow Framework Backwards Compatibility Tests
run: |
echo "Running backwards compatibility tests ..."
./gradlew bwcTestSuite -Dtests.security.manager=false
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.1.0/)
### Enhancements
- Substitute REST path or body parameters in Workflow Steps ([#525](https://github.com/opensearch-project/flow-framework/pull/525))
- Added an optional workflow_step param to the get workflow steps API ([#538](https://github.com/opensearch-project/flow-framework/pull/538))
- Add created, updated, and provisioned timestamps to saved template ([#551](https://github.com/opensearch-project/flow-framework/pull/551))
- Enable Flow Framework by default ([#553](https://github.com/opensearch-project/flow-framework/pull/553))

### Bug Fixes
Expand Down
192 changes: 192 additions & 0 deletions build.gradle
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import java.nio.file.Files
import org.opensearch.gradle.testclusters.OpenSearchCluster
import org.opensearch.gradle.testclusters.StandaloneRestIntegTestTask
import org.opensearch.gradle.test.RestIntegTestTask
import java.util.concurrent.Callable
import java.nio.file.Paths
Expand All @@ -23,6 +24,16 @@ buildscript {
opensearch_no_snapshot = opensearch_build.replace("-SNAPSHOT","")
System.setProperty('tests.security.manager', 'false')
common_utils_version = System.getProperty("common_utils.version", opensearch_build)

bwcVersionShort = "2.12.0"
bwcVersion = bwcVersionShort + ".0"
bwcOpenSearchFFDownload = 'https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/' + bwcVersionShort + '/latest/linux/x64/tar/builds/' +
'opensearch/plugins/opensearch-flow-framework-' + bwcVersion + '.zip'
baseName = "ffBwcCluster"
bwcFilePath = "src/test/resources/org/opensearch/flowframework/bwc/"
bwcFlowFrameworkPath = bwcFilePath + "flowframework/"

isSameMajorVersion = opensearch_version.split("\\.")[0] == bwcVersionShort.split("\\.")[0]
}

repositories {
Expand Down Expand Up @@ -78,6 +89,9 @@ dependencyLicenses.enabled = false
// This requires an additional Jar not published as part of build-tools
loggerUsageCheck.enabled = false
thirdPartyAudit.enabled = false
// Allow test cases to be named Tests without having to be inherited from LuceneTestCase.
// see https://github.com/elastic/elasticsearch/blob/323f312bbc829a63056a79ebe45adced5099f6e6/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/TestingConventionsTasks.java
testingConventions.enabled = false

// No need to validate pom, as we do not upload to maven/sonatype
validateNebulaPom.enabled = false
Expand Down Expand Up @@ -192,6 +206,12 @@ jacocoTestReport {
}
tasks.named("check").configure { dependsOn(jacocoTestReport) }

tasks.named("yamlRestTest").configure {
filter {
excludeTestsMatching "org.opensearch.flowframework.rest.*IT"
excludeTestsMatching "org.opensearch.flowframework.bwc.*IT"
}
}

// Set up integration tests
task integTest(type: RestIntegTestTask) {
Expand Down Expand Up @@ -231,6 +251,13 @@ integTest {
}
}

// Exclude BWC tests, run separately
if (System.getProperty("tests.rest.bwcsuite") == null) {
filter {
excludeTestsMatching "org.opensearch.flowframework.bwc.*IT"
}
}

// Exclude integration tests that require security plugin
if (System.getProperty("https") == null || System.getProperty("https") == "false") {
filter {
Expand Down Expand Up @@ -425,6 +452,166 @@ task integTestRemote(type: RestIntegTestTask) {
}
}

2.times {i ->
testClusters {
"${baseName}$i" {
testDistribution = "ARCHIVE"
versions = [bwcVersionShort, opensearch_version]
numberOfNodes = 3
plugin(provider(new Callable<RegularFile>(){
@Override
RegularFile call() throws Exception {
return new RegularFile() {
@Override
File getAsFile() {
if (new File("$project.rootDir/$bwcFilePath/flow-framework/$bwcVersion").exists()) {
project.delete(files("$project.rootDir/$bwcFilePath/flow-framework/$bwcVersion"))
}
project.mkdir bwcFlowFrameworkPath + bwcVersion
ant.get(src: bwcOpenSearchFFDownload,
dest: bwcFlowFrameworkPath + bwcVersion,
httpusecaches: false)
return fileTree(bwcFlowFrameworkPath + bwcVersion).getSingleFile()
}
}
}
}))
setting 'path.repo', "${buildDir}/cluster/shared/repo/${baseName}"
setting 'http.content_type.required', 'true'
}
}
}

List<Provider<RegularFile>> plugins = [
provider(new Callable<RegularFile>(){
@Override
RegularFile call() throws Exception {
return new RegularFile() {
@Override
File getAsFile() {
return configurations.zipArchive.asFileTree.getSingleFile()
}
}
}
}),
provider(new Callable<RegularFile>(){
@Override
RegularFile call() throws Exception {
return new RegularFile() {
@Override
File getAsFile() {
return fileTree(bwcFilePath + "flow-framework/" + project.version).getSingleFile()
}
}
}
})
]

// Creates 2 test clusters with 3 nodes of the old version.
2.times {i ->
task "${baseName}#oldVersionClusterTask$i"(type: StandaloneRestIntegTestTask) {
onlyIf { isSameMajorVersion || (i == 1) }
useCluster testClusters."${baseName}$i"
filter {
includeTestsMatching "org.opensearch.flowframework.bwc.*IT"
}
systemProperty 'tests.rest.bwcsuite', 'old_cluster'
systemProperty 'tests.rest.bwcsuite_round', 'old'
systemProperty 'tests.plugin_bwc_version', bwcVersion
nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}$i".allHttpSocketURI.join(",")}")
nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}$i".getName()}")
}
}

// Upgrades one node of the old cluster to new OpenSearch version with upgraded plugin version
// This results in a mixed cluster with 2 nodes on the old version and 1 upgraded node.
// This is also used as a one third upgraded cluster for a rolling upgrade.
task "${baseName}#mixedClusterTask"(type: StandaloneRestIntegTestTask) {
onlyIf { isSameMajorVersion }
useCluster testClusters."${baseName}0"
dependsOn "${baseName}#oldVersionClusterTask0"
doFirst {
testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
}
filter {
includeTestsMatching "org.opensearch.flowframework.bwc.*IT"
}
systemProperty 'tests.rest.bwcsuite', 'mixed_cluster'
systemProperty 'tests.rest.bwcsuite_round', 'first'
systemProperty 'tests.plugin_bwc_version', bwcVersion
nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}0".allHttpSocketURI.join(",")}")
nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}0".getName()}")
}

// Upgrades the second node to new OpenSearch version with upgraded plugin version after the first node is upgraded.
// This results in a mixed cluster with 1 node on the old version and 2 upgraded nodes.
// This is used for rolling upgrade.
task "${baseName}#twoThirdsUpgradedClusterTask"(type: StandaloneRestIntegTestTask) {
onlyIf { isSameMajorVersion }
dependsOn "${baseName}#mixedClusterTask"
useCluster testClusters."${baseName}0"
doFirst {
testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
}
filter {
includeTestsMatching "org.opensearch.flowframework.bwc.*IT"
}
systemProperty 'tests.rest.bwcsuite', 'mixed_cluster'
systemProperty 'tests.rest.bwcsuite_round', 'second'
systemProperty 'tests.plugin_bwc_version', bwcVersion
nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}0".allHttpSocketURI.join(",")}")
nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}0".getName()}")
}

// Upgrades the third node to new OpenSearch version with upgraded plugin version after the second node is upgraded.
// This results in a fully upgraded cluster.
// This is used for rolling upgrade.
task "${baseName}#rollingUpgradeClusterTask"(type: StandaloneRestIntegTestTask) {
onlyIf { isSameMajorVersion }
dependsOn "${baseName}#twoThirdsUpgradedClusterTask"
useCluster testClusters."${baseName}0"
doFirst {
testClusters."${baseName}0".upgradeNodeAndPluginToNextVersion(plugins)
}
filter {
includeTestsMatching "org.opensearch.flowframework.bwc.*IT"
}
mustRunAfter "${baseName}#mixedClusterTask"
systemProperty 'tests.rest.bwcsuite', 'mixed_cluster'
systemProperty 'tests.rest.bwcsuite_round', 'third'
systemProperty 'tests.plugin_bwc_version', bwcVersion
nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}0".allHttpSocketURI.join(",")}")
nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}0".getName()}")
}

// Upgrades all the nodes of the old cluster to new OpenSearch version with upgraded plugin version
// at the same time resulting in a fully upgraded cluster.
task "${baseName}#fullRestartClusterTask"(type: StandaloneRestIntegTestTask) {
dependsOn "${baseName}#oldVersionClusterTask1"
useCluster testClusters."${baseName}1"
doFirst {
testClusters."${baseName}1".upgradeAllNodesAndPluginsToNextVersion(plugins)
}
filter {
includeTestsMatching "org.opensearch.flowframework.bwc.*IT"
}
systemProperty 'tests.rest.bwcsuite', 'upgraded_cluster'
systemProperty 'tests.plugin_bwc_version', bwcVersion
nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}1".allHttpSocketURI.join(",")}")
nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}1".getName()}")
}

// A bwc test suite which runs all the bwc tasks combined.
task bwcTestSuite(type: StandaloneRestIntegTestTask) {
filter {
excludeTestsMatching '**.*Test*'
excludeTestsMatching '**.*IT*'
setFailOnNoMatchingTests(false)
}
dependsOn tasks.named("${baseName}#mixedClusterTask")
dependsOn tasks.named("${baseName}#rollingUpgradeClusterTask")
dependsOn tasks.named("${baseName}#fullRestartClusterTask")
}

// test retry configuration
allprojects {
Expand All @@ -438,6 +625,11 @@ allprojects {
}
}
}
// Needed for Gradle 9.0
tasks.withType(StandaloneRestIntegTestTask).configureEach {
testClassesDirs = sourceSets.test.output.classesDirs
classpath = sourceSets.test.runtimeClasspath
}
}

// Automatically sets up the integration test cluster locally
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,12 @@ private CommonValue() {}
public static final String CREATE_TIME = "create_time";
/** The template field name for the user who created the workflow **/
public static final String USER_FIELD = "user";
/** The created time field */
public static final String CREATED_TIME = "created_time";
/** The last updated time field */
public static final String LAST_UPDATED_TIME_FIELD = "last_updated_time";
/** The last provisioned time field */
public static final String LAST_PROVISIONED_TIME_FIELD = "last_provisioned_time";

/*
* Constants associated with Rest or Transport actions
Expand Down Expand Up @@ -156,10 +162,6 @@ private CommonValue() {}
public static final String APP_TYPE_FIELD = "app_type";
/** To include field for an agent response */
public static final String INCLUDE_OUTPUT_IN_AGENT_RESPONSE = "include_output_in_agent_response";
/** The created time field for an agent */
public static final String CREATED_TIME = "created_time";
/** The last updated time field for an agent */
public static final String LAST_UPDATED_TIME_FIELD = "last_updated_time";
/** HttpHost */
public static final String HTTP_HOST_FIELD = "http_host";
/** Http scheme */
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -395,6 +395,22 @@ public void putInitialStateToWorkflowState(String workflowId, User user, ActionL
* @param listener action listener
*/
public void updateTemplateInGlobalContext(String documentId, Template template, ActionListener<IndexResponse> listener) {
updateTemplateInGlobalContext(documentId, template, listener, false);
}

/**
* Replaces a document in the global context index
* @param documentId the document Id
* @param template the use-case template
* @param listener action listener
* @param ignoreNotStartedCheck if set true, ignores the requirement that the provisioning is not started
*/
public void updateTemplateInGlobalContext(
String documentId,
Template template,
ActionListener<IndexResponse> listener,
boolean ignoreNotStartedCheck
) {
if (!doesIndexExist(GLOBAL_CONTEXT_INDEX)) {
String errorMessage = "Failed to update template for workflow_id : " + documentId + ", global_context index does not exist.";
logger.error(errorMessage);
Expand All @@ -404,7 +420,7 @@ public void updateTemplateInGlobalContext(String documentId, Template template,
doesTemplateExist(documentId, templateExists -> {
if (templateExists) {
isWorkflowNotStarted(documentId, workflowIsNotStarted -> {
if (workflowIsNotStarted) {
if (workflowIsNotStarted || ignoreNotStartedCheck) {
IndexRequest request = new IndexRequest(GLOBAL_CONTEXT_INDEX).id(documentId);
try (
XContentBuilder builder = XContentFactory.jsonBuilder();
Expand Down
Loading

0 comments on commit 2707210

Please sign in to comment.