Debug

Signed-off-by: Daniel Widdis <[email protected]>
dbwiddis · Jan 28, 2024 · cd5ca71 · cd5ca71
1 parent 693b71d
commit cd5ca71
Show file tree

Hide file tree

Showing 6 changed files with 14 additions and 79 deletions.
diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml
@@ -9,83 +9,12 @@ on:
     types: [opened, synchronize, reopened]
 
 jobs:
-  spotless:
-    if: github.repository == 'opensearch-project/flow-framework'
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      # Spotless requires JDK 17+
-      - uses: actions/setup-java@v4
-        with:
-          java-version: 17
-          distribution: temurin
-      - name: Spotless Check
-        run: ./gradlew spotlessCheck
-  javadoc:
-    if: github.repository == 'opensearch-project/flow-framework'
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - name: Javadoc CheckStyle
-        run: ./gradlew checkstyleMain
-      - name: Javadoc Check
-        run: ./gradlew javadoc
-  build:
-    needs: [spotless, javadoc]
-    strategy:
-      matrix:
-        os: [ubuntu-latest, macos-latest, windows-latest]
-        java: [11, 21]
-        include:
-          - os: ubuntu-latest
-            java: 17
-            codecov: yes
-    name: Test JDK${{ matrix.java }}, ${{ matrix.os }}
-    runs-on: ${{ matrix.os }}
-    steps:
-      - uses: actions/checkout@v4
-      - name: Set up JDK ${{ matrix.java }}
-        uses: actions/setup-java@v4
-        with:
-          java-version: ${{ matrix.java }}
-          distribution: temurin
-      - name: Build and Run Tests
-        run: |
-          ./gradlew check -x integTest -x yamlRestTest -x spotlessJava
-      - name: Upload Coverage Report
-        if: ${{ matrix.codecov }}
-        uses: codecov/codecov-action@v3
-        with:
-          file: ./build/reports/jacoco/test/jacocoTestReport.xml
-  integTest:
-    needs: [spotless, javadoc]
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ubuntu-latest, macos-latest, windows-latest]
-        java: [11, 21]
-        include:
-          - os: ubuntu-latest
-            java: 17
-    name: Integ Test JDK${{ matrix.java }}, ${{ matrix.os }}
-    runs-on: ${{ matrix.os }}
-    steps:
-      - uses: actions/checkout@v4
-      - name: Set up JDK ${{ matrix.java }}
-        uses: actions/setup-java@v4
-        with:
-          java-version: ${{ matrix.java }}
-          distribution: temurin
-      - name: Build and Run Tests
-        run: |
-          ./gradlew integTest yamlRestTest
   integMultiNodeTest:
-    needs: [spotless, javadoc]
     strategy:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        java: [21]
+        java: [11, 17, 18, 19, 20, 21]
     name: Multi-Node Integ Test JDK${{ matrix.java }}, ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     steps:

diff --git a/src/main/java/org/opensearch/flowframework/FlowFrameworkPlugin.java b/src/main/java/org/opensearch/flowframework/FlowFrameworkPlugin.java
@@ -185,7 +185,7 @@ public List<ExecutorBuilder<?>> getExecutorBuilders(Settings settings) {
             new ScalingExecutorBuilder(
                 WORKFLOW_THREAD_POOL,
                 1,
-                OpenSearchExecutors.allocatedProcessors(settings),
+                Math.max(1, OpenSearchExecutors.allocatedProcessors(settings) / 2),
                 TimeValue.timeValueMinutes(5),
                 FLOW_FRAMEWORK_THREAD_POOL_PREFIX + WORKFLOW_THREAD_POOL
             )

diff --git a/src/main/java/org/opensearch/flowframework/model/WorkflowNode.java b/src/main/java/org/opensearch/flowframework/model/WorkflowNode.java
@@ -52,7 +52,7 @@ public class WorkflowNode implements ToXContentObject {
     /** The field defining the timeout value for this node */
     public static final String NODE_TIMEOUT_FIELD = "node_timeout";
     /** The default timeout value if the template doesn't override it */
-    public static final TimeValue NODE_TIMEOUT_DEFAULT_VALUE = new TimeValue(10, SECONDS);
+    public static final TimeValue NODE_TIMEOUT_DEFAULT_VALUE = new TimeValue(30, SECONDS);
 
     private final String id; // unique id
     private final String type; // maps to a WorkflowStep

diff --git a/src/main/resources/mappings/workflow-steps.json b/src/main/resources/mappings/workflow-steps.json
@@ -19,7 +19,8 @@
         ],
         "required_plugins":[
             "opensearch-ml"
-        ]
+        ],
+        "timeout": "60s"
     },
     "delete_connector": {
         "inputs": [

diff --git a/src/test/java/org/opensearch/flowframework/TestHelpers.java b/src/test/java/org/opensearch/flowframework/TestHelpers.java
@@ -113,6 +113,11 @@ public static Response makeRequest(
         if (entity != null) {
             request.setEntity(entity);
         }
+        try {
+            return client.performRequest(request);
+        } catch (IOException e) {
+            // In restricted resource cluster, this may time out and fail
+        }
         return client.performRequest(request);
     }
 

diff --git a/src/test/java/org/opensearch/flowframework/rest/FlowFrameworkRestApiIT.java b/src/test/java/org/opensearch/flowframework/rest/FlowFrameworkRestApiIT.java
@@ -88,7 +88,7 @@ public void testFailedUpdateWorkflow() throws Exception {
         // Ensure Ml config index is initialized as creating a connector requires this, then hit Provision API and assert status
         Response provisionResponse;
         if (!indexExistsWithAdminClient(".plugins-ml-config")) {
-            assertBusy(() -> assertTrue(indexExistsWithAdminClient(".plugins-ml-config")), 40, TimeUnit.SECONDS);
+            assertBusy(() -> assertTrue(indexExistsWithAdminClient(".plugins-ml-config")), 60, TimeUnit.SECONDS);
             provisionResponse = provisionWorkflow(client(), workflowId);
         } else {
             provisionResponse = provisionWorkflow(client(), workflowId);
@@ -218,7 +218,7 @@ public void testCreateAndProvisionRemoteModelWorkflow() throws Exception {
 
         // Ensure Ml config index is initialized as creating a connector requires this, then hit Provision API and assert status
         if (!indexExistsWithAdminClient(".plugins-ml-config")) {
-            assertBusy(() -> assertTrue(indexExistsWithAdminClient(".plugins-ml-config")), 40, TimeUnit.SECONDS);
+            assertBusy(() -> assertTrue(indexExistsWithAdminClient(".plugins-ml-config")), 60, TimeUnit.SECONDS);
             response = provisionWorkflow(client(), workflowId);
         } else {
             response = provisionWorkflow(client(), workflowId);
@@ -246,7 +246,7 @@ public void testCreateAndProvisionAgentFrameworkWorkflow() throws Exception {
         // Hit Create Workflow API to create agent-framework template, with template validation check and provision parameter
         Response response;
         if (!indexExistsWithAdminClient(".plugins-ml-config")) {
-            assertBusy(() -> assertTrue(indexExistsWithAdminClient(".plugins-ml-config")), 40, TimeUnit.SECONDS);
+            assertBusy(() -> assertTrue(indexExistsWithAdminClient(".plugins-ml-config")), 60, TimeUnit.SECONDS);
             response = createWorkflowWithProvision(client(), template);
         } else {
             response = createWorkflowWithProvision(client(), template);
@@ -257,7 +257,7 @@ public void testCreateAndProvisionAgentFrameworkWorkflow() throws Exception {
         // wait and ensure state is completed/done
         assertBusy(
             () -> { getAndAssertWorkflowStatus(client(), workflowId, State.COMPLETED, ProvisioningProgress.DONE); },
-            30,
+            120,
             TimeUnit.SECONDS
         );