[Backport 2.x] Add Feature Filtering in Model Validation (#1261)

* Consider feature filter in model validation (#1258) Signed-off-by: Kaituo Li <[email protected]> * follow opensearch-project/k-NN#1795 Signed-off-by: Kaituo Li <[email protected]> --------- Signed-off-by: Kaituo Li <[email protected]>
opensearch-project · Jul 8, 2024 · e2c821f · e2c821f
1 parent a64a859
commit e2c821f
Show file tree

Hide file tree

Showing 43 changed files with 2,516 additions and 569 deletions.
diff --git a/.github/workflows/add-untriaged.yml b/.github/workflows/add-untriaged.yml
@@ -4,6 +4,9 @@ on:
   issues:
     types: [opened, reopened, transferred]
 
+env:
+  ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
+
 jobs:
   apply-label:
     runs-on: ubuntu-latest

diff --git a/.github/workflows/auto-release.yml b/.github/workflows/auto-release.yml
@@ -5,6 +5,9 @@ on:
     tags:
       - '*'
 
+env:
+  ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
+
 jobs:
   build:
     runs-on: ubuntu-latest

diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml
@@ -5,6 +5,9 @@ on:
       - closed
       - labeled
 
+env:
+  ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
+
 jobs:
   backport:
     if: github.event.pull_request.merged == true

diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
@@ -7,6 +7,9 @@ on:
     branches:
       - "*"
 
+env:
+  ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
+
 jobs:
   Get-CI-Image-Tag:
     uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main

diff --git a/.github/workflows/delete_backport_branch.yml b/.github/workflows/delete_backport_branch.yml
@@ -3,7 +3,10 @@ on:
   pull_request:
     types:
       - closed
-
+
+env:
+  ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
+
 jobs:
   delete-branch:
     runs-on: ubuntu-latest

diff --git a/.github/workflows/draft-release-notes-workflow.yml b/.github/workflows/draft-release-notes-workflow.yml
@@ -5,6 +5,9 @@ on:
     branches:
       - main
 
+env:
+  ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
+
 jobs:
   update_release_draft:
     name: Update draft release notes

diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml
@@ -6,6 +6,9 @@ on:
     types:
       - opened
 
+env:
+  ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
+
 jobs:
   label:
     runs-on: ubuntu-latest

diff --git a/.github/workflows/link-check-workflow.yml b/.github/workflows/link-check-workflow.yml
@@ -5,6 +5,9 @@ on:
   pull_request:
     branches: [main]
 
+env:
+  ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
+
 jobs:
   linkchecker:
     runs-on: ubuntu-latest

diff --git a/.github/workflows/maven-publish.yml b/.github/workflows/maven-publish.yml
@@ -8,6 +8,9 @@ on:
         - '1.*'
         - '2.*'
 
+env:
+  ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
+
 jobs:
   build-and-publish-snapshots:
     strategy:

diff --git a/.github/workflows/test_build_multi_platform.yml b/.github/workflows/test_build_multi_platform.yml
@@ -7,6 +7,9 @@ on:
     branches:
       - "*"
 
+env:
+  ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
+
 jobs:
   Get-CI-Image-Tag:
     uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main
@@ -53,19 +56,14 @@ jobs:
       - name: Multi Nodes Integration Testing
         run: |
           ./gradlew integTest -PnumNodes=3
-      - name: Upload Coverage Report
-        uses: codecov/codecov-action@v3
-        with:
-          file: ./build/reports/jacoco/test/jacocoTestReport.xml
-          flags: plugin
 
   Build-ad-linux:
     needs: [Get-CI-Image-Tag, spotless]
     strategy:
       matrix:
         java: [11, 17, 21]
       fail-fast: false
-    name: Build and Test Anomaly detection Plugin
+    name: Build and Test Anomaly detection Plugin on Linux
     runs-on: ubuntu-latest
     container:
       # using the same image which is used by opensearch-build team to build the OpenSearch Distribution
@@ -94,11 +92,6 @@ jobs:
                                ./gradlew build -x spotlessJava &&
                                ./gradlew publishToMavenLocal &&
                                ./gradlew integTest -PnumNodes=3"
-      - name: Upload Coverage Report
-        uses: codecov/codecov-action@v3
-        with:
-          file: ./build/reports/jacoco/test/jacocoTestReport.xml
-          flags: plugin
 
   Build-ad-macos:
     needs: spotless
@@ -107,7 +100,7 @@ jobs:
         java: [11,17,21]
       fail-fast: false
 
-    name: Build and Test Anomaly detection Plugin
+    name: Build and Test Anomaly detection Plugin on MacOS
     runs-on: macos-13
     env:
       JENKINS_URL: build.ci.opensearch.org
@@ -128,14 +121,17 @@ jobs:
       - name: Build and Run Tests
         run: |
           ./gradlew build -x spotlessJava
+      # coverage.gradle is only applied in single local node test
+      - name: Upload Coverage Report
+        uses: codecov/codecov-action@v4
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+        with:
+          file: ./build/reports/jacoco/test/jacocoTestReport.xml
+          flags: plugin
       - name: Publish to Maven Local
         run: |
           ./gradlew publishToMavenLocal
       - name: Multi Nodes Integration Testing
         run: |
           ./gradlew integTest -PnumNodes=3
-      - name: Upload Coverage Report
-        uses: codecov/codecov-action@v3
-        with:
-          file: ./build/reports/jacoco/test/jacocoTestReport.xml
-          flags: plugin
diff --git a/.github/workflows/test_bwc.yml b/.github/workflows/test_bwc.yml
@@ -7,6 +7,9 @@ on:
     branches:
       - "*"
 
+env:
+  ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
+
 jobs:
   Get-CI-Image-Tag:
     uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main

diff --git a/.github/workflows/test_security.yml b/.github/workflows/test_security.yml
@@ -7,6 +7,9 @@ on:
     branches:
       - "*"
 
+env:
+  ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
+
 jobs:
   Build-ad:
     strategy:

diff --git a/src/main/java/org/opensearch/ad/constant/ADCommonMessages.java b/src/main/java/org/opensearch/ad/constant/ADCommonMessages.java
@@ -25,7 +25,6 @@ public class ADCommonMessages {
     public static String DETECTOR_IS_RUNNING = "Detector is already running";
     public static String DETECTOR_MISSING = "Detector is missing";
     public static String AD_TASK_ACTION_MISSING = "AD task action is missing";
-    public static final String INDEX_NOT_FOUND = "index does not exist";
     public static final String UNSUPPORTED_PROFILE_TYPE = "Unsupported profile types";
 
     public static final String REQUEST_THROTTLED_MSG = "Request throttled. Please try again later.";

diff --git a/src/main/java/org/opensearch/ad/rest/handler/ADModelValidationActionHandler.java b/src/main/java/org/opensearch/ad/rest/handler/ADModelValidationActionHandler.java
@@ -17,6 +17,7 @@
 import org.opensearch.core.xcontent.NamedXContentRegistry;
 import org.opensearch.timeseries.AnalysisType;
 import org.opensearch.timeseries.feature.SearchFeatureDao;
+import org.opensearch.timeseries.model.ValidationIssueType;
 import org.opensearch.timeseries.rest.handler.ModelValidationActionHandler;
 import org.opensearch.timeseries.transport.ValidateConfigResponse;
 import org.opensearch.timeseries.util.SecurityClientUtil;
@@ -50,7 +51,8 @@ public ADModelValidationActionHandler(
             clock,
             settings,
             user,
-            AnalysisType.AD
+            AnalysisType.AD,
+            ValidationIssueType.DETECTION_INTERVAL
         );
     }
 

diff --git a/src/main/java/org/opensearch/ad/transport/ValidateAnomalyDetectorTransportAction.java b/src/main/java/org/opensearch/ad/transport/ValidateAnomalyDetectorTransportAction.java
@@ -28,6 +28,7 @@
 import org.opensearch.rest.RestRequest;
 import org.opensearch.timeseries.feature.SearchFeatureDao;
 import org.opensearch.timeseries.model.Config;
+import org.opensearch.timeseries.model.ValidationAspect;
 import org.opensearch.timeseries.rest.handler.Processor;
 import org.opensearch.timeseries.transport.BaseValidateConfigTransportAction;
 import org.opensearch.timeseries.transport.ValidateConfigRequest;
@@ -61,7 +62,8 @@ public ValidateAnomalyDetectorTransportAction(
             actionFilters,
             transportService,
             searchFeatureDao,
-            AD_FILTER_BY_BACKEND_ROLES
+            AD_FILTER_BY_BACKEND_ROLES,
+            ValidationAspect.DETECTOR
         );
     }
 

diff --git a/src/main/java/org/opensearch/forecast/rest/handler/ForecastModelValidationActionHandler.java b/src/main/java/org/opensearch/forecast/rest/handler/ForecastModelValidationActionHandler.java
@@ -17,6 +17,7 @@
 import org.opensearch.forecast.model.Forecaster;
 import org.opensearch.timeseries.AnalysisType;
 import org.opensearch.timeseries.feature.SearchFeatureDao;
+import org.opensearch.timeseries.model.ValidationIssueType;
 import org.opensearch.timeseries.rest.handler.ModelValidationActionHandler;
 import org.opensearch.timeseries.transport.ValidateConfigResponse;
 import org.opensearch.timeseries.util.SecurityClientUtil;
@@ -50,7 +51,8 @@ public ForecastModelValidationActionHandler(
             clock,
             settings,
             user,
-            AnalysisType.FORECAST
+            AnalysisType.FORECAST,
+            ValidationIssueType.FORECAST_INTERVAL
         );
     }
 

diff --git a/src/main/java/org/opensearch/forecast/transport/ValidateForecasterTransportAction.java b/src/main/java/org/opensearch/forecast/transport/ValidateForecasterTransportAction.java
@@ -22,6 +22,7 @@
 import org.opensearch.rest.RestRequest;
 import org.opensearch.timeseries.feature.SearchFeatureDao;
 import org.opensearch.timeseries.model.Config;
+import org.opensearch.timeseries.model.ValidationAspect;
 import org.opensearch.timeseries.rest.handler.Processor;
 import org.opensearch.timeseries.transport.BaseValidateConfigTransportAction;
 import org.opensearch.timeseries.transport.ValidateConfigRequest;
@@ -55,7 +56,8 @@ public ValidateForecasterTransportAction(
             actionFilters,
             transportService,
             searchFeatureDao,
-            FORECAST_FILTER_BY_BACKEND_ROLES
+            FORECAST_FILTER_BY_BACKEND_ROLES,
+            ValidationAspect.FORECASTER
         );
     }
 

diff --git a/src/main/java/org/opensearch/timeseries/constant/CommonMessages.java b/src/main/java/org/opensearch/timeseries/constant/CommonMessages.java
@@ -54,9 +54,9 @@ public static String getTooManyCategoricalFieldErr(int limit) {
     public static String TIME_FIELD_NOT_ENOUGH_HISTORICAL_DATA =
         "There isn't enough historical data found with current timefield selected.";
     public static String CATEGORY_FIELD_TOO_SPARSE =
-        "Data is most likely too sparse with the given category fields. Consider revising category field/s or ingesting more data ";
+        "Data is most likely too sparse with the given category fields. Consider revising category field/s or ingesting more data.";
     public static String WINDOW_DELAY_REC =
-        "Latest seen data point is at least %d minutes ago, consider changing window delay to at least %d minutes.";
+        "Latest seen data point is at least %d minutes ago. Consider changing window delay to at least %d minutes.";
     public static String INTERVAL_REC = "The selected interval might collect sparse data. Consider changing interval length to: ";
     public static String RAW_DATA_TOO_SPARSE =
         "Source index data is potentially too sparse for model training. Consider changing interval length or ingesting more data";
@@ -65,13 +65,14 @@ public static String getTooManyCategoricalFieldErr(int limit) {
     public static String CATEGORY_FIELD_NO_DATA =
         "No entity was found with the given categorical fields. Consider revising category field/s or ingesting more data";
     public static String FEATURE_QUERY_TOO_SPARSE =
-        "Data is most likely too sparse when given feature queries are applied. Consider revising feature queries.";
+        "Data is most likely too sparse when given feature queries are applied. Consider revising feature queries";
     public static String TIMEOUT_ON_INTERVAL_REC = "Timed out getting interval recommendation";
     public static final String NOT_EXISTENT_VALIDATION_TYPE = "The given validation type doesn't exist";
     public static final String NOT_EXISTENT_SUGGEST_TYPE = "The given suggest type doesn't exist";
     public static final String DESCRIPTION_LENGTH_TOO_LONG = "Description length is too long. Max length is "
         + TimeSeriesSettings.MAX_DESCRIPTION_LENGTH
         + " characters.";
+    public static final String INDEX_NOT_FOUND = "index does not exist";
 
     // ======================================
     // Index message