Skip to content

Commit

Permalink
Merge branch 'master' into INLONG-10940
Browse files Browse the repository at this point in the history
  • Loading branch information
MOONSakura0614 committed Sep 4, 2024
2 parents d95eb06 + edf93bd commit 13f38dd
Show file tree
Hide file tree
Showing 100 changed files with 4,402 additions and 339 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/ci_ut.yml
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ jobs:
CI: false

- name: Unit test with Maven
run: mvn --batch-mode --update-snapshots -e -V test -pl !:sort-end-to-end-tests-v1.15,!:sort-end-to-end-tests-v1.13
run: mvn --batch-mode --update-snapshots -e -V test -pl !:sort-end-to-end-tests-v1.15,!:sort-end-to-end-tests-v1.13,!:sort-end-to-end-tests-v1.18
env:
CI: false

Expand Down
96 changes: 96 additions & 0 deletions .github/workflows/ci_ut_flink18.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

name:
InLong Unit Test For Flink 1.18

on:
push:
paths:
- '.github/workflows/ci_ut_flink18.yml'
- 'inlong-sort/**'
- '!**.md'

pull_request:
paths:
- '.github/workflows/ci_ut_flink18.yml'
- 'inlong-sort/**'
- '!**.md'

jobs:
unit-test:
name: Unit Test
runs-on: ubuntu-22.04
steps:
- name: Checkout
uses: actions/checkout@v4

# Release space size
- name: Remove unnecessary packages
run: |
echo "=== Before pruning ==="
df -h
sudo rm -rf /usr/share/dotnet
sudo rm -rf /usr/local/lib/android
sudo rm -rf /opt/ghc
sudo rm -rf /opt/hostedtoolcache
echo "=== After pruning ==="
df -h
- name: Set up JDK
uses: actions/setup-java@v4
with:
java-version: 8
distribution: adopt

- name: Cache Maven packages
uses: actions/cache@v4
with:
path: |
~/.m2/repository/*/*/*
!~/.m2/repository/org/apache/inlong
key: ${{ runner.os }}-inlong-flink18-${{ hashFiles('**/pom.xml') }}
restore-keys: ${{ runner.os }}-inlong-flink18

- name: Build for Flink 1.18 with Maven
run: mvn --update-snapshots -e -V clean install -U -pl :sort-end-to-end-tests-v1.18 -am -Pv1.18 -DskipTests -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=120 -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000
env:
CI: false

- name: Unit test for Flink 1.18 with Maven
run: mvn --update-snapshots -e -V verify -pl :sort-end-to-end-tests-v1.18 -am -Pv1.18
env:
CI: false

- name: Upload unit test results
if: ${{ failure() }}
uses: actions/upload-artifact@v4
with:
name: surefire-reports
path: ./**/target/surefire-reports/
if-no-files-found: ignore

- name: Upload integration test results
if: ${{ failure() }}
uses: actions/upload-artifact@v4
with:
name: failsafe-reports
path: ./**/target/failsafe-reports/
if-no-files-found: ignore

- name: Clean up build packages
run: mvn clean
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
* limitations under the License.
*/

package org.apache.inlong.audit.config;
package org.apache.inlong.audit.consts;

/**
* Open api constants
Expand Down Expand Up @@ -54,8 +54,8 @@ public class OpenApiConstants {
public static final String PARAMS_END_TIME = "endTime";
public static final String PARAMS_AUDIT_ID = "auditId";
public static final String PARAMS_AUDIT_TAG = "auditTag";
public static final String PARAMS_INLONG_GROUP_Id = "inlongGroupId";
public static final String PARAMS_INLONG_STREAM_Id = "inlongStreamId";
public static final String PARAMS_INLONG_GROUP_ID = "inlongGroupId";
public static final String PARAMS_INLONG_STREAM_ID = "inlongStreamId";
public static final String PARAMS_IP = "ip";
public static final String PARAMS_AUDIT_CYCLE = "auditCycle";
public static final String KEY_HTTP_BODY_SUCCESS = "success";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,11 +37,11 @@
import java.util.concurrent.TimeUnit;

import static org.apache.inlong.audit.config.ConfigConstants.DATE_FORMAT;
import static org.apache.inlong.audit.config.OpenApiConstants.DEFAULT_API_CACHE_EXPIRED_HOURS;
import static org.apache.inlong.audit.config.OpenApiConstants.DEFAULT_API_CACHE_MAX_SIZE;
import static org.apache.inlong.audit.config.OpenApiConstants.KEY_API_CACHE_EXPIRED_HOURS;
import static org.apache.inlong.audit.config.OpenApiConstants.KEY_API_CACHE_MAX_SIZE;
import static org.apache.inlong.audit.consts.ConfigConstants.DEFAULT_AUDIT_TAG;
import static org.apache.inlong.audit.consts.OpenApiConstants.DEFAULT_API_CACHE_EXPIRED_HOURS;
import static org.apache.inlong.audit.consts.OpenApiConstants.DEFAULT_API_CACHE_MAX_SIZE;
import static org.apache.inlong.audit.consts.OpenApiConstants.KEY_API_CACHE_EXPIRED_HOURS;
import static org.apache.inlong.audit.consts.OpenApiConstants.KEY_API_CACHE_MAX_SIZE;

/**
* Abstract cache.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,14 +51,14 @@
import static org.apache.inlong.audit.config.ConfigConstants.KEY_DATASOURCE_MAX_IDLE_CONNECTIONS;
import static org.apache.inlong.audit.config.ConfigConstants.KEY_DATASOURCE_MAX_TOTAL_CONNECTIONS;
import static org.apache.inlong.audit.config.ConfigConstants.KEY_DATASOURCE_MIN_IDLE_CONNECTIONS;
import static org.apache.inlong.audit.config.OpenApiConstants.DEFAULT_API_THREAD_POOL_SIZE;
import static org.apache.inlong.audit.config.OpenApiConstants.KEY_API_THREAD_POOL_SIZE;
import static org.apache.inlong.audit.config.SqlConstants.DEFAULT_SOURCE_QUERY_IDS_SQL;
import static org.apache.inlong.audit.config.SqlConstants.DEFAULT_SOURCE_QUERY_IPS_SQL;
import static org.apache.inlong.audit.config.SqlConstants.DEFAULT_SOURCE_QUERY_MINUTE_SQL;
import static org.apache.inlong.audit.config.SqlConstants.KEY_SOURCE_QUERY_IDS_SQL;
import static org.apache.inlong.audit.config.SqlConstants.KEY_SOURCE_QUERY_IPS_SQL;
import static org.apache.inlong.audit.config.SqlConstants.KEY_SOURCE_QUERY_MINUTE_SQL;
import static org.apache.inlong.audit.consts.OpenApiConstants.DEFAULT_API_THREAD_POOL_SIZE;
import static org.apache.inlong.audit.consts.OpenApiConstants.KEY_API_THREAD_POOL_SIZE;

/**
* Real time query data from audit source.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,42 +48,42 @@
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

import static org.apache.inlong.audit.config.OpenApiConstants.DEFAULT_API_BACKLOG_SIZE;
import static org.apache.inlong.audit.config.OpenApiConstants.DEFAULT_API_DAY_PATH;
import static org.apache.inlong.audit.config.OpenApiConstants.DEFAULT_API_GET_AUDIT_PROXY_PATH;
import static org.apache.inlong.audit.config.OpenApiConstants.DEFAULT_API_GET_IDS_PATH;
import static org.apache.inlong.audit.config.OpenApiConstants.DEFAULT_API_GET_IPS_PATH;
import static org.apache.inlong.audit.config.OpenApiConstants.DEFAULT_API_HOUR_PATH;
import static org.apache.inlong.audit.config.OpenApiConstants.DEFAULT_API_MINUTES_PATH;
import static org.apache.inlong.audit.config.OpenApiConstants.DEFAULT_API_REAL_LIMITER_QPS;
import static org.apache.inlong.audit.config.OpenApiConstants.DEFAULT_API_THREAD_POOL_SIZE;
import static org.apache.inlong.audit.config.OpenApiConstants.DEFAULT_HTTP_SERVER_BIND_PORT;
import static org.apache.inlong.audit.config.OpenApiConstants.HTTP_RESPOND_CODE;
import static org.apache.inlong.audit.config.OpenApiConstants.KEY_API_BACKLOG_SIZE;
import static org.apache.inlong.audit.config.OpenApiConstants.KEY_API_DAY_PATH;
import static org.apache.inlong.audit.config.OpenApiConstants.KEY_API_GET_AUDIT_PROXY_PATH;
import static org.apache.inlong.audit.config.OpenApiConstants.KEY_API_GET_IDS_PATH;
import static org.apache.inlong.audit.config.OpenApiConstants.KEY_API_GET_IPS_PATH;
import static org.apache.inlong.audit.config.OpenApiConstants.KEY_API_HOUR_PATH;
import static org.apache.inlong.audit.config.OpenApiConstants.KEY_API_MINUTES_PATH;
import static org.apache.inlong.audit.config.OpenApiConstants.KEY_API_REAL_LIMITER_QPS;
import static org.apache.inlong.audit.config.OpenApiConstants.KEY_API_THREAD_POOL_SIZE;
import static org.apache.inlong.audit.config.OpenApiConstants.KEY_HTTP_BODY_ERR_DATA;
import static org.apache.inlong.audit.config.OpenApiConstants.KEY_HTTP_BODY_ERR_MSG;
import static org.apache.inlong.audit.config.OpenApiConstants.KEY_HTTP_BODY_SUCCESS;
import static org.apache.inlong.audit.config.OpenApiConstants.KEY_HTTP_HEADER_CONTENT_TYPE;
import static org.apache.inlong.audit.config.OpenApiConstants.KEY_HTTP_SERVER_BIND_PORT;
import static org.apache.inlong.audit.config.OpenApiConstants.PARAMS_AUDIT_COMPONENT;
import static org.apache.inlong.audit.config.OpenApiConstants.PARAMS_AUDIT_CYCLE;
import static org.apache.inlong.audit.config.OpenApiConstants.PARAMS_AUDIT_ID;
import static org.apache.inlong.audit.config.OpenApiConstants.PARAMS_AUDIT_TAG;
import static org.apache.inlong.audit.config.OpenApiConstants.PARAMS_END_TIME;
import static org.apache.inlong.audit.config.OpenApiConstants.PARAMS_INLONG_GROUP_Id;
import static org.apache.inlong.audit.config.OpenApiConstants.PARAMS_INLONG_STREAM_Id;
import static org.apache.inlong.audit.config.OpenApiConstants.PARAMS_IP;
import static org.apache.inlong.audit.config.OpenApiConstants.PARAMS_START_TIME;
import static org.apache.inlong.audit.config.OpenApiConstants.VALUE_HTTP_HEADER_CONTENT_TYPE;
import static org.apache.inlong.audit.consts.ConfigConstants.DEFAULT_AUDIT_TAG;
import static org.apache.inlong.audit.consts.OpenApiConstants.DEFAULT_API_BACKLOG_SIZE;
import static org.apache.inlong.audit.consts.OpenApiConstants.DEFAULT_API_DAY_PATH;
import static org.apache.inlong.audit.consts.OpenApiConstants.DEFAULT_API_GET_AUDIT_PROXY_PATH;
import static org.apache.inlong.audit.consts.OpenApiConstants.DEFAULT_API_GET_IDS_PATH;
import static org.apache.inlong.audit.consts.OpenApiConstants.DEFAULT_API_GET_IPS_PATH;
import static org.apache.inlong.audit.consts.OpenApiConstants.DEFAULT_API_HOUR_PATH;
import static org.apache.inlong.audit.consts.OpenApiConstants.DEFAULT_API_MINUTES_PATH;
import static org.apache.inlong.audit.consts.OpenApiConstants.DEFAULT_API_REAL_LIMITER_QPS;
import static org.apache.inlong.audit.consts.OpenApiConstants.DEFAULT_API_THREAD_POOL_SIZE;
import static org.apache.inlong.audit.consts.OpenApiConstants.DEFAULT_HTTP_SERVER_BIND_PORT;
import static org.apache.inlong.audit.consts.OpenApiConstants.HTTP_RESPOND_CODE;
import static org.apache.inlong.audit.consts.OpenApiConstants.KEY_API_BACKLOG_SIZE;
import static org.apache.inlong.audit.consts.OpenApiConstants.KEY_API_DAY_PATH;
import static org.apache.inlong.audit.consts.OpenApiConstants.KEY_API_GET_AUDIT_PROXY_PATH;
import static org.apache.inlong.audit.consts.OpenApiConstants.KEY_API_GET_IDS_PATH;
import static org.apache.inlong.audit.consts.OpenApiConstants.KEY_API_GET_IPS_PATH;
import static org.apache.inlong.audit.consts.OpenApiConstants.KEY_API_HOUR_PATH;
import static org.apache.inlong.audit.consts.OpenApiConstants.KEY_API_MINUTES_PATH;
import static org.apache.inlong.audit.consts.OpenApiConstants.KEY_API_REAL_LIMITER_QPS;
import static org.apache.inlong.audit.consts.OpenApiConstants.KEY_API_THREAD_POOL_SIZE;
import static org.apache.inlong.audit.consts.OpenApiConstants.KEY_HTTP_BODY_ERR_DATA;
import static org.apache.inlong.audit.consts.OpenApiConstants.KEY_HTTP_BODY_ERR_MSG;
import static org.apache.inlong.audit.consts.OpenApiConstants.KEY_HTTP_BODY_SUCCESS;
import static org.apache.inlong.audit.consts.OpenApiConstants.KEY_HTTP_HEADER_CONTENT_TYPE;
import static org.apache.inlong.audit.consts.OpenApiConstants.KEY_HTTP_SERVER_BIND_PORT;
import static org.apache.inlong.audit.consts.OpenApiConstants.PARAMS_AUDIT_COMPONENT;
import static org.apache.inlong.audit.consts.OpenApiConstants.PARAMS_AUDIT_CYCLE;
import static org.apache.inlong.audit.consts.OpenApiConstants.PARAMS_AUDIT_ID;
import static org.apache.inlong.audit.consts.OpenApiConstants.PARAMS_AUDIT_TAG;
import static org.apache.inlong.audit.consts.OpenApiConstants.PARAMS_END_TIME;
import static org.apache.inlong.audit.consts.OpenApiConstants.PARAMS_INLONG_GROUP_ID;
import static org.apache.inlong.audit.consts.OpenApiConstants.PARAMS_INLONG_STREAM_ID;
import static org.apache.inlong.audit.consts.OpenApiConstants.PARAMS_IP;
import static org.apache.inlong.audit.consts.OpenApiConstants.PARAMS_START_TIME;
import static org.apache.inlong.audit.consts.OpenApiConstants.VALUE_HTTP_HEADER_CONTENT_TYPE;
import static org.apache.inlong.audit.entities.ApiType.DAY;
import static org.apache.inlong.audit.entities.ApiType.GET_AUDIT_PROXY;
import static org.apache.inlong.audit.entities.ApiType.GET_IDS;
Expand Down Expand Up @@ -208,14 +208,14 @@ private boolean checkNecessaryParams(Map<String, String> params) {
return params.containsKey(PARAMS_START_TIME)
&& params.containsKey(PARAMS_END_TIME)
&& params.containsKey(PARAMS_AUDIT_ID)
&& params.containsKey(PARAMS_INLONG_GROUP_Id)
&& params.containsKey(PARAMS_INLONG_STREAM_Id);
&& params.containsKey(PARAMS_INLONG_GROUP_ID)
&& params.containsKey(PARAMS_INLONG_STREAM_ID);
case MINUTES:
return params.containsKey(PARAMS_START_TIME)
&& params.containsKey(PARAMS_END_TIME)
&& params.containsKey(PARAMS_AUDIT_ID)
&& params.containsKey(PARAMS_INLONG_GROUP_Id)
&& params.containsKey(PARAMS_INLONG_STREAM_Id)
&& params.containsKey(PARAMS_INLONG_GROUP_ID)
&& params.containsKey(PARAMS_INLONG_STREAM_ID)
&& params.containsKey(PARAMS_AUDIT_CYCLE);
case GET_IDS:
return params.containsKey(PARAMS_START_TIME)
Expand Down Expand Up @@ -249,8 +249,8 @@ private void handleLegalParams(JsonObject responseJson, Map<String, String> para
case HOUR:
statData = HourCache.getInstance().getData(params.get(PARAMS_START_TIME),
params.get(PARAMS_END_TIME),
params.get(PARAMS_INLONG_GROUP_Id),
params.get(PARAMS_INLONG_STREAM_Id),
params.get(PARAMS_INLONG_GROUP_ID),
params.get(PARAMS_INLONG_STREAM_ID),
params.get(PARAMS_AUDIT_ID),
params.get(PARAMS_AUDIT_TAG));
responseJson.add(KEY_HTTP_BODY_ERR_DATA, gson.toJsonTree(statData));
Expand All @@ -259,8 +259,8 @@ private void handleLegalParams(JsonObject responseJson, Map<String, String> para
statData = DayCache.getInstance().getData(
params.get(PARAMS_START_TIME),
params.get(PARAMS_END_TIME),
params.get(PARAMS_INLONG_GROUP_Id),
params.get(PARAMS_INLONG_STREAM_Id),
params.get(PARAMS_INLONG_GROUP_ID),
params.get(PARAMS_INLONG_STREAM_ID),
params.get(PARAMS_AUDIT_ID));
responseJson.add(KEY_HTTP_BODY_ERR_DATA, gson.toJsonTree(statData));
break;
Expand All @@ -276,8 +276,8 @@ private void handleLegalParams(JsonObject responseJson, Map<String, String> para
statData = RealTimeQuery.getInstance().queryIpsById(
params.get(PARAMS_START_TIME),
params.get(PARAMS_END_TIME),
params.get(PARAMS_INLONG_GROUP_Id),
params.get(PARAMS_INLONG_STREAM_Id),
params.get(PARAMS_INLONG_GROUP_ID),
params.get(PARAMS_INLONG_STREAM_ID),
params.get(PARAMS_AUDIT_ID));
responseJson.add(KEY_HTTP_BODY_ERR_DATA, gson.toJsonTree(statData));
break;
Expand All @@ -303,22 +303,22 @@ private List<StatData> handleMinutesApi(Map<String, String> params) {
case MINUTE:
statData = RealTimeQuery.getInstance().queryLogTs(params.get(PARAMS_START_TIME),
params.get(PARAMS_END_TIME),
params.get(PARAMS_INLONG_GROUP_Id),
params.get(PARAMS_INLONG_STREAM_Id),
params.get(PARAMS_INLONG_GROUP_ID),
params.get(PARAMS_INLONG_STREAM_ID),
params.get(PARAMS_AUDIT_ID));
break;
case MINUTE_10:
statData = TenMinutesCache.getInstance().getData(params.get(PARAMS_START_TIME),
params.get(PARAMS_END_TIME),
params.get(PARAMS_INLONG_GROUP_Id),
params.get(PARAMS_INLONG_STREAM_Id), params.get(PARAMS_AUDIT_ID),
params.get(PARAMS_INLONG_GROUP_ID),
params.get(PARAMS_INLONG_STREAM_ID), params.get(PARAMS_AUDIT_ID),
params.get(PARAMS_AUDIT_TAG));
break;
case MINUTE_30:
statData = HalfHourCache.getInstance().getData(params.get(PARAMS_START_TIME),
params.get(PARAMS_END_TIME),
params.get(PARAMS_INLONG_GROUP_Id),
params.get(PARAMS_INLONG_STREAM_Id), params.get(PARAMS_AUDIT_ID),
params.get(PARAMS_INLONG_GROUP_ID),
params.get(PARAMS_INLONG_STREAM_ID), params.get(PARAMS_AUDIT_ID),
params.get(PARAMS_AUDIT_TAG));
break;
default:
Expand Down
1 change: 1 addition & 0 deletions inlong-dashboard/src/ui/locales/en.json
Original file line number Diff line number Diff line change
Expand Up @@ -838,6 +838,7 @@
"pages.Clusters.Node.Status": "Status",
"pages.Clusters.Node.Status.Normal": "Normal",
"pages.Clusters.Node.Status.Timeout": "Timeout",
"pages.Clusters.Node.LastModifier": "Last modifier",
"pages.Clusters.Node.Creator": "Creator",
"pages.Clusters.Node.Create": "Create",
"pages.Clusters.Node.IpRule": "Please enter the IP address correctly",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -69,9 +69,9 @@ const Comp: React.FC<Props> = ({ inlongGroupId, inlongStreamId, ...modalProps })
exitsId = true;
}
const width =
(cur['fieldName'].length > cur['fieldValue'].length
? cur['fieldName'].length
: cur['fieldValue'].length) * 10;
(cur['fieldName']?.length > cur['fieldValue']?.length
? cur['fieldName']?.length
: cur['fieldValue']?.length) * 10;
acc.push({
title: cur['fieldName'],
key: cur['fieldName'],
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -350,13 +350,12 @@ export const getTableColumns = (source, dim) => {
title: item.auditName,
dataIndex: item.auditId,
render: text => {
let color = 'black';
if (text?.includes('+')) {
color = 'red';
return <span style={{ color: 'red' }}>{text}</span>;
} else if (text?.includes('-')) {
color = 'green';
return <span style={{ color: 'green' }}>{text}</span>;
}
return <span style={{ color: color }}>{text}</span>;
return <span>{text}</span>;
},
}));
return [
Expand Down
Loading

0 comments on commit 13f38dd

Please sign in to comment.